class SelectEventString(object): def __init__(self): self.dbhost = rj.get_db_host() self.dbuser = rj.get_db_user() self.dbpwd = rj.get_db_pwd() self.dbport = rj.get_db_port() self.dbname = rj.get_db_name() self.db = MySQLDatabase(self.dbuser, self.dbpwd, self.dbname, self.dbhost, self.dbport) self.Session = self.db.mysql_session() def _check_event_string(self): '''Check the event string matches roughly YYYYMMDDTHHMMSS format''' try: event_folders = glob(rsync_path + '/*') event_strings = [os.path.basename(i)[0:15] for i in event_folders] output_list = [] for i in event_strings: if i[0].isdigit() and i[8] == 'T': output_list.append(i) else: pass return output_list except Exception as e: msg = "String event selection excep - CheckEventString._check_event_string --" log.error("{0}{1}".format(msg,e)) def _filter_event(self, event_string): '''Check if an event string is found in database''' try: rows = Queries(self.Session, DataFile, event_string).match_event() if not rows: return event_string except Exception as e: msg = "Query on database excep - SelectEventString._filter_event --" log.error("{0}{1}".format(msg,e)) def get_selected_events_list(self): '''Create list of event strings with YYYYMMDDTHHMMSS format and not found in database''' try: checked_list = self._check_event_string() selected_list = [] for i in checked_list: if self._filter_event(i) is not None: selected_list.append(self._filter_event(i)) return selected_list except Exception as e: msg = "Creation of events already in db excep - SelectEventString.get_selected_events_list --" log.error("{0}{1}".format(msg,e))
def main(): dbuser = rj.get_db_user(); dbpwd = rj.get_db_pwd() dbhost = rj.get_db_host(); dbname = rj.get_db_name() dbport = rj.get_db_port(); dbtables = rj.get_db_tables() sender = rj.get_sender(); smtphost = rj.get_smtp_host() recipient = rj.get_recipient() db = MySQLDatabase(dbuser, dbpwd, dbname, dbhost, dbport) Session = db.mysql_session() files_list = StorageFilesList().create_selected_list() for file_path in files_list: cksgz_stg = md5Checksum(file_path).calculate_checksum() cks_stg = md5Checksum(file_path).get_checksum_gz() head, fname_gz = os.path.split(file_path) for tbl in dbtables: db_element = db_query(tbl, Session, fname_gz) if db_element is not None: cks_db = db_element[1] cksgz_db = db_element[2] if cks_db is not None and cksgz_db is not None: if cksgz_stg == cksgz_db and cks_stg == cks_db: try: print(file_path) #os.remove(file_path) except Exception as e: msg = "File removal exception --" log.error("{0}{1}".format(msg,e)) else: message = 'Severe alert - storage and DB file checksums DO NOT MATCH' SendEmail(message,recipient,smtphost,sender).send_email() else: message = 'Severe alert - checksum not calculated' SendEmail(message,recipient,smtphost,sender).send_email() else: pass db.close_session()
from multiprocessing import Pool from utilities import LoggingClass from database import MySQLDatabase log = LoggingClass('', True).get_logger() rj = ReadJson() dbuser = rj.get_db_user() dbpwd = rj.get_db_pwd() dbname = rj.get_db_name() dbhost = rj.get_db_host() dbport = rj.get_db_port() thr_nr = rj.get_threads_number() db = MySQLDatabase(dbuser, dbpwd, dbname, dbhost, dbport) Session = db.mysql_session() def _query_filename(filename): try: rows = Queries(Session, DataFile, filename).match_filename() if not rows: return filename except Exception as e: msg = "Query on database excep - _query_filename --" log.error("{0}{1}".format(msg, e)) finally: db.close_session() pool = multiprocessing.Pool(multiprocessing.cpu_count())