class SelectEventString(object):
def __init__(self):
self.dbhost = rj.get_db_host()
self.dbuser = rj.get_db_user()
self.dbpwd = rj.get_db_pwd()
self.dbport = rj.get_db_port()
self.dbname = rj.get_db_name()
self.db = MySQLDatabase(self.dbuser, self.dbpwd, self.dbname, self.dbhost, self.dbport)
self.Session = self.db.mysql_session()
def _check_event_string(self):
'''Check the event string matches roughly YYYYMMDDTHHMMSS format'''
try:
event_folders = glob(rsync_path + '/*')
event_strings = [os.path.basename(i)[0:15] for i in event_folders]
output_list = []
for i in event_strings:
if i[0].isdigit() and i[8] == 'T':
output_list.append(i)
else:
pass
return output_list
except Exception as e:
msg = "String event selection excep - CheckEventString._check_event_string --"
log.error("{0}{1}".format(msg,e))
def _filter_event(self, event_string):
'''Check if an event string is found in database'''
try:
rows = Queries(self.Session, DataFile, event_string).match_event()
if not rows:
return event_string
except Exception as e:
msg = "Query on database excep - SelectEventString._filter_event --"
log.error("{0}{1}".format(msg,e))
def get_selected_events_list(self):
'''Create list of event strings with YYYYMMDDTHHMMSS format and not found in database'''
try:
checked_list = self._check_event_string()
selected_list = []
for i in checked_list:
if self._filter_event(i) is not None:
selected_list.append(self._filter_event(i))
return selected_list
except Exception as e:
msg = "Creation of events already in db excep - SelectEventString.get_selected_events_list --"
log.error("{0}{1}".format(msg,e))
def main():
dbuser = rj.get_db_user(); dbpwd = rj.get_db_pwd()
dbhost = rj.get_db_host(); dbname = rj.get_db_name()
dbport = rj.get_db_port(); dbtables = rj.get_db_tables()
sender = rj.get_sender(); smtphost = rj.get_smtp_host()
recipient = rj.get_recipient()
db = MySQLDatabase(dbuser, dbpwd, dbname, dbhost, dbport)
Session = db.mysql_session()
files_list = StorageFilesList().create_selected_list()
for file_path in files_list:
cksgz_stg = md5Checksum(file_path).calculate_checksum()
cks_stg = md5Checksum(file_path).get_checksum_gz()
head, fname_gz = os.path.split(file_path)
for tbl in dbtables:
db_element = db_query(tbl, Session, fname_gz)
if db_element is not None:
cks_db = db_element[1]
cksgz_db = db_element[2]
if cks_db is not None and cksgz_db is not None:
if cksgz_stg == cksgz_db and cks_stg == cks_db:
try:
print(file_path)
#os.remove(file_path)
except Exception as e:
msg = "File removal exception --"
log.error("{0}{1}".format(msg,e))
else:
message = 'Severe alert - storage and DB file checksums DO NOT MATCH'
SendEmail(message,recipient,smtphost,sender).send_email()
else:
message = 'Severe alert - checksum not calculated'
SendEmail(message,recipient,smtphost,sender).send_email()
else:
pass
db.close_session()
from multiprocessing import Pool
from utilities import LoggingClass
from database import MySQLDatabase
log = LoggingClass('', True).get_logger()
rj = ReadJson()
dbuser = rj.get_db_user()
dbpwd = rj.get_db_pwd()
dbname = rj.get_db_name()
dbhost = rj.get_db_host()
dbport = rj.get_db_port()
thr_nr = rj.get_threads_number()
db = MySQLDatabase(dbuser, dbpwd, dbname, dbhost, dbport)
Session = db.mysql_session()
def _query_filename(filename):
try:
rows = Queries(Session, DataFile, filename).match_filename()
if not rows:
return filename
except Exception as e:
msg = "Query on database excep - _query_filename --"
log.error("{0}{1}".format(msg, e))
finally:
db.close_session()
pool = multiprocessing.Pool(multiprocessing.cpu_count())