def new_alerts_present(username): """Check for new alerts based on the last access time for the specified user.""" ret = False try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from admin_alerts where user = "******"' % username entry, err = db.get_single_row(db_path, query) if err: raise Exception(err) if not entry: ret = True # print entry query = 'select count(*) as count from alerts where last_update_time > "%d"' % entry[ 'last_refresh_time'] # print query entry, err = db.get_single_row(db_path, query) # print entry, err if err: raise Exception(err) if entry and int(entry['count']) > 0: ret = True except Exception, e: return False, 'Error checking for new alerts : %s' % str(e)
def new_alerts_present(username): """Check for new alerts based on the last access time for the specified user.""" ret = False try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from admin_alerts where user = "******"' % username entry, err = db.get_single_row(db_path, query) if err: raise Exception(err) if not entry: ret = True # print entry query = 'select count(*) as count from alerts where last_update_time > "%d"' % entry[ 'last_refresh_time'] # print query entry, err = db.get_single_row(db_path, query) # print entry, err if err: raise Exception(err) if entry and int(entry['count']) > 0: ret = True except Exception, e: return False, 'Error checking for new alerts : %s' % str(e)
def get_org_info(): """Return organization's information args: None returns: dict with keys- 'org_name', 'unit_name', 'unit_id', 'subunit_name', 'subunit_id' """ ret_dict = {} try: db_path, err = config.get_db_path() if err: raise Exception(err) cmd = "select * from org_info" ret, err = db.get_single_row(db_path, cmd) if err: raise Exception(err) if ret: # return the dict which has all the fields ret_dict = ret except Exception, e: return None, 'Could not fetch details of the organisation: %s' % e
def get_event_notification_trigger(ent_id): """Get the trigger entry corresponding to the passed trigger id """ return_dict = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from event_notification_triggers where ent_id=%d' % int( ent_id) return_dict, err = db.get_single_row(db_path, query) if err: raise Exception(err) if return_dict: cron_list, err = scheduler_utils.get_cron_tasks( return_dict['cron_task_id']) if err: raise Exception(err) if cron_list: return_dict['schedule_description'] = cron_list[0][ 'schedule_description'] return_dict['description'] = cron_list[0]['description'] except Exception, e: return None, 'Error retrieving event notification trigger : %s' % str( e)
def get_log_level(): d = None conn = None log_level = None try: d, err = db.get_single_row(db_path, "select * from global_params where id=1") if err: raise Exception(err) if d and "logging_level" in d: log_level = d["logging_level"] else: # Not yet set so insert the default and return it cmd_list = [] cmd = [ "insert into global_params (logging_level, id) values(?,?)", ( logging.INFO, 1, ) ] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) log_level = logging.INFO except Exception, e: return None, "Error getting log level : %s" % str(e)
def generate_alert_email_body(alert_id): """Given an alert id, generate the appropriate email message body for that alert """ msg = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from alerts where alert_id = "%s"' % alert_id entry, err = db.get_single_row(db_path, query) # print entry if err: raise Exception(err) fat, err = datetime_utils.convert_from_epoch( entry['first_alert_time'], return_format='str', str_format='%c', to='local') if err: raise Exception(err) lut, err = datetime_utils.convert_from_epoch( entry['last_update_time'], return_format='str', str_format='%c', to='local') if err: raise Exception(err) msg = 'Alert time: %s\nAlert message: %s.' % (lut, entry['alert_str']) if entry['repeat_count'] > 1: msg += ' This alert has been generated %d times since %s.' % ( entry['repeat_count'], fat) except Exception, e: return None, 'Error generating alert email message body : %s' % str(e)
def get_scan_configurations(db_location = None, scan_configuration_id = None, standalone = False, include_deleted=False): configurations = None try: if not db_location: db_location, err = get_db_location(standalone) if err: raise Exception(err) if scan_configuration_id: query = 'select * from scan_configurations where id="%d"'%scan_configuration_id else: query = 'select * from scan_configurations' if not include_deleted: if scan_configuration_id: query += ' and status_id != -1' else: query += ' where status_id != -1' #print query configurations, err = db.get_multiple_rows(db_location, query) if err: raise Exception(err) for c in configurations: query = 'select count(*) as num_files, sum(size) as total_size, count(distinct(extension)) as num_extensions from file_info where scan_configuration_id="%d"'%c['id'] row, err = db.get_single_row(db_location, query) if err: raise Exception(err) row['total_size_human_readable'] = '0' if row: if not row['total_size']: row['total_size'] = 0 row['total_size_human_readable'] = filesize.get_naturalsize(row['total_size']) c.update(row) except Exception, e: return None, 'Error loading Storage Insights configurations : %s'%str(e)
def save_email_settings(d): """Save the email server settings """ conn = None try: db_path, err = config.get_db_path() if err: raise Exception(err) d1, err = db.get_single_row(db_path, "select * from email_config") if err: raise Exception(err) if d1: # Config exists so update ret, err = db.execute_iud(db_path, [["update email_config set server=?, port=?, username=?, pswd=?, tls=? where id = ?", ( d["server"], d["port"], d["username"], d["pswd"], d["tls"], 1,)]]) if err: raise Exception(err) else: # No config exists so insert ret, err = db.execute_iud(db_path, [["insert into email_config (server, port, username, pswd, tls, id) values (?,?,?,?,?,?)", ( d["server"], d["port"], d["username"], d["pswd"], d["tls"], 1, )]]) if err: raise Exception(err) except Exception, e: return False, 'Error saving email settings : %s' % str(e)
def set_log_level(level): try: db_path, err = config.get_db_path() if err: raise Exception(err) if level not in [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]: logger.setLevel(logging.INFO) else: d1, err = db.get_single_row( db_path, "select * from global_params") if err: raise Exception(err) cmd_list = [] if d1: cmd = [ "update global_params set logging_level=? where id = ?", (level, 1,)] else: cmd = [ "insert into global_params (logging_level, id) values(?,?)", (level, 1,)] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) logger.setLevel(level) except Exception, e: return False, 'Error setting log level : %s' % str(e)
def get_file_info(param, param_type='id', db_location = None, standalone = False): result = None try: if not param: raise Exception('Unspecified param') if param_type not in ['id', 'path']: raise Exception('Invalid param type') if param_type == 'id': query = 'select * from file_info where id="%s"'%param elif param_type == 'path': query = 'select * from file_info where path="%s"'%param if not db_location: db_location, err = scan_utils.get_db_location(standalone) if err: raise Exception(err) result, err = db.get_single_row(db_location, query) if err: raise Exception(err) if 'last_modify_time' in result: tm_str, err = datetime_utils.convert_from_epoch(result['last_modify_time'], return_format='str', str_format='%c', to='local') if err: raise Exception(err) result['last_modify_time_str'] = tm_str if 'size' in result: result['size_human_readable'] = filesize.get_naturalsize(result['size']) except Exception, e: return None, 'Error retrieving duplicate file sets : %s'%str(e)
def generate_alert_email_body(alert_id): """Given an alert id, generate the appropriate email message body for that alert """ msg = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from alerts where alert_id = "%s"' % alert_id entry, err = db.get_single_row(db_path, query) # print entry if err: raise Exception(err) fat, err = datetime_utils.convert_from_epoch(entry['first_alert_time'], return_format='str', str_format='%c', to='local') if err: raise Exception(err) lut, err = datetime_utils.convert_from_epoch(entry['last_update_time'], return_format='str', str_format='%c', to='local') if err: raise Exception(err) msg = 'Alert time: %s\nAlert message: %s.' % (lut, entry['alert_str']) if entry['repeat_count'] > 1: msg += ' This alert has been generated %d times since %s.' % ( entry['repeat_count'], fat) except Exception, e: return None, 'Error generating alert email message body : %s' % str(e)
def _get_repeat_entry(alert_dict, past_x_seconds=900): """Check the db to see if the passed alert is a repeat of an existing alert entry in the past x seconds """ result = None try: db_path, err = config.get_db_path() if err: raise Exception(err) now, err = datetime_utils.get_epoch(when='now', num_previous_days=0) if alert_dict['component']: query = "select * from alerts where severity_type_id = '%d' and subsystem_type_id = '%d' and component = '%s' and alert_str=\"%s\" and first_alert_time >= %d;" % ( alert_dict['severity_type_id'], alert_dict['subsystem_type_id'], alert_dict['component'], alert_dict['alert_str'], now - past_x_seconds) else: query = "select * from alerts where severity_type_id = '%d' and subsystem_type_id = '%d' and alert_str=\"%s\" and first_alert_time >= %d);" % ( alert_dict['severity_type_id'], alert_dict['subsystem_type_id'], alert_dict['alert_str'], now - past_x_seconds) # print query result, err = db.get_single_row(db_path, query) if err: raise Exception(err) # print result, err except Exception, e: return None, 'Error checking for repeats : %s' % str(e)
def get_file_info_row(db_path, path): row = False try: query = 'select * from file_info where path="%s"'%path row, err = db.get_single_row(db_path, query) if err: raise Exception(err) except Exception, e: return None, 'Error retrieving file info row : %s'%str(e)
def get_file_info_row(db_path, path): row = False try: query = 'select * from file_info where path="%s"' % path row, err = db.get_single_row(db_path, query) if err: raise Exception(err) except Exception, e: return None, 'Error retrieving file info row : %s' % str(e)
def get_auth_settings(): """Get the current authentication settings from the db.""" d = None try: db_path, err = config.get_db_path() if err: raise Exception(err) d, err = db.get_single_row( db_path, "select * from samba_global_common where id=1") if err: raise Exception(err) if d and 'security' in d and d['security'] == "ads": d1, err = db.get_single_row( db_path, "select * from samba_global_ad where id=1") if err: raise Exception(err) if d1: d.update(d1) except Exception, e: return None, 'Error loading authentication settings : %s' % str(e)
def get_auth_settings(): """Get the current authentication settings from the db.""" d = None try: db_path, err = config.get_db_path() if err: raise Exception(err) d, err = db.get_single_row( db_path, "select * from samba_global_common where id=1") if err: raise Exception(err) if d and 'security' in d and d['security'] == "ads": d1, err = db.get_single_row( db_path, "select * from samba_global_ad where id=1") if err: raise Exception(err) if d1: d.update(d1) except Exception, e: return None, 'Error loading authentication settings : %s' % str(e)
def get_rsync_share_details(name): share = None try: db_path, err = config.get_db_path() if err: raise Exception(err) share, err = db.get_single_row( db_path, "select * from rsync_shares where name='%s'" % name) if not share: raise Exception("Specified share not found ") except Exception, e: return False, 'Error deleting the share: %s' % str(e)
def get_event_notification_configuration(enc_id): """Get a particular email configuration.. """ ret = None try: db_path, err = config.get_db_path() if err: raise Exception(err) ret, err = db.get_single_row( db_path, "select * from event_notification_configuration_email where enc_id = %d" % enc_id) if err: raise Exception(err) except Exception, e: return None, 'Error retrieving email notification configuration : %s' % str(e)
def load_email_settings(): """Load the email server settings """ conn = None d = None try: db_path, err = config.get_db_path() if err: raise Exception(err) d, err = db.get_single_row( db_path, "select * from email_config where id = 1") if err: raise Exception(err) except Exception, e: return None, 'Error loading email settings : %s' % str(e)
def get_task(task_id): """Get a particular entry with the passed task_id from the tasks table.""" task = None try: db_path, err = config.get_db_path() if err: raise Exception(err) cmd = "select * from tasks where task_id=='%d'" % int(task_id) task, err = db.get_single_row(db_path, cmd) if err: raise Exception(err) if not task: raise Exception('Selected task not found') except Exception, e: return None, 'Error retrieving task details : %s' % e
def get_task(task_id): """Get a particular entry with the passed task_id from the tasks table.""" task = None try: db_path, err = config.get_db_path() if err: raise Exception(err) cmd = "select * from tasks where task_id=='%d'" % int(task_id) task, err = db.get_single_row(db_path, cmd) if err: raise Exception(err) if not task: raise Exception('Selected task not found') except Exception, e: return None, 'Error retrieving task details : %s' % e
def get_share_info(mode, index): """Get the info for a share either based on name or by db id""" d = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = None if mode == "by_id": query = "select * from samba_shares where share_id = %s" % index else: query = "select * from samba_shares where name = %s" % index d, err = db.get_single_row(db_path, query) if err: raise Exception(err) except Exception, e: return None, 'Error loading CIFS share information : %s' % str(e)
def get_share_info(mode, index): """Get the info for a share either based on name or by db id""" d = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = None if mode == "by_id": query = "select * from samba_shares where share_id = %s" % index else: query = "select * from samba_shares where name = %s" % index d, err = db.get_single_row(db_path, query) if err: raise Exception(err) except Exception, e: return None, 'Error loading CIFS share information : %s' % str(e)
def get_db_details(standalone = False): db_details = None try: db_details = {} db_location, err = get_db_location(standalone) if err: raise Exception(err) size = os.path.getsize(db_location) db_details['size'] = size db_details['size_str'] = filesize.get_naturalsize(size) query = 'select count(*) as count from file_info' row, err = db.get_single_row(db_location, query) if err: raise Exception(err) db_details['num_files'] = row['count'] except Exception, e: return None, 'Error loading Storage Insights database details : %s'%str(e)
def get_db_details(standalone=False): db_details = None try: db_details = {} db_location, err = get_db_location(standalone) if err: raise Exception(err) size = os.path.getsize(db_location) db_details['size'] = size db_details['size_str'] = filesize.get_naturalsize(size) query = 'select count(*) as count from file_info' row, err = db.get_single_row(db_location, query) if err: raise Exception(err) db_details['num_files'] = row['count'] except Exception, e: return None, 'Error loading Storage Insights database details : %s' % str( e)
def delete_rsync_share(name): try: db_path, err = config.get_db_path() if err: raise Exception(err) cmd_list = [] check, err = db.get_single_row( db_path, "select * from rsync_shares where name='%s'" % name) if not check: raise Exception("Specified share not found ") cmd = ["delete from rsync_shares where name='%s'" % name] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) conf, err = _generate_rsync_config() if err: raise Exception(err) except Exception, e: return False, 'Error deleting the share: %s' % str(e)
def get_count(action='READ', past_x_seconds=60): count = -1 try: config_dir, err = config.get_config_dir() if err: raise Exception(err) if action != 'ALL': query = "select count(*) as count from logs where timestamp >= Datetime('now', '-%d seconds');" % past_x_seconds else: query = "select count(*) as count from logs where actions = '%s' and timestamp >= Datetime('now', '-%d seconds');" % ( action.upper(), past_x_seconds) # print query db_path = '%s/db/inotify.db' % config_dir ret, err = db.get_single_row(db_path, query) if err: print err count = ret['count'] # print ret except Exception, e: return -1, 'Error getting counts : %s' % str(e)
def get_count(action='READ', past_x_seconds=60): count = -1 try: config_dir, err = config.get_config_dir() if err: raise Exception(err) if action != 'ALL': query = "select count(*) as count from logs where timestamp >= Datetime('now', '-%d seconds');" % past_x_seconds else: query = "select count(*) as count from logs where actions = '%s' and timestamp >= Datetime('now', '-%d seconds');" % ( action.upper(), past_x_seconds) # print query db_path = '%s/db/inotify.db' % config_dir ret, err = db.get_single_row(db_path, query) if err: print err count = ret['count'] # print ret except Exception, e: return -1, 'Error getting counts : %s' % str(e)
def get_scan_configurations(db_location=None, scan_configuration_id=None, standalone=False, include_deleted=False): configurations = None try: if not db_location: db_location, err = get_db_location(standalone) if err: raise Exception(err) if scan_configuration_id: query = 'select * from scan_configurations where id="%d"' % scan_configuration_id else: query = 'select * from scan_configurations' if not include_deleted: if scan_configuration_id: query += ' and status_id != -1' else: query += ' where status_id != -1' #print query configurations, err = db.get_multiple_rows(db_location, query) if err: raise Exception(err) for c in configurations: query = 'select count(*) as num_files, sum(size) as total_size, count(distinct(extension)) as num_extensions from file_info where scan_configuration_id="%d"' % c[ 'id'] row, err = db.get_single_row(db_location, query) if err: raise Exception(err) row['total_size_human_readable'] = '0' if row: if not row['total_size']: row['total_size'] = 0 row['total_size_human_readable'] = filesize.get_naturalsize( row['total_size']) c.update(row) except Exception, e: return None, 'Error loading Storage Insights configurations : %s' % str( e)
def get_integralstor_uuid(): """Return Integralstor's UUID fields args: None returns: dict with keys 'uuid_hex' & 'uuid_str' """ id_dict = None try: db_path, err = config.get_db_path() if err: raise Exception(err) cmd = "select * from integralstor_info" ret, err = db.get_single_row(db_path, cmd) if err: raise Exception(err) if ret: # return the dict which has all the fields id_dict = ret except Exception, e: return None, 'Could not fetch UUID: %s' % e
def set_log_level(level): try: db_path, err = config.get_db_path() if err: raise Exception(err) if level not in [ logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL ]: logger.setLevel(logging.INFO) else: d1, err = db.get_single_row(db_path, "select * from global_params") if err: raise Exception(err) cmd_list = [] if d1: cmd = [ "update global_params set logging_level=? where id = ?", ( level, 1, ) ] else: cmd = [ "insert into global_params (logging_level, id) values(?,?)", ( level, 1, ) ] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) logger.setLevel(level) except Exception, e: return False, 'Error setting log level : %s' % str(e)
def _get_repeat_entry(alert_dict, past_x_seconds=900): """Check the db to see if the passed alert is a repeat of an existing alert entry in the past x seconds """ result = None try: db_path, err = config.get_db_path() if err: raise Exception(err) now, err = datetime_utils.get_epoch(when='now', num_previous_days=0) if alert_dict['component']: query = "select * from alerts where severity_type_id = '%d' and subsystem_type_id = '%d' and component = '%s' and alert_str=\"%s\" and first_alert_time >= %d;" % ( alert_dict['severity_type_id'], alert_dict['subsystem_type_id'], alert_dict['component'], alert_dict['alert_str'], now - past_x_seconds) else: query = "select * from alerts where severity_type_id = '%d' and subsystem_type_id = '%d' and alert_str=\"%s\" and first_alert_time >= %d);" % ( alert_dict['severity_type_id'], alert_dict['subsystem_type_id'], alert_dict['alert_str'], now - past_x_seconds) # print query result, err = db.get_single_row(db_path, query) if err: raise Exception(err) # print result, err except Exception, e: return None, 'Error checking for repeats : %s' % str(e)
def create_rsync_share(name, path, comment, list, readonly, uid, gid): try: db_path, err = config.get_db_path() if err: raise Exception(err) check, err = db.get_single_row( db_path, "select * from rsync_shares where name='%s'" % name) if check: raise Exception("Share already exists.Use a different share name") cmd_list = [] cmd = ["insert into rsync_shares (name,path,comment,list,readonly,uid,gid) values(?,?,?,?,?,?,?)", ( name, path, comment, list, readonly, uid, gid)] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) conf, err = _generate_rsync_config() if err: raise Exception(err) except Exception, e: return False, 'Error saving rsync config settings : %s' % str(e)
def get_log_level(): d = None conn = None log_level = None try: d, err = db.get_single_row( db_path, "select * from global_params where id=1") if err: raise Exception(err) if d and "logging_level" in d: log_level = d["logging_level"] else: # Not yet set so insert the default and return it cmd_list = [] cmd = [ "insert into global_params (logging_level, id) values(?,?)", (logging.INFO, 1,)] cmd_list.append(cmd) ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) log_level = logging.INFO except Exception, e: return None, "Error getting log level : %s" % str(e)
def update_auth_settings(d): """Update the authentication settings in the db to what has been passed in the dict""" try: db_path, err = config.get_db_path() if err: raise Exception(err) auth_settings, err = get_auth_settings() if err: raise Exception(err) if not auth_settings: cmd = ["insert into samba_global_common (id, workgroup, netbios_name, security, include_homes_section) values (?, ?, ?, ?, ?)", (1, d["workgroup"], d["netbios_name"], d["security"], True,)] else: cmd = ["update samba_global_common set workgroup=?, netbios_name=?, security=?, include_homes_section=? where id = ?", (d["workgroup"], d["netbios_name"], d["security"], True, 1,)] cmd_list = [] cmd_list.append(cmd) if d["security"] == "ads": d1, err = db.get_single_row( db_path, "select * from samba_global_ad") if err: raise Exception(err) if d1: cmd = ["update samba_global_ad set realm=?, password_server=?, ad_schema_mode=?, id_map_min=?, id_map_max=?, password_server_ip=? where id = ?", (d["realm"], d["password_server"], 'rfc2307', 16777216, 33554431, d["password_server_ip"], 1, )] cmd_list.append(cmd) else: cmd = ["insert into samba_global_ad (realm, password_server, ad_schema_mode, id_map_min, id_map_max, password_server_ip, id) values(?,?,?,?,?,?,?)", ( d["realm"], d["password_server"], 'rfc2307', 16777216, 33554431, d["password_server_ip"], 1,)] cmd_list.append(cmd) # print cmd_list ret, err = db.execute_iud(db_path, cmd_list) if err: raise Exception(err) except Exception, e: return False, 'Error saving authentication settings : %s' % str(e)
def get_event_notification_trigger(ent_id): """Get the trigger entry corresponding to the passed trigger id """ return_dict = None try: db_path, err = config.get_db_path() if err: raise Exception(err) query = 'select * from event_notification_triggers where ent_id=%d' % int( ent_id) return_dict, err = db.get_single_row(db_path, query) if err: raise Exception(err) if return_dict: cron_list, err = scheduler_utils.get_cron_tasks( return_dict['cron_task_id']) if err: raise Exception(err) if cron_list: return_dict['schedule_description'] = cron_list[0]['schedule_description'] return_dict['description'] = cron_list[0]['description'] except Exception, e: return None, 'Error retrieving event notification trigger : %s' % str(e)
def export_old_alerts(older_than_days=1): """Move all alerts older than the older_than_days into a file in /var/log/integralstor/logs/exported dir """ try: cutoff_seconds, err = datetime_utils.get_epoch( when='now', num_previous_days=older_than_days) if err: raise Exception(err) #query = "select * from alerts where last_update_time < Datetime('now', '-%d days') order by alert_id;"%older_than_days query = "select * from alerts where last_update_time < %d order by alert_id;" % cutoff_seconds full_alerts_list, err = _get_and_parse_alerts(query) if err: raise Exception(err) alerts_list = [] # print 'full', full_alerts_list db_path, err = config.get_db_path() if err: raise Exception(err) for a in full_alerts_list: # If it is still in the holding table then skip it so it can get # processed.. query = "select * from event_notifications_holding where event_id=%d;" % int( a['alert_id']) ret, err = db.get_single_row(db_path, query) # print ret, err if err: raise Exception(err) if not ret: alerts_list.append(a) # print 'to export', alerts_list if alerts_list: delete_commands = [] export_dir_name, err = config.get_exported_logs_dir_path() if err: raise Exception(err) if not os.path.exists(export_dir_name): os.makedirs(export_dir_name) now, err = datetime_utils.get_epoch(when='now', num_previous_days=0) if err: raise Exception(err) now_str, err = datetime_utils.convert_from_epoch( now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') export_filename = 'alerts_%s' % now_str # print export_filename with open('%s/%s' % (export_dir_name, export_filename), 'w') as f: f.write( 'First alert time(UTC) | Last update time(UTC) | Repeat count | Subsystem | Severity | Alert message\n' ) f.write( '-------------------------------------------------------------------------------------------\n' ) for al in alerts_list: f.write('%s | %s | %d | %s | %s | %s\n\n' % (al['first_alert_time'], al['last_update_time'], al['repeat_count'], al['subsystem'], al['severity'], al['alert_str'])) delete_commands.append([ 'delete from alerts where alert_id="%d"' % int(al['alert_id']) ]) # print delete_commands db_path, err = config.get_db_path() if err: raise Exception(err) ret, err = db.execute_iud(db_path, delete_commands) if err: raise Exception(err) except Exception, e: return False, 'Error exporting old alerts : %s' % str(e)
def export_old_audits(min_to_export=1000, export_count=500): """Export the oldest export_count audits if the total number of audits exceeds min_to_export """ try: # print min_to_export, export_count db_path, err = config.get_db_path() if err: raise Exception(err) ret, err = db.get_single_row( db_path, 'select count(*) as count from audit') if err: raise Exception(err) if ret['count'] > int(min_to_export): query = "select * from audit order by audit_id limit %d;" % int( export_count) full_audit_list, err = db.get_multiple_rows(db_path, query) if err: raise Exception(err) # print full_audit_list audit_list = [] for a in full_audit_list: # If it is still in the holding table then skip it so it can # get processed.. query = "select * from event_notifications_holding where event_id=%d;" % int( a['audit_id']) ret, err = db.get_single_row(db_path, query) # print ret, err if err: raise Exception(err) if not ret: audit_list.append(a) # print audit_list if audit_list: delete_commands = [] export_dir_name, err = config.get_exported_logs_dir_path() if not os.path.exists(export_dir_name): os.makedirs(export_dir_name) now, err = datetime_utils.get_epoch( when='now', num_previous_days=0) if err: raise Exception(err) now_str, err = datetime_utils.convert_from_epoch( now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') export_filename = 'audits_%s' % now_str with open('%s/%s' % (export_dir_name, export_filename), 'w') as f: f.write( 'Audit time(UTC) | Audit type | Performed by | Performed from | Audit message\n') f.write( '-------------------------------------------------------------------------------------------\n') for entry in audit_list: # print entry aud, err = _parse_audit_entry(entry) if err: raise Exception(err) # print aud, err f.write('%s | %s | %s | %s | %s\n\n' % ( aud['time'], aud['action'], aud['username'], aud['ip'], aud['action_str'])) delete_commands.append( ['delete from audit where audit_id="%d"' % int(aud['audit_id'])]) # print delete_commands ret, err = db.execute_iud(db_path, delete_commands) if err: raise Exception(err) except Exception, e: return False, 'Error exporting old audits : %s' % str(e)
def export_old_alerts(older_than_days=1): """Move all alerts older than the older_than_days into a file in /var/log/integralstor/logs/exported dir """ try: cutoff_seconds, err = datetime_utils.get_epoch( when='now', num_previous_days=older_than_days) if err: raise Exception(err) #query = "select * from alerts where last_update_time < Datetime('now', '-%d days') order by alert_id;"%older_than_days query = "select * from alerts where last_update_time < %d order by alert_id;" % cutoff_seconds full_alerts_list, err = _get_and_parse_alerts(query) if err: raise Exception(err) alerts_list = [] # print 'full', full_alerts_list db_path, err = config.get_db_path() if err: raise Exception(err) for a in full_alerts_list: # If it is still in the holding table then skip it so it can get # processed.. query = "select * from event_notifications_holding where event_id=%d;" % int( a['alert_id']) ret, err = db.get_single_row(db_path, query) # print ret, err if err: raise Exception(err) if not ret: alerts_list.append(a) # print 'to export', alerts_list if alerts_list: delete_commands = [] export_dir_name, err = config.get_exported_logs_dir_path() if err: raise Exception(err) if not os.path.exists(export_dir_name): os.makedirs(export_dir_name) now, err = datetime_utils.get_epoch( when='now', num_previous_days=0) if err: raise Exception(err) now_str, err = datetime_utils.convert_from_epoch( now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') export_filename = 'alerts_%s' % now_str # print export_filename with open('%s/%s' % (export_dir_name, export_filename), 'w') as f: f.write( 'First alert time(UTC) | Last update time(UTC) | Repeat count | Subsystem | Severity | Alert message\n') f.write( '-------------------------------------------------------------------------------------------\n') for al in alerts_list: f.write('%s | %s | %d | %s | %s | %s\n\n' % ( al['first_alert_time'], al['last_update_time'], al['repeat_count'], al['subsystem'], al['severity'], al['alert_str'])) delete_commands.append( ['delete from alerts where alert_id="%d"' % int(al['alert_id'])]) # print delete_commands db_path, err = config.get_db_path() if err: raise Exception(err) ret, err = db.execute_iud(db_path, delete_commands) if err: raise Exception(err) except Exception, e: return False, 'Error exporting old alerts : %s' % str(e)
def export_old_audits(min_to_export=1000, export_count=500): """Export the oldest export_count audits if the total number of audits exceeds min_to_export """ try: # print min_to_export, export_count db_path, err = config.get_db_path() if err: raise Exception(err) ret, err = db.get_single_row( db_path, 'select count(*) as count from audit') if err: raise Exception(err) if ret['count'] > int(min_to_export): query = "select * from audit order by audit_id limit %d;" % int( export_count) full_audit_list, err = db.get_multiple_rows(db_path, query) if err: raise Exception(err) # print full_audit_list audit_list = [] for a in full_audit_list: # If it is still in the holding table then skip it so it can # get processed.. query = "select * from event_notifications_holding where event_id=%d;" % int( a['audit_id']) ret, err = db.get_single_row(db_path, query) # print ret, err if err: raise Exception(err) if not ret: audit_list.append(a) # print audit_list if audit_list: delete_commands = [] export_dir_name, err = config.get_exported_logs_dir_path() if not os.path.exists(export_dir_name): os.makedirs(export_dir_name) now, err = datetime_utils.get_epoch( when='now', num_previous_days=0) if err: raise Exception(err) now_str, err = datetime_utils.convert_from_epoch( now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') export_filename = 'audits_%s' % now_str with open('%s/%s' % (export_dir_name, export_filename), 'w') as f: f.write( 'Audit time(UTC) | Audit type | Performed by | Performed from | Audit message\n') f.write( '-------------------------------------------------------------------------------------------\n') for entry in audit_list: # print entry aud, err = _parse_audit_entry(entry) if err: raise Exception(err) # print aud, err f.write('%s | %s | %s | %s | %s\n\n' % ( aud['time'], aud['action'], aud['username'], aud['ip'], aud['action_str'])) delete_commands.append( ['delete from audit where audit_id="%d"' % int(aud['audit_id'])]) # print delete_commands ret, err = db.execute_iud(db_path, delete_commands) if err: raise Exception(err) except Exception, e: return False, 'Error exporting old audits : %s' % str(e)
def initiate_scan(scan_configuration_id, standalone=False): scan_id = 0 db_location = None conn = None error_list = [] successful_creation_modification_transactions_count = 0 failed_creation_modification_transactions_count = 0 successful_deletion_transactions_count = 0 failed_deletion_transactions_count = 0 scanned_files_count = 0 scanned_dirs_count = 0 new_files_count = 0 modified_files_count = 0 deleted_files_count = 0 try: global scan_killed scan_killed = False print '-----------------------------SETTINGS---------------------------------' db_location, err = get_db_location(standalone) if err: raise Exception(err) configs, err = get_scan_configurations( db_location=db_location, scan_configuration_id=scan_configuration_id, standalone=standalone, include_deleted=False) if err: raise Exception(err) if not configs: raise Exception('Specified configuration does not exist') scan_dir = configs[0]['scan_dir'] exclude_dirs = configs[0]['exclude_dirs'] generate_checksum = configs[0]['generate_checksum'] db_transaction_size = configs[0]['db_transaction_size'] record_history = configs[0]['record_history'] if generate_checksum: print 'Generating file checksums' else: print 'Not generating file checksums' if record_history: print 'Recording file event history' else: print 'Not recording file event history' dir_str = 'Collecting statistics for the folder : %s..\n' % scan_dir exclude_dirs_list = None if exclude_dirs: dir_str += '..but excluding the following directories : "%s"' % exclude_dirs exclude_dirs_list = [] components = exclude_dirs.split(',') for component in components: exclude_dirs_list.append( '%s/%s' % (scan_dir, component.strip().lstrip('/'))) print dir_str print '----------------------------------------------------------------------' pid = os.getpid() scan_id, err = log_scan_start(db_location, configs[0], pid) if err: raise Exception(err) counter = 0 cmd_list = [] transaction_file_list = [] if not os.path.exists(scan_dir): raise Exception('Specified scan directory %s does not exist.' % scan_dir) print 'Collecting information about the directory structure for %s..' % scan_dir for root, dirs, files in os.walk(unicode(scan_dir)): if scan_killed: break scanned_dirs_count += 1 #print 'Processing directory %s'%root if exclude_dirs_list and root in exclude_dirs_list: #Config says exclude this dir so skip it. print 'Skipping excluded directory : %s' % root continue for file in files: full_path = os.path.normpath('%s/%s' % (root, file)) #print full_path try: db_to_be_updated = False scanned_files_count += 1 transaction_file_list.append(full_path) if os.path.islink(full_path): continue extension = None if full_path: rt, extension = os.path.splitext(full_path) mtime = os.path.getmtime(full_path) #print 'mtime', mtime size = os.path.getsize(full_path) chksum = None if generate_checksum: chksum, err = checksum_utils.generate_checksum( full_path, algorithm='sha256') if err: error_list.append((full_path, err)) print '!!!!', err continue #First check if we have some recorded info about this file. query = 'select * from file_info where path = "%s"' % full_path file_info_row, err = db.get_single_row(db_location, query) if err: raise Exception(err) #Now decide whether to insert/update the DB.. insert_update_file_info = False update_file_info = False if not file_info_row: #Info not in the DB so insert.. #Insert it if the path does not exist or else ignore.. cmd = [ 'insert or ignore into file_info(path, extension, size, checksum, last_modify_time, last_access_time, last_scan_id, scan_configuration_id) values (?,?,?,?,?,?,?,?)', ( full_path, extension, size, chksum, int(mtime), int(mtime), scan_id, scan_configuration_id, ) ] cmd_list.append(cmd) new_files_count += 1 db_to_be_updated = True else: #Exists in DB but has anything changed with the file?? If so, update if file_info_row['last_modify_time'] != int(mtime): #File mtime does not match whats in the DB so modify update_file_info = True if generate_checksum: if file_info_row['checksum'] != chksum: #File checksum does not match whats in the DB so modify update_file_info = True if update_file_info: modified_files_count += 1 #This file has NOT been processed already in a previous run of the same scan_id if generate_checksum: update_cmd = [ 'update file_info set size=?, last_modify_time=?, last_access_time=?, extension=?, checksum=?, last_scan_id = ?, scan_configuration_id=? where path = ?', ( size, int(mtime), int(mtime), extension, chksum, scan_id, scan_configuration_id, full_path, ) ] else: update_cmd = [ 'update file_info set size=?, last_modify_time=?, last_access_time=?, extension=?, last_scan_id=?, scan_configuration_id=? where path = ?', ( size, int(mtime), int(mtime), extension, scan_id, scan_configuration_id, full_path, ) ] cmd_list.append(update_cmd) db_to_be_updated = True if record_history: cmd = [ 'insert or ignore into file_events_history(file_info_id, path, events, event_time, last_scan_id, scan_configuration_id) values ((select id from file_info where path=?),?,?,?,?, ?)', ( full_path, full_path, 'MODIFY', int(mtime), scan_id, scan_configuration_id, ) ] cmd_list.append(cmd) db_to_be_updated = True if db_to_be_updated: counter += 1 if cmd_list and (counter != 0) and ( counter % db_transaction_size == 0): print 'Scanned %d files' % scanned_files_count #print cmd_list ret, err = db.execute_iud(db_location, cmd_list, get_rowid=False) #print ret, err if err: failed_creation_modification_transactions_count += counter for transaction_file in transaction_file_list: error_list.append(( transaction_file, 'Error inserting/updating into the database : %s' % err)) else: successful_creation_modification_transactions_count += counter pd = { 'scan_id': scan_id, 'status_id': 1, 'status_str': 'Scanned %d directories and %d files. Processed %d creations/modifilcation transactions successfully with %d errors. New files detected : %d, files modified since last scan : %d' % (scanned_dirs_count, scanned_files_count, successful_creation_modification_transactions_count, failed_creation_modification_transactions_count, new_files_count, modified_files_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count': successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count': failed_creation_modification_transactions_count, 'successful_deletion_transactions_count': successful_deletion_transactions_count, 'failed_deletion_transactions_count': failed_deletion_transactions_count, 'new_files_count': new_files_count, 'modified_files_count': modified_files_count, 'deleted_files_count': deleted_files_count } ret, err = log_scan_progress(db_location, pd) cmd_list = [] transaction_file_list = [] counter = 0 except Exception, e: #print e error_list.append((full_path, str(e))) if not scan_killed: if cmd_list: print 'Processing the last batch of %d files.' % counter #print cmd_list #Still have unprocessed files so insert them! ret, err = db.execute_iud(db_location, cmd_list, get_rowid=False) if err: failed_creation_modification_transactions_count += counter for transaction_file in transaction_file_list: error_list.append( (transaction_file, 'Error inserting into the database : %s' % err)) else: successful_creation_modification_transactions_count += counter pd = { 'scan_id': scan_id, 'status_id': 1, 'status_str': 'Processed all creations and modifications. Now processing deletions. Scanned %d directories and %d files. Processed %d creation/modification transactions successfully with %d errors' % (scanned_dirs_count, scanned_files_count, successful_creation_modification_transactions_count, failed_creation_modification_transactions_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count': successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count': failed_creation_modification_transactions_count, 'successful_deletion_transactions_count': successful_deletion_transactions_count, 'failed_deletion_transactions_count': failed_deletion_transactions_count, 'new_files_count': new_files_count, 'modified_files_count': modified_files_count, 'deleted_files_count': deleted_files_count } ret, err = log_scan_progress(db_location, pd) #Now scan for deleted files that are in our db but have actually been deleted.. print 'scanning for deletes' query = 'select * from file_info where scan_configuration_id="%d" and last_scan_id != "%d"' % ( scan_configuration_id, scan_id) (cur, conn), err = db.get_query_cursor_and_connection( db_location, query) if err: raise Exception(err) file_info_row, err = db.get_next_row(cur) if err: raise Exception(err) cmd_list = [] transaction_file_list = [] while file_info_row: if not os.path.isfile(file_info_row['path']): #File no longer exists so delete the file_info entry and put in a delete into the file_events_history table deleted_files_count += 1 transaction_file_list.append(file_info_row['path']) if record_history: #!!!!!!!!!!!!CHANGE TO USE INTEGRALSTOR's CALLS now_time = int(time.time()) #cmd_list.append(['insert into file_events_history(path, events, event_time, last_scan_id, scan_configuration_id) values (?,?,?,?, ?)', (full_path, 'DELETE', int(now_time), scan_id, scan_configuration_id, )]) cmd_list.append([ 'insert or ignore into file_events_history(file_info_id, path, events, event_time, last_scan_id, scan_configuration_id) values ((select id from file_info where path=?),?,?,?,?, ?)', ( file_info_row['path'], file_info_row['path'], 'DELETE', int(now_time), scan_id, scan_configuration_id, ) ]) cmd_list.append([ 'delete from file_info where id="%d"' % file_info_row['id'] ]) #Now read the next row file_info_row, err = db.get_next_row(cur) if err: raise Exception(err) if conn: db.close_connection(conn) #print cmd_list if cmd_list: print cmd_list ret, err = db.execute_iud(db_location, cmd_list) print ret, err if err: failed_deletion_transactions_count += counter for transaction_file in transaction_file_list: error_list.append( (transaction_file, 'Error inserting into the database : %s' % err)) raise Exception(err) else: successful_deletion_transactions_count += counter if failed_creation_modification_transactions_count > 0 or failed_deletion_transactions_count > 0: status_id = 3 else: status_id = 2 pd = { 'scan_id': scan_id, 'status_id': status_id, 'status_str': 'Processed all creation/modification/deletion changes. %d file creations, %d file modifications and %d file deletions detected. %d failed creation/modification transactions and %d failed deletion transactions. ' % (new_files_count, modified_files_count, deleted_files_count, failed_creation_modification_transactions_count, failed_deletion_transactions_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count': successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count': failed_creation_modification_transactions_count, 'successful_deletion_transactions_count': successful_deletion_transactions_count, 'failed_deletion_transactions_count': failed_deletion_transactions_count, 'new_files_count': new_files_count, 'modified_files_count': modified_files_count, 'deleted_files_count': deleted_files_count } print 'a' ret, err = log_scan_progress(db_location, pd) print ret, err print 'aa' else: time_str = time.strftime('%a, %d %b %Y %H:%M:%S') pd = { 'scan_id': scan_id, 'status_id': 4, 'status_str': 'Paused at %s' % time_str } ret, err = log_scan_progress(db_location, pd)
def initiate_scan(scan_configuration_id, standalone = False): scan_id = 0 db_location = None conn = None error_list = [] successful_creation_modification_transactions_count = 0 failed_creation_modification_transactions_count = 0 successful_deletion_transactions_count = 0 failed_deletion_transactions_count = 0 scanned_files_count = 0 scanned_dirs_count = 0 new_files_count = 0 modified_files_count = 0 deleted_files_count = 0 try: global scan_killed scan_killed = False print '-----------------------------SETTINGS---------------------------------' db_location, err = get_db_location(standalone) if err: raise Exception(err) configs, err = get_scan_configurations(db_location = db_location, scan_configuration_id = scan_configuration_id, standalone = standalone, include_deleted=False) if err: raise Exception(err) if not configs: raise Exception('Specified configuration does not exist') scan_dir = configs[0]['scan_dir'] exclude_dirs = configs[0]['exclude_dirs'] generate_checksum = configs[0]['generate_checksum'] db_transaction_size = configs[0]['db_transaction_size'] record_history = configs[0]['record_history'] if generate_checksum: print 'Generating file checksums' else: print 'Not generating file checksums' if record_history: print 'Recording file event history' else: print 'Not recording file event history' dir_str = 'Collecting statistics for the folder : %s..\n'%scan_dir exclude_dirs_list = None if exclude_dirs: dir_str += '..but excluding the following directories : "%s"'%exclude_dirs exclude_dirs_list = [] components = exclude_dirs.split(',') for component in components: exclude_dirs_list.append('%s/%s'%(scan_dir, component.strip().lstrip('/'))) print dir_str print '----------------------------------------------------------------------' pid = os.getpid() scan_id, err = log_scan_start(db_location, configs[0], pid) if err: raise Exception(err) counter = 0 cmd_list = [] transaction_file_list = [] if not os.path.exists(scan_dir): raise Exception('Specified scan directory %s does not exist.'%scan_dir) print 'Collecting information about the directory structure for %s..'%scan_dir for root, dirs, files in os.walk(unicode(scan_dir)): if scan_killed: break scanned_dirs_count += 1 #print 'Processing directory %s'%root if exclude_dirs_list and root in exclude_dirs_list: #Config says exclude this dir so skip it. print 'Skipping excluded directory : %s'%root continue for file in files: full_path = os.path.normpath('%s/%s'%(root, file)) #print full_path try: db_to_be_updated = False scanned_files_count += 1 transaction_file_list.append(full_path) if os.path.islink(full_path): continue extension = None if full_path: rt, extension = os.path.splitext(full_path) mtime = os.path.getmtime(full_path) #print 'mtime', mtime size = os.path.getsize(full_path) chksum = None if generate_checksum: chksum, err = checksum_utils.generate_checksum(full_path, algorithm = 'sha256') if err: error_list.append((full_path, err)) print '!!!!', err continue #First check if we have some recorded info about this file. query = 'select * from file_info where path = "%s"'%full_path file_info_row, err = db.get_single_row(db_location, query) if err: raise Exception(err) #Now decide whether to insert/update the DB.. insert_update_file_info = False update_file_info = False if not file_info_row: #Info not in the DB so insert.. #Insert it if the path does not exist or else ignore.. cmd = ['insert or ignore into file_info(path, extension, size, checksum, last_modify_time, last_access_time, last_scan_id, scan_configuration_id) values (?,?,?,?,?,?,?,?)', (full_path, extension, size, chksum, int(mtime), int(mtime),scan_id, scan_configuration_id,)] cmd_list.append(cmd) new_files_count += 1 db_to_be_updated = True else: #Exists in DB but has anything changed with the file?? If so, update if file_info_row['last_modify_time'] != int(mtime): #File mtime does not match whats in the DB so modify update_file_info = True if generate_checksum: if file_info_row['checksum'] != chksum: #File checksum does not match whats in the DB so modify update_file_info = True if update_file_info: modified_files_count += 1 #This file has NOT been processed already in a previous run of the same scan_id if generate_checksum: update_cmd = ['update file_info set size=?, last_modify_time=?, last_access_time=?, extension=?, checksum=?, last_scan_id = ?, scan_configuration_id=? where path = ?', (size, int(mtime), int(mtime), extension, chksum, scan_id, scan_configuration_id, full_path,)] else: update_cmd = ['update file_info set size=?, last_modify_time=?, last_access_time=?, extension=?, last_scan_id=?, scan_configuration_id=? where path = ?', (size, int(mtime), int(mtime), extension, scan_id, scan_configuration_id, full_path,)] cmd_list.append(update_cmd) db_to_be_updated = True if record_history: cmd = ['insert or ignore into file_events_history(file_info_id, path, events, event_time, last_scan_id, scan_configuration_id) values ((select id from file_info where path=?),?,?,?,?, ?)', (full_path, full_path, 'MODIFY', int(mtime), scan_id, scan_configuration_id, )] cmd_list.append(cmd) db_to_be_updated = True if db_to_be_updated: counter += 1 if cmd_list and (counter != 0) and (counter % db_transaction_size == 0): print 'Scanned %d files'%scanned_files_count #print cmd_list ret, err = db.execute_iud(db_location, cmd_list, get_rowid=False) #print ret, err if err: failed_creation_modification_transactions_count += counter for transaction_file in transaction_file_list: error_list.append((transaction_file, 'Error inserting/updating into the database : %s'%err)) else: successful_creation_modification_transactions_count += counter pd = {'scan_id': scan_id, 'status_id':1, 'status_str':'Scanned %d directories and %d files. Processed %d creations/modifilcation transactions successfully with %d errors. New files detected : %d, files modified since last scan : %d'%(scanned_dirs_count, scanned_files_count, successful_creation_modification_transactions_count, failed_creation_modification_transactions_count, new_files_count, modified_files_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count':successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count':failed_creation_modification_transactions_count, 'successful_deletion_transactions_count' : successful_deletion_transactions_count, 'failed_deletion_transactions_count':failed_deletion_transactions_count, 'new_files_count':new_files_count, 'modified_files_count':modified_files_count, 'deleted_files_count' : deleted_files_count} ret, err = log_scan_progress(db_location, pd) cmd_list = [] transaction_file_list = [] counter = 0 except Exception, e: #print e error_list.append((full_path, str(e))) if not scan_killed: if cmd_list: print 'Processing the last batch of %d files.'%counter #print cmd_list #Still have unprocessed files so insert them! ret, err = db.execute_iud(db_location, cmd_list, get_rowid=False) if err: failed_creation_modification_transactions_count += counter for transaction_file in transaction_file_list: error_list.append((transaction_file, 'Error inserting into the database : %s'%err)) else: successful_creation_modification_transactions_count += counter pd = {'scan_id': scan_id, 'status_id':1, 'status_str':'Processed all creations and modifications. Now processing deletions. Scanned %d directories and %d files. Processed %d creation/modification transactions successfully with %d errors'%(scanned_dirs_count, scanned_files_count, successful_creation_modification_transactions_count, failed_creation_modification_transactions_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count':successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count':failed_creation_modification_transactions_count, 'successful_deletion_transactions_count' : successful_deletion_transactions_count, 'failed_deletion_transactions_count':failed_deletion_transactions_count, 'new_files_count':new_files_count, 'modified_files_count':modified_files_count, 'deleted_files_count' : deleted_files_count} ret, err = log_scan_progress(db_location, pd) #Now scan for deleted files that are in our db but have actually been deleted.. print 'scanning for deletes' query = 'select * from file_info where scan_configuration_id="%d" and last_scan_id != "%d"'%(scan_configuration_id, scan_id) (cur, conn), err = db.get_query_cursor_and_connection(db_location, query) if err: raise Exception(err) file_info_row, err = db.get_next_row(cur) if err: raise Exception(err) cmd_list = [] transaction_file_list = [] while file_info_row: if not os.path.isfile(file_info_row['path']): #File no longer exists so delete the file_info entry and put in a delete into the file_events_history table deleted_files_count += 1 transaction_file_list.append(file_info_row['path']) if record_history: #!!!!!!!!!!!!CHANGE TO USE INTEGRALSTOR's CALLS now_time = int(time.time()) #cmd_list.append(['insert into file_events_history(path, events, event_time, last_scan_id, scan_configuration_id) values (?,?,?,?, ?)', (full_path, 'DELETE', int(now_time), scan_id, scan_configuration_id, )]) cmd_list.append(['insert or ignore into file_events_history(file_info_id, path, events, event_time, last_scan_id, scan_configuration_id) values ((select id from file_info where path=?),?,?,?,?, ?)', (file_info_row['path'], file_info_row['path'], 'DELETE', int(now_time), scan_id, scan_configuration_id, )]) cmd_list.append(['delete from file_info where id="%d"'%file_info_row['id']]) #Now read the next row file_info_row, err = db.get_next_row(cur) if err: raise Exception(err) if conn: db.close_connection(conn) #print cmd_list if cmd_list: print cmd_list ret, err = db.execute_iud(db_location, cmd_list) print ret, err if err: failed_deletion_transactions_count += counter for transaction_file in transaction_file_list: error_list.append((transaction_file, 'Error inserting into the database : %s'%err)) raise Exception(err) else: successful_deletion_transactions_count += counter if failed_creation_modification_transactions_count > 0 or failed_deletion_transactions_count > 0: status_id = 3 else: status_id = 2 pd = {'scan_id': scan_id, 'status_id':status_id, 'status_str':'Processed all creation/modification/deletion changes. %d file creations, %d file modifications and %d file deletions detected. %d failed creation/modification transactions and %d failed deletion transactions. '%(new_files_count, modified_files_count, deleted_files_count, failed_creation_modification_transactions_count, failed_deletion_transactions_count), 'scanned_dirs_count': scanned_dirs_count, 'scanned_files_count': scanned_files_count, 'successful_creation_modification_transactions_count':successful_creation_modification_transactions_count, 'failed_creation_modification_transactions_count':failed_creation_modification_transactions_count, 'successful_deletion_transactions_count' : successful_deletion_transactions_count, 'failed_deletion_transactions_count':failed_deletion_transactions_count, 'new_files_count':new_files_count, 'modified_files_count':modified_files_count, 'deleted_files_count' : deleted_files_count} print 'a' ret, err = log_scan_progress(db_location, pd) print ret, err print 'aa' else: time_str = time.strftime('%a, %d %b %Y %H:%M:%S') pd = {'scan_id': scan_id, 'status_id':4, 'status_str':'Paused at %s'%time_str} ret, err = log_scan_progress(db_location, pd)