result = list() filter_logs = [] if os.path.isdir('/var/log/filter'): filter_logs = list( sorted(glob.glob("/var/log/filter/filter_*.log"), reverse=True)) if os.path.isfile('/var/log/filter.log'): filter_logs.append('/var/log/filter.log') for filter_log in filter_logs: do_exit = False try: filename = fetch_clog(filter_log) except Exception as e: filename = filter_log for record in reverse_log_reader(filename): if record['line'].find('filterlog') > -1: rule = dict() metadata = dict() # rule metadata (unique hash, hostname, timestamp) if re.search('filterlog\[\d*\]:', record['line']): # rfc3164 format log_ident = re.split('filterlog[^:]*:', record['line']) tmp = log_ident[0].split() metadata['__host__'] = tmp.pop() metadata['__timestamp__'] = ' '.join(tmp) rulep = log_ident[1].strip().split(',') else: # rfc5424 format tmp = record['line'].split() metadata['__timestamp__'] = tmp[1].split('+')[0]
data_filters_comp[filterField] = re.compile(filter_regexp) except sre_constants.error: # remove illegal expression # del data_filters[filterField] data_filters_comp[filterField] = re.compile('.*') # filter one specific log line if 'filepos' in data_filters and data_filters['filepos'].isdigit(): log_start_pos = int(data_filters['filepos']) else: log_start_pos = None # query suricata eve log result = {'filters': data_filters, 'rows': [], 'total_rows': 0, 'origin': suricata_log.split('/')[-1]} if os.path.exists(suricata_log): for line in reverse_log_reader(filename=suricata_log, start_pos=log_start_pos): try: record = ujson.loads(line['line']) except ValueError: # can not handle line record = {} # only process valid alert items if 'alert' in record: # add position in file record['filepos'] = line['pos'] record['fileid'] = parameters['fileid'] # flatten structure record['alert_sid'] = record['alert']['signature_id'] record['alert_action'] = record['alert']['action'] record['alert'] = record['alert']['signature']
# XXX happens on rdr (ID is not unique) or when no label is found result[line_id] = {'label': 'XXX'} return result if __name__ == '__main__': # read parameters parameters = {'limit': '0', 'digest': ''} update_params(parameters) parameters['limit'] = int(parameters['limit']) # parse current running config running_conf_descr = fetch_rules_descriptions() result = list() for record in reverse_log_reader(fetch_clog(filter_log)): if record['line'].find('filterlog') > -1: rule = dict() metadata = dict() # rule metadata (unique hash, hostname, timestamp) tmp = record['line'].split('filterlog:')[0].split() metadata['__digest__'] = md5.new(record['line']).hexdigest() metadata['__host__'] = tmp.pop() metadata['__timestamp__'] = ' '.join(tmp) rulep = record['line'].split('filterlog:')[1].strip().split(',') update_rule(rule, metadata, rulep, fields_general) if 'version' in rule: if rule['version'] == '4': update_rule(rule, metadata, rulep, fields_ipv4) if 'proto' in rule:
result[line_id] = {'rid': None, 'label': rid} return result if __name__ == '__main__': # read parameters parameters = {'limit': '0', 'digest': ''} update_params(parameters) parameters['limit'] = int(parameters['limit']) # parse current running config running_conf_descr = fetch_rule_details() result = list() for record in reverse_log_reader(fetch_clog(filter_log)): if record['line'].find('filterlog') > -1: rule = dict() metadata = dict() # rule metadata (unique hash, hostname, timestamp) tmp = record['line'].split('filterlog:')[0].split() metadata['__digest__'] = md5(record['line'].encode()).hexdigest() metadata['__host__'] = tmp.pop() metadata['__timestamp__'] = ' '.join(tmp) rulep = record['line'].split('filterlog:')[1].strip().split(',') update_rule(rule, metadata, rulep, fields_general) if 'version' in rule: if rule['version'] == '4': update_rule(rule, metadata, rulep, fields_ipv4) if 'proto' in rule:
# filter one specific log line if 'filepos' in data_filters and data_filters['filepos'].isdigit(): log_start_pos = int(data_filters['filepos']) else: log_start_pos = None # query suricata eve log result = { 'filters': data_filters, 'rows': [], 'total_rows': 0, 'origin': suricata_log.split('/')[-1] } if os.path.exists(suricata_log): for line in reverse_log_reader(filename=suricata_log, start_pos=log_start_pos): try: record = ujson.loads(line['line']) except ValueError: # can not handle line record = {} # only process valid alert items if 'alert' in record: # add position in file record['filepos'] = line['pos'] record['fileid'] = parameters['fileid'] # flatten structure record['alert_sid'] = record['alert']['signature_id'] record['alert_action'] = record['alert']['action'] record['alert'] = record['alert']['signature']
import time import datetime from lib import suricata_alert_log from log_helper import reverse_log_reader if __name__ == '__main__': result = [] for filename in sorted(glob.glob('%s*' % suricata_alert_log)): row = dict() row['size'] = os.stat(filename).st_size # always list first file and non empty next. if row['size'] > 0 or filename.split('/')[-1].count('.') == 1: row['modified'] = os.stat(filename).st_mtime row['filename'] = filename.split('/')[-1] # try to find actual timestamp from file for line in reverse_log_reader(filename=filename): if line['line'] != '': try: record = ujson.loads(line['line']) except ValueError: continue if 'timestamp' in record: row['modified'] = int( time.mktime( datetime.datetime.strptime( record['timestamp'].split('.')[0], "%Y-%m-%dT%H:%M:%S").timetuple())) break ext = filename.split('.')[-1] if ext.isdigit():
import time import datetime from lib import suricata_alert_log from log_helper import reverse_log_reader if __name__ == '__main__': result = [] for filename in sorted(glob.glob('%s*' % suricata_alert_log)): row = dict() row['size'] = os.stat(filename).st_size # always list first file and non empty next. if row['size'] > 0 or filename.split('/')[-1].count('.') == 1: row['modified'] = os.stat(filename).st_mtime row['filename'] = filename.split('/')[-1] # try to find actual timestamp from file for line in reverse_log_reader(filename=filename): if line['line'] != '': try: record = ujson.loads(line['line']) except ValueError: continue if 'timestamp' in record: row['modified'] = int(time.mktime(datetime.datetime.strptime(record['timestamp'].split('.')[0], "%Y-%m-%dT%H:%M:%S").timetuple())) break ext = filename.split('.')[-1] if ext.isdigit(): row['sequence'] = int(ext) else: row['sequence'] = None