result = list()
filter_logs = []
if os.path.isdir('/var/log/filter'):
filter_logs = list(
sorted(glob.glob("/var/log/filter/filter_*.log"), reverse=True))
if os.path.isfile('/var/log/filter.log'):
filter_logs.append('/var/log/filter.log')
for filter_log in filter_logs:
do_exit = False
try:
filename = fetch_clog(filter_log)
except Exception as e:
filename = filter_log
for record in reverse_log_reader(filename):
if record['line'].find('filterlog') > -1:
rule = dict()
metadata = dict()
# rule metadata (unique hash, hostname, timestamp)
if re.search('filterlog\[\d*\]:', record['line']):
# rfc3164 format
log_ident = re.split('filterlog[^:]*:', record['line'])
tmp = log_ident[0].split()
metadata['__host__'] = tmp.pop()
metadata['__timestamp__'] = ' '.join(tmp)
rulep = log_ident[1].strip().split(',')
else:
# rfc5424 format
tmp = record['line'].split()
metadata['__timestamp__'] = tmp[1].split('+')[0]
data_filters_comp[filterField] = re.compile(filter_regexp)
except sre_constants.error:
# remove illegal expression
# del data_filters[filterField]
data_filters_comp[filterField] = re.compile('.*')
# filter one specific log line
if 'filepos' in data_filters and data_filters['filepos'].isdigit():
log_start_pos = int(data_filters['filepos'])
else:
log_start_pos = None
# query suricata eve log
result = {'filters': data_filters, 'rows': [], 'total_rows': 0, 'origin': suricata_log.split('/')[-1]}
if os.path.exists(suricata_log):
for line in reverse_log_reader(filename=suricata_log, start_pos=log_start_pos):
try:
record = ujson.loads(line['line'])
except ValueError:
# can not handle line
record = {}
# only process valid alert items
if 'alert' in record:
# add position in file
record['filepos'] = line['pos']
record['fileid'] = parameters['fileid']
# flatten structure
record['alert_sid'] = record['alert']['signature_id']
record['alert_action'] = record['alert']['action']
record['alert'] = record['alert']['signature']
# XXX happens on rdr (ID is not unique) or when no label is found
result[line_id] = {'label': 'XXX'}
return result
if __name__ == '__main__':
# read parameters
parameters = {'limit': '0', 'digest': ''}
update_params(parameters)
parameters['limit'] = int(parameters['limit'])
# parse current running config
running_conf_descr = fetch_rules_descriptions()
result = list()
for record in reverse_log_reader(fetch_clog(filter_log)):
if record['line'].find('filterlog') > -1:
rule = dict()
metadata = dict()
# rule metadata (unique hash, hostname, timestamp)
tmp = record['line'].split('filterlog:')[0].split()
metadata['__digest__'] = md5.new(record['line']).hexdigest()
metadata['__host__'] = tmp.pop()
metadata['__timestamp__'] = ' '.join(tmp)
rulep = record['line'].split('filterlog:')[1].strip().split(',')
update_rule(rule, metadata, rulep, fields_general)
if 'version' in rule:
if rule['version'] == '4':
update_rule(rule, metadata, rulep, fields_ipv4)
if 'proto' in rule:
result[line_id] = {'rid': None, 'label': rid}
return result
if __name__ == '__main__':
# read parameters
parameters = {'limit': '0', 'digest': ''}
update_params(parameters)
parameters['limit'] = int(parameters['limit'])
# parse current running config
running_conf_descr = fetch_rule_details()
result = list()
for record in reverse_log_reader(fetch_clog(filter_log)):
if record['line'].find('filterlog') > -1:
rule = dict()
metadata = dict()
# rule metadata (unique hash, hostname, timestamp)
tmp = record['line'].split('filterlog:')[0].split()
metadata['__digest__'] = md5(record['line'].encode()).hexdigest()
metadata['__host__'] = tmp.pop()
metadata['__timestamp__'] = ' '.join(tmp)
rulep = record['line'].split('filterlog:')[1].strip().split(',')
update_rule(rule, metadata, rulep, fields_general)
if 'version' in rule:
if rule['version'] == '4':
update_rule(rule, metadata, rulep, fields_ipv4)
if 'proto' in rule:
# filter one specific log line
if 'filepos' in data_filters and data_filters['filepos'].isdigit():
log_start_pos = int(data_filters['filepos'])
else:
log_start_pos = None
# query suricata eve log
result = {
'filters': data_filters,
'rows': [],
'total_rows': 0,
'origin': suricata_log.split('/')[-1]
}
if os.path.exists(suricata_log):
for line in reverse_log_reader(filename=suricata_log,
start_pos=log_start_pos):
try:
record = ujson.loads(line['line'])
except ValueError:
# can not handle line
record = {}
# only process valid alert items
if 'alert' in record:
# add position in file
record['filepos'] = line['pos']
record['fileid'] = parameters['fileid']
# flatten structure
record['alert_sid'] = record['alert']['signature_id']
record['alert_action'] = record['alert']['action']
record['alert'] = record['alert']['signature']
import time
import datetime
from lib import suricata_alert_log
from log_helper import reverse_log_reader
if __name__ == '__main__':
result = []
for filename in sorted(glob.glob('%s*' % suricata_alert_log)):
row = dict()
row['size'] = os.stat(filename).st_size
# always list first file and non empty next.
if row['size'] > 0 or filename.split('/')[-1].count('.') == 1:
row['modified'] = os.stat(filename).st_mtime
row['filename'] = filename.split('/')[-1]
# try to find actual timestamp from file
for line in reverse_log_reader(filename=filename):
if line['line'] != '':
try:
record = ujson.loads(line['line'])
except ValueError:
continue
if 'timestamp' in record:
row['modified'] = int(
time.mktime(
datetime.datetime.strptime(
record['timestamp'].split('.')[0],
"%Y-%m-%dT%H:%M:%S").timetuple()))
break
ext = filename.split('.')[-1]
if ext.isdigit():
import time
import datetime
from lib import suricata_alert_log
from log_helper import reverse_log_reader
if __name__ == '__main__':
result = []
for filename in sorted(glob.glob('%s*' % suricata_alert_log)):
row = dict()
row['size'] = os.stat(filename).st_size
# always list first file and non empty next.
if row['size'] > 0 or filename.split('/')[-1].count('.') == 1:
row['modified'] = os.stat(filename).st_mtime
row['filename'] = filename.split('/')[-1]
# try to find actual timestamp from file
for line in reverse_log_reader(filename=filename):
if line['line'] != '':
try:
record = ujson.loads(line['line'])
except ValueError:
continue
if 'timestamp' in record:
row['modified'] = int(time.mktime(datetime.datetime.strptime(record['timestamp'].split('.')[0],
"%Y-%m-%dT%H:%M:%S").timetuple()))
break
ext = filename.split('.')[-1]
if ext.isdigit():
row['sequence'] = int(ext)
else:
row['sequence'] = None
