def parse(source, message):
# define interval (TODO from source)
# TODO if nothing test all
start = 0
end = len(Message.patterns)
if Message.debug_parsing:
log.info('Parsing message: %s%s%s' %
(log.COLOR_GREY, message, log.COLOR_NONE))
for category, pattern, assertion, add in Message.patterns[start:end]:
if not assertion(source):
continue
parsed = pattern.match(message)
if parsed:
if Message.debug_parsing:
log.info(' Matches %s' % pattern.pattern)
if category == 'UNKNOWN':
log.warn(' which is \'UNKNOWN\' format.')
for k, v in add.items():
parsed[k] = v
if Message.debug_parsing:
print(parsed)
print()
""" return new Parser object """
return Message(source, category, message, pattern.pattern,
parsed)
if Message.debug_parsing:
log.warn(' No match, no timestamp!')
return None
def __init__(self, host, port, username, subfolder=None):
super().__init__()
self.host = host
self.port = port
self.username = username
self.subfolder = subfolder
# connect to server
import paramiko
password = None
while True:
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
self.client.connect(self.host, port=self.port, username=self.username, password=password, timeout=5)
break
except paramiko.ssh_exception.SSHException as e:
if not password:
import getpass
password = getpass.getpass('Password: '******'No authentication methods available.')
return
except paramiko.ssh_exception.AuthenticationException:
log.err('Authentication failed.')
return
except paramiko.ssh_exception.NoValidConnectionsError:
log.err('Cannot connect to the host over SSH.')
return
except:
traceback.print_exc()
return
log.info('Connected to %s.' % self.host)
self.os = self.determine_os()
def plot(events, display_filter_str): # TODO start, end
display_filter = Filter.parse(display_filter_str)
severities = ('UNKNOWN', 'none', 'info', 'notice', 'warning', 'critical')
matches = display_filter.run() if display_filter else events.keys()
if matches:
log.info('Will plot %d events.' % len(matches))
else:
log.warn('No events match criteria.')
return
# TODO fix ticks
#plt.style.use('dark_background')
fig, ax = plt.subplots(1, 1, figsize=(8, 20))
plt.xticks(rotation=30)
by_severity = OrderedDict([(s, []) for s in severities])
for db_id in matches or []:
event = events[db_id]
by_severity[event.severity].append(event.timestamp)
ax.hist([mdates.date2num(by_severity[s]) for s in severities],
bins=50,
stacked=True,
color=('darkgrey', 'lightgrey', 'yellowgreen', 'gold', 'orange',
'crimson'),
label=severities)
locator = mdates.AutoDateLocator()
ax.xaxis.set_major_locator(locator)
#ax.xaxis.set_major_locator(ticker.MultipleLocator(10))
ax.xaxis.set_major_formatter(mdates.AutoDateFormatter(locator))
handles, labels = ax.get_legend_handles_labels()
plt.legend(handles[::-1], labels[::-1])
plt.show()
def reload_config():
log.info('Loading config file...')
# read lines from conf file
with open(os.path.join(os.path.dirname(sys.argv[0]), 'files/ensa.conf'),
'r') as f:
for line in f.readlines():
line = line.strip()
# skip empty and comments
if len(line) == 0 or line.startswith('#'):
continue
# get keys and values
k, _, v = line.partition('=')
k = k.strip()
v = v.strip()
if k in ensa.censore_keys:
log.debug_config(' line: *********')
else:
log.debug_config(' line: \'%s\'' % (line))
# cast to correct type and save into weber.config
if k in ensa.config.keys():
ensa.config[k].value = v
else:
ensa.config['@' + k] = ensa.Option(v, str)
if k in ensa.censore_keys:
v = '*********'
log.debug_config(' parsed: %s = %s (%s)' % (k, v, str(type(v))))
def determine_os(self):
try:
with open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r') as f:
pass
log.info('Determined Linux-based OS.')
return Linux()
except:
traceback.print_exc()
log.err('Could not determine OS.')
return None
def determine_os(self):
sftp = self.client.open_sftp()
# TODO support hierarchical search (e.g. Linux -> Android)
try:
f = sftp.open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r')
log.info('Determined Linux-based OS.')
return Linux()
except:
traceback.print_exc()
log.err('Could not determine OS.')
finally:
sftp.close()
return None
def analyze(self):
for attribute, reformatter, keys in Message.attributes:
for key in keys:
if key in self.parsed.keys():
value = self.parsed[key]
if value is None:
continue
# save time as self.timestamp,
# other as attributes
if attribute == 'timestamp':
self.timestamp = reformatter(value)
#if self.timestamp > lib.normalize_datetime(): # TODO mtime, subtracted...
# self.timestamp += relativedelta(years=-1)
if Message.debug_parsing:
log.info(' Added timestamp: %s' % self.timestamp)
else:
if Message.debug_parsing:
log.info(' Adding %s: %s as %s attribute' %
(key, value, attribute))
self.attributes[attribute] = reformatter(value)
if self.category == 'kernel':
self.attributes['service'] = 'kernel'
elif self.category == 'sudo':
if 'error' in self.attributes.keys():
self.score = 9
Message.mark_suspicious(self, 'sudo failed')
else:
self.score = 3
elif self.category == 'su':
if self.attributes['result'] == 'FAILED':
self.score = 9
Message.mark_suspicious(self, 'su failed')
else:
self.score = 3
del self.attributes['result']
elif self.category in ('groupadd', 'useradd', 'chsh', 'passwd',
'chage', 'chfn', 'userdel', 'groupdel'):
self.score = 4
elif self.category in ('auth', ):
if 'result' in self.attributes.keys():
if self.attributes['result'] == 'Failed':
self.score = 9
Message.mark_suspicious(self, '\'SSH\' auth failed')
else:
self.score = 2
del self.attributes['result']
elif ('event' in self.attributes.keys()
and 'failure' in self.attributes['event']):
self.score = 9
else:
self.score = 2
elif self.category == 'cron-session':
self.score = 2
elif self.category == 'kernel':
if 'entered promiscuous mode' in self.message:
self.score = 4
elif 'segfault at ' in self.message:
self.score = 7
elif self.category == 'daemon':
if self.attributes['service'] == 'login':
if re.search(r'(A|a)uthentication failure', self.message):
self.score = 8
Message.mark_suspicious(self, '\'login\' auth failed')
else:
self.score = 2
elif self.attributes['service'] in ('dhclient', 'ntpdate'):
self.score = 2
elif self.attributes['service'] == 'init':
self.score = 4
elif self.attributes['service'] in ('su', 'sudo'):
self.score = 2
elif self.attributes['service'] == 'mysqld_safe':
if re.search(r'PLEASE REMEMBER TO SET A PASSWORD',
self.message):
self.score = 6
elif self.attributes['service'] == 'sshd':
if 'error' in self.attributes.keys():
self.score = 7
if re.search(r'error: Bind to port \d+ on .+ failed',
self.message):
self.score = 4
elif re.search('session (?:opened|closed) for user (\w+) by',
self.message):
user = re.search(
'session (?:opened|closed) for user (\w+) by',
self.message).group(1)
self.attributes['user'] = user
self.score = 2
elif re.search('Server listening on .+ port (\d+).',
self.message):
port = re.search('Server listening on .+ port (\d+).',
self.message).group(1)
self.score = 1 if port == 22 else 4
# TODO mysql
# TODO ntpd
elif 'Timezone set to' in self.message:
self.score = 4
elif self.category == 'apache-access':
if 400 <= self.attributes['response'] < 500:
self.score = 5
elif 500 <= self.attributes['response']:
self.score = 6
if 'other' in self.attributes.keys(): # unknown data appended
self.score = 4
if (self.attributes['response'] == 200 # WP failed login
and self.attributes['method'] == 'POST'
and self.attributes['request'].endswith('/wp-login.php')):
self.score = 7
Message.mark_suspicious(self, '\'WordPress\' auth failed')
elif self.category == 'apache-error':
if 'other' in self.attributes.keys(): # unknown data appended
self.score = 4
if 'error reading the headers' in self.message:
self.score = 5
#
# set severity based on score
if self.score >= 8:
self.severity = 'critical'
elif self.score >= 5:
self.severity = 'warning'
elif self.score >= 3:
self.severity = 'notice'
elif self.score >= 1:
self.severity = 'info'
elif self.category == 'UNKNOWN':
self.severity = 'UNKNOWN'
else:
self.severity = 'none'
def parse(string):
"""
x
x == ?
x != ?
x < ?
x <= ?
x > ?
x >= ?
x and y
x or y
not x
x contains ?
x matches ?
(x, y)
()
" "
' '
suspicious x
x: timestamp, score, severity, source, category, message, <attr>
"""
pattern = r'(".*?"|\'.*?\'|==|!=|<=|<|>=|>| or | and |not |\(|\)|contains|matches|suspicious|,)'
parts = [x.strip() for x in re.split(pattern, string) if x.strip()]
#print(parts)
""" for each element compute its level """
unique_levels = set()
bracket_count = 0
levels = []
for part in parts:
if bracket_count < 0:
log.err('Bad bracket order!')
break
if part == '(':
bracket_count += 1
levels.append(-1)
elif part == ')':
bracket_count -= 1
levels.append(-1)
else:
level = bracket_count + (Filter.operators.get(part) or 0.9)
unique_levels.add(level)
levels.append(level)
#print('levels:', levels)
#print('uniq levels:', unique_levels)
if bracket_count:
log.err('No matching brackets!')
""" from topmost level, create objects and put them into a pool on same position """
pool = [None for _ in parts]
for level in sorted(unique_levels, reverse=True):
if Filter.debug_filter:
log.info('Looking for lvl %s elements' % level)
leveled = [(parts[i], i) for i in range(len(parts))
if levels[i] == level]
if Filter.debug_filter:
log.info(' Matching leveled:', leveled)
for part, i in leveled:
""" convert to objects """
operands = []
if Filter.debug_filter:
log.info(' Dealing with "%s" at %d...' % (part, i))
# collapse left of i if greater or equal level, skip None
# same for right
for direction, index_inc, border_cmp, level_cmp in [
('left', -1, lambda x: x >= 0,
lambda x, y: x >= y or x == -1),
('right', 1, lambda x: x < len(parts),
lambda x, y: x > y or x == -1),
]:
collapse_index = i
while True:
collapse_index += index_inc
if not border_cmp(collapse_index):
break
if level_cmp(levels[collapse_index], level):
if pool[collapse_index] is None: # already processed element
continue
if Filter.debug_filter:
log.info(' Collapsing %s...' % direction)
operands.append(
pool[collapse_index]
) # TODO continue with multior or what?
pool[collapse_index] = None
break
else:
break
pool[i] = Filter(part, level, *operands)
if Filter.debug_filter:
log.info(' new pool[%d]: %s' % (i, str(pool[i])))
if Filter.debug_filter:
log.info('pool at the end:', pool)
""" only 1 should be left in the pool (or 0 if no filter) """
remains = list(filter(None, pool))
if len(remains) == 0:
return None
elif len(remains) == 1:
return remains[0]
else:
log.err('Bad filter (not fully collapsed)!')
return None
from source import ensa
from source import commands
lib.reload_config()
db_password = (ensa.config['db.password'].value or
getpass(log.question('DB password: '******'Cannot connect to DB!')
lib.exit_program(None, None)
rings = ensa.db.get_rings()
if rings:
log.info(
'Welcome! Create new ring using `ra` or choose an '
'existing one with `rs <name>`.')
ring_lens = commands.get_format_len_ring(rings)
log.info('Existing rings:')
for ring in rings:
log.info(' '+commands.format_ring(*ring, *ring_lens))
else:
log.info(
'Welcome! It looks that you do not have any rings created. '
'To do this, use `ra`.')
while True:
# get command
try:
cmd = input(log.prompt).strip()