def normalize_datetime(time=None, silent=False):
if not time:
return datetime.now()
if type(time) == str:
variant = 0
while True:
try:
return dateutil_parse(time).replace(tzinfo=None)
except ValueError:
variant += 1
if variant == 1: # apache-access format
time = time.replace(':', ' ', 1)
else:
if not silent:
log.err(' Cannot unify datetime:', time)
return None
elif type(time) == datetime:
return time.replace(tzinfo=None)
elif type(time) in (int, float):
# probably UNIX timestamp
return datetime.fromtimestamp(time)
else:
if not silent:
log.err(' Unknown time type:', time, type(time))
return None
def __err(self, *args):
info = 'entityRecognize_pytorch '
for a in args:
info += str(a)
log.err(info)
if self.trainloger is not None:
self.trainloger.info(info)
def dump(self):
"""
Read interesting files, run interesting commands.
"""
if not self.os:
log.err('Cannot dump unknown OS.')
return None
result = {}
# read files
for key, filename in self.os.files.items():
log.debug('Reading %s...' % filename)
result[key] = self.read(filename)
# run commands
for key, command in self.os.commands.items():
log.debug('Trying to run \'%s\'...' % command)
result[key] = self.execute(command)
# run file finders to know which files to read
for key, command in self.os.file_finders.items():
log.debug('Trying to locate files for \'%s\'...' % key)
files = self.execute(command).decode().split()
for f in files:
log.debug(' Reading %s...' % f)
result[key + '_' + re.sub(r'[\\\/:]', '_', f)] = self.read(f)
# determine ownership and permissions of files/directories
# TODO
###
return result
def determine_os(self):
try:
with open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r') as f:
pass
log.info('Determined Linux-based OS.')
return Linux()
except:
traceback.print_exc()
log.err('Could not determine OS.')
return None
def venn(events, *filters):
if len(filters) > 6:
log.err('Too many filters.')
return
filters = {f: Filter.parse(f) for f in filters}
to_show = {
f: set(e for e in events if F.test(e))
for f, F in filters.items()
}
venn_diagram(to_show)
plt.show()
def determine_os(self):
sftp = self.client.open_sftp()
# TODO support hierarchical search (e.g. Linux -> Android)
try:
f = sftp.open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r')
log.info('Determined Linux-based OS.')
return Linux()
except:
traceback.print_exc()
log.err('Could not determine OS.')
finally:
sftp.close()
return None
def create_target(path):
# TODO parse path to determine type
if path.startswith('/'):
return Local(subfolder=(None if len(path) == 1 else path))
else:
# SSH
r = re.match(r'(\w+)@([\w\-.]+):?([\w\/]+)?$', path)
if r:
username, host, subfolder = r.groups()
port = 22 # TODO some support for different
return SSH(host, port, username, subfolder)
else:
log.err('Unsupported target.')
return None
def create_target(path):
# TODO parse path to determine type
if path.startswith('/'):
return Local(subfolder=(None if len(path) == 1 else path))
elif path.startswith('ssh://'):
r = re.match(r'ssh:\/\/(\w+)@([\w\-.]+):?([\w\/]+)?$', path)
if not r:
log.err('Invalid SSH connection string.')
return None
username, host, subfolder = r.groups()
port = 22 # TODO some support for different
return SSH(host, port, username, subfolder)
else:
log.err('Unsupported target.')
return None
def dump(self):
"""
Read interesting files, run interesting commands.
"""
if not self.os:
log.err('Cannot dump unknown OS.')
return None
result = {}
# read files
for key, filename in self.os.files.items():
log.debug('Reading %s...' % filename)
result[key] = self.read(filename)
# run commands
for key, command in self.os.commands.items():
log.debug('Trying to run \'%s\'...' % command)
result[key] = self.execute(command)
return result
def get_map(points, labels, image_size=(1024, 768)):
if not points:
log.err('Cannot show map without point.')
return None
border_const = 0.005
min_lat = min(p[0] for p in points) - border_const
max_lat = max(p[0] for p in points) + border_const
min_lon = min(p[1] for p in points) - border_const
max_lon = max(p[1] for p in points) + border_const
center = ((min_lon + max_lon) / 2, (min_lat + max_lat) / 2)
""" count appropriate zoom """
# https://wiki.openstreetmap.org/wiki/Zoom_levels
if len(points) == 1:
zoom = 18
else:
zoom = 0
tmp_tile_width = 360
while (tmp_tile_width > abs(max_lon - min_lon)
and tmp_tile_width/2 > abs(max_lat - min_lat)):
#print('Zoom', zoom, 'TTW', tmp_tile_width, 'latdiff', abs(max_lat - min_lat), 'londiff', abs(max_lon - min_lon))
tmp_tile_width /= 2
zoom += 1
zoom = min(18, max(1, zoom + 1))
#print('Center:', center)
#print('Zoom:', zoom)
fig = plt.figure(figsize=(20, 15), frameon=False)
ax = plt.subplot(111)
mm = geotiler.Map(center=center, size=image_size,
zoom=zoom, provider='osm')
img = geotiler.render_map(mm)
geo_points = [mm.rev_geocode(p[::-1]) for p in points]
X, Y = zip(*geo_points)
ax.axis('off') # TODO remove border
ax.imshow(img)
ax.scatter(X, Y, marker='p', c='darkgreen',
edgecolor='none', s=500, alpha=0.8)
for x, y, label in zip(X, Y, labels):
#ax.annotate(label, (x, y), (x+5, y-5))
ax.text(x+5, y-5, label, fontsize=30)
# TODO change positioning if overlap is expected
return fig
def __init__(self, host, port, username, subfolder=None):
super().__init__()
self.host = host
self.port = port
self.username = username
self.subfolder = subfolder
# connect to server
import paramiko
password = None
while True:
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
self.client.connect(self.host, port=self.port, username=self.username, password=password, timeout=5)
break
except paramiko.ssh_exception.SSHException as e:
if not password:
import getpass
password = getpass.getpass('Password: '******'No authentication methods available.')
return
except paramiko.ssh_exception.AuthenticationException:
log.err('Authentication failed.')
return
except paramiko.ssh_exception.NoValidConnectionsError:
log.err('Cannot connect to the host over SSH.')
return
except:
traceback.print_exc()
return
log.info('Connected to %s.' % self.host)
self.os = self.determine_os()
def datetime_from_str(string,
only_date=False,
only_time=False,
also_return_type=False):
if type(string) == datetime:
string = datetime_to_str(string)
tries = [
('dt', '%Y-%m-%d %H:%M:%S.%f', not only_date and not only_time),
('dt', '%Y-%m-%d %H:%M:%S', not only_date and not only_time),
('dt', '%Y-%m-%d %H:%M', not only_date and not only_time),
('d', '%Y-%m-%d', not only_time),
('d', '%Y', not only_time),
('t', '%H:%M:%S', not only_date),
('t', '%H:%M', not only_date),
]
for format_type, format_string, condition in tries:
try:
if not condition:
continue
result = datetime.strptime(string, format_string)
""" add reference date if only time is parsed """
if format_type == 't':
reference = ensa.variables['reference_time']
result = result.replace(year=reference.year,
month=reference.month,
day=reference.day)
if also_return_type:
result = (format_type, result)
return result
except:
#traceback.print_exc()
continue
""" no match """
log.err('Cannot parse datetime \'%s\'' %
string) # TODO comment, `tlt` might use this as feature
if also_return_type:
return (None, None)
else:
return None
def parse(string):
"""
x
x == ?
x != ?
x < ?
x <= ?
x > ?
x >= ?
x and y
x or y
not x
x contains ?
x matches ?
(x, y)
()
" "
' '
suspicious x
x: timestamp, score, severity, source, category, message, <attr>
"""
pattern = r'(".*?"|\'.*?\'|==|!=|<=|<|>=|>| or | and |not |\(|\)|contains|matches|suspicious|,)'
parts = [x.strip() for x in re.split(pattern, string) if x.strip()]
#print(parts)
""" for each element compute its level """
unique_levels = set()
bracket_count = 0
levels = []
for part in parts:
if bracket_count < 0:
log.err('Bad bracket order!')
break
if part == '(':
bracket_count += 1
levels.append(-1)
elif part == ')':
bracket_count -= 1
levels.append(-1)
else:
level = bracket_count + (Filter.operators.get(part) or 0.9)
unique_levels.add(level)
levels.append(level)
#print('levels:', levels)
#print('uniq levels:', unique_levels)
if bracket_count:
log.err('No matching brackets!')
""" from topmost level, create objects and put them into a pool on same position """
pool = [None for _ in parts]
for level in sorted(unique_levels, reverse=True):
if Filter.debug_filter:
log.info('Looking for lvl %s elements' % level)
leveled = [(parts[i], i) for i in range(len(parts))
if levels[i] == level]
if Filter.debug_filter:
log.info(' Matching leveled:', leveled)
for part, i in leveled:
""" convert to objects """
operands = []
if Filter.debug_filter:
log.info(' Dealing with "%s" at %d...' % (part, i))
# collapse left of i if greater or equal level, skip None
# same for right
for direction, index_inc, border_cmp, level_cmp in [
('left', -1, lambda x: x >= 0,
lambda x, y: x >= y or x == -1),
('right', 1, lambda x: x < len(parts),
lambda x, y: x > y or x == -1),
]:
collapse_index = i
while True:
collapse_index += index_inc
if not border_cmp(collapse_index):
break
if level_cmp(levels[collapse_index], level):
if pool[collapse_index] is None: # already processed element
continue
if Filter.debug_filter:
log.info(' Collapsing %s...' % direction)
operands.append(
pool[collapse_index]
) # TODO continue with multior or what?
pool[collapse_index] = None
break
else:
break
pool[i] = Filter(part, level, *operands)
if Filter.debug_filter:
log.info(' new pool[%d]: %s' % (i, str(pool[i])))
if Filter.debug_filter:
log.info('pool at the end:', pool)
""" only 1 should be left in the pool (or 0 if no filter) """
remains = list(filter(None, pool))
if len(remains) == 0:
return None
elif len(remains) == 1:
return remains[0]
else:
log.err('Bad filter (not fully collapsed)!')
return None
def run(self):
# comparison? return ids from select
#print(self.x1, self.value, self.x2)
#pdb.set_trace()
try:
if self.value in ('==', '!=', '<', '<=', '>', '>=', 'contains',
'matches'):
to_replace = ('?', '?')
operator = self.value
# contains and matches are slightly different
if self.value == 'contains':
to_replace = ('?', "'%' || ? || '%'")
operator = 'LIKE'
if self.value == 'matches':
operator = 'REGEXP'
if self.x1.value in ('timestamp', 'source', 'category',
'score', 'severity', 'message'):
# run non-attribute queries
return [
x[0] for x in lib.db.query(
lib.rreplace(
Filter.queries[self.x1.value].format(operator),
*to_replace, 1), (self.x2.value, ))
]
else:
# attribute query
return [
x[0] for x in lib.db.query(
lib.rreplace(
Filter.queries['attribute'].format(operator), *
to_replace, 1), (self.x1.value, self.x2.value))
]
# suspicious? use WHERE IN query, return entry ids
if self.value == 'suspicious':
# get values
suspects = Message.suspicious.get(self.x1.value)
if not suspects:
return []
# update placeholders in query, use suspects
return [
x[0] for x in lib.db.query(
Filter.queries['suspicious'].
replace('#', ','.join(
'?'
for _ in suspects)), (self.x1.value, *suspects))
]
# boolean? return intersection/union of ids
elif self.value == 'or':
x1_result = self.x1.run()
x2_result = self.x2.run()
return list(set(x1_result + x2_result))
elif self.value == 'and':
x1_result = set(self.x1.run())
x2_result = set(self.x2.run())
return [x for x in x1_result if x in x2_result]
elif self.value == 'not':
x1_result = self.x1.run()
all_entries = [
x[0] for x in lib.db.query("SELECT entry_id FROM entry")
]
return list(set(all_entries) - set(x1_result))
except:
traceback.print_exc()
log.err('Invalid filter.')
return []
Ensa is collaborative tool for human information management.
"""
from getpass import getpass
import readline
from source import log
from source import lib
from source import ensa
from source import commands
lib.reload_config()
db_password = (ensa.config['db.password'].value or
getpass(log.question('DB password: '******'Cannot connect to DB!')
lib.exit_program(None, None)
rings = ensa.db.get_rings()
if rings:
log.info(
'Welcome! Create new ring using `ra` or choose an '
'existing one with `rs <name>`.')
ring_lens = commands.get_format_len_ring(rings)
log.info('Existing rings:')
for ring in rings:
log.info(' '+commands.format_ring(*ring, *ring_lens))
else:
log.info(
'Welcome! It looks that you do not have any rings created. '
'To do this, use `ra`.')
