def normalize_datetime(time=None, silent=False): if not time: return datetime.now() if type(time) == str: variant = 0 while True: try: return dateutil_parse(time).replace(tzinfo=None) except ValueError: variant += 1 if variant == 1: # apache-access format time = time.replace(':', ' ', 1) else: if not silent: log.err(' Cannot unify datetime:', time) return None elif type(time) == datetime: return time.replace(tzinfo=None) elif type(time) in (int, float): # probably UNIX timestamp return datetime.fromtimestamp(time) else: if not silent: log.err(' Unknown time type:', time, type(time)) return None
def __err(self, *args): info = 'entityRecognize_pytorch ' for a in args: info += str(a) log.err(info) if self.trainloger is not None: self.trainloger.info(info)
def dump(self): """ Read interesting files, run interesting commands. """ if not self.os: log.err('Cannot dump unknown OS.') return None result = {} # read files for key, filename in self.os.files.items(): log.debug('Reading %s...' % filename) result[key] = self.read(filename) # run commands for key, command in self.os.commands.items(): log.debug('Trying to run \'%s\'...' % command) result[key] = self.execute(command) # run file finders to know which files to read for key, command in self.os.file_finders.items(): log.debug('Trying to locate files for \'%s\'...' % key) files = self.execute(command).decode().split() for f in files: log.debug(' Reading %s...' % f) result[key + '_' + re.sub(r'[\\\/:]', '_', f)] = self.read(f) # determine ownership and permissions of files/directories # TODO ### return result
def determine_os(self): try: with open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r') as f: pass log.info('Determined Linux-based OS.') return Linux() except: traceback.print_exc() log.err('Could not determine OS.') return None
def venn(events, *filters): if len(filters) > 6: log.err('Too many filters.') return filters = {f: Filter.parse(f) for f in filters} to_show = { f: set(e for e in events if F.test(e)) for f, F in filters.items() } venn_diagram(to_show) plt.show()
def determine_os(self): sftp = self.client.open_sftp() # TODO support hierarchical search (e.g. Linux -> Android) try: f = sftp.open(os.path.join(self.subfolder or '/', 'etc/passwd'), 'r') log.info('Determined Linux-based OS.') return Linux() except: traceback.print_exc() log.err('Could not determine OS.') finally: sftp.close() return None
def create_target(path): # TODO parse path to determine type if path.startswith('/'): return Local(subfolder=(None if len(path) == 1 else path)) else: # SSH r = re.match(r'(\w+)@([\w\-.]+):?([\w\/]+)?$', path) if r: username, host, subfolder = r.groups() port = 22 # TODO some support for different return SSH(host, port, username, subfolder) else: log.err('Unsupported target.') return None
def create_target(path): # TODO parse path to determine type if path.startswith('/'): return Local(subfolder=(None if len(path) == 1 else path)) elif path.startswith('ssh://'): r = re.match(r'ssh:\/\/(\w+)@([\w\-.]+):?([\w\/]+)?$', path) if not r: log.err('Invalid SSH connection string.') return None username, host, subfolder = r.groups() port = 22 # TODO some support for different return SSH(host, port, username, subfolder) else: log.err('Unsupported target.') return None
def dump(self): """ Read interesting files, run interesting commands. """ if not self.os: log.err('Cannot dump unknown OS.') return None result = {} # read files for key, filename in self.os.files.items(): log.debug('Reading %s...' % filename) result[key] = self.read(filename) # run commands for key, command in self.os.commands.items(): log.debug('Trying to run \'%s\'...' % command) result[key] = self.execute(command) return result
def get_map(points, labels, image_size=(1024, 768)): if not points: log.err('Cannot show map without point.') return None border_const = 0.005 min_lat = min(p[0] for p in points) - border_const max_lat = max(p[0] for p in points) + border_const min_lon = min(p[1] for p in points) - border_const max_lon = max(p[1] for p in points) + border_const center = ((min_lon + max_lon) / 2, (min_lat + max_lat) / 2) """ count appropriate zoom """ # https://wiki.openstreetmap.org/wiki/Zoom_levels if len(points) == 1: zoom = 18 else: zoom = 0 tmp_tile_width = 360 while (tmp_tile_width > abs(max_lon - min_lon) and tmp_tile_width/2 > abs(max_lat - min_lat)): #print('Zoom', zoom, 'TTW', tmp_tile_width, 'latdiff', abs(max_lat - min_lat), 'londiff', abs(max_lon - min_lon)) tmp_tile_width /= 2 zoom += 1 zoom = min(18, max(1, zoom + 1)) #print('Center:', center) #print('Zoom:', zoom) fig = plt.figure(figsize=(20, 15), frameon=False) ax = plt.subplot(111) mm = geotiler.Map(center=center, size=image_size, zoom=zoom, provider='osm') img = geotiler.render_map(mm) geo_points = [mm.rev_geocode(p[::-1]) for p in points] X, Y = zip(*geo_points) ax.axis('off') # TODO remove border ax.imshow(img) ax.scatter(X, Y, marker='p', c='darkgreen', edgecolor='none', s=500, alpha=0.8) for x, y, label in zip(X, Y, labels): #ax.annotate(label, (x, y), (x+5, y-5)) ax.text(x+5, y-5, label, fontsize=30) # TODO change positioning if overlap is expected return fig
def __init__(self, host, port, username, subfolder=None): super().__init__() self.host = host self.port = port self.username = username self.subfolder = subfolder # connect to server import paramiko password = None while True: self.client = paramiko.SSHClient() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: self.client.connect(self.host, port=self.port, username=self.username, password=password, timeout=5) break except paramiko.ssh_exception.SSHException as e: if not password: import getpass password = getpass.getpass('Password: '******'No authentication methods available.') return except paramiko.ssh_exception.AuthenticationException: log.err('Authentication failed.') return except paramiko.ssh_exception.NoValidConnectionsError: log.err('Cannot connect to the host over SSH.') return except: traceback.print_exc() return log.info('Connected to %s.' % self.host) self.os = self.determine_os()
def datetime_from_str(string, only_date=False, only_time=False, also_return_type=False): if type(string) == datetime: string = datetime_to_str(string) tries = [ ('dt', '%Y-%m-%d %H:%M:%S.%f', not only_date and not only_time), ('dt', '%Y-%m-%d %H:%M:%S', not only_date and not only_time), ('dt', '%Y-%m-%d %H:%M', not only_date and not only_time), ('d', '%Y-%m-%d', not only_time), ('d', '%Y', not only_time), ('t', '%H:%M:%S', not only_date), ('t', '%H:%M', not only_date), ] for format_type, format_string, condition in tries: try: if not condition: continue result = datetime.strptime(string, format_string) """ add reference date if only time is parsed """ if format_type == 't': reference = ensa.variables['reference_time'] result = result.replace(year=reference.year, month=reference.month, day=reference.day) if also_return_type: result = (format_type, result) return result except: #traceback.print_exc() continue """ no match """ log.err('Cannot parse datetime \'%s\'' % string) # TODO comment, `tlt` might use this as feature if also_return_type: return (None, None) else: return None
def parse(string): """ x x == ? x != ? x < ? x <= ? x > ? x >= ? x and y x or y not x x contains ? x matches ? (x, y) () " " ' ' suspicious x x: timestamp, score, severity, source, category, message, <attr> """ pattern = r'(".*?"|\'.*?\'|==|!=|<=|<|>=|>| or | and |not |\(|\)|contains|matches|suspicious|,)' parts = [x.strip() for x in re.split(pattern, string) if x.strip()] #print(parts) """ for each element compute its level """ unique_levels = set() bracket_count = 0 levels = [] for part in parts: if bracket_count < 0: log.err('Bad bracket order!') break if part == '(': bracket_count += 1 levels.append(-1) elif part == ')': bracket_count -= 1 levels.append(-1) else: level = bracket_count + (Filter.operators.get(part) or 0.9) unique_levels.add(level) levels.append(level) #print('levels:', levels) #print('uniq levels:', unique_levels) if bracket_count: log.err('No matching brackets!') """ from topmost level, create objects and put them into a pool on same position """ pool = [None for _ in parts] for level in sorted(unique_levels, reverse=True): if Filter.debug_filter: log.info('Looking for lvl %s elements' % level) leveled = [(parts[i], i) for i in range(len(parts)) if levels[i] == level] if Filter.debug_filter: log.info(' Matching leveled:', leveled) for part, i in leveled: """ convert to objects """ operands = [] if Filter.debug_filter: log.info(' Dealing with "%s" at %d...' % (part, i)) # collapse left of i if greater or equal level, skip None # same for right for direction, index_inc, border_cmp, level_cmp in [ ('left', -1, lambda x: x >= 0, lambda x, y: x >= y or x == -1), ('right', 1, lambda x: x < len(parts), lambda x, y: x > y or x == -1), ]: collapse_index = i while True: collapse_index += index_inc if not border_cmp(collapse_index): break if level_cmp(levels[collapse_index], level): if pool[collapse_index] is None: # already processed element continue if Filter.debug_filter: log.info(' Collapsing %s...' % direction) operands.append( pool[collapse_index] ) # TODO continue with multior or what? pool[collapse_index] = None break else: break pool[i] = Filter(part, level, *operands) if Filter.debug_filter: log.info(' new pool[%d]: %s' % (i, str(pool[i]))) if Filter.debug_filter: log.info('pool at the end:', pool) """ only 1 should be left in the pool (or 0 if no filter) """ remains = list(filter(None, pool)) if len(remains) == 0: return None elif len(remains) == 1: return remains[0] else: log.err('Bad filter (not fully collapsed)!') return None
def run(self): # comparison? return ids from select #print(self.x1, self.value, self.x2) #pdb.set_trace() try: if self.value in ('==', '!=', '<', '<=', '>', '>=', 'contains', 'matches'): to_replace = ('?', '?') operator = self.value # contains and matches are slightly different if self.value == 'contains': to_replace = ('?', "'%' || ? || '%'") operator = 'LIKE' if self.value == 'matches': operator = 'REGEXP' if self.x1.value in ('timestamp', 'source', 'category', 'score', 'severity', 'message'): # run non-attribute queries return [ x[0] for x in lib.db.query( lib.rreplace( Filter.queries[self.x1.value].format(operator), *to_replace, 1), (self.x2.value, )) ] else: # attribute query return [ x[0] for x in lib.db.query( lib.rreplace( Filter.queries['attribute'].format(operator), * to_replace, 1), (self.x1.value, self.x2.value)) ] # suspicious? use WHERE IN query, return entry ids if self.value == 'suspicious': # get values suspects = Message.suspicious.get(self.x1.value) if not suspects: return [] # update placeholders in query, use suspects return [ x[0] for x in lib.db.query( Filter.queries['suspicious']. replace('#', ','.join( '?' for _ in suspects)), (self.x1.value, *suspects)) ] # boolean? return intersection/union of ids elif self.value == 'or': x1_result = self.x1.run() x2_result = self.x2.run() return list(set(x1_result + x2_result)) elif self.value == 'and': x1_result = set(self.x1.run()) x2_result = set(self.x2.run()) return [x for x in x1_result if x in x2_result] elif self.value == 'not': x1_result = self.x1.run() all_entries = [ x[0] for x in lib.db.query("SELECT entry_id FROM entry") ] return list(set(all_entries) - set(x1_result)) except: traceback.print_exc() log.err('Invalid filter.') return []
Ensa is collaborative tool for human information management. """ from getpass import getpass import readline from source import log from source import lib from source import ensa from source import commands lib.reload_config() db_password = (ensa.config['db.password'].value or getpass(log.question('DB password: '******'Cannot connect to DB!') lib.exit_program(None, None) rings = ensa.db.get_rings() if rings: log.info( 'Welcome! Create new ring using `ra` or choose an ' 'existing one with `rs <name>`.') ring_lens = commands.get_format_len_ring(rings) log.info('Existing rings:') for ring in rings: log.info(' '+commands.format_ring(*ring, *ring_lens)) else: log.info( 'Welcome! It looks that you do not have any rings created. ' 'To do this, use `ra`.')