def exec_inst_phase2(self, inst: Instruction, t_id: int): if isinstance(inst, str): return [None, None] if inst.action == ACTION.RETRIEVE_BY_ID: result = self.read_id(inst.table_name, inst.record_id) return [LogEntry(result, result, inst, t_id), result] if inst.action == ACTION.RETRIEVE_BY_AREA_CODE: result = self.read_area_code(inst.table_name, inst.record_id) return [LogEntry(result, result, inst, t_id), result] if inst.action == ACTION.WRITE_RECORD: before_image = self.read_id(inst.table_name, inst.tuple_data.id) result = self.write(inst.table_name, inst.tuple_data) after_image = self.read_id(inst.table_name, inst.tuple_data.id) return [LogEntry(before_image, after_image, inst, t_id), result] if inst.action == ACTION.DELETE_RECORD: before_image = self.read_id(inst.table_name, inst.record_id) result = self.delete_record(inst.table_name, inst.record_id) after_image = self.read_id(inst.table_name, inst.record_id) return [LogEntry(before_image, after_image, inst, t_id), result] if inst.action == ACTION.DELETE_TABLE: self.mem.flush() before_image = self.disk.get_table_storage(inst.table_name) result = self.delete_table(inst.table_name) return [LogEntry(before_image, None, inst, t_id), result] return [None, None]
def test_task3(self): expectedEntries = ( LogEntry('4 12:50 CEG303 561 5065 0'), LogEntry('4 19:17 CEG308 552 27998 1') ) result = self.log.findentriesbyday(4) self.assertEqual(len(expectedEntries), 2) self.assertEqual(expectedEntries[0], result[0])
def _read_log_lines_then_yield(yielder, first_line): """ Read all provided log lines from the given yielder. """ first_entry = LogEntry.from_log_string(first_line) log_entries = [first_entry] for line in yielder: log_entry = LogEntry.from_log_string(line) log_entries.append(log_entry) ids_entry_dict = IdsConverter().log_entries_to_ids_entries_dict( log_entries) for _, app_entries in ids_entry_dict.items(): for ids_entry in app_entries: yield ids_entry
def run(self): for line in fileinput.input(self.watch): l = LogEntry(string = line) self.logs.append(l) return self.logs self.sumarize()
def run(experiment): log_entries = [] for line in Dir.yield_lines(experiment.file_path, ITEM_LIMIT): log_entry = LogEntry.from_log_string(line) log_entries.append(log_entry) experiment.entries = log_entries # Exp 1: map time_before_map = time.time() OneHotVsMapping.handle_log_entries("MAP", OneHotVsMappingConverter(), log_entries, experiment) # Exp 2: one-hot time_after_map_before_one_hot = time.time() OneHotVsMapping.handle_log_entries("OHOT", IdsConverter(), log_entries, experiment) time_after_all = time.time() time_for_map = time_after_map_before_one_hot - time_before_map time_for_one_hot = time_after_all - time_after_map_before_one_hot timing_lines = [ "Benchmark result | %s entries processed | OneClassSVM classifier" % len(log_entries), "", "Mapping: %s" % util.fmtr.format_time_passed(time_for_map), "One-hot: %s" % util.fmtr.format_time_passed(time_for_one_hot) ] experiment.add_result_file("time_map_vs_onehot", timing_lines)
def proc_log_string(orig_log_string): """ Process a line. """ if "COLOUR" not in orig_log_string: return orig_log_string log_entry = LogEntry.from_log_string(orig_log_string) if log_entry.data[LogEntry.APP_ID_FIELD] != "COLOUR": return orig_log_string message = log_entry.data[LogEntry.LOG_MESSAGE_FIELD] r, g, b = [float(x) for x in message.split(",")] if any([num not in range(0, 256) for num in (r, g, b)]): raise RuntimeError("FIX THIS SHIT! File corrupted.") if (r, g, b) in [(150, 140, 200), (170, 250, 140), (120, 180, 130), (120, 180, 200)]: assert (log_entry.intrusion == "normal") return orig_log_string elif (r, g, b) in [(255, 0, 0), (200, 50, 50), (170, 80, 80)]: log_entry.set_any(intrusion="red") new_log_string = log_entry.get_log_string() return new_log_string else: raise NotImplementedError("Unexpected colour in line: %s" % orig_log_string)
def _run_server(self): from log_entry import LogEntry LogEntry(message='Web server started.') if self.debug: self._run_wsgi_debug() else: self._run_wsgi()
def _run_wsgi(self): self._setup_server_cert() import cherrypy.wsgiserver import cherrypy.wsgiserver.ssl_builtin from log_entry import LogEntry logger.info('Starting server...') if self.auto_start_servers: from pritunl.server import Server for server in Server.get_servers(): if server.get_orgs(): server.start() server = cherrypy.wsgiserver.CherryPyWSGIServer( (self.bind_addr, self.port), self.app) server.ssl_adapter = cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter( self._server_cert_path, self._server_key_path) try: server.start() except (KeyboardInterrupt, SystemExit), exc: signal.signal(signal.SIGINT, signal.SIG_IGN) LogEntry(message='Web server stopped.') self.interrupt = True logger.info('Stopping server...') server.stop() self._close_db()
def test_not_formatted_entry(self): """Checks that non formatted input are handled correctly""" print("********************************") print("test_not_formatted_entry()") entryLine = "This is not a log entry!" logEntry = LogEntry(entryLine) self.assertFalse(logEntry.parsed)
def force_stop(self): if not self.status: return logger.info('Forcing stop server. %r' % { 'server_id': self.id, }) _process[self.id].send_signal(signal.SIGKILL) LogEntry(message='Stopped server "%s".' % self.name)
def stop(self): if not self.status: return logger.debug('Stopping server. %r' % { 'server_id': self.id, }) _process[self.id].send_signal(signal.SIGINT) LogEntry(message='Stopped server "%s".' % self.name)
def run(self): for line in fileinput.input(self.watch): if line.strip().startswith("ModSecurity"): l = LogEntry(string=line) self.logs.append(l) return self.logs self.sumarize()
def test_normalize_table_with_double_date(self): table_dict = { 'projectId': 'p', 'datasetId': 'd', 'tableId': 'table_v1_20180407_201804072200' } table_id = LogEntry.normalize_table_id(table_dict) self.assertEqual("p.d.table_v1_*", table_id)
def reload(self): if not self.status: self.start() logger.debug('Reloading server. %r' % { 'server_id': self.id, }) self.generate_ca_cert() _process[self.id].send_signal(signal.SIGUSR1) LogEntry(message='Reloaded server "%s".' % self.name)
def remove(self): for server in self.get_servers(): if server.status: server.stop() server.remove_org(self.id) shutil.rmtree(self.path) LogEntry(message='Deleted organization.') Event(type=ORGS_UPDATED)
def tail_and_process(self): for line in self._log_file.tail(): try: log_entry = LogEntry(line) for profile in self._profiles.filter(log_entry): print "got knock attempt: %r" % log_entry iptables_table, open_duration, port = profile.verify(log_entry) if not port: continue source_ip = log_entry.get_source_ip() self._port_opener.open(source_ip, port, open_duration, iptables_table) syslog.syslog("Received authenticated port-knock for port " + str(port) + " from " + source_ip) except: syslog.syslog("knocknock skipping unrecognized line: %s" % line[:120])
def test_normalize_sharded_table(self): table_dict = { 'projectId': 'p', 'datasetId': 'd', 'tableId': 't_20180407' } table_id = LogEntry.normalize_table_id(table_dict) self.assertEqual("p.d.t_*", table_id)
def _initialize(self): self._create_ssl_conf() self._cert_request() self._generate_otp_secret() self.commit(0600) self._cert_create() self._delete_ssl_conf() LogEntry(message='Created new user.') Event(type=USERS_UPDATED)
def _get_log_entries_from_file(file_path, limit): """ Read up to <limit> number of log entries from the given file. """ log_entries = [] for line in Dir.yield_lines(file_path, limit): log_entries.append(LogEntry.from_log_string(line)) return log_entries
def test_normalize_partition(self): table_dict = { 'projectId': 'p', 'datasetId': 'd', 'tableId': 't$partition' } table_id = LogEntry.normalize_table_id(table_dict) self.assertEqual("p.d.t", table_id)
def test_drop_old_entries(self): """Test the removal of entries older than the monitored period""" print("********************************") print("test_drop_old_entries()") # Create an old entry object in the log oldEntryStr = self.entryGenerator.generate_entry(datetime.now() - timedelta(hours=10)) oldEntry = LogEntry(oldEntryStr) # Create a recent entry newEntryString = self.entryGenerator.generate_entry(datetime.now()) newEntry = LogEntry(newEntryString) # Add them manually to the LogHandler self.logHandler.add_entry(oldEntry) self.logHandler.add_entry(newEntry) self.assertEqual(len(self.logHandler.log), 2) self.logHandler.drop_old_entries() # Only one of the entries should be left self.assertEqual(len(self.logHandler.log), 1) self.assertEqual(self.logHandler.hits, 1)
def _detect_type(first_line): """ Detect the file type from the first line. """ if first_line == HEADER: return FileType.IDSE_FILE try: _ = LogEntry.from_log_string(first_line) return FileType.LOG_FILE except ValueError: raise ValueError("Invalid file given! Can't parse:\n%s" % first_line)
def __list_entries(): json_files = [ y for x in os.walk("logs") for y in glob(os.path.join(x[0], '*.json')) ] for json_file in json_files: logging.info("Parsing log entries from {}".format(json_file)) with open(json_file) as data_file: json_dict = json.load(data_file) for entry in json_dict['entries']: yield LogEntry(entry)
def test_add_entry(self): """Tests adding entries to the LogHandler""" print("********************************") print("test_add_entry()") unparsedEntry = self.entryGenerator.generate_entry(datetime.now()) logEntry = LogEntry(unparsedEntry) self.logHandler.add_entry(logEntry) # Check that all the attributes of the entry have been processed self.assertEqual(self.logHandler.log[0], logEntry) self.assertEqual(self.logHandler.hits, 1) self.assertEqual(self.logHandler.size, logEntry.size) self.assertEqual(self.logHandler.sections, {logEntry.section: 1}) self.assertEqual(self.logHandler.ips, {logEntry.ip: 1}) self.assertEqual(self.logHandler.methods, {logEntry.method: 1}) self.assertEqual(self.logHandler.codes, {logEntry.code: 1}) # Put an entry that is not formatted # to check if it's correctly dropped logEntry = LogEntry("This is not a formatted entry\n") self.logHandler.add_entry(logEntry) self.assertEqual(len(self.logHandler.log), 1) self.assertEqual(self.logHandler.hits, 1)
def test_delete_entry(self): """Tests deleting entries from the LogHandler""" print("********************************") print("test_delete_entry()") # Add 2 entries and delete the oldest unparsedEntry = self.entryGenerator.generate_entry(datetime.now()) logEntry1 = LogEntry(unparsedEntry) unparsedEntry = self.entryGenerator.generate_entry(datetime.now()) logEntry2 = LogEntry(unparsedEntry) self.logHandler.add_entry(logEntry1) self.logHandler.add_entry(logEntry2) self.logHandler.delete_entry() # logEntry1 should be deleted (oldest) # Check that only logEntry2 is left self.assertEqual(self.logHandler.log[0], logEntry2) self.assertEqual(self.logHandler.hits, 1) self.assertEqual(self.logHandler.size, logEntry2.size) self.assertEqual(self.logHandler.sections, {logEntry2.section: 1}) self.assertEqual(self.logHandler.ips, {logEntry2.ip: 1}) self.assertEqual(self.logHandler.methods, {logEntry2.method: 1}) self.assertEqual(self.logHandler.codes, {logEntry2.code: 1})
def convert_log_entries(simulation_output): """Converts a list of log entries to LogEntry class instances and saves objects in list.""" log_entry_list = list() for line in simulation_output: level = line[0] timestamp = convert_log_timestamp(line) ip_addr_host = line[3] program = line[4] message = ' '.join(line[5:]) ip_addr_external = analyze_log_message_ip(message) log_entry_list.append(LogEntry(level, timestamp, ip_addr_host, program, message, ip_addr_external)) print('Log entries converted') return log_entry_list
def _initialize(self): logger.info('Initialize new server. %r' % { 'server_id': self.id, }) os.makedirs(os.path.join(self.path, TEMP_DIR)) try: self._generate_dh_param() self.commit() LogEntry(message='Created new server.') except: logger.exception('Failed to create server. %r' % { 'server_id': self.id, }) shutil.rmtree(self.path) raise
def reservoir_sample_limit(item_generator, sample_size, limit_to): """ Sample with 'Reservoir Sampling' from the given generator the given number of elements. *limit_to: List of data types to limit to. """ if any([l not in ids_data.get_app_ids() for l in limit_to]): raise ValueError("Given limits are invalid: %s" % limit_to) limited_generator = ( line for line in item_generator # Convert line to LogEntry, check that app_id is in the allowed limits if log_entry_to_app_id(LogEntry.from_log_string(line)) in limit_to) return reservoir_sample(limited_generator, sample_size)
def remove(self): logger.info('Removing server. %r' % { 'server_id': self.id, }) if self.status: self.force_stop() for i in xrange(20): if not self.status: break time.sleep(0.1) if self.status: self.force_stop() time.sleep(0.5) self._remove_primary_user() shutil.rmtree(self.path) LogEntry(message='Deleted server.') Event(type=SERVERS_UPDATED)
def test_log_entry(self): """Tests creation of a LogEntry instance from one line of the log""" print("********************************") print("test_log_entry()") now = datetime.now() now = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second) # Truncate current datetime to remove microseconds # (for the test to succeed) entryLine = '127.0.0.1 - - [%s +1000] "GET /icons/blank.gif HTTP/1.1" \ 403 301' % now.strftime("%d/%b/%Y:%H:%M:%S") logEntry = LogEntry(entryLine) self.assertEqual(logEntry.ip, "127.0.0.1") self.assertEqual(logEntry.time, now) self.assertEqual(logEntry.method, "GET") self.assertEqual(logEntry.section, "icons") self.assertEqual(logEntry.size, 301)
def generate_log_entries(number): """ Generate <number> LogEntry objects. """ result = [] vins = [ chr(random.choice(range(65, 91))) + str(x) for x in random.sample(range(100000, 900000), int(number)) ] colr_gen = lambda: random.randint(0, 255) tsp_gen = lambda: random.randint(0, 499) log_msg_gens = [ (ids_data.get_generators(), lambda: str(float(random.randint(-3, 2)))), (ids_data.get_colours(), lambda: "{},{},{}".format(colr_gen(), colr_gen(), colr_gen())), (ids_data.POSE_CC, lambda: random.choice(["DE", "AT", "CH", "FR"])), (ids_data.POSE_POI, (lambda: random.choice(ids_data.get_poi_types()) + "," + random.choice(ids_data.get_poi_results()))), (ids_data.POSE_TSP, lambda: "{},{},{},{}".format( tsp_gen(), tsp_gen(), tsp_gen(), tsp_gen())) ] for i in range(0, int(number)): vin = vins[i] app_id = random.choice(ids_data.get_app_ids()) level = random.choice(ids_data.get_levels()) gps_position = "{},{}".format(tsp_gen(), tsp_gen()) log_message = None for keys, gen in log_msg_gens: if app_id in keys: log_message = gen() if not log_message: raise ValueError("You suck!") intrusion = random.choice(ids_data.get_labels()) result.append( LogEntry(vin=vin, app_id=app_id, level=level, gps_position=gps_position, log_message=log_message, intrusion=intrusion)) return result