def exec_inst_phase2(self, inst: Instruction, t_id: int):
if isinstance(inst, str):
return [None, None]
if inst.action == ACTION.RETRIEVE_BY_ID:
result = self.read_id(inst.table_name, inst.record_id)
return [LogEntry(result, result, inst, t_id), result]
if inst.action == ACTION.RETRIEVE_BY_AREA_CODE:
result = self.read_area_code(inst.table_name, inst.record_id)
return [LogEntry(result, result, inst, t_id), result]
if inst.action == ACTION.WRITE_RECORD:
before_image = self.read_id(inst.table_name, inst.tuple_data.id)
result = self.write(inst.table_name, inst.tuple_data)
after_image = self.read_id(inst.table_name, inst.tuple_data.id)
return [LogEntry(before_image, after_image, inst, t_id), result]
if inst.action == ACTION.DELETE_RECORD:
before_image = self.read_id(inst.table_name, inst.record_id)
result = self.delete_record(inst.table_name, inst.record_id)
after_image = self.read_id(inst.table_name, inst.record_id)
return [LogEntry(before_image, after_image, inst, t_id), result]
if inst.action == ACTION.DELETE_TABLE:
self.mem.flush()
before_image = self.disk.get_table_storage(inst.table_name)
result = self.delete_table(inst.table_name)
return [LogEntry(before_image, None, inst, t_id), result]
return [None, None]
def test_task3(self):
expectedEntries = (
LogEntry('4 12:50 CEG303 561 5065 0'),
LogEntry('4 19:17 CEG308 552 27998 1')
)
result = self.log.findentriesbyday(4)
self.assertEqual(len(expectedEntries), 2)
self.assertEqual(expectedEntries[0], result[0])
def _read_log_lines_then_yield(yielder, first_line):
""" Read all provided log lines from the given yielder. """
first_entry = LogEntry.from_log_string(first_line)
log_entries = [first_entry]
for line in yielder:
log_entry = LogEntry.from_log_string(line)
log_entries.append(log_entry)
ids_entry_dict = IdsConverter().log_entries_to_ids_entries_dict(
log_entries)
for _, app_entries in ids_entry_dict.items():
for ids_entry in app_entries:
yield ids_entry
def run(self):
for line in fileinput.input(self.watch):
l = LogEntry(string = line)
self.logs.append(l)
return self.logs
self.sumarize()
def run(experiment):
log_entries = []
for line in Dir.yield_lines(experiment.file_path, ITEM_LIMIT):
log_entry = LogEntry.from_log_string(line)
log_entries.append(log_entry)
experiment.entries = log_entries
# Exp 1: map
time_before_map = time.time()
OneHotVsMapping.handle_log_entries("MAP", OneHotVsMappingConverter(),
log_entries, experiment)
# Exp 2: one-hot
time_after_map_before_one_hot = time.time()
OneHotVsMapping.handle_log_entries("OHOT", IdsConverter(), log_entries,
experiment)
time_after_all = time.time()
time_for_map = time_after_map_before_one_hot - time_before_map
time_for_one_hot = time_after_all - time_after_map_before_one_hot
timing_lines = [
"Benchmark result | %s entries processed | OneClassSVM classifier"
% len(log_entries), "",
"Mapping: %s" % util.fmtr.format_time_passed(time_for_map),
"One-hot: %s" % util.fmtr.format_time_passed(time_for_one_hot)
]
experiment.add_result_file("time_map_vs_onehot", timing_lines)
def proc_log_string(orig_log_string):
""" Process a line. """
if "COLOUR" not in orig_log_string:
return orig_log_string
log_entry = LogEntry.from_log_string(orig_log_string)
if log_entry.data[LogEntry.APP_ID_FIELD] != "COLOUR":
return orig_log_string
message = log_entry.data[LogEntry.LOG_MESSAGE_FIELD]
r, g, b = [float(x) for x in message.split(",")]
if any([num not in range(0, 256) for num in (r, g, b)]):
raise RuntimeError("FIX THIS SHIT! File corrupted.")
if (r, g, b) in [(150, 140, 200), (170, 250, 140), (120, 180, 130),
(120, 180, 200)]:
assert (log_entry.intrusion == "normal")
return orig_log_string
elif (r, g, b) in [(255, 0, 0), (200, 50, 50), (170, 80, 80)]:
log_entry.set_any(intrusion="red")
new_log_string = log_entry.get_log_string()
return new_log_string
else:
raise NotImplementedError("Unexpected colour in line: %s" %
orig_log_string)
def _run_server(self):
from log_entry import LogEntry
LogEntry(message='Web server started.')
if self.debug:
self._run_wsgi_debug()
else:
self._run_wsgi()
def _run_wsgi(self):
self._setup_server_cert()
import cherrypy.wsgiserver
import cherrypy.wsgiserver.ssl_builtin
from log_entry import LogEntry
logger.info('Starting server...')
if self.auto_start_servers:
from pritunl.server import Server
for server in Server.get_servers():
if server.get_orgs():
server.start()
server = cherrypy.wsgiserver.CherryPyWSGIServer(
(self.bind_addr, self.port), self.app)
server.ssl_adapter = cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter(
self._server_cert_path, self._server_key_path)
try:
server.start()
except (KeyboardInterrupt, SystemExit), exc:
signal.signal(signal.SIGINT, signal.SIG_IGN)
LogEntry(message='Web server stopped.')
self.interrupt = True
logger.info('Stopping server...')
server.stop()
self._close_db()
def test_not_formatted_entry(self):
"""Checks that non formatted input are handled correctly"""
print("********************************")
print("test_not_formatted_entry()")
entryLine = "This is not a log entry!"
logEntry = LogEntry(entryLine)
self.assertFalse(logEntry.parsed)
def force_stop(self):
if not self.status:
return
logger.info('Forcing stop server. %r' % {
'server_id': self.id,
})
_process[self.id].send_signal(signal.SIGKILL)
LogEntry(message='Stopped server "%s".' % self.name)
def stop(self):
if not self.status:
return
logger.debug('Stopping server. %r' % {
'server_id': self.id,
})
_process[self.id].send_signal(signal.SIGINT)
LogEntry(message='Stopped server "%s".' % self.name)
def run(self):
for line in fileinput.input(self.watch):
if line.strip().startswith("ModSecurity"):
l = LogEntry(string=line)
self.logs.append(l)
return self.logs
self.sumarize()
def test_normalize_table_with_double_date(self):
table_dict = {
'projectId': 'p',
'datasetId': 'd',
'tableId': 'table_v1_20180407_201804072200'
}
table_id = LogEntry.normalize_table_id(table_dict)
self.assertEqual("p.d.table_v1_*", table_id)
def reload(self):
if not self.status:
self.start()
logger.debug('Reloading server. %r' % {
'server_id': self.id,
})
self.generate_ca_cert()
_process[self.id].send_signal(signal.SIGUSR1)
LogEntry(message='Reloaded server "%s".' % self.name)
def remove(self):
for server in self.get_servers():
if server.status:
server.stop()
server.remove_org(self.id)
shutil.rmtree(self.path)
LogEntry(message='Deleted organization.')
Event(type=ORGS_UPDATED)
def tail_and_process(self):
for line in self._log_file.tail():
try:
log_entry = LogEntry(line)
for profile in self._profiles.filter(log_entry):
print "got knock attempt: %r" % log_entry
iptables_table, open_duration, port = profile.verify(log_entry)
if not port:
continue
source_ip = log_entry.get_source_ip()
self._port_opener.open(source_ip, port, open_duration, iptables_table)
syslog.syslog("Received authenticated port-knock for port " + str(port) + " from " + source_ip)
except:
syslog.syslog("knocknock skipping unrecognized line: %s" % line[:120])
def test_normalize_sharded_table(self):
table_dict = {
'projectId': 'p',
'datasetId': 'd',
'tableId': 't_20180407'
}
table_id = LogEntry.normalize_table_id(table_dict)
self.assertEqual("p.d.t_*", table_id)
def _initialize(self):
self._create_ssl_conf()
self._cert_request()
self._generate_otp_secret()
self.commit(0600)
self._cert_create()
self._delete_ssl_conf()
LogEntry(message='Created new user.')
Event(type=USERS_UPDATED)
def _get_log_entries_from_file(file_path, limit):
""" Read up to <limit> number of log entries from the given file. """
log_entries = []
for line in Dir.yield_lines(file_path, limit):
log_entries.append(LogEntry.from_log_string(line))
return log_entries
def test_normalize_partition(self):
table_dict = {
'projectId': 'p',
'datasetId': 'd',
'tableId': 't$partition'
}
table_id = LogEntry.normalize_table_id(table_dict)
self.assertEqual("p.d.t", table_id)
def test_drop_old_entries(self):
"""Test the removal of entries older than the monitored period"""
print("********************************")
print("test_drop_old_entries()")
# Create an old entry object in the log
oldEntryStr = self.entryGenerator.generate_entry(datetime.now() -
timedelta(hours=10))
oldEntry = LogEntry(oldEntryStr)
# Create a recent entry
newEntryString = self.entryGenerator.generate_entry(datetime.now())
newEntry = LogEntry(newEntryString)
# Add them manually to the LogHandler
self.logHandler.add_entry(oldEntry)
self.logHandler.add_entry(newEntry)
self.assertEqual(len(self.logHandler.log), 2)
self.logHandler.drop_old_entries()
# Only one of the entries should be left
self.assertEqual(len(self.logHandler.log), 1)
self.assertEqual(self.logHandler.hits, 1)
def _detect_type(first_line):
""" Detect the file type from the first line. """
if first_line == HEADER:
return FileType.IDSE_FILE
try:
_ = LogEntry.from_log_string(first_line)
return FileType.LOG_FILE
except ValueError:
raise ValueError("Invalid file given! Can't parse:\n%s" % first_line)
def __list_entries():
json_files = [
y for x in os.walk("logs")
for y in glob(os.path.join(x[0], '*.json'))
]
for json_file in json_files:
logging.info("Parsing log entries from {}".format(json_file))
with open(json_file) as data_file:
json_dict = json.load(data_file)
for entry in json_dict['entries']:
yield LogEntry(entry)
def test_add_entry(self):
"""Tests adding entries to the LogHandler"""
print("********************************")
print("test_add_entry()")
unparsedEntry = self.entryGenerator.generate_entry(datetime.now())
logEntry = LogEntry(unparsedEntry)
self.logHandler.add_entry(logEntry)
# Check that all the attributes of the entry have been processed
self.assertEqual(self.logHandler.log[0], logEntry)
self.assertEqual(self.logHandler.hits, 1)
self.assertEqual(self.logHandler.size, logEntry.size)
self.assertEqual(self.logHandler.sections, {logEntry.section: 1})
self.assertEqual(self.logHandler.ips, {logEntry.ip: 1})
self.assertEqual(self.logHandler.methods, {logEntry.method: 1})
self.assertEqual(self.logHandler.codes, {logEntry.code: 1})
# Put an entry that is not formatted
# to check if it's correctly dropped
logEntry = LogEntry("This is not a formatted entry\n")
self.logHandler.add_entry(logEntry)
self.assertEqual(len(self.logHandler.log), 1)
self.assertEqual(self.logHandler.hits, 1)
def test_delete_entry(self):
"""Tests deleting entries from the LogHandler"""
print("********************************")
print("test_delete_entry()")
# Add 2 entries and delete the oldest
unparsedEntry = self.entryGenerator.generate_entry(datetime.now())
logEntry1 = LogEntry(unparsedEntry)
unparsedEntry = self.entryGenerator.generate_entry(datetime.now())
logEntry2 = LogEntry(unparsedEntry)
self.logHandler.add_entry(logEntry1)
self.logHandler.add_entry(logEntry2)
self.logHandler.delete_entry() # logEntry1 should be deleted (oldest)
# Check that only logEntry2 is left
self.assertEqual(self.logHandler.log[0], logEntry2)
self.assertEqual(self.logHandler.hits, 1)
self.assertEqual(self.logHandler.size, logEntry2.size)
self.assertEqual(self.logHandler.sections, {logEntry2.section: 1})
self.assertEqual(self.logHandler.ips, {logEntry2.ip: 1})
self.assertEqual(self.logHandler.methods, {logEntry2.method: 1})
self.assertEqual(self.logHandler.codes, {logEntry2.code: 1})
def convert_log_entries(simulation_output):
"""Converts a list of log entries to LogEntry class instances and saves objects in list."""
log_entry_list = list()
for line in simulation_output:
level = line[0]
timestamp = convert_log_timestamp(line)
ip_addr_host = line[3]
program = line[4]
message = ' '.join(line[5:])
ip_addr_external = analyze_log_message_ip(message)
log_entry_list.append(LogEntry(level, timestamp, ip_addr_host, program, message, ip_addr_external))
print('Log entries converted')
return log_entry_list
def _initialize(self):
logger.info('Initialize new server. %r' % {
'server_id': self.id,
})
os.makedirs(os.path.join(self.path, TEMP_DIR))
try:
self._generate_dh_param()
self.commit()
LogEntry(message='Created new server.')
except:
logger.exception('Failed to create server. %r' % {
'server_id': self.id,
})
shutil.rmtree(self.path)
raise
def reservoir_sample_limit(item_generator, sample_size, limit_to):
"""
Sample with 'Reservoir Sampling' from the given generator the given number of elements.
*limit_to: List of data types to limit to.
"""
if any([l not in ids_data.get_app_ids() for l in limit_to]):
raise ValueError("Given limits are invalid: %s" % limit_to)
limited_generator = (
line for line in item_generator
# Convert line to LogEntry, check that app_id is in the allowed limits
if log_entry_to_app_id(LogEntry.from_log_string(line)) in limit_to)
return reservoir_sample(limited_generator, sample_size)
def remove(self):
logger.info('Removing server. %r' % {
'server_id': self.id,
})
if self.status:
self.force_stop()
for i in xrange(20):
if not self.status:
break
time.sleep(0.1)
if self.status:
self.force_stop()
time.sleep(0.5)
self._remove_primary_user()
shutil.rmtree(self.path)
LogEntry(message='Deleted server.')
Event(type=SERVERS_UPDATED)
def test_log_entry(self):
"""Tests creation of a LogEntry instance from one line of the log"""
print("********************************")
print("test_log_entry()")
now = datetime.now()
now = datetime(now.year, now.month, now.day, now.hour, now.minute,
now.second)
# Truncate current datetime to remove microseconds
# (for the test to succeed)
entryLine = '127.0.0.1 - - [%s +1000] "GET /icons/blank.gif HTTP/1.1" \
403 301' % now.strftime("%d/%b/%Y:%H:%M:%S")
logEntry = LogEntry(entryLine)
self.assertEqual(logEntry.ip, "127.0.0.1")
self.assertEqual(logEntry.time, now)
self.assertEqual(logEntry.method, "GET")
self.assertEqual(logEntry.section, "icons")
self.assertEqual(logEntry.size, 301)
def generate_log_entries(number):
""" Generate <number> LogEntry objects. """
result = []
vins = [
chr(random.choice(range(65, 91))) + str(x)
for x in random.sample(range(100000, 900000), int(number))
]
colr_gen = lambda: random.randint(0, 255)
tsp_gen = lambda: random.randint(0, 499)
log_msg_gens = [
(ids_data.get_generators(), lambda: str(float(random.randint(-3, 2)))),
(ids_data.get_colours(),
lambda: "{},{},{}".format(colr_gen(), colr_gen(), colr_gen())),
(ids_data.POSE_CC, lambda: random.choice(["DE", "AT", "CH", "FR"])),
(ids_data.POSE_POI, (lambda: random.choice(ids_data.get_poi_types()) +
"," + random.choice(ids_data.get_poi_results()))),
(ids_data.POSE_TSP, lambda: "{},{},{},{}".format(
tsp_gen(), tsp_gen(), tsp_gen(), tsp_gen()))
]
for i in range(0, int(number)):
vin = vins[i]
app_id = random.choice(ids_data.get_app_ids())
level = random.choice(ids_data.get_levels())
gps_position = "{},{}".format(tsp_gen(), tsp_gen())
log_message = None
for keys, gen in log_msg_gens:
if app_id in keys:
log_message = gen()
if not log_message:
raise ValueError("You suck!")
intrusion = random.choice(ids_data.get_labels())
result.append(
LogEntry(vin=vin,
app_id=app_id,
level=level,
gps_position=gps_position,
log_message=log_message,
intrusion=intrusion))
return result
