def range_by(self, start_tsc: int, end_tsc: int, ltsc: bool) -> 'LiteLog': ''' Find log entries whose tsc: start_tsc < tsc < end_tsc When left_one_more is True, add one more log whose tsc may be less then start_tsc ltsc = true : we compare with the starting time-stamp ltsc = false: we compare with the finishing time-stamp ''' left_key = LogEntry.TscCompare(start_tsc, ltsc) right_key = LogEntry.TscCompare(end_tsc, ltsc) left_index = bisect.bisect_right(self.log_list_, left_key) right_index = bisect.bisect_left(self.log_list_, right_key) ''' if left_one_more: if left_index > 0: left_index -= 1 if right_one_less: if right_index > left_index: right_index -= 1 ''' log = LiteLog() log.log_list_ = self.log_list_[left_index: right_index] return log
def get(self, action): if action == 'unblock_ips': BlockedIP.remove_outdated() self.response.out.write('Done!') elif action == 'checkjobs': WatchJob.check_all_jobs() self.response.out.write('Done!') if action == 'log_cleanup': LogEntry.cleanup() self.response.out.write('Done!')
def ingestLogFile(self): if self.validated and not self.ingested: logEntries = list() date = datetime.strptime( time.ctime(os.path.getctime(self.filename)), "%a %b %d %H:%M:%S %Y") date = date.strftime("%m/%d/%Y %I:%M %p") if date[0] == "0": date = date[1:] firstHalf = date[:date.index(" ") + 1] secondHalf = date[date.index(" ") + 1:] if secondHalf[0] == "0": secondHalf = secondHalf[1:] date = firstHalf + secondHalf lineNumber = 0 for line in self.lines: logEntry = LogEntry() logEntry.date = date logEntry.description = line logEntry.creator = self.creator logEntry.eventType = self.eventType logEntry.artifact = self.filename logEntry.lineNumber = lineNumber logEntry.id = logEntry.artifact + "_" + str( logEntry.lineNumber) logEntries.append(logEntry) lineNumber += 1 self.ingested = True return logEntries return None
def save_entry(entry, log_entry_object): if entry["light_reading"] == "NaN": return False new_entry = LogEntry(**entry) try: new_entry.save() except sqlite3.IntegrityError: if args.show_errors: print("Entry skipped: {}".format(entry["timestamp"])) return False return True
def check(self): # check if job is overdue if self.last_seen + timedelta(minutes=self.interval) < datetime.utcnow(): self.status = 'offline' LogEntry.log_event(self.key(), 'Error', 'job is overdue') # perform all actions for action_key in self.timeout_actions: try: db.get(action_key).perform_action() except Exception as exp: logging.error('Error executing timeout action: ' + str(exp)) self.put()
def tailAndProcess(self): for line in self.logFile.tail(): try: logEntry = LogEntry(line) profile = self.profiles.getProfileForPort(logEntry.getDestinationPort()) if (profile != None): try: ciphertext = logEntry.getEncryptedData() port = profile.decrypt(ciphertext, self.config.getWindow()) sourceIP = logEntry.getSourceIP() self.portOpener.open(sourceIP, port) syslog.syslog("Received authenticated port-knock for port " + str(port) + " from " + sourceIP) #knock-ack profile = self.getProfile(sourceIP) syslog.syslog("Profile obtained for" + sourceIP) packetData = profile.encrypt(port) syslog.syslog("Payload for knock-ack prepared") knockPort = profile.getKnockPort() (idField, seqField, ackField, winField) = unpack('!HIIH', packetData) hping = existsInPath("hping3") if hping is None: syslog.syslog("Error, you must install hping3 first.") sys.exit(2) command = [hping, "-S", "-c", "1", "-p", str(knockPort), "-N", str(idField), "-w", str(winField), "-M", str(seqField), "-L", str(ackField), host] try: subprocess.call(command, shell=False, stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT) syslog.syslog("Knock-ack sent for port " + str(port) + " to " + sourceIP) #print "Knock-ack sent." # knock-ack end except OSError: syslog.syslog("Error: Do you have hping3 installed?") sys.exit(3) except MacFailedException: pass except Exception as e: # print "Unexpected error:", sys.exc_info() syslog.syslog("knocknock skipping unrecognized line" + e)
def get_accelerometer_data(self): # Sort the data all_data = sorted(LogEntry.select(), key=lambda row: datetime.strptime(row.timestamp, self.time_stamp_format)) accelerometer_data = [{"timestamp": row.timestamp, "light": row.light_reading, "proximity": row.proximity_reading, "x": row.x_reading, "y": row.y_reading, "z": row.z_reading} for row in all_data] return accelerometer_data
def update(self): """Updates the registry : removes the old entries, adds the new ones with addNewEntry""" with open(self.file, 'r') as log: count = 0 for line in log: # We only read lines that have not yet been read if (count >= self.lastLine): # We turn them into log entries l = LogEntry(line) # And add them to the registry self.addNewEntry(l) count += 1 # Updates the last read line self.lastLine = count # Now we have to remove the old entries time = datetime.now() while len(self.registry) != 0 and abs( (self.registry[0].date.replace(tzinfo=None) - time).total_seconds()) > 120: self.deleteEntry() # Now we handle the alerts if (self.avgHits > self.threshold and not self.alert): self.addAlert() elif (self.avgHits < self.threshold and self.alert): self.removeAlert()
def parse_file(): file_path = dir_path + "/" + filename fp = open(file_path, 'r') cnt = 1 with open(file_path) as fp: line = fp.readline() while line: (server, user_count, organism_count) = parse_line(line) log_entry = LogEntry(server=server, num_users=user_count, num_organisms=organism_count) # print "log entry " # print log_entry.server + " " + str(log_entry.num_organisms) + " " + str(log_entry.num_users) log_container.add_entry(log_entry) # print "server: " + server # print "user_count: " + str(user_count) # print "organism_count: " + str(organism_count) cnt += 1 line = fp.readline() # print cnt # if cnt > 5: # break # cnt = 1 # while line: # print("Line {}: {}".format(cnt, line.strip())) # line = fp.readline() # cnt += 1 fp.close()
def create_parsed_log_entry_object(parsed_log_entry): """ Creates object from parsed log entry :param parsed_log_entry: list --- list with log entry artifacts. len(parsed_log_entry) == 9 :return: LogEntry object """ lgr.debug("Trying to create parsed log entry object from {} ...".format( parsed_log_entry)) if len(parsed_log_entry) == 9: lgr.debug("Creating log entry object from ") asctime = parsed_log_entry[0] msecs = parsed_log_entry[1] process = parsed_log_entry[2] levelname = parsed_log_entry[3] name = parsed_log_entry[4] request_ids = parsed_log_entry[5] user_identitys = parsed_log_entry[6] instance = parsed_log_entry[7] message = parsed_log_entry[8] log_entry_object = LogEntry(asctime, msecs, process, levelname, name, request_ids, user_identitys, instance, message) return log_entry_object else: lgr.debug("Log entry {} does not match \ a requirement, skipped...".format(parsed_log_entry))
def get(self, job_name): template = jinja_environment.get_template('templates/log_template.htm') job_id = WatchJob.all().filter('name =', job_name).get() job_logs = LogEntry.all().filter('job =', job_id).order('-event_time').run(limit=100) self.response.out.write(template.render(name=job_name, logging=job_logs))
def __init__(self): self.logEntries = dict() self.logEntriesInTable = list() self.searchLogEntryTableWidget = None self.colNamesInSearchLogsTable = list() self.vectorManager = None self.nextAvailableId = 5 ids = [0, 1, 2, 3, 4] dates = ["1/26/20", "1/26/20", "1/26/20", "1/26/20", "1/26/20"] teams = [ LogEntry.BLUE_TEAM, LogEntry.WHITE_TEAM, LogEntry.BLUE_TEAM, LogEntry.RED_TEAM, LogEntry.BLUE_TEAM ] descriptions = [ "Blue Team Defender Turns on Computer.", "White Team Analyst Starts Taking Notes.", "SQL Injection attack from Red Team.", "Cross-Site Scripting Attack from Red Team.", "Blue Team Defender turns off computer." ] artifacts = [ "blue_log.csv", "white_recording.png", "red_attack.txt", "red_escalation.txt", "blue_response.csv" ] for i in range(len(descriptions)): logEntry = LogEntry() logEntry.date = dates[i] logEntry.description = descriptions[i] logEntry.creator = teams[i] logEntry.id = ids[i] logEntry.artifact = artifacts[i] self.logEntries[ids[i]] = logEntry self.logEntriesInTable = list(self.logEntries.values())
def entry_from_dict(self, dict_entry): """Creates a new log entry given the dictionary gathered from SPLUNK.""" log_entry = LogEntry(serial=int(dict_entry['_cd'].replace(":", "")), timestamp=dict_entry['_time'], content=dict_entry['_raw'], host=dict_entry['host'], source=dict_entry['source'], sourcetype=dict_entry['sourcetype']) return log_entry
def tailAndProcess(self): for line in self.logFile.tail(): try: logEntry = LogEntry(line) profile = self.profiles.getProfileForPort(logEntry.getDestinationPort()) if (profile != None): try: ciphertext = logEntry.getEncryptedData() port = profile.decrypt(ciphertext, self.config.getWindow()) sourceIP = logEntry.getSourceIP() self.portOpener.open(sourceIP, port) syslog.syslog("Received authenticated port-knock for port " + str(port) + " from " + sourceIP) except MacFailedException: pass except: # print "Unexpected error:", sys.exc_info() syslog.syslog("knocknock skipping unrecognized line.")
def test_entry(self): entry = LogEntry( "80", "192.1.1.1", "00-11-22-33-44-55", "81", "192.1.1.2", "AA-BB-CC-DD-EE-FF", "TCP", "20", True, ) self.assertEqual("80", entry.sourcePortNumber) self.assertTrue(type(entry.timestamp) is int)
def tailAndProcess(self): for line in self.logFile.tail(): try: logEntry = LogEntry(line) profile = self.profiles.getProfileForPort( logEntry.getDestinationPort()) if (profile != None): try: ciphertext = logEntry.getEncryptedData() port = profile.decrypt(ciphertext, self.config.getWindow()) sourceIP = logEntry.getSourceIP() self.portOpener.open(sourceIP, port) syslog.syslog( "Received authenticated port-knock for port " + str(port) + " from " + sourceIP) except MacFailedException: pass except: # print "Unexpected error:", sys.exc_info() syslog.syslog("knocknock skipping unrecognized line.")
def setup_tables(): try: LogEntry.create_table() except (sqlite3.OperationalError): pass if args.reset: LogEntry.drop_table() LogEntry.create_table()
def get_accelerometer_data(self): # Sort the data all_data = sorted(LogEntry.select(), key=lambda row: datetime.strptime( row.timestamp, self.time_stamp_format)) accelerometer_data = [{ "timestamp": row.timestamp, "light": row.light_reading, "proximity": row.proximity_reading, "x": row.x_reading, "y": row.y_reading, "z": row.z_reading } for row in all_data] return accelerometer_data
def post(self): try: # Get the body, which should be a JSON Array of JSON Objects for multiple log entries or only a JSON # object for single entries data = request.get_json() dictionary_list = data if isinstance(data, list) else [data] # For each JSON Object, try to create a log entry # Store the successful log entries in a list and error messages from invalid logs in another list logs = [] errors = [] for dictionary in dictionary_list: log = LogEntry(dictionary) if log.valid: logs.append(log) else: errors.append(log.get_error_message()) # TODO - Connect this to logstash and/or database # Return the results as a dictionary (Flask will JSONify the dict) and the respective HTTP Status Code return self.generate_response(logs, errors) except TypeError as te: return "Error:" + str(te), 418 except AttributeError as ae: error = ae.with_traceback() return "Error:" + str(ae), 418
def load_log(logpath: str) -> 'LiteLog': log = LiteLog() my_list = [] with open(logpath) as fd: for line in fd: try: entry = LogEntry.parse(line) my_list.append(entry) except: print('error when loading',line) print(' in ', logpath) raise log.log_list_ = np.array(my_list) return log
def ingestLogFile(self): if self.validated and not self.ingested: logEntries = list() lineNumber = 0 for line in self.lines: logEntry = LogEntry() timestamp = self.timestamps[lineNumber] timestampAsDate = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S") formattedDate = timestampAsDate.strftime("%m/%d/%Y %I:%M %p") logEntry.date = formattedDate logEntry.description = line logEntry.creator = self.creator logEntry.eventType = self.eventType logEntry.artifact = self.filename logEntry.lineNumber = lineNumber logEntry.id = logEntry.artifact + "_" + str( logEntry.lineNumber) logEntries.append(logEntry) lineNumber += 1 self.ingested = True return logEntries return None
def plot_data(): format_string = '%H:%M:%S %m/%d/%Y' log_data = LogEntry.select().order_by(LogEntry.timestamp) # log_data = [row for row in log_data] log_data = sorted(log_data, key=lambda row: datetime.strptime(row.timestamp, format_string)) light_readings = [row.light_reading for row in log_data] light_readings = [min(reading, 100.0) for reading in light_readings] timestamps = [datetime.strptime(row.timestamp, format_string) for row in log_data] fig = plt.figure() # plt.figure(1, figsize=(10, 6)) plt.clf() plt.cla() plt.plot_date(timestamps, light_readings, 'b') fig.autofmt_xdate() plt.show()
def store(self, log_bundles): log.debug('store the logs by calling the model layer') log.debug(log_bundles) log_entries = log_bundles[0]['logEntries'] #.logEntries.tolist() log.info(log_entries) for log_entry in log_entries: log.info(log_entry) log.info(log_entry) ndb_log_entry = LogEntry(type='debug') ndb_log_entry.type = log_entry['type'] ndb_log_entry.color = log_entry['color'] ndb_log_entry.encoded_data = log_entry['encodedData'] ndb_log_entry.pathname = log_entry['pathname'] #ndb_log_entry.timestamp = log_entry['timestamp'] #log_entries=[]) ndb_log_entry.put() #def get(self, log_id): #log.debug('retrieve the logs for logid ' + log_id)
def handleAddNode(self): if self.vectorComboBoxTable.count() > 0: vectorName = self.vectorComboBoxTable.currentText() vector = self.clientHandler.vectorManager.vectors[vectorName] logEntry = LogEntry() logEntry.creator = logEntry.WHITE_TEAM logEntry.eventType = logEntry.WHITE_TEAM logEntry.id = "-1" logEntry.date = (datetime.datetime.today() ).strftime("%m/%d/%Y %I:%M %p").lstrip("0") logEntry.associatedVectors.append( self.vectorComboBoxTable.currentText()) vector.addSignificantEventFromLogEntry(logEntry) self.updateVectorTable(vector) self.updateVectorGraph(vector) self.updateVectorGraph(vector)
def save_data(log_data): num_entries = len(log_data) ten_percent = num_entries / 10 records_added = 0 records_skipped = 0 log_entry_object = LogEntry() # Run as a single transaction for speed with db.transaction(): for index, entry in enumerate(log_data): if save_entry(entry, log_entry_object): records_added += 1 else: records_skipped += 1 # Print progress to screen if index % ten_percent == 0: print("{:0.0%} done".format(index / float(num_entries))) print_save_results(num_entries, records_added, records_skipped)
def update(self, remote_ip, uptime): self.last_seen = datetime.utcnow() if self.last_ip != remote_ip: LogEntry.log_event(self.key(), 'Info', 'IP changed - new IP: ' + remote_ip) self.last_ip = remote_ip if uptime is not None: if self.update is not None and self.uptime > uptime: LogEntry.log_event(self.key(), 'Reboot', 'Reboot - Previous uptime: ' + str(timedelta(seconds=self.uptime))) for action_key in self.reboot_actions: try: db.get(action_key).perform_action() except Exception as exp: logging.error('Error executing reboot action: ' + str(exp)) self.uptime = uptime self.put() # job got back online if self.status == 'offline': self.status = 'online' LogEntry.log_event(self.key(), 'Info', 'Job back online - IP: ' + remote_ip) # perform all back_online actions for action_key in self.backonline_actions: try: db.get(action_key).perform_action() except Exception as exp: logging.error('Error executing backonline action: ' + str(exp)) # delete previous (waiting) task if self.task_name is not None: logging.debug('old task: ' + self.task_name) Queue.delete_tasks(Queue(), Task(name=self.task_name)) task_name = self.name + '_' + datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f') # create task to be executed in updated no called in interval minutes taskqueue.add(name=task_name, url='/task', params={'key': self.key()}, countdown=(self.interval + 2) * 60) self.task_name = task_name self.put()
def get_light_data(): light_data = LogEntry.select() return light_data
def load_light(): light_data = LogEntry.select() return sorted(light_data, key=lambda row: datetime.strptime(row.timestamp, format_string))
def save_packet(self, packet): """ Function for recording a packet during sniff runtime packet = the packet passed through the sniff function """ # TODO: make this work with layer 2, for now just skip filtering those packets if not packet.haslayer("IP"): return # timestamp used for port scan detection currentTime = int(datetime.now().timestamp()) if self.portScanTimeout is None: self.portScanTimeout = currentTime # how to tell if we need to reset our port scan record if currentTime > self.portScanTimeout + self.scan_window: self.portScanTimeout = currentTime self.PS_RECORD = dict() # A bunch of packet data, collected to be stored sourceMAC = packet.src destMAC = packet.dst ipLayer = packet.getlayer("IP") # IP where this came from srcIP = ipLayer.src dstIP = ipLayer.dst destPort = ipLayer.dport if hasattr(ipLayer, "dport") else None srcPort = ipLayer.sport if hasattr(ipLayer, "sport") else None if (not ipLayer.haslayer("TCP") and not ipLayer.haslayer("UDP") and not ipLayer.haslayer("ICMP")): return # Whitelist check if srcIP not in self.whitelist: # Testing config - does not utilize a database # isTest = self.config == "onlyUDP" or self.config == "testing" trafficType = ("TCP" if ipLayer.haslayer("TCP") else "UDP" if ipLayer.haslayer("UDP") else "ICMP" if ipLayer.haslayer("ICMP") else "Other") # Log Entry object we're saving log = LogEntry( srcPort, srcIP, sourceMAC, destPort, dstIP, destMAC, trafficType, ipLayer.len, destPort in self.openPorts, ) # self.RECORD is where we save logs for easy testing if srcIP in self.RECORD.keys(): self.RECORD[srcIP].append(log) else: self.RECORD[srcIP] = [log] # saving the database ID in case of port scan detection if self.config == "base": dbID = self.db.save(log) else: return # self.PS_RECORD is a separate dictionary used for port scan detection if srcIP not in self.PS_RECORD.keys(): self.PS_RECORD[srcIP] = dict() self.PS_RECORD[srcIP][log.destPortNumber] = dbID else: self.PS_RECORD[srcIP][log.destPortNumber] = dbID # Sending out the port scan alert if len(self.PS_RECORD[srcIP]) > self.scan_sensitivity: self.db.alert( Alert( variant="alert", message="Port scan detected from IP {}".format( srcIP), references=list(self.PS_RECORD[srcIP].values()), )) self.PS_RECORD[srcIP] = dict()
def retrieveLogEntryDb(self, logEntryId): query = {"id": str(logEntryId)} logEntry = None for entry in self.col.find(query): logEntry = LogEntry() logEntry.id = entry["_id"] logEntry.associatedVectors = eval(entry["vectors"]) logEntry.location = entry["location"] logEntry.eventType = entry["eventType"] logEntry.description = entry["description"] logEntry.creator = entry["creator"] logEntry.date = entry["date"] logEntry.artifact = entry["artifact"] logEntry.lineNumber = entry["lineNumber"] return logEntry
def retrieveLogEntriesDb(self): self.logEntries.clear() for entry in self.col.find(): logEntry = LogEntry() logEntry.id = entry["_id"] logEntry.associatedVectors = eval(entry["vectors"]) logEntry.location = entry["location"] logEntry.eventType = entry["eventType"] logEntry.description = entry["description"] logEntry.creator = entry["creator"] logEntry.date = entry["date"] logEntry.artifact = entry["artifact"] logEntry.lineNumber = entry["lineNumber"] self.logEntries[logEntry.id] = logEntry
def test_get_grade(self): assert LogEntry.get_grade(None," DIESEL ") == Grade.Diesel
def test_get_tank(self): assert LogEntry.get_tank_num(None," TANK 1") == 1
def load_light(): light_data = LogEntry.select() return sorted( light_data, key=lambda row: datetime.strptime(row.timestamp, format_string))
def test_get_capacity(self): assert LogEntry.get_capacity(None," 26380") != None assert LogEntry.get_capacity(None," 26380") == 26380
def test_get_pump(self): assert LogEntry.get_pump_num(None," PUMP 6 ") == 6