def _get_database_(self, indb, quiet): db = Database(Config.get_database(indb)) if not quiet: db.print_statistics() else: db.get_db_stat() return db
def run(self, db=False, print_reboots=False, sort_by_host=False, **kwargs): if not db: print("no database specified") exit(1) db = Config.get_database(db) db = Database(db) db.print_statistics() print("") hostdf, hostdic = db.get_hostnames(True) pprint(hostdf) # pprint(hostdic); exit(1) if print_reboots: dr = DatabaseReader4(db, "") reboots = dr.find_reboots() reboots.drop([PID, PROCESS_NAME, SEQUENCE_ID], axis=1, inplace=True) if sort_by_host: reboots.sort_values(['ct_units_id', 'time'], ascending=[True, True], inplace=True) print("") print(reboots)
#!/usr/bin/env python3 from pprint import pprint from util.config import Config from protoapi.moduleapi2 import ModuleApi2 #indb1 = 'ng4'; inrange1 = [8, 3997625, 8606957] #indb2 = 'ng4'; inrange2 = [9, 3997624, 8606957] db1 = Config.get_database('ng4') db2 = Config.get_database('ng4') # get net instance mod = ModuleApi2() # setup database connection mod.connect("", db1, 5, db2, 6) mod.consume(1069, 8606957, 1069, 8606957) #mod.connect("", db1, 8, db2, 9) #mod.consume(3997625, 8606957, # 3997624, 8606957) # #mod.consume(3997625, 4100000, # 3997624, 4100000) #mod.consume(4100001, 4500000, # 4100001, 4500000) mod.run_evaluation(1) mod.run_evaluation(2)
def run(self, indb=False, print_reboots=False, sort_by_host=False, **kwargs): if not indb: print("no database specified") exit(1) db = Config.get_database(indb) db = Database(db) db.print_statistics() print("") QUERY = " SELECT "+ \ " events.id AS id,"+ \ " events.sequence_id,"+ \ " events."+db.hostname_in_events+","+ \ " events.type_id,"+ \ " events.pid,"+ \ " events.process_name,"+ \ " parent.id AS parent_id,"+ \ " parent.process_name AS parent_process_name,"+ \ " grandparent.process_name AS grandparent_process_name,"+ \ " grandparent.id AS grandparent_id,"+ \ " events.time,"+ \ " process_events.domain_name,"+ \ " process_events.user_name,"+ \ " process_events.command_line,"+ \ " file_events.type AS fileop, file_events.src_file_name, file_events.dst_file_name,"+ \ " registry_events.type_string, registry_events.path, registry_events.key, registry_events.data,"+ \ " network_events.is_connection_outgoing, network_events.protocol_id,"+ \ " network_events.local_ip_address, network_events.local_port, network_events.remote_ip_address, network_events.remote_port"+ \ " FROM events"+ \ " LEFT OUTER JOIN process_events USING (id)"+ \ " LEFT OUTER JOIN file_events USING (id)"+ \ " LEFT OUTER JOIN registry_events USING (id)" + \ " LEFT OUTER JOIN network_events USING (id)" + \ " LEFT OUTER JOIN events AS parent ON events.parent_id = parent.id"+ \ " LEFT OUTER JOIN events AS grandparent ON parent.parent_id = grandparent.id"+ \ " WHERE events.type_id = 4"+ \ " LIMIT 5" df = pd.read_sql(QUERY, db.engine) print("got", len(df), "lines") pprint(df) print("") print("") resprox = db.conn.execute(QUERY) rows = int(resprox.rowcount) print("got", rows, "lines") while rows > 0: line = row2dict(resprox.fetchone()) pprint(line) rows -= 1 print("done")
def run(self, db=False, inrange=None, inhost=None, print_proc=False, print_parent=False, print_cmd=False, print_file=False, print_reg=False, filter=False, evid=False, plot=False, quiet=False, robert=None, nopath=False, nochildpath=False, **kwargs): db = Config.get_database(db) db = Database(db) if not quiet: db.print_statistics() else: db.get_db_stat() # # find hostnames with valid events # hostdf, hostdict = db.get_hostnames(count_events=False) # # print(hostdf) # # print(hostdict) # # # is supplied host a valid hosts? # hostkey = None # if int(inhost[0]) in hostdict: # hostkey = inhost[0] # else: # print(inhost[0], "not in database?") # exit(1) # range if not inrange: fromid = int(db.stat[MIN_ID]) toid = int(db.stat[MAX_ID]) else: fromid = int(inrange[0]) toid = int(inrange[1]) if not quiet: print("\nprocessing", fromid, "-", toid) dr4 = DatabaseReader4(db, inhost[0]) time_start = time.time() resprox = dr4.read_sql(str(fromid), str(toid)) if not quiet: print("after SELECT: ", resprox.rowcount, "events", "("+str(int(resprox.rowcount/ (time.time()-time_start))), "events/s)") print("") time_start = time.time() events_processed = 0 while True: listofrows = resprox.fetchmany(size=400000) if len(listofrows)==0: break for rowprox in listofrows: # if events_processed % 10000 == 0 and (not quiet): # print(".",end='',flush=True) row = row2dict(rowprox) if filter: Filter.run_filters(row) if robert: self.do_robert(row, nopath, nochildpath) else: if row[TYPE_ID] == PROCESS: if print_proc: print("P"+row[PROCESS_NAME]) if print_cmd: print("C"+row[COMMAND_LINE]) if print_parent and row[PARENT_PROCESS_NAME]: print("p"+row[PARENT_PROCESS_NAME]) elif print_file and row[TYPE_ID] == FILE: if evid: print(row[ID]+"^"+row[SRC_FILE_NAME]) else: print("F"+row[SRC_FILE_NAME]) if row[TYPE] == RENAME: if evid: print(row[ID]+"^"+row[DST_FILE_NAME]) else: print("F"+row[DST_FILE_NAME]) elif print_reg and row[TYPE_ID] == REGISTRY: print("R"+row[PATH]+'⣿'+row[KEY]) elif plot and row[TYPE_ID] == FILE: # if row[PARENT_PROCESS_NAME] is None: # row[PARENT_PROCESS_NAME] = "UNKNOWN" if row[GRANDPARENT_PROCESS_NAME] is None: row[GRANDPARENT_PROCESS_NAME] = "UNKNOWN" # if row[TYPE_ID] != RENAME: # row[DST_FILE_NAME] print(row[ID]+"°"+row[GRANDPARENT_PROCESS_NAME]+ "°"+row[PARENT_PROCESS_NAME]+ "°"+row[SRC_FILE_NAME]+"°"+str(len(row[SRC_FILE_NAME]))+ "°"+row[DST_FILE_NAME]+"°"+str(len(row[DST_FILE_NAME])) ) else: pass events_processed +=1 if not quiet: print("\n..last event:", row[ID], row[TIME]) print("-->", events_processed, "events", "("+str(int(events_processed/ (time.time()-time_start))), "events/s)")