def _get_database_(self, indb, quiet):
db = Database(Config.get_database(indb))
if not quiet:
db.print_statistics()
else:
db.get_db_stat()
return db
def run(self, db=False, print_reboots=False, sort_by_host=False, **kwargs):
if not db:
print("no database specified")
exit(1)
db = Config.get_database(db)
db = Database(db)
db.print_statistics()
print("")
hostdf, hostdic = db.get_hostnames(True)
pprint(hostdf)
# pprint(hostdic); exit(1)
if print_reboots:
dr = DatabaseReader4(db, "")
reboots = dr.find_reboots()
reboots.drop([PID, PROCESS_NAME, SEQUENCE_ID],
axis=1,
inplace=True)
if sort_by_host:
reboots.sort_values(['ct_units_id', 'time'],
ascending=[True, True],
inplace=True)
print("")
print(reboots)
#!/usr/bin/env python3
from pprint import pprint
from util.config import Config
from protoapi.moduleapi2 import ModuleApi2
#indb1 = 'ng4'; inrange1 = [8, 3997625, 8606957]
#indb2 = 'ng4'; inrange2 = [9, 3997624, 8606957]
db1 = Config.get_database('ng4')
db2 = Config.get_database('ng4')
# get net instance
mod = ModuleApi2()
# setup database connection
mod.connect("", db1, 5, db2, 6)
mod.consume(1069, 8606957, 1069, 8606957)
#mod.connect("", db1, 8, db2, 9)
#mod.consume(3997625, 8606957,
# 3997624, 8606957)
#
#mod.consume(3997625, 4100000,
# 3997624, 4100000)
#mod.consume(4100001, 4500000,
# 4100001, 4500000)
mod.run_evaluation(1)
mod.run_evaluation(2)
def run(self,
indb=False,
print_reboots=False,
sort_by_host=False,
**kwargs):
if not indb:
print("no database specified")
exit(1)
db = Config.get_database(indb)
db = Database(db)
db.print_statistics()
print("")
QUERY = " SELECT "+ \
" events.id AS id,"+ \
" events.sequence_id,"+ \
" events."+db.hostname_in_events+","+ \
" events.type_id,"+ \
" events.pid,"+ \
" events.process_name,"+ \
" parent.id AS parent_id,"+ \
" parent.process_name AS parent_process_name,"+ \
" grandparent.process_name AS grandparent_process_name,"+ \
" grandparent.id AS grandparent_id,"+ \
" events.time,"+ \
" process_events.domain_name,"+ \
" process_events.user_name,"+ \
" process_events.command_line,"+ \
" file_events.type AS fileop, file_events.src_file_name, file_events.dst_file_name,"+ \
" registry_events.type_string, registry_events.path, registry_events.key, registry_events.data,"+ \
" network_events.is_connection_outgoing, network_events.protocol_id,"+ \
" network_events.local_ip_address, network_events.local_port, network_events.remote_ip_address, network_events.remote_port"+ \
" FROM events"+ \
" LEFT OUTER JOIN process_events USING (id)"+ \
" LEFT OUTER JOIN file_events USING (id)"+ \
" LEFT OUTER JOIN registry_events USING (id)" + \
" LEFT OUTER JOIN network_events USING (id)" + \
" LEFT OUTER JOIN events AS parent ON events.parent_id = parent.id"+ \
" LEFT OUTER JOIN events AS grandparent ON parent.parent_id = grandparent.id"+ \
" WHERE events.type_id = 4"+ \
" LIMIT 5"
df = pd.read_sql(QUERY, db.engine)
print("got", len(df), "lines")
pprint(df)
print("")
print("")
resprox = db.conn.execute(QUERY)
rows = int(resprox.rowcount)
print("got", rows, "lines")
while rows > 0:
line = row2dict(resprox.fetchone())
pprint(line)
rows -= 1
print("done")
def run(self, db=False, inrange=None, inhost=None,
print_proc=False, print_parent=False, print_cmd=False, print_file=False, print_reg=False,
filter=False, evid=False, plot=False, quiet=False, robert=None, nopath=False, nochildpath=False, **kwargs):
db = Config.get_database(db)
db = Database(db)
if not quiet:
db.print_statistics()
else:
db.get_db_stat()
# # find hostnames with valid events
# hostdf, hostdict = db.get_hostnames(count_events=False)
# # print(hostdf)
# # print(hostdict)
#
# # is supplied host a valid hosts?
# hostkey = None
# if int(inhost[0]) in hostdict:
# hostkey = inhost[0]
# else:
# print(inhost[0], "not in database?")
# exit(1)
# range
if not inrange:
fromid = int(db.stat[MIN_ID])
toid = int(db.stat[MAX_ID])
else:
fromid = int(inrange[0])
toid = int(inrange[1])
if not quiet:
print("\nprocessing", fromid, "-", toid)
dr4 = DatabaseReader4(db, inhost[0])
time_start = time.time()
resprox = dr4.read_sql(str(fromid), str(toid))
if not quiet:
print("after SELECT: ", resprox.rowcount, "events", "("+str(int(resprox.rowcount/ (time.time()-time_start))), "events/s)")
print("")
time_start = time.time()
events_processed = 0
while True:
listofrows = resprox.fetchmany(size=400000)
if len(listofrows)==0:
break
for rowprox in listofrows:
# if events_processed % 10000 == 0 and (not quiet):
# print(".",end='',flush=True)
row = row2dict(rowprox)
if filter:
Filter.run_filters(row)
if robert:
self.do_robert(row, nopath, nochildpath)
else:
if row[TYPE_ID] == PROCESS:
if print_proc:
print("P"+row[PROCESS_NAME])
if print_cmd:
print("C"+row[COMMAND_LINE])
if print_parent and row[PARENT_PROCESS_NAME]:
print("p"+row[PARENT_PROCESS_NAME])
elif print_file and row[TYPE_ID] == FILE:
if evid:
print(row[ID]+"^"+row[SRC_FILE_NAME])
else:
print("F"+row[SRC_FILE_NAME])
if row[TYPE] == RENAME:
if evid:
print(row[ID]+"^"+row[DST_FILE_NAME])
else:
print("F"+row[DST_FILE_NAME])
elif print_reg and row[TYPE_ID] == REGISTRY:
print("R"+row[PATH]+'⣿'+row[KEY])
elif plot and row[TYPE_ID] == FILE:
# if row[PARENT_PROCESS_NAME] is None:
# row[PARENT_PROCESS_NAME] = "UNKNOWN"
if row[GRANDPARENT_PROCESS_NAME] is None:
row[GRANDPARENT_PROCESS_NAME] = "UNKNOWN"
# if row[TYPE_ID] != RENAME:
# row[DST_FILE_NAME]
print(row[ID]+"°"+row[GRANDPARENT_PROCESS_NAME]+
"°"+row[PARENT_PROCESS_NAME]+
"°"+row[SRC_FILE_NAME]+"°"+str(len(row[SRC_FILE_NAME]))+
"°"+row[DST_FILE_NAME]+"°"+str(len(row[DST_FILE_NAME])) )
else:
pass
events_processed +=1
if not quiet:
print("\n..last event:", row[ID], row[TIME])
print("-->", events_processed, "events", "("+str(int(events_processed/ (time.time()-time_start))), "events/s)")
