def analyze(self, msg):
routing, event, mtd = msg.data
self.log('storing new event')
agent = AgentId(routing['agentid'])
aid = agent.invariableToString()
ts = _x_(event, '?/base.TIMESTAMP')
if ts is None or ts > (2 * time.time()):
ts = _x_(event, 'base.TIMESTAMP')
if ts is None:
ts = time_uuid.utctime()
else:
ts = int(ts)
eid = routing['event_id']
self.db.execute_async(self.statements['events'].bind(
(eid,
base64.b64encode(
msgpack.packb({
'routing': routing,
'event': event
})), aid)))
self.db.execute_async(self.statements['timeline'].bind(
(aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
self.db.execute_async(self.statements['timeline_by_type'].bind(
(aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
self.db.execute_async(self.statements['recent'].bind((aid, )))
self.db.execute_async(self.statements['last'].bind(
(eid, aid, routing['event_type'])))
inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
if inv_id is not None and inv_id != '':
self.db.execute_async(self.statements['investigation'].bind(
(inv_id.upper(), time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
self.log('storing objects')
new_objects = mtd['obj']
new_relations = mtd['rel']
for ignored in self.ignored_objects:
if ignored in new_objects:
del (new_objects[ignored])
for k in new_relations.keys():
if ignored in k:
del (new_relations[k])
if 0 != len(new_objects) or 0 != len(new_relations):
self._ingestObjects(aid, ts, new_objects, new_relations)
self.log('finished storing objects: %s / %s' %
(len(new_objects), len(new_relations)))
return (True, )
def analyze( self, msg ):
routing, event, mtd = msg.data
self.log( 'storing new event' )
agent = AgentId( routing[ 'agentid' ] )
aid = agent.invariableToString()
ts = _x_( event, '?/base.TIMESTAMP' )
if ts is None or ts > ( 2 * time.time() ):
ts = _x_( event, 'base.TIMESTAMP' )
if ts is None:
ts = time_uuid.utctime()
else:
ts = int( ts )
eid = routing[ 'event_id' ]
self.db.execute_async( self.statements[ 'events' ].bind( ( eid,
base64.b64encode( msgpack.packb( { 'routing' : routing, 'event' : event } ) ),
aid ) ) )
self.db.execute_async( self.statements[ 'timeline' ].bind( ( aid,
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
self.db.execute_async( self.statements[ 'timeline_by_type' ].bind( ( aid,
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
self.db.execute_async( self.statements[ 'recent' ].bind( ( aid, ) ) )
self.db.execute_async( self.statements[ 'last' ].bind( ( eid,
aid,
routing[ 'event_type' ] ) ) )
inv_id = _x_( event, '?/hbs.INVESTIGATION_ID' )
if inv_id is not None and inv_id != '':
self.db.execute_async( self.statements[ 'investigation' ].bind( ( inv_id.upper(),
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
self.log( 'storing objects' )
new_objects = mtd[ 'obj' ]
new_relations = mtd[ 'rel' ]
for ignored in self.ignored_objects:
if ignored in new_objects:
del( new_objects[ ignored ] )
for k in new_relations.keys():
if ignored in k:
del( new_relations[ k ] )
if 0 != len( new_objects ) or 0 != len( new_relations ):
self._ingestObjects( aid, ts, new_objects, new_relations )
self.log( 'finished storing objects: %s / %s' % ( len( new_objects ), len( new_relations )) )
return ( True, )
def event():
event = {}
event['event_id'] = str(
time_uuid.TimeUUID.with_timestamp(time_uuid.utctime()))
event['event_name'] = 'EventName' + str(random.randint(1000, 9999))
event['bucket_id'] = str(fake.date(pattern="%Y-%m-%d"))
event['payload'] = payload()
return json.dumps(event)
def generate_token(username):
if username is None:
raise InvalidRequestData('Invalid username.')
conn = get_connection()
cursor = conn.cursor()
new_token = str(uuid.with_timestamp(utctime()))
cursor.execute('UPDATE auth SET token = ? WHERE user = ?',
(new_token, username))
conn.commit()
conn.close()
return new_token
def analyze(self, msg):
routing, event, mtd = msg.data
self.processedCounter += 1
if 0 == (self.processedCounter % 1000):
self.log('MOD_IN %s' % self.processedCounter)
if 0 == (self.processedCounter % 5000):
self.org_ttls = {}
agent = AgentId(routing['aid'])
sid = agent.sensor_id
ts = _x_(event, '?/base.TIMESTAMP')
ttls = self.getOrgTtls(agent.org_id)
if ts is not None:
ts = float(ts) / 1000
if ts is None or ts > (2 * time.time()):
ts = _x_(event, 'base.TIMESTAMP')
if ts is None:
ts = time_uuid.utctime()
else:
ts = float(ts) / 1000
eid = uuid.UUID(routing['event_id'])
if 1 < self.modelingLevel:
self.asyncInsert(
self.stmt_events.bind((eid,
base64.b64encode(
msgpack.packb({
'routing': routing,
'event': event
})), sid, ttls['events'])))
self.asyncInsert(
self.stmt_timeline.bind(
(sid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'], ttls['events'])))
self.nWrites += 2
self.asyncInsert(
self.stmt_timeline_by_type.bind(
(sid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'], ttls['events'])))
self.nWrites += 1
if 2 <= self.modelingLevel:
self.asyncInsert(
self.stmt_last.bind(
(ttls['events'], eid, sid, routing['event_type'])))
self.nWrites += 1
if 5 <= self.modelingLevel:
this_atom = _x_(event, '?/hbs.THIS_ATOM')
parent_atom = _x_(event, '?/hbs.PARENT_ATOM')
null_atom = "\x00" * 16
if this_atom is not None:
if this_atom == null_atom:
this_atom = None
else:
try:
this_atom = uuid.UUID(bytes=str(this_atom))
except:
self.log('invalid atom: %s / %s ( %s )' %
(this_atom, type(this_atom),
traceback.format_exc()))
this_atom = None
if parent_atom is not None:
if parent_atom == null_atom:
parent_atom = None
else:
try:
parent_atom = uuid.UUID(bytes=str(parent_atom))
except:
self.log('invalid atom: %s / %s ( %s )' %
(parent_atom, type(parent_atom),
traceback.format_exc()))
parent_atom = None
if this_atom is not None:
self.asyncInsert(
self.stmt_atoms_lookup.bind(
(this_atom, eid, ttls['atoms'])))
self.nWrites += 1
if this_atom is not None and parent_atom is not None:
self.asyncInsert(
self.stmt_atoms_children.bind(
(parent_atom,
this_atom if this_atom is not None else uuid.UUID(
bytes=null_atom), eid, ttls['atoms'])))
self.nWrites += 1
if 2 <= self.modelingLevel:
inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
if inv_id is not None and inv_id != '':
self.asyncInsert(
self.stmt_investigation.bind(
(inv_id.upper().split('//')[0],
time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'], ttls['detections'])))
self.nWrites += 1
# Only deal with Objects at level 7 and above.
if 7 <= self.modelingLevel:
new_objects = mtd['obj']
new_relations = mtd['rel']
for ignored in self.ignored_objects:
if ignored in new_objects:
del (new_objects[ignored])
for k in new_relations.keys():
if ignored in k:
del (new_relations[k])
if 0 != len(new_objects) or 0 != len(new_relations):
self._ingestObjects(ttls, sid, ts, new_objects, new_relations,
agent.org_id, eid)
#self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
return (True, )
def generate_time_uuid():
t_id = str(time_uuid.TimeUUID.with_timestamp(time_uuid.utctime()))
formatted_id = t_id[14:19] + t_id[9:14] + t_id[0:9] + t_id[19:]
return formatted_id[0:18] + str(uuid.uuid4())[18:]
def make_identifier() -> str:
return TimeUUID.with_timestamp(utctime())
def analyze(self, msg):
routing, event, mtd = msg.data
self.processedCounter += 1
if 0 == (self.processedCounter % 50):
self.log('MOD_IN %s' % self.processedCounter)
agent = AgentId(routing['agentid'])
aid = agent.invariableToString()
ts = _x_(event, '?/base.TIMESTAMP')
if ts is not None:
ts = float(ts) / 1000
if ts is None or ts > (2 * time.time()):
ts = _x_(event, 'base.TIMESTAMP')
if ts is None:
ts = time_uuid.utctime()
else:
ts = float(ts) / 1000
eid = routing['event_id']
self.db.execute_async(self.statements['events'].bind(
(eid,
base64.b64encode(
msgpack.packb({
'routing': routing,
'event': event
})), aid)))
self.db.execute_async(self.statements['timeline'].bind(
(aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
self.db.execute_async(self.statements['timeline_by_type'].bind(
(aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
self.db.execute_async(self.statements['recent'].bind((aid, )))
self.db.execute_async(self.statements['last'].bind(
(eid, aid, routing['event_type'])))
this_atom = _x_(event, '?/hbs.THIS_ATOM')
parent_atom = _x_(event, '?/hbs.PARENT_ATOM')
null_atom = "\x00" * 16
if this_atom is not None:
if this_atom == null_atom:
this_atom = None
else:
try:
this_atom = uuid.UUID(bytes=str(this_atom))
except:
self.log(
'invalid atom: %s / %s ( %s )' %
(this_atom, type(this_atom), traceback.format_exc()))
this_atom = None
if parent_atom is not None:
if parent_atom == null_atom:
parent_atom = None
else:
try:
parent_atom = uuid.UUID(bytes=str(parent_atom))
except:
self.log('invalid atom: %s / %s ( %s )' %
(parent_atom, type(parent_atom),
traceback.format_exc()))
parent_atom = None
if this_atom is not None:
self.db.execute_async(self.statements['atoms_lookup'].bind(
(this_atom, eid)))
if this_atom is not None and parent_atom is not None:
self.db.execute_async(self.statements['atoms_children'].bind(
(parent_atom,
this_atom if this_atom is not None else uuid.UUID(
bytes=null_atom), eid)))
inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
if inv_id is not None and inv_id != '':
self.db.execute_async(self.statements['investigation'].bind(
(inv_id.upper().split('//')[0],
time_uuid.TimeUUID.with_timestamp(ts), eid,
routing['event_type'])))
new_objects = mtd['obj']
new_relations = mtd['rel']
for ignored in self.ignored_objects:
if ignored in new_objects:
del (new_objects[ignored])
for k in new_relations.keys():
if ignored in k:
del (new_relations[k])
if 0 != len(new_objects) or 0 != len(new_relations):
self._ingestObjects(aid, ts, new_objects, new_relations)
#self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
return (True, )
def analyze( self, msg ):
routing, event, mtd = msg.data
self.processedCounter += 1
if 0 == ( self.processedCounter % 50 ):
self.log( 'MOD_IN %s' % self.processedCounter )
agent = AgentId( routing[ 'agentid' ] )
aid = agent.invariableToString()
ts = _x_( event, '?/base.TIMESTAMP' )
if ts is not None:
ts = float( ts ) / 1000
if ts is None or ts > ( 2 * time.time() ):
ts = _x_( event, 'base.TIMESTAMP' )
if ts is None:
ts = time_uuid.utctime()
else:
ts = float( ts ) / 1000
eid = routing[ 'event_id' ]
self.db.execute_async( self.statements[ 'events' ].bind( ( eid,
base64.b64encode( msgpack.packb( { 'routing' : routing, 'event' : event } ) ),
aid ) ) )
self.db.execute_async( self.statements[ 'timeline' ].bind( ( aid,
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
self.db.execute_async( self.statements[ 'timeline_by_type' ].bind( ( aid,
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
self.db.execute_async( self.statements[ 'recent' ].bind( ( aid, ) ) )
self.db.execute_async( self.statements[ 'last' ].bind( ( eid,
aid,
routing[ 'event_type' ] ) ) )
this_atom = _x_( event, '?/hbs.THIS_ATOM' )
parent_atom = _x_( event, '?/hbs.PARENT_ATOM' )
null_atom = "\x00" * 16
if this_atom is not None:
if this_atom == null_atom:
this_atom = None
else:
try:
this_atom = uuid.UUID( bytes = str( this_atom ) )
except:
self.log( 'invalid atom: %s / %s ( %s )' % ( this_atom, type( this_atom ), traceback.format_exc() ) )
this_atom = None
if parent_atom is not None:
if parent_atom == null_atom:
parent_atom = None
else:
try:
parent_atom = uuid.UUID( bytes = str( parent_atom ) )
except:
self.log( 'invalid atom: %s / %s ( %s )' % ( parent_atom, type( parent_atom ), traceback.format_exc() ) )
parent_atom = None
if this_atom is not None:
self.db.execute_async( self.statements[ 'atoms_lookup' ].bind( ( this_atom,
eid ) ) )
if this_atom is not None and parent_atom is not None:
self.db.execute_async( self.statements[ 'atoms_children' ].bind( ( parent_atom,
this_atom if this_atom is not None else uuid.UUID( bytes = null_atom ),
eid ) ) )
inv_id = _x_( event, '?/hbs.INVESTIGATION_ID' )
if inv_id is not None and inv_id != '':
self.db.execute_async( self.statements[ 'investigation' ].bind( ( inv_id.upper().split( '//' )[ 0 ],
time_uuid.TimeUUID.with_timestamp( ts ),
eid,
routing[ 'event_type' ] ) ) )
new_objects = mtd[ 'obj' ]
new_relations = mtd[ 'rel' ]
for ignored in self.ignored_objects:
if ignored in new_objects:
del( new_objects[ ignored ] )
for k in new_relations.keys():
if ignored in k:
del( new_relations[ k ] )
if 0 != len( new_objects ) or 0 != len( new_relations ):
self._ingestObjects( aid, ts, new_objects, new_relations )
#self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
return ( True, )