def analyze(self, msg):
        routing, event, mtd = msg.data

        self.log('storing new event')

        agent = AgentId(routing['agentid'])
        aid = agent.invariableToString()
        ts = _x_(event, '?/base.TIMESTAMP')

        if ts is None or ts > (2 * time.time()):
            ts = _x_(event, 'base.TIMESTAMP')
            if ts is None:
                ts = time_uuid.utctime()
            else:
                ts = int(ts)

        eid = routing['event_id']

        self.db.execute_async(self.statements['events'].bind(
            (eid,
             base64.b64encode(
                 msgpack.packb({
                     'routing': routing,
                     'event': event
                 })), aid)))

        self.db.execute_async(self.statements['timeline'].bind(
            (aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
             routing['event_type'])))

        self.db.execute_async(self.statements['timeline_by_type'].bind(
            (aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
             routing['event_type'])))

        self.db.execute_async(self.statements['recent'].bind((aid, )))

        self.db.execute_async(self.statements['last'].bind(
            (eid, aid, routing['event_type'])))

        inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
        if inv_id is not None and inv_id != '':
            self.db.execute_async(self.statements['investigation'].bind(
                (inv_id.upper(), time_uuid.TimeUUID.with_timestamp(ts), eid,
                 routing['event_type'])))
        self.log('storing objects')
        new_objects = mtd['obj']
        new_relations = mtd['rel']

        for ignored in self.ignored_objects:
            if ignored in new_objects:
                del (new_objects[ignored])
            for k in new_relations.keys():
                if ignored in k:
                    del (new_relations[k])

        if 0 != len(new_objects) or 0 != len(new_relations):
            self._ingestObjects(aid, ts, new_objects, new_relations)
        self.log('finished storing objects: %s / %s' %
                 (len(new_objects), len(new_relations)))
        return (True, )
    def analyze( self, msg ):
        routing, event, mtd = msg.data

        self.log( 'storing new event' )

        agent = AgentId( routing[ 'agentid' ] )
        aid = agent.invariableToString()
        ts = _x_( event, '?/base.TIMESTAMP' )

        if ts is None or ts > ( 2 * time.time() ):
            ts = _x_( event, 'base.TIMESTAMP' )
            if ts is None:
                ts = time_uuid.utctime()
            else:
                ts = int( ts )

        eid = routing[ 'event_id' ]

        self.db.execute_async( self.statements[ 'events' ].bind( ( eid,
                                                                   base64.b64encode( msgpack.packb( { 'routing' : routing, 'event' : event } ) ),
                                                                   aid ) ) )

        self.db.execute_async( self.statements[ 'timeline' ].bind( ( aid,
                                                                     time_uuid.TimeUUID.with_timestamp( ts ),
                                                                     eid,
                                                                     routing[ 'event_type' ] ) ) )

        self.db.execute_async( self.statements[ 'timeline_by_type' ].bind( ( aid,
                                                                             time_uuid.TimeUUID.with_timestamp( ts ),
                                                                             eid,
                                                                             routing[ 'event_type' ] ) ) )

        self.db.execute_async( self.statements[ 'recent' ].bind( ( aid, ) ) )

        self.db.execute_async( self.statements[ 'last' ].bind( ( eid,
                                                                 aid,
                                                                 routing[ 'event_type' ] ) ) )

        inv_id = _x_( event, '?/hbs.INVESTIGATION_ID' )
        if inv_id is not None and inv_id != '':
            self.db.execute_async( self.statements[ 'investigation' ].bind( ( inv_id.upper(),
                                                                              time_uuid.TimeUUID.with_timestamp( ts ),
                                                                              eid,
                                                                              routing[ 'event_type' ] ) ) )
        self.log( 'storing objects' )
        new_objects = mtd[ 'obj' ]
        new_relations = mtd[ 'rel' ]

        for ignored in self.ignored_objects:
            if ignored in new_objects:
                del( new_objects[ ignored ] )
            for k in new_relations.keys():
                if ignored in k:
                    del( new_relations[ k ] )

        if 0 != len( new_objects ) or 0 != len( new_relations ):
            self._ingestObjects( aid, ts, new_objects, new_relations )
        self.log( 'finished storing objects: %s / %s' % ( len( new_objects ), len( new_relations )) )
        return ( True, )
def event():
    event = {}
    event['event_id'] = str(
        time_uuid.TimeUUID.with_timestamp(time_uuid.utctime()))
    event['event_name'] = 'EventName' + str(random.randint(1000, 9999))
    event['bucket_id'] = str(fake.date(pattern="%Y-%m-%d"))
    event['payload'] = payload()
    return json.dumps(event)
Exemple #4
0
def generate_token(username):
    if username is None:
        raise InvalidRequestData('Invalid username.')
    conn = get_connection()
    cursor = conn.cursor()
    new_token = str(uuid.with_timestamp(utctime()))
    cursor.execute('UPDATE auth SET token = ? WHERE user = ?',
                   (new_token, username))
    conn.commit()
    conn.close()
    return new_token
Exemple #5
0
    def analyze(self, msg):
        routing, event, mtd = msg.data

        self.processedCounter += 1

        if 0 == (self.processedCounter % 1000):
            self.log('MOD_IN %s' % self.processedCounter)
            if 0 == (self.processedCounter % 5000):
                self.org_ttls = {}

        agent = AgentId(routing['aid'])
        sid = agent.sensor_id
        ts = _x_(event, '?/base.TIMESTAMP')

        ttls = self.getOrgTtls(agent.org_id)

        if ts is not None:
            ts = float(ts) / 1000

        if ts is None or ts > (2 * time.time()):
            ts = _x_(event, 'base.TIMESTAMP')
            if ts is None:
                ts = time_uuid.utctime()
            else:
                ts = float(ts) / 1000

        eid = uuid.UUID(routing['event_id'])

        if 1 < self.modelingLevel:
            self.asyncInsert(
                self.stmt_events.bind((eid,
                                       base64.b64encode(
                                           msgpack.packb({
                                               'routing': routing,
                                               'event': event
                                           })), sid, ttls['events'])))

            self.asyncInsert(
                self.stmt_timeline.bind(
                    (sid, time_uuid.TimeUUID.with_timestamp(ts), eid,
                     routing['event_type'], ttls['events'])))
            self.nWrites += 2

        self.asyncInsert(
            self.stmt_timeline_by_type.bind(
                (sid, time_uuid.TimeUUID.with_timestamp(ts), eid,
                 routing['event_type'], ttls['events'])))
        self.nWrites += 1

        if 2 <= self.modelingLevel:
            self.asyncInsert(
                self.stmt_last.bind(
                    (ttls['events'], eid, sid, routing['event_type'])))
            self.nWrites += 1

        if 5 <= self.modelingLevel:
            this_atom = _x_(event, '?/hbs.THIS_ATOM')
            parent_atom = _x_(event, '?/hbs.PARENT_ATOM')
            null_atom = "\x00" * 16

            if this_atom is not None:
                if this_atom == null_atom:
                    this_atom = None
                else:
                    try:
                        this_atom = uuid.UUID(bytes=str(this_atom))
                    except:
                        self.log('invalid atom: %s / %s ( %s )' %
                                 (this_atom, type(this_atom),
                                  traceback.format_exc()))
                        this_atom = None

            if parent_atom is not None:
                if parent_atom == null_atom:
                    parent_atom = None
                else:
                    try:
                        parent_atom = uuid.UUID(bytes=str(parent_atom))
                    except:
                        self.log('invalid atom: %s / %s ( %s )' %
                                 (parent_atom, type(parent_atom),
                                  traceback.format_exc()))
                        parent_atom = None

            if this_atom is not None:
                self.asyncInsert(
                    self.stmt_atoms_lookup.bind(
                        (this_atom, eid, ttls['atoms'])))
                self.nWrites += 1

            if this_atom is not None and parent_atom is not None:
                self.asyncInsert(
                    self.stmt_atoms_children.bind(
                        (parent_atom,
                         this_atom if this_atom is not None else uuid.UUID(
                             bytes=null_atom), eid, ttls['atoms'])))
                self.nWrites += 1

        if 2 <= self.modelingLevel:
            inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
            if inv_id is not None and inv_id != '':
                self.asyncInsert(
                    self.stmt_investigation.bind(
                        (inv_id.upper().split('//')[0],
                         time_uuid.TimeUUID.with_timestamp(ts), eid,
                         routing['event_type'], ttls['detections'])))
                self.nWrites += 1

        # Only deal with Objects at level 7 and above.
        if 7 <= self.modelingLevel:
            new_objects = mtd['obj']
            new_relations = mtd['rel']

            for ignored in self.ignored_objects:
                if ignored in new_objects:
                    del (new_objects[ignored])
                for k in new_relations.keys():
                    if ignored in k:
                        del (new_relations[k])

            if 0 != len(new_objects) or 0 != len(new_relations):
                self._ingestObjects(ttls, sid, ts, new_objects, new_relations,
                                    agent.org_id, eid)
            #self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
        return (True, )
Exemple #6
0
def generate_time_uuid():
    t_id = str(time_uuid.TimeUUID.with_timestamp(time_uuid.utctime()))
    formatted_id = t_id[14:19] + t_id[9:14] + t_id[0:9] + t_id[19:]
    return formatted_id[0:18] + str(uuid.uuid4())[18:]
Exemple #7
0
def make_identifier() -> str:
    return TimeUUID.with_timestamp(utctime())
    def analyze(self, msg):
        routing, event, mtd = msg.data

        self.processedCounter += 1

        if 0 == (self.processedCounter % 50):
            self.log('MOD_IN %s' % self.processedCounter)

        agent = AgentId(routing['agentid'])
        aid = agent.invariableToString()
        ts = _x_(event, '?/base.TIMESTAMP')

        if ts is not None:
            ts = float(ts) / 1000

        if ts is None or ts > (2 * time.time()):
            ts = _x_(event, 'base.TIMESTAMP')
            if ts is None:
                ts = time_uuid.utctime()
            else:
                ts = float(ts) / 1000

        eid = routing['event_id']

        self.db.execute_async(self.statements['events'].bind(
            (eid,
             base64.b64encode(
                 msgpack.packb({
                     'routing': routing,
                     'event': event
                 })), aid)))

        self.db.execute_async(self.statements['timeline'].bind(
            (aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
             routing['event_type'])))

        self.db.execute_async(self.statements['timeline_by_type'].bind(
            (aid, time_uuid.TimeUUID.with_timestamp(ts), eid,
             routing['event_type'])))

        self.db.execute_async(self.statements['recent'].bind((aid, )))

        self.db.execute_async(self.statements['last'].bind(
            (eid, aid, routing['event_type'])))

        this_atom = _x_(event, '?/hbs.THIS_ATOM')
        parent_atom = _x_(event, '?/hbs.PARENT_ATOM')
        null_atom = "\x00" * 16

        if this_atom is not None:
            if this_atom == null_atom:
                this_atom = None
            else:
                try:
                    this_atom = uuid.UUID(bytes=str(this_atom))
                except:
                    self.log(
                        'invalid atom: %s / %s ( %s )' %
                        (this_atom, type(this_atom), traceback.format_exc()))
                    this_atom = None

        if parent_atom is not None:
            if parent_atom == null_atom:
                parent_atom = None
            else:
                try:
                    parent_atom = uuid.UUID(bytes=str(parent_atom))
                except:
                    self.log('invalid atom: %s / %s ( %s )' %
                             (parent_atom, type(parent_atom),
                              traceback.format_exc()))
                    parent_atom = None

        if this_atom is not None:
            self.db.execute_async(self.statements['atoms_lookup'].bind(
                (this_atom, eid)))

        if this_atom is not None and parent_atom is not None:
            self.db.execute_async(self.statements['atoms_children'].bind(
                (parent_atom,
                 this_atom if this_atom is not None else uuid.UUID(
                     bytes=null_atom), eid)))

        inv_id = _x_(event, '?/hbs.INVESTIGATION_ID')
        if inv_id is not None and inv_id != '':
            self.db.execute_async(self.statements['investigation'].bind(
                (inv_id.upper().split('//')[0],
                 time_uuid.TimeUUID.with_timestamp(ts), eid,
                 routing['event_type'])))
        new_objects = mtd['obj']
        new_relations = mtd['rel']

        for ignored in self.ignored_objects:
            if ignored in new_objects:
                del (new_objects[ignored])
            for k in new_relations.keys():
                if ignored in k:
                    del (new_relations[k])

        if 0 != len(new_objects) or 0 != len(new_relations):
            self._ingestObjects(aid, ts, new_objects, new_relations)
        #self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
        return (True, )
    def analyze( self, msg ):
        routing, event, mtd = msg.data

        self.processedCounter += 1

        if 0 == ( self.processedCounter % 50 ):
            self.log( 'MOD_IN %s' % self.processedCounter )

        agent = AgentId( routing[ 'agentid' ] )
        aid = agent.invariableToString()
        ts = _x_( event, '?/base.TIMESTAMP' )

        if ts is not None:
            ts = float( ts ) / 1000

        if ts is None or ts > ( 2 * time.time() ):
            ts = _x_( event, 'base.TIMESTAMP' )
            if ts is None:
                ts = time_uuid.utctime()
            else:
                ts = float( ts ) / 1000

        eid = routing[ 'event_id' ]

        self.db.execute_async( self.statements[ 'events' ].bind( ( eid,
                                                                   base64.b64encode( msgpack.packb( { 'routing' : routing, 'event' : event } ) ),
                                                                   aid ) ) )

        self.db.execute_async( self.statements[ 'timeline' ].bind( ( aid,
                                                                     time_uuid.TimeUUID.with_timestamp( ts ),
                                                                     eid,
                                                                     routing[ 'event_type' ] ) ) )

        self.db.execute_async( self.statements[ 'timeline_by_type' ].bind( ( aid,
                                                                             time_uuid.TimeUUID.with_timestamp( ts ),
                                                                             eid,
                                                                             routing[ 'event_type' ] ) ) )

        self.db.execute_async( self.statements[ 'recent' ].bind( ( aid, ) ) )

        self.db.execute_async( self.statements[ 'last' ].bind( ( eid,
                                                                 aid,
                                                                 routing[ 'event_type' ] ) ) )

        this_atom = _x_( event, '?/hbs.THIS_ATOM' )
        parent_atom = _x_( event, '?/hbs.PARENT_ATOM' )
        null_atom = "\x00" * 16

        if this_atom is not None:
            if this_atom == null_atom:
                this_atom = None
            else:
                try:
                    this_atom = uuid.UUID( bytes = str( this_atom ) )
                except:
                    self.log( 'invalid atom: %s / %s ( %s )' % ( this_atom, type( this_atom ), traceback.format_exc() ) )
                    this_atom = None

        if parent_atom is not None:
            if parent_atom == null_atom:
                parent_atom = None
            else:
                try:
                    parent_atom = uuid.UUID( bytes = str( parent_atom ) )
                except:
                    self.log( 'invalid atom: %s / %s ( %s )' % ( parent_atom, type( parent_atom ), traceback.format_exc() ) )
                    parent_atom = None

        if this_atom is not None:
            self.db.execute_async( self.statements[ 'atoms_lookup' ].bind( ( this_atom,
                                                                             eid ) ) )

        if this_atom is not None and parent_atom is not None:
            self.db.execute_async( self.statements[ 'atoms_children' ].bind( ( parent_atom,
                                                                               this_atom if this_atom is not None else uuid.UUID( bytes = null_atom ),
                                                                               eid ) ) )

        inv_id = _x_( event, '?/hbs.INVESTIGATION_ID' )
        if inv_id is not None and inv_id != '':
            self.db.execute_async( self.statements[ 'investigation' ].bind( ( inv_id.upper().split( '//' )[ 0 ],
                                                                              time_uuid.TimeUUID.with_timestamp( ts ),
                                                                              eid,
                                                                              routing[ 'event_type' ] ) ) )
        new_objects = mtd[ 'obj' ]
        new_relations = mtd[ 'rel' ]

        for ignored in self.ignored_objects:
            if ignored in new_objects:
                del( new_objects[ ignored ] )
            for k in new_relations.keys():
                if ignored in k:
                    del( new_relations[ k ] )

        if 0 != len( new_objects ) or 0 != len( new_relations ):
            self._ingestObjects( aid, ts, new_objects, new_relations )
        #self.log( 'finished storing objects %s: %s / %s' % ( routing[ 'event_type' ], len( new_objects ), len( new_relations )) )
        return ( True, )