def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): logger.info('Handling attribute') # check if jsonattr is an attribute object if 'Attribute' in jsonobj: jsonattr = jsonobj['Attribute'] #Add trending categName = jsonattr['category'] timestamp = jsonattr.get('timestamp', int(time.time())) trendings_helper.addTrendingCateg(categName, timestamp) tags = [] for tag in jsonattr.get('Tag', []): try: tags.append(tag) except KeyError: pass trendings_helper.addTrendingTags(tags, timestamp) to_push = [] for field in json.loads(cfg.get('Dashboard', 'fieldname_order')): if type(field) is list: to_join = [] for subField in field: to_join.append(str(getFields(jsonobj, subField))) to_add = cfg.get('Dashboard', 'char_separator').join(to_join) else: to_add = getFields(jsonobj, field) to_push.append(to_add) #try to get coord from ip if jsonattr['category'] == "Network activity": geo_helper.getCoordFromIpAndPublish(jsonattr['value'], jsonattr['category']) #try to get coord from ip if jsonattr['type'] == "phone-number": geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category']) if not hasAlreadyBeenContributed: eventLabeled = len(jsonobj.get('EventTag', [])) > 0 action = jsonobj.get('action', None) contributor_helper.handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], 'Attribute', jsonattr['category'], action, isLabeled=eventLabeled) # Push to log publish_log(zmq_name, 'Attribute', to_push)
def handler_event(zmq_name, jsonobj): logger.info('Handling event') #fields: threat_level_id, id, info jsonevent = jsonobj['Event'] #Add trending eventName = jsonevent['info'] timestamp = jsonevent['timestamp'] trendings_helper.addTrendingEvent(eventName, timestamp) tags = [] for tag in jsonevent.get('Tag', []): tags.append(tag) trendings_helper.addTrendingTags(tags, timestamp) #redirect to handler_attribute if 'Attribute' in jsonevent: attributes = jsonevent['Attribute'] if type(attributes) is list: for attr in attributes: jsoncopy = copy.deepcopy(jsonobj) jsoncopy['Attribute'] = attr handler_attribute(zmq_name, jsoncopy) else: handler_attribute(zmq_name, attributes) if 'Object' in jsonevent: objects = jsonevent['Object'] if type(objects) is list: for obj in objects: jsoncopy = copy.deepcopy(jsonobj) jsoncopy['Object'] = obj handler_object(zmq_name, jsoncopy) else: handler_object(zmq_name, objects) action = jsonobj.get('action', None) eventLabeled = len(jsonobj.get('EventTag', [])) > 0 org = jsonobj.get('Orgc', {}).get('name', None) if org is not None: contributor_helper.handleContribution(zmq_name, org, 'Event', None, action, isLabeled=eventLabeled)
def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False, parentObject=False): logger.info('Handling attribute') # check if jsonattr is an attribute object if 'Attribute' in jsonobj: jsonattr = jsonobj['Attribute'] else: jsonattr = jsonobj attributeType = 'Attribute' if jsonattr[ 'object_id'] == '0' else 'ObjectAttribute' #Add trending categName = jsonattr['category'] timestamp = jsonattr.get('timestamp', int(time.time())) trendings_helper.addTrendingCateg(categName, timestamp) tags = [] for tag in jsonattr.get('Tag', []): tags.append(tag) trendings_helper.addTrendingTags(tags, timestamp) #try to get coord from ip if jsonattr['category'] == "Network activity": geo_helper.getCoordFromIpAndPublish(jsonattr['value'], jsonattr['category']) #try to get coord from ip if jsonattr['type'] == "phone-number": geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category']) if not hasAlreadyBeenContributed: eventLabeled = len(jsonobj.get('EventTag', [])) > 0 action = jsonobj.get('action', None) contributor_helper.handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], attributeType, jsonattr['category'], action, isLabeled=eventLabeled) # Push to log live_helper.publish_log(zmq_name, attributeType, jsonobj)
def test(): flag_error = False today = datetime.datetime.now() now = time.time # Events event1 = 'test_event_1' event2 = 'test_event_2' trendings_helper.addTrendingEvent(event1, now()) trendings_helper.addTrendingEvent(event1, now() + 5) trendings_helper.addTrendingEvent(event2, now() + 10) expected_result = [[int(now()), [[event1, 2.0], [event2, 1.0]]]] rep = trendings_helper.getTrendingEvents(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingEvents result not matching') flag_error = True # Tags tag1 = {'id': 'tag1', 'colour': 'blue', 'name': 'tag1Name'} tag2 = {'id': 'tag2', 'colour': 'red', 'name': 'tag2Name'} trendings_helper.addTrendingTags([tag1], now()) trendings_helper.addTrendingTags([tag1], now() + 5) trendings_helper.addTrendingTags([tag2], now() + 10) expected_result = [[int(now()), [[tag1, 2.0], [tag2, 1.0]]]] rep = trendings_helper.getTrendingTags(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingTags result not matching') flag_error = True # Sightings trendings_helper.addSightings(now()) trendings_helper.addSightings(now()) trendings_helper.addFalsePositive(now()) expected_result = [[1512636256, {'sightings': 2, 'false_positive': 1}]] rep = trendings_helper.getTrendingSightings(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingSightings result not matching') flag_error = True return flag_error