def execute_anb_event(cid, sources): # The inner dictionary is for keeping track of object IDs we have # already seen. The strings are for holding the CSV data. data = { 'seen_objects': {}, 'emails': '', 'samples': '', 'objects': '', 'events': '', 'domains': '', 'indicators': '', 'ips': '' } crits_event = Event.objects(id=cid, source__name__in=sources).first() if not crits_event: return data # Pre-populate with our event. data['seen_objects'][str(crits_event.id)] = crits_event data['events'] += "%s,%s,%s\r\n" % ('None', crits_event.id, crits_event.title) generate_anb_event_data('Event', crits_event.id, data, sources) # No need to pass this back to the view. del data['seen_objects'] return data
def execute_anb_event(cid, sources): # The inner dictionary is for keeping track of object IDs we have # already seen. The strings are for holding the CSV data. data = { 'seen_objects': {}, 'emails': '', 'samples': '', 'objects': '', 'events': '', 'domains': '', 'indicators': '', 'ips': '' } crits_event = Event.objects(id=cid, source__name__in=sources).first() if not crits_event: return data # Pre-populate with our event. data['seen_objects'][str(crits_event.id)] = crits_event data['events'] += "%s,%s,%s\r\n" % ( 'None', crits_event.id, crits_event.title) generate_anb_event_data('Event', crits_event.id, data, sources) # No need to pass this back to the view. del data['seen_objects'] return data
def update_event_description(event_id, description, analyst): """ Update event description. :param event_id: The ObjectId of the Event to update. :type event_id: str :param description: The new description. :type description: str :param analyst: The user updating this Event. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) """ if not description: return {'success': False, 'message': "No description to change"} event = Event.objects(id=event_id).first() if not event: return {'success': False, 'message': "No event found"} # Have to unescape the submitted data. Use unescape() to escape # < and friends. Use urllib2.unquote() to escape %3C and friends. h = HTMLParser() description = h.unescape(description) event.description = description try: event.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Target': return Target.objects(email_address=value).first() else: return None
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save()
def event_remove(_id, username): """ Remove an event from CRITs. :param _id: The ObjectId of the Event to remove. :type _id: str :param username: The user removing this Event. :type username: str :returns: dict with keys "success" (boolean) and "message" (str) """ event = Event.objects(id=_id).first() if event: event.delete(username=username) return {'success':True}
def event_remove(_id, username): """ Remove an event from CRITs. :param _id: The ObjectId of the Event to remove. :type _id: str :param username: The user removing this Event. :type username: str :returns: dict with keys "success" (boolean) and "message" (str) """ event = Event.objects(id=_id).first() if event: event.delete(username=username) return {'success': True}
def execute_anb_event(cid, sources): data = { 'emails': '', 'samples': '', 'objects': '', 'events': '', 'domains': '', 'indicators': '', 'ips': '' } crits_event = Event.objects(id=cid, source__name__in=sources).first() if not crits_event: return data generate_anb_event_data('Event', crits_event.id, data, sources) return data
def update_event_description(event_id, description, analyst): """ Update event description. :param event_id: The ObjectId of the Event to update. :type event_id: str :param description: The new description. :type description: str :param analyst: The user updating this Event. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) """ if not description: return {'success': False, 'message': "No description to change"} event = Event.objects(id=event_id).first() event.description = description try: event.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def update_event_type(event_id, type_, analyst): """ Update event type. :param event_id: The ObjectId of the Event to update. :type event_id: str :param type_: The new type. :type type_: str :param analyst: The user updating this Event. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) """ if not type_: return {'success': False, 'message': "No event type to change"} event = Event.objects(id=event_id).first() event.set_event_type(type_) try: event.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save()
def campaign_heatmap(request): campaigns = Campaign.objects().only('name', 'aliases', 'locations') events = Event.objects().only('title', 'created', 'locations', 'campaign') emails = Email.objects().only('created', 'locations', 'campaign') # list of countries in alphabetical order. set 0 for the amount of campaign # associated with this country for the next step. country_list = [] for c in campaigns: if len(c.locations): for l in c.locations: if [l.location,0] not in country_list: country_list.append([l.location,0]) country_list.sort() # For those campaigns with no location assigned, have an Unknown location. country_list.append(['Unknown', 0]) # list of campaigns in order of country, then alphabetical by name campaign_list = [] # for each country we build a tmp list, find all campaigns for that country, # sort the list, then append it to the campaign list. bump the count so we # know how many columns to span. for c in country_list: tmp = [] for cam in campaigns: if len(cam.locations): for l in cam.locations: if l.location == c[0]: c[1] += 1 if cam.name not in tmp: tmp.append(cam.name) break else: # Assuming we are checking the Unknown location, if this # campaign has no location assigned, add it to Unknown. if c[0] == 'Unknown': c[1] += 1 if cam.name not in tmp: tmp.append(cam.name) # If we haven't added a campaign to this location, show "No Campaigns". # This also prevents a left-shift in the counting and header rows. if len(tmp) == 0: tmp.append("No Campaigns") tmp.sort() campaign_list += tmp # list of the months going back in history and the activity of each campaign # during that month month_list = [] # for each campaign, find associated events and emails. For each event and # email, use the created date to put it into the appropriate list. month_d = {} idx = 0 # this is a default row in the heatmap with all values set to 0. pad_list = [0 for _ in range(len(campaign_list))] for c in campaign_list: build_month_d(pad_list, month_d, c, idx, events) build_month_d(pad_list, month_d, c, idx, emails) idx += 1 # sort the months in reverse order for descending display. for key in sorted(month_d, reverse=True): month_list.append([key, month_d[key]]) final_data = { 'country_list': country_list, 'campaign_list': campaign_list, 'month_list': month_list, } return final_data
def get_event_details(event_id, user): """ Generate the data to render the Event details template. :param event_id: The ObjectId of the Event to get details for. :type event_id: str :param user: The user requesting this information. :type user: str :returns: template (str), arguments (dict) """ template = None sources = user_sources(user) event = Event.objects(id=event_id, source__name__in=sources).first() if not user.check_source_tlp(event): event = None if not event: template = "error.html" args = {'error': "ID does not exist or insufficient privs for source"} return template, args event.sanitize("%s" % user) campaign_form = CampaignForm() download_form = DownloadFileForm(initial={ "obj_type": 'Event', "obj_id": event_id }) # remove pending notifications for user remove_user_from_notification("%s" % user, event.id, 'Event') # subscription subscription = { 'type': 'Event', 'id': event.id, 'subscribed': is_user_subscribed("%s" % user, 'Event', event.id), } #objects objects = event.sort_objects() #relationships relationships = event.sort_relationships("%s" % user, meta=True) # Get count of related Events for each related Indicator for ind in relationships.get('Indicator', []): count = Event.objects(relationships__object_id=ind['id'], source__name__in=sources).count() ind['rel_ind_events'] = count # Get count of related Events for each related Sample for smp in relationships.get('Sample', []): count = Event.objects(relationships__object_id=smp['id'], source__name__in=sources).count() smp['rel_smp_events'] = count # relationship relationship = {'type': 'Event', 'value': event.id} #comments comments = {'comments': event.get_comments(), 'url_key': event.id} #screenshots screenshots = event.get_screenshots(user) # favorites favorite = is_user_favorite("%s" % user, 'Event', event.id) # services service_list = get_supported_services('Event') # analysis results service_results = event.get_analysis_results() args = { 'service_list': service_list, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'screenshots': screenshots, 'event': event, 'campaign_form': campaign_form, 'service_results': service_results, 'download_form': download_form, 'EventACL': EventACL } return template, args
def _scan(self, context): #TODO: not sure if this should come after we make the TAXII message # so the check is closer to actual submission time? if not resolve_taxii_server(self.hostname): self._error("Cannot contact TAXII Server: %s" % self.hostname) return else: self._info("TAXII Server Online: %s" % self.hostname) self._notify() client = tc.HttpClient() client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({'key_file': self.keyfile, 'cert_file': self.certfile}) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS) event_list = Event.objects(id=context._id) if len(event_list) < 1: self._info("Could not locate event in the database") self._notify() else: event_data = event_list[0] (stix_doc, final_sources, final_objects) = event_data.to_stix(context.username) if len(final_sources) < 1: self._error("No sources to send to! Ensure all related content is marked as releasable!") return final_objects.append(event_data) # collect the list of data feeds to send this message to destination_feeds = [] for crtfile in self.certfiles: (source, feed, filepath) = crtfile.split(',') if source.strip() in final_sources: destination_feeds.append((source.strip(), feed.strip(), filepath.strip())) self._info("Generating STIX document(s).") self._notify() inbox_messages = [] # generate inbox messages # for now we will send one message per feed to isolate failures to one # feed submission and not prevent other messages from being sent. for feed in destination_feeds: # Create encrypted block encrypted_block = encrypt_block( tm.ContentBlock( content_binding = t.CB_STIX_XML_10, content = stix_doc.to_xml()).to_xml(), feed[2] ) # Wrap encrypted block in content block content_block = tm.ContentBlock( content_binding = "SMIME", content = encrypted_block ) # Create inbox message inbox_message = tm.InboxMessage( message_id = tm.generate_message_id(), content_blocks = [content_block], extended_headers = {'TargetFeed': feed[1]} ) inbox_messages.append((feed[0], inbox_message)) self._info("Sending TAXII message(s)") self._notify() # send messages for (src, inbox_msg) in inbox_messages: response = client.callTaxiiService2(self.hostname, "/inbox/", t.VID_TAXII_XML_10, inbox_message.to_xml()) taxii_message = t.get_message_from_http_response(response, inbox_message.message_id) if taxii_message.status_type == tm.ST_SUCCESS: # update releasability for objects date = datetime.datetime.now() instance = Releasability.ReleaseInstance(analyst=context.username, date=date) for idx in enumerate(final_objects): final_objects[idx[0]].add_releasability_instance(name=src, instance=instance) self._add_result(self.name, "Success", {'recipient': src}) else: self._add_result(self.name, "Failure", {'recipient': src}) # save releasability to database self._info("Updated releasability status for all related content.") self._notify() for obj in final_objects: obj.save() return
def get_event_details(event_id, user): """ Generate the data to render the Event details template. :param event_id: The ObjectId of the Event to get details for. :type event_id: str :param user: The user requesting this information. :type user: str :returns: template (str), arguments (dict) """ template = None sources = user_sources(user) event = Event.objects(id=event_id, source__name__in=sources).first() if not user.check_source_tlp(event): event = None if not event: template = "error.html" args = {'error': "ID does not exist or insufficient privs for source"} return template, args event.sanitize("%s" % user) campaign_form = CampaignForm() download_form = DownloadFileForm(initial={"obj_type": 'Event', "obj_id": event_id}) # remove pending notifications for user remove_user_from_notification("%s" % user, event.id, 'Event') # subscription subscription = { 'type': 'Event', 'id': event.id, 'subscribed': is_user_subscribed("%s" % user, 'Event', event.id), } #objects objects = event.sort_objects() #relationships relationships = event.sort_relationships("%s" % user, meta=True) # Get count of related Events for each related Indicator for ind in relationships.get('Indicator', []): count = Event.objects(relationships__object_id=ind['id'], source__name__in=sources).count() ind['rel_ind_events'] = count # Get count of related Events for each related Sample for smp in relationships.get('Sample', []): count = Event.objects(relationships__object_id=smp['id'], source__name__in=sources).count() smp['rel_smp_events'] = count # relationship relationship = { 'type': 'Event', 'value': event.id } #comments comments = {'comments': event.get_comments(), 'url_key': event.id} #screenshots screenshots = event.get_screenshots(user) # favorites favorite = is_user_favorite("%s" % user, 'Event', event.id) # services service_list = get_supported_services('Event') # analysis results service_results = event.get_analysis_results() args = {'service_list': service_list, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'screenshots': screenshots, 'event': event, 'campaign_form': campaign_form, 'service_results': service_results, 'download_form': download_form, 'EventACL': EventACL} return template, args
def add_sample_for_event(event_id, data, analyst, filedata=None, filename=None, md5=None): """ Add a sample related to this Event. :param event_id: The ObjectId of the Event to associate with. :type event_id: str :param data: The form data. :type data: dict :param analyst: The user adding this Sample. :type analyst: str :param filedata: The sample data. :type filedata: file handle. :param filename: The name of the file. :type filename: str :param md5: The MD5 of the file. :type md5: str :returns: dict with keys "success" (boolean) and "error" (str) """ sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=sources).first() if not event: error = "No matching event found" return {'success': False, 'error': error} source = data['source'] campaign = data['campaign'] confidence = data['confidence'] reference = data['source_reference'] file_format = data['file_format'] password = data['password'] bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] ticket = data[form_consts.Common.TICKET_VARIABLE_NAME] try: if filedata: sample_md5 = handle_uploaded_file(filedata, source, reference, file_format, password, analyst, campaign=campaign, confidence=confidence, bucket_list=bucket_list, ticket=ticket) else: if not filename or not md5: error = "Need a file, or a filename and an md5" return {'success': False, 'error': error} else: sample_md5 = handle_uploaded_file(None, source, reference, file_format, password, analyst, campaign=campaign, confidence=confidence, bucket_list=bucket_list, ticket=ticket, filename=filename.strip(), md5=md5.strip()) except ZipFileError, zfe: return {'success': False, 'error': zfe.value}
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'Action': return Action.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'Signature': return Signature.objects(id=_id).first() elif type_ == 'SignatureType': return SignatureType.objects(id=_id).first() elif type_ == 'SignatureDependency': return SignatureDependency.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import RelationshipType from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event, EventType from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.objects.object_type import ObjectType from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.backdoor import Backdoor from crits.samples.exploit import Exploit from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) if type_ == 'Backdoor': return Backdoor.objects(id=_id).first() if type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'EventType': return EventType.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'ObjectType': return ObjectType.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'RelationshipType': return RelationshipType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def add_sample_for_event(event_id, data, analyst, filedata=None, filename=None, md5=None, email_addr=None, inherit_sources=False): """ Add a sample related to this Event. :param event_id: The ObjectId of the Event to associate with. :type event_id: str :param data: The form data. :type data: dict :param analyst: The user adding this Sample. :type analyst: str :param filedata: The sample data. :type filedata: file handle. :param filename: The name of the file. :type filename: str :param md5: The MD5 of the file. :type md5: str :param email_addr: Email address to which to email the sample :type email_addr: str :param inherit_sources: 'True' if Sample should inherit Event's Source(s) :type inherit_sources: bool :returns: dict with keys "success" (boolean) and "message" (str) """ response = {'success': False, 'message': 'Unknown error; unable to upload file.'} users_sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=users_sources).first() if not event: return {'success': False, 'message': "No matching event found"} source = data['source'] reference = data['reference'] file_format = data['file_format'] campaign = data['campaign'] confidence = data['confidence'] bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] ticket = data[form_consts.Common.TICKET_VARIABLE_NAME] method = data['method'] if filename: filename = filename.strip() # If selected, new sample inherits the campaigns of the related event. if data['inherit_campaigns']: if campaign: event.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = event.campaign inherited_source = event.source if inherit_sources else None try: if filedata: result = handle_uploaded_file(filedata, source, method, reference, file_format, data['password'], analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source) else: if md5: md5 = md5.strip().lower() result = handle_uploaded_file(None, source, method, reference, file_format, None, analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, md5=md5, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source, is_return_only_md5=False) except ZipFileError, zfe: return {'success': False, 'message': zfe.value}
def create_indicator_from_tlo( tlo_type, tlo, analyst, source_name=None, tlo_id=None, ind_type=None, value=None, update_existing=True, add_domain=True, ): """ Create an indicator from a Top-Level Object (TLO). :param tlo_type: The CRITs type of the parent TLO. :type tlo_type: str :param tlo: A CRITs parent TLO class object :type tlo: class - some CRITs TLO :param analyst: The user creating this indicator. :type analyst: str :param source_name: The source name for the new source instance that records this indicator being added. :type source_name: str :param tlo_id: The ObjectId of the parent TLO. :type tlo_id: str :param ind_type: The indicator type, if TLO is not Domain or IP. :type ind_type: str :param value: The value of the indicator, if TLO is not Domain or IP. :type value: str :param update_existing: If Indicator already exists, update it :type update_existing: boolean :param add_domain: If new indicator contains a domain/ip, add a matching Domain or IP TLO :type add_domain: boolean :returns: dict with keys: "success" (boolean), "message" (str), "value" (str), "indicator" :class:`crits.indicators.indicator.Indicator` """ if not tlo: tlo = class_from_id(tlo_type, tlo_id) if not tlo: return {"success": False, "message": "Could not find %s" % tlo_type} source = tlo.source campaign = tlo.campaign bucket_list = tlo.bucket_list tickets = tlo.tickets # If value and ind_type provided, use them instead of defaults if tlo_type == "Domain": value = value or tlo.domain ind_type = ind_type or IndicatorTypes.DOMAIN elif tlo_type == "IP": value = value or tlo.ip ind_type = ind_type or tlo.ip_type elif tlo_type == "Indicator": value = value or tlo.value ind_type = ind_type or tlo.ind_type if not value or not ind_type: # if not provided & no default return {"success": False, "message": "Indicator value & type must be provided" "for TLO of type %s" % tlo_type} # check if indicator already exists if Indicator.objects(ind_type=ind_type, value=value).first() and not update_existing: return {"success": False, "message": "Indicator already exists"} result = handle_indicator_ind( value, source, ctype=ind_type, threat_type=IndicatorThreatTypes.UNKNOWN, attack_type=IndicatorAttackTypes.UNKNOWN, analyst=analyst, add_domain=add_domain, add_relationship=True, campaign=campaign, bucket_list=bucket_list, ticket=tickets, ) if result["success"]: ind = Indicator.objects(id=result["objectid"]).first() if ind: if source_name: # add source to show when indicator was created/updated ind.add_source( source=source_name, method="Indicator created/updated " "from %s with ID %s" % (tlo_type, tlo.id), date=datetime.datetime.now(), analyst=analyst, ) tlo.add_relationship(ind, RelationshipTypes.RELATED_TO, analyst=analyst) tlo.save(username=analyst) for rel in tlo.relationships: if rel.rel_type == "Event": # Get event object to pass in. rel_item = Event.objects(id=rel.object_id).first() if rel_item: ind.add_relationship(rel_item, RelationshipTypes.RELATED_TO, analyst=analyst) ind.save(username=analyst) tlo.reload() rels = tlo.sort_relationships("%s" % analyst, meta=True) return {"success": True, "message": rels, "value": tlo.id, "indicator": ind} else: return {"success": False, "message": "Failed to create Indicator"} else: return result
def get_event_details(event_id, analyst): """ Generate the data to render the Event details template. :param event_id: The ObjectId of the Event to get details for. :type event_id: str :param analyst: The user requesting this information. :type analyst: str :returns: template (str), arguments (dict) """ template = None sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=sources).first() if not event: template = "error.html" args = {'error': "ID does not exist or insufficient privs for source"} return template, args event.sanitize("%s" % analyst) campaign_form = CampaignForm() download_form = DownloadFileForm(initial={ "obj_type": 'Event', "obj_id": event_id }) # remove pending notifications for user remove_user_from_notification("%s" % analyst, event.id, 'Event') # subscription subscription = { 'type': 'Event', 'id': event.id, 'subscribed': is_user_subscribed("%s" % analyst, 'Event', event.id), } #objects objects = event.sort_objects() #relationships relationships = event.sort_relationships("%s" % analyst, meta=True) # relationship relationship = {'type': 'Event', 'value': event.id} #comments comments = {'comments': event.get_comments(), 'url_key': event.id} #screenshots screenshots = event.get_screenshots(analyst) # favorites favorite = is_user_favorite("%s" % analyst, 'Event', event.id) # services service_list = get_supported_services('Event') # analysis results service_results = event.get_analysis_results() args = { 'service_list': service_list, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'screenshots': screenshots, 'event': event, 'campaign_form': campaign_form, 'service_results': service_results, 'download_form': download_form } return template, args
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # Quick fail if not _id or not type_: return None # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode("utf8")): return None if type_ == "Actor": return Actor.objects(id=_id).first() elif type_ == "Backdoor": return Backdoor.objects(id=_id).first() elif type_ == "ActorThreatIdentifier": return ActorThreatIdentifier.objects(id=_id).first() elif type_ == "Campaign": return Campaign.objects(id=_id).first() elif type_ == "Certificate": return Certificate.objects(id=_id).first() elif type_ == "Comment": return Comment.objects(id=_id).first() elif type_ == "Domain": return Domain.objects(id=_id).first() elif type_ == "Email": return Email.objects(id=_id).first() elif type_ == "Event": return Event.objects(id=_id).first() elif type_ == "Exploit": return Exploit.objects(id=_id).first() elif type_ == "Indicator": return Indicator.objects(id=_id).first() elif type_ == "Action": return Action.objects(id=_id).first() elif type_ == "IP": return IP.objects(id=_id).first() elif type_ == "PCAP": return PCAP.objects(id=_id).first() elif type_ == "RawData": return RawData.objects(id=_id).first() elif type_ == "RawDataType": return RawDataType.objects(id=_id).first() elif type_ == "Sample": return Sample.objects(id=_id).first() elif type_ == "Signature": return Signature.objects(id=_id).first() elif type_ == "SignatureType": return SignatureType.objects(id=_id).first() elif type_ == "SignatureDependency": return SignatureDependency.objects(id=_id).first() elif type_ == "SourceAccess": return SourceAccess.objects(id=_id).first() elif type_ == "Screenshot": return Screenshot.objects(id=_id).first() elif type_ == "Target": return Target.objects(id=_id).first() elif type_ == "UserRole": return UserRole.objects(id=_id).first() else: return None
def get_event_details(event_id, analyst): """ Generate the data to render the Event details template. :param event_id: The ObjectId of the Event to get details for. :type event_id: str :param analyst: The user requesting this information. :type analyst: str :returns: template (str), arguments (dict) """ template = None sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=sources).first() if not event: template = "error.html" args = {'error': "ID does not exist or insufficient privs for source"} return template, args event.sanitize("%s" % analyst) campaign_form = CampaignForm() download_form = DownloadFileForm(initial={"obj_type": 'Event', "obj_id": event_id}) # remove pending notifications for user remove_user_from_notification("%s" % analyst, event.id, 'Event') # subscription subscription = { 'type': 'Event', 'id': event.id, 'subscribed': is_user_subscribed("%s" % analyst, 'Event', event.id), } #objects objects = event.sort_objects() #relationships relationships = event.sort_relationships("%s" % analyst, meta=True) # relationship relationship = { 'type': 'Event', 'value': event.id } #comments comments = {'comments': event.get_comments(), 'url_key': event.id} #screenshots screenshots = event.get_screenshots(analyst) # favorites favorite = is_user_favorite("%s" % analyst, 'Event', event.id) # services service_list = get_supported_services('Event') # analysis results service_results = event.get_analysis_results() args = {'service_list': service_list, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'screenshots': screenshots, 'event': event, 'campaign_form': campaign_form, 'service_results': service_results, 'download_form': download_form} return template, args
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature from crits.targets.target import Target # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in [ 'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator', 'Screenshot' ] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Actor': return Actor.objects(name=value).first() if type_ == 'Backdoor': return Backdoor.objects(id=value).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(name=value).first() elif type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(message_id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Exploit': return Exploit.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Signature': return Signature.objects(md5=value).first() elif type_ == 'Target': target = Target.objects(email_address=value).first() if target: return target else: return Target.objects(email_address__iexact=value).first() else: return None
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit', 'Indicator', 'Screenshot'] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Actor': return Actor.objects(name=value).first() if type_ == 'Backdoor': return Backdoor.objects(id=value).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(name=value).first() elif type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Exploit': return Exploit.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Target': target = Target.objects(email_address=value).first() if target: return target else: return Target.objects(email_address__iexact=value).first() else: return None
def _scan(self, context): #TODO: not sure if this should come after we make the TAXII message # so the check is closer to actual submission time? if not resolve_taxii_server(self.hostname): self._error("Cannot contact TAXII Server: %s" % self.hostname) return else: self._info("TAXII Server Online: %s" % self.hostname) self._notify() client = tc.HttpClient() client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({ 'key_file': self.keyfile, 'cert_file': self.certfile }) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS) event_list = Event.objects(id=context._id) if len(event_list) < 1: self._info("Could not locate event in the database") self._notify() else: event_data = event_list[0] (stix_doc, final_sources, final_objects) = event_data.to_stix(context.username) if len(final_sources) < 1: self._error( "No sources to send to! Ensure all related content is marked as releasable!" ) return final_objects.append(event_data) # collect the list of data feeds to send this message to destination_feeds = [] for crtfile in self.certfiles: (source, feed, filepath) = crtfile.split(',') if source.strip() in final_sources: destination_feeds.append( (source.strip(), feed.strip(), filepath.strip())) self._info("Generating STIX document(s).") self._notify() inbox_messages = [] # generate inbox messages # for now we will send one message per feed to isolate failures to one # feed submission and not prevent other messages from being sent. for feed in destination_feeds: # Create encrypted block encrypted_block = encrypt_block( tm.ContentBlock(content_binding=t.CB_STIX_XML_10, content=stix_doc.to_xml()).to_xml(), feed[2]) # Wrap encrypted block in content block content_block = tm.ContentBlock(content_binding="SMIME", content=encrypted_block) # Create inbox message inbox_message = tm.InboxMessage( message_id=tm.generate_message_id(), content_blocks=[content_block], extended_headers={'TargetFeed': feed[1]}) inbox_messages.append((feed[0], inbox_message)) self._info("Sending TAXII message(s)") self._notify() # send messages for (src, inbox_msg) in inbox_messages: response = client.callTaxiiService2( self.hostname, "/inbox/", t.VID_TAXII_XML_10, inbox_message.to_xml()) taxii_message = t.get_message_from_http_response( response, inbox_message.message_id) if taxii_message.status_type == tm.ST_SUCCESS: # update releasability for objects date = datetime.datetime.now() instance = Releasability.ReleaseInstance( analyst=context.username, date=date) for idx in enumerate(final_objects): final_objects[idx[0]].add_releasability_instance( name=src, instance=instance) self._add_result(self.name, "Success", {'recipient': src}) else: self._add_result(self.name, "Failure", {'recipient': src}) # save releasability to database self._info( "Updated releasability status for all related content.") self._notify() for obj in final_objects: obj.save() return
def add_sample_for_event(event_id, data, analyst, filedata=None, filename=None, md5=None, email_addr=None, inherit_sources=False): """ Add a sample related to this Event. :param event_id: The ObjectId of the Event to associate with. :type event_id: str :param data: The form data. :type data: dict :param analyst: The user adding this Sample. :type analyst: str :param filedata: The sample data. :type filedata: file handle. :param filename: The name of the file. :type filename: str :param md5: The MD5 of the file. :type md5: str :param email_addr: Email address to which to email the sample :type email_addr: str :param inherit_sources: 'True' if Sample should inherit Event's Source(s) :type inherit_sources: bool :returns: dict with keys "success" (boolean) and "message" (str) """ response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } users_sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=users_sources).first() if not event: return {'success': False, 'message': "No matching event found"} source = data['source'] reference = data['reference'] file_format = data['file_format'] campaign = data['campaign'] confidence = data['confidence'] bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] ticket = data[form_consts.Common.TICKET_VARIABLE_NAME] method = data['method'] if filename: filename = filename.strip() # If selected, new sample inherits the campaigns of the related event. if data['inherit_campaigns']: if campaign: event.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = event.campaign inherited_source = event.source if inherit_sources else None try: if filedata: result = handle_uploaded_file(filedata, source, method, reference, file_format, data['password'], analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source) else: if md5: md5 = md5.strip().lower() result = handle_uploaded_file(None, source, method, reference, file_format, None, analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, md5=md5, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source, is_return_only_md5=False) except ZipFileError, zfe: return {'success': False, 'message': zfe.value}
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def campaign_heatmap(request): campaigns = Campaign.objects().only('name', 'aliases', 'locations') events = Event.objects().only('title', 'created', 'locations', 'campaign') emails = Email.objects().only('created', 'locations', 'campaign') # list of countries in alphabetical order. set 0 for the amount of campaign # associated with this country for the next step. country_list = [] for c in campaigns: if len(c.locations): for l in c.locations: if [l.location, 0] not in country_list: country_list.append([l.location, 0]) country_list.sort() # For those campaigns with no location assigned, have an Unknown location. country_list.append(['Unknown', 0]) # list of campaigns in order of country, then alphabetical by name campaign_list = [] # for each country we build a tmp list, find all campaigns for that country, # sort the list, then append it to the campaign list. bump the count so we # know how many columns to span. for c in country_list: tmp = [] for cam in campaigns: if len(cam.locations): for l in cam.locations: if l.location == c[0]: c[1] += 1 if cam.name not in tmp: tmp.append(cam.name) break else: # Assuming we are checking the Unknown location, if this # campaign has no location assigned, add it to Unknown. if c[0] == 'Unknown': c[1] += 1 if cam.name not in tmp: tmp.append(cam.name) # If we haven't added a campaign to this location, show "No Campaigns". # This also prevents a left-shift in the counting and header rows. if len(tmp) == 0: tmp.append("No Campaigns") tmp.sort() campaign_list += tmp # list of the months going back in history and the activity of each campaign # during that month month_list = [] # for each campaign, find associated events and emails. For each event and # email, use the created date to put it into the appropriate list. month_d = {} idx = 0 # this is a default row in the heatmap with all values set to 0. pad_list = [0 for _ in range(len(campaign_list))] for c in campaign_list: build_month_d(pad_list, month_d, c, idx, events) build_month_d(pad_list, month_d, c, idx, emails) idx += 1 # sort the months in reverse order for descending display. for key in sorted(month_d, reverse=True): month_list.append([key, month_d[key]]) final_data = { 'country_list': country_list, 'campaign_list': campaign_list, 'month_list': month_list, } return final_data