def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]: return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "} is_new_indicator = False dmain = None ip = None rank = { 'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4, } indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True if 'campaign' in ind: if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0: confidence = ind.get('campaign_confidence', 'low') ind['campaign'] = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if isinstance(ind['campaign'], EmbeddedCampaign): indicator.add_campaign(ind['campaign']) elif isinstance(ind['campaign'], list): for campaign in ind['campaign']: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value url_contains_ip = False if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain_or_ip = urlparse.urlparse(ind_value).hostname elif ind_type == "URI - Domain Name": domain_or_ip = ind_value (sdomain, fqdn) = get_domain(domain_or_ip) if sdomain == "no_tld_found_error" and ind_type == "URI - URL": try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass if not url_contains_ip: success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success['object'] if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = 'Address - ipv4-addr' except DjangoValidationError: ind_type = 'Address - ipv6-addr' success = None if add_domain: success = ip_add_update(ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] indicator.save(username=analyst) if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {'success': True, 'objectid': str(indicator.id), 'is_new_indicator': is_new_indicator, 'object': indicator}
def create_indicator_and_ip(type_, id_, ip, analyst): """ Add indicators for an IP address. :param type_: The CRITs top-level object we are getting this IP from. :type type_: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param id_: The ObjectId of the top-level object to search for. :type id_: str :param ip: The IP address to generate an indicator out of. :type ip: str :param analyst: The user adding this indicator. :type analyst: str :returns: dict with keys: "success" (boolean), "message" (str), "value" (str) """ obj_class = class_from_id(type_, id_) if obj_class: ip_class = IP.objects(ip=ip).first() ind_type = "Address - ipv4-addr" ind_class = Indicator.objects(ind_type=ind_type, value=ip).first() # setup IP if ip_class: ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) else: ip_class = IP() ip_class.ip = ip ip_class.source = obj_class.source ip_class.save(username=analyst) ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) # setup Indicator message = "" if ind_class: message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) else: ind_class = Indicator() ind_class.source = obj_class.source ind_class.ind_type = ind_type ind_class.value = ip ind_class.save(username=analyst) message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) # save try: obj_class.save(username=analyst) ip_class.save(username=analyst) ind_class.save(username=analyst) if message['success']: rels = obj_class.sort_relationships("%s" % analyst, meta=True) return {'success': True, 'message': rels, 'value': obj_class.id} else: return {'success': False, 'message': message['message']} except Exception, e: return {'success': False, 'message': e}
def create_indicator_and_ip(type_, id_, ip, analyst): """ Add indicators for an IP address. :param type_: The CRITs top-level object we are getting this IP from. :type type_: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param id_: The ObjectId of the top-level object to search for. :type id_: str :param ip: The IP address to generate an indicator out of. :type ip: str :param analyst: The user adding this indicator. :type analyst: str :returns: dict with keys: "success" (boolean), "message" (str), "value" (str) """ obj_class = class_from_id(type_, id_) if obj_class: ip_class = IP.objects(ip=ip).first() ind_type = "Address - ipv4-addr" ind_class = Indicator.objects(ind_type=ind_type, value=ip).first() # setup IP if ip_class: ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) else: ip_class = IP() ip_class.ip = ip ip_class.source = obj_class.source ip_class.save(username=analyst) ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) # setup Indicator message = "" if ind_class: message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) else: ind_class = Indicator() ind_class.source = obj_class.source ind_class.ind_type = ind_type ind_class.value = ip ind_class.save(username=analyst) message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) # save try: obj_class.save(username=analyst) ip_class.save(username=analyst) ind_class.save(username=analyst) if message['success']: rels = obj_class.sort_relationships("%s" % analyst, meta=True) return { 'success': True, 'message': rels, 'value': obj_class.id } else: return {'success': False, 'message': message['message']} except Exception, e: return {'success': False, 'message': e}
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ is_new_indicator = False rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4} indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True ec = None if 'campaign' in ind: confidence = 'low' if 'campaign_confidence' in ind: confidence = ind['campaign_confidence'] ec = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get( indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get( indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if ec: indicator.add_campaign(ec) indicator.save(username=analyst) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain = ind_value.split("/")[2] elif ind_type == "URI - Domain Name": domain = ind_value #try: (sdomain, fqdn) = get_domain(domain) success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain).first() else: dmain = success['object'] if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) indicator.save(username=analyst) elif ind_type.startswith( "Address - ip") or ind_type == "Address - cidr": success = None if add_domain: success = ip_add_update(indicator.value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(None, indicator, analyst) return { 'success': True, 'objectid': indicator.id, 'is_new_indicator': is_new_indicator, 'object': indicator }
def handle_indicator_insert( ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={} ): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind["type"] not in IndicatorTypes.values(): return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]} if ind["threat_type"] not in IndicatorThreatTypes.values(): return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]} if ind["attack_type"] not in IndicatorAttackTypes.values(): return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]} (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"]) if error: return {"success": False, "message": error} is_new_indicator = False dmain = None ip = None rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4} if ind.get("status", None) is None or len(ind.get("status", "")) < 1: ind["status"] = Status.NEW indicator = Indicator.objects( ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"] ).first() if not indicator: indicator = Indicator() indicator.ind_type = ind["type"] indicator.threat_type = ind["threat_type"] indicator.attack_type = ind["attack_type"] indicator.value = ind["value"] indicator.lower = ind["lower"] indicator.description = ind["description"] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) indicator.status = ind["status"] is_new_indicator = True else: if ind["status"] != Status.NEW: indicator.status = ind["status"] add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"]) if indicator.description is None: indicator.description = add_desc else: indicator.description += add_desc if "campaign" in ind: if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0: confidence = ind.get("campaign_confidence", "low") ind["campaign"] = EmbeddedCampaign( name=ind["campaign"], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now(), ) if isinstance(ind["campaign"], EmbeddedCampaign): indicator.add_campaign(ind["campaign"]) elif isinstance(ind["campaign"], list): for campaign in ind["campaign"]: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind["confidence"] indicator.confidence.analyst = analyst if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind["impact"] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.lower url_contains_ip = False if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI): if ind_type == IndicatorTypes.URI: domain_or_ip = urlparse.urlparse(ind_value).hostname try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass else: domain_or_ip = ind_value if not url_contains_ip: success = None if add_domain: success = upsert_domain( domain_or_ip, indicator.source, username="******" % analyst, campaign=indicator.campaign, bucket_list=bucket_list, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success["object"] if ind_type in IPTypes.values() or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = IndicatorTypes.IPV4_ADDRESS except DjangoValidationError: ind_type = IndicatorTypes.IPV6_ADDRESS success = None if add_domain: success = ip_add_update( ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: ip = IP.objects(ip=indicator.value).first() else: ip = success["object"] indicator.save(username=analyst) if dmain: dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}