def set_indicator_attack_type(id_, attack_type, user, **kwargs): """ Set the Indicator attack type. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param attack_type: The new indicator attack type. :type attack_type: str :param user: The user updating the indicator. :type user: str :returns: dict with key "success" (boolean) """ # check to ensure we're not duping an existing indicator indicator = Indicator.objects(id=id_).first() value = indicator.value ind_check = Indicator.objects(attack_type=attack_type, value=value).first() if ind_check: # we found a dupe return {"success": False, "message": "Duplicate would exist making this change."} elif attack_type not in IndicatorAttackTypes.values(): return {"success": False, "message": "Not a valid Attack Type."} else: try: indicator.attack_type = attack_type indicator.save(username=user) return {"success": True} except ValidationError: return {"success": False}
def run(self, obj, config): if isinstance(obj, RawData): data = obj.data elif isinstance(obj, Sample): samp_data = obj.filedata.read() data = make_ascii_strings(data=samp_data) if not data: self._debug("Could not find sample data to parse.") return else: self._debug("This type is not supported by this service.") return ips = extract_ips(data) for ip in ips: tdict = {'Type': "IP Address"} id_ = Indicator.objects(value=ip).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential IP Address', ip, tdict) domains = extract_domains(data) for domain in domains: tdict = {'Type': "Domain"} id_ = Indicator.objects(value=domain).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Domains', domain, tdict) emails = extract_emails(data) for email in emails: tdict = {'Type': "Email"} id_ = Indicator.objects(value=email).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Emails', email, tdict)
def set_indicator_type(indicator_id, itype, username): """ Set the Indicator type. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param itype: The new indicator type. :type itype: str :param username: The user updating the indicator. :type username: str :returns: dict with key "success" (boolean) """ # check to ensure we're not duping an existing indicator indicator = Indicator.objects(id=indicator_id).first() value = indicator.value ind_check = Indicator.objects(ind_type=itype, value=value).first() if ind_check: # we found a dupe return {'success': False} else: try: indicator.ind_type = itype indicator.save(username=username) return {'success': True} except ValidationError: return {'success': False}
def run(self, obj, config): if isinstance(obj, Event): data = obj.description elif isinstance(obj, RawData): data = obj.data elif isinstance(obj, Sample): samp_data = obj.filedata.read() data = make_ascii_strings(data=samp_data) if not data: self._debug("Could not find sample data to parse.") return else: self._debug("This type is not supported by this service.") return ips = extract_ips(data) for ip in ips: tdict = {'Type': IndicatorTypes.IPV4_ADDRESS} id_ = Indicator.objects(value=ip).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential IP Address', ip, tdict) domains = extract_domains(data) for domain in domains: tdict = {'Type': IndicatorTypes.DOMAIN} id_ = Indicator.objects(value=domain).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Domains', domain, tdict) emails = extract_emails(data) for email in emails: tdict = {'Type': IndicatorTypes.EMAIL_ADDRESS} id_ = Indicator.objects(value=email).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Emails', email, tdict) hashes = extract_hashes(data) for hash_ in hashes: type_ = hash_[0] val = hash_[1] tdict = {'Type': type_} if type_ == IndicatorTypes.MD5: id_ = Sample.objects(md5=val).only('id').first() elif type_ == IndicatorTypes.SHA1: id_ = Sample.objects(sha1=val).only('id').first() elif type_ == IndicatorTypes.SHA256: id_ = Sample.objects(sha256=val).only('id').first() elif type_ == IndicatorTypes.SSDEEP: id_ = Sample.objects(ssdeep=val).only('id').first() else: id_ = None if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Samples', val, tdict)
def ci_search(itype, confidence, impact, actions): """ Find indicators based on type, confidence, impact, and/or actions. :param itype: The indicator type to search for. :type itype: str :param confidence: The confidence level(s) to search for. :type confidence: str :param impact: The impact level(s) to search for. :type impact: str :param actions: The action(s) to search for. :type actions: str :returns: :class:`crits.core.crits_mongoengine.CritsQuerySet` """ query = {} if confidence: item_list = confidence.replace(' ', '').split(',') query["confidence.rating"] = {"$in": item_list} if impact: item_list = impact.replace(' ', '').split(',') query["impact.rating"] = {"$in": item_list} if actions: item_list = actions.split(',') query["actions.action_type"] = {"$in": item_list} query["type"] = "%s" % itype.strip() result_filter = ('type', 'value', 'confidence', 'impact', 'actions') results = Indicator.objects(__raw__=query).only(*result_filter) return results
def ci_update(indicator_id, ci_type, value, analyst): """ Update confidence or impact for an indicator. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param ci_type: What we are updating. :type ci_type: str ("confidence" or "impact") :param value: The value to set. :type value: str ("unknown", "benign", "low", "medium", "high") :param analyst: The user updating this indicator. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ indicator = Indicator.objects(id=indicator_id).first() if not indicator: return {'success': False, 'message': 'Could not find Indicator'} if ci_type == "confidence" or ci_type == "impact": try: if ci_type == "confidence": indicator.set_confidence(analyst, value) else: indicator.set_impact(analyst, value) indicator.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, "message": e}
def activity_update(indicator_id, activity): """ Update activity for an Indicator. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param activity: The activity information. :type activity: dict :returns: dict with keys: "success" (boolean), "message" (str) if failed, "object" (dict) if successful. """ sources = user_sources(activity['analyst']) indicator = Indicator.objects(id=indicator_id, source__name__in=sources).first() if not indicator: return {'success': False, 'message': 'Could not find Indicator'} try: indicator.edit_activity(activity['analyst'], activity['start_date'], activity['end_date'], activity['description'], activity['date']) indicator.save(username=activity['analyst']) return {'success': True, 'object': activity} except ValidationError, e: return {'success': False, 'message': e}
def action_add(indicator_id, action): """ Add an action to an indicator. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param action: The information about the action. :type action: dict :returns: dict with keys: "success" (boolean), "message" (str) if failed, "object" (dict) if successful. """ sources = user_sources(action['analyst']) indicator = Indicator.objects(id=indicator_id, source__name__in=sources).first() if not indicator: return {'success': False, 'message': 'Could not find Indicator'} try: indicator.add_action(action['action_type'], action['active'], action['analyst'], action['begin_date'], action['end_date'], action['performed_date'], action['reason'], action['date']) indicator.save(username=action['analyst']) return {'success': True, 'object': action} except ValidationError, e: return {'success': False, 'message': e}
def activity_update(id_, activity, user=None, **kwargs): """ Update activity for an Indicator. :param id_: The ObjectId of the indicator to update. :type id_: str :param activity: The activity information. :type activity: dict :param user: The user updating the activity. :type user: str :returns: dict with keys: "success" (boolean), "message" (str) if failed, "object" (dict) if successful. """ sources = user_sources(user) indicator = Indicator.objects(id=id_, source__name__in=sources).first() if not indicator: return {"success": False, "message": "Could not find Indicator"} try: activity = datetime_parser(activity) activity["analyst"] = user indicator.edit_activity( activity["analyst"], activity["start_date"], activity["end_date"], activity["description"], activity["date"] ) indicator.save(username=user) return {"success": True, "object": activity} except ValidationError, e: return {"success": False, "message": e}
def create_indicator_from_obj(ind_type, obj_type, id_, value, analyst): """ Add indicators from CRITs object. :param ind_type: The indicator type to add. :type ind_type: str :param obj_type: The CRITs type of the parent object. :type obj_type: str :param id_: The ObjectId of the parent object. :type id_: str :param value: The value of the indicator to add. :type value: str :param analyst: The user adding this indicator. :type analyst: str :returns: dict with keys: "success" (boolean), "message" (str), "value" (str) """ obj = class_from_id(obj_type, id_) if not obj: return {'success': False, 'message': 'Could not find object.'} source = obj.source bucket_list = obj.bucket_list campaign = None campaign_confidence = None if len(obj.campaign) > 0: campaign = obj.campaign[0].name campaign_confidence = obj.campaign[0].confidence result = handle_indicator_ind(value, source, reference=None, ctype=ind_type, analyst=analyst, add_domain=True, add_relationship=True, campaign=campaign, campaign_confidence=campaign_confidence, bucket_list=bucket_list) if result['success']: ind = Indicator.objects(id=result['objectid']).first() if ind: obj.add_relationship(rel_item=ind, rel_type="Related_To", analyst=analyst) obj.save(username=analyst) for rel in obj.relationships: if rel.rel_type == "Event": ind.add_relationship(rel_id=rel.object_id, type_=rel.rel_type, rel_type="Related_To", analyst=analyst) ind.save(username=analyst) obj.reload() rels = obj.sort_relationships("%s" % analyst, meta=True) return {'success': True, 'message': rels, 'value': id_} else: return {'success': False, 'message': result['message']}
def create_indicator_from_raw(type_, id_, value, analyst): """ Add indicators from raw data. :param type_: The indicator type to add. :type type_: str :param id_: The ObjectId of the RawData object. :type id_: str :param value: The value of the indicator to add. :type value: str :param analyst: The user adding this indicator. :type analyst: str :returns: dict with keys: "success" (boolean), "message" (str), "value" (str) """ raw_data = RawData.objects(id=id_).first() if not raw_data: return {'success': False, 'message': 'Could not find raw data'} source = raw_data.source bucket_list = raw_data.bucket_list campaign = None campaign_confidence = None if len(raw_data.campaign) > 0: campaign = raw_data.campaign[0].name campaign_confidence = raw_data.campaign[0].confidence result = handle_indicator_ind(value, source, reference=None, ctype=type_, analyst=analyst, add_domain=True, add_relationship=True, campaign=campaign, campaign_confidence=campaign_confidence, bucket_list=bucket_list) if result['success']: ind = Indicator.objects(id=result['objectid']).first() if ind: raw_data.add_relationship(rel_item=ind, rel_type="Related_To", analyst=analyst) raw_data.save(username=analyst) for rel in raw_data.relationships: if rel.rel_type == "Event": ind.add_relationship(rel_id=rel.object_id, type_=rel.rel_type, rel_type="Related_To", analyst=analyst) ind.save(username=analyst) raw_data.reload() rels = raw_data.sort_relationships("%s" % analyst, meta=True) return {'success': True, 'message': rels, 'value': id_} else: return {'success': False, 'message': result['message']}
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Target': return Target.objects(email_address=value).first() else: return None
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save()
def _scan(self, context): if isinstance(context, RawDataContext): raw_data = RawData.objects(id=context.identifier).first() if not raw_data: self._debug("Could not find raw data to parse.") return data = raw_data.data elif isinstance(context, SampleContext): data = make_ascii_strings(md5=context.identifier) if not data: self._debug("Could not find sample data to parse.") return else: self._debug("This type is not supported by this service.") return ips = extract_ips(data) for ip in ips: tdict = {'Type': "IP Address"} id_ = Indicator.objects(value=ip).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential IP Address', ip, tdict) domains = extract_domains(data) for domain in domains: tdict = {'Type': "Domain"} id_ = Indicator.objects(value=domain).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Domains', domain, tdict) emails = extract_emails(data) for email in emails: tdict = {'Type': "Email"} id_ = Indicator.objects(value=email).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Emails', email, tdict)
def _scan(self, context): if isinstance(context, RawDataContext): raw_data = RawData.objects(id=context.identifier).first() if not raw_data: self._debug("Could not find raw data to parse.") return data = raw_data.data elif isinstance(context, SampleContext): data = make_ascii_strings(md5=context.identifier) if not data: self._debug("Could not find sample data to parse.") return else: self._debug("This type is not supported by this service.") return ips = extract_ips(data) for ip in ips: tdict = {"Type": "IP Address"} id_ = Indicator.objects(value=ip).only("id").first() if id_: tdict["exists"] = str(id_.id) self._add_result("Potential IP Address", ip, tdict) domains = extract_domains(data) for domain in domains: tdict = {"Type": "Domain"} id_ = Indicator.objects(value=domain).only("id").first() if id_: tdict["exists"] = str(id_.id) self._add_result("Potential Domains", domain, tdict) emails = extract_emails(data) for email in emails: tdict = {"Type": "Email"} id_ = Indicator.objects(value=email).only("id").first() if id_: tdict["exists"] = str(id_.id) self._add_result("Potential Emails", email, tdict)
def indicator_remove(_id, username): """ Remove an Indicator from CRITs. :param _id: The ObjectId of the indicator to remove. :type _id: str :param username: The user removing the indicator. :type username: str :returns: dict with keys "success" (boolean) and "message" (list) if failed. """ if is_admin(username): indicator = Indicator.objects(id=_id).first() if indicator: indicator.delete(username=username) return {'success': True} else: return {'success': False, 'message': ['Cannot find Indicator']} else: return {'success': False, 'message': ['Must be an admin to delete']}
def action_remove(indicator_id, date, analyst): """ Remove an action from an indicator. :param indicator_id: The ObjectId of the indicator to update. :type indicator_id: str :param date: The date of the action to remove. :type date: datetime.datetime :param analyst: The user removing the action. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ indicator = Indicator.objects(id=indicator_id).first() if not indicator: return {'success': False, 'message': 'Could not find Indicator'} try: indicator.delete_action(date) indicator.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save()
def activity_remove(id_, date, user, **kwargs): """ Remove activity from an Indicator. :param id_: The ObjectId of the indicator to update. :type id_: str :param date: The date of the activity to remove. :type date: datetime.datetime :param user: The user removing this activity. :type user: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ indicator = Indicator.objects(id=id_).first() if not indicator: return {"success": False, "message": "Could not find Indicator"} try: date = datetime_parser(date) indicator.delete_activity(date) indicator.save(username=user) return {"success": True} except ValidationError, e: return {"success": False, "message": e}
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import RelationshipType from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event, EventType from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.objects.object_type import ObjectType from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.backdoor import Backdoor from crits.samples.exploit import Exploit from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) if type_ == 'Backdoor': return Backdoor.objects(id=_id).first() if type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'EventType': return EventType.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'ObjectType': return ObjectType.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'RelationshipType': return RelationshipType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'Action': return Action.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'Signature': return Signature.objects(id=_id).first() elif type_ == 'SignatureType': return SignatureType.objects(id=_id).first() elif type_ == 'SignatureDependency': return SignatureDependency.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def to_stix(self, username=None): """ Converts a CRITs event to a STIX document. The resulting document includes all related emails, samples, and indicators converted to CybOX Observable objects. Returns the STIX document and releasability constraints. (NOTE: the following statement is untrue until the releasability checking is finished, which includes setting releasability on all CRITs objects.) Raises UnreleasableEventError if the releasability on the relationships and the event do not share any common releasability sources. """ from crits.emails.email import Email from crits.samples.sample import Sample from crits.indicators.indicator import Indicator from cybox.common import Time, ToolInformationList, ToolInformation from cybox.core import Observables from stix.common import StructuredText from stix.core import STIXPackage, STIXHeader from stix.common import InformationSource from stix.common.identity import Identity stix_indicators = [] stix_observables = [] final_objects = [] # create a list of sources to send as part of the results. # list should be limited to the sources this user is allowed to use. # this list should be used along with the list of objects to set the # appropriate source's 'released' key to True for each object. final_sources = [] user_source_list = user_sources(username) for f in self.releasability: if f.name in user_source_list: final_sources.append(f.name) final_sources = set(final_sources) # TODO: eventually we can use class_from_id instead of the if block # but only once we support all CRITs types. for r in self.relationships: obj = None if r.rel_type == Email._meta['crits_type']: obj = Email.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_cybox() stix_observables.append(ind[0]) elif r.rel_type == Sample._meta['crits_type']: obj = Sample.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_cybox() for i in ind: stix_observables.append(i) elif r.rel_type == Indicator._meta['crits_type']: #NOTE: Currently this will raise an exception if there # are multiple indicators with the same value. # Should be fixed automatically once we transition # indicators to be related based on ObjectId rather # than value. obj = Indicator.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_stix_indicator() stix_indicators.append(ind) else: continue #Create a releasability list that is the intersection of # each related item's releasability with the event's # releasability. If the resulting set is empty, raise exception #TODO: Set releasability on all objects so that we actually # get results here instead of always raising an exception. if obj: releas_sources = set([rel.name for rel in releas]) final_sources = final_sources.intersection(releas_sources) #TODO: uncomment the following lines when objects have # releasability set. #if not final_sources: # raise UnreleasableEventError(r.value) # add to the final_objects list to send as part of the results final_objects.append(obj) tool_list = ToolInformationList() tool = ToolInformation("CRITs", "MITRE") tool.version = settings.CRITS_VERSION tool_list.append(tool) i_s = InformationSource( time=Time(produced_time= datetime.datetime.now()), identity = Identity(name=settings.COMPANY_NAME), tools = tool_list ) description = StructuredText(value=self.description) header = STIXHeader(information_source=i_s, description=description, package_intent=self.event_type, title=self.title) return (STIXPackage(indicators=stix_indicators, observables=Observables(stix_observables), stix_header=header, id_=self.event_id), final_sources, final_objects)
def parse_result(self, result_extract, acl_write, md5_parent): stream_md5 = None if type(result_extract) is dict: #make reccursion extract each file embbed if 'FileMD5' in result_extract and result_extract['FileMD5']: tmp_dict = {} b_yara = False b_ioc = False #extract info no_info = [ 'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile', 'FileMD5', 'RootFileType', 'TempDirExtract' ] for key, value in result_extract.iteritems(): if not key in no_info: self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Info', key, {'value': str(value)}) #extract yara match if result_extract['Yara']: for item_v in result_extract['Yara']: for key, value in item_v.iteritems(): self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Signatures yara matched', key, value) b_yara = True #extract IOC if result_extract['ExtractInfo']: for item_v in result_extract['ExtractInfo']: for key, value in item_v.iteritems(): self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Extract potential IOC', key, {'value': str(value)}) b_ioc = True #add_sample if 'PathFile' in result_extract and type( result_extract['PathFile']) is list and len( result_extract['PathFile']) > 0: if os.path.isfile(str(result_extract['PathFile'][0])): with open(str(result_extract['PathFile'][0]), 'r') as content_file_tmp: content_tmp = content_file_tmp.read() stream_md5 = hashlib.md5(content_tmp).hexdigest() name = str(stream_md5).decode('ascii', errors='ignore') id_ = Sample.objects(md5=stream_md5).only('id').first() if id_: self._info( 'Add relationship with sample existed:' + str(stream_md5)) #make relationship id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes.CONTAINED_WITHIN, rel_date=datetime.now(), analyst=self.current_task.user.username) else: if acl_write and ( self.config['import_file'] or (self.config['import_file_yara'] and b_yara) or (self.config['import_file_ioc'] and b_ioc)): obj_parent = None if md5_parent: obj_parent = Sample.objects( md5=md5_parent).only('id').first() if not obj_parent: sample = handle_file( name, content_tmp, self.obj.source, related_id=str(self.obj.id), related_type=str( self.obj._meta['crits_type']), campaign=self.obj.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) else: sample = handle_file( name, content_tmp, obj_parent.source, related_id=str(obj_parent.id), related_type=str( obj_parent._meta['crits_type']), campaign=obj_parent.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) self._info('Add sample ' + str(stream_md5)) else: #add IOC if not add sample if self.current_task.user.has_access_to( IndicatorACL.WRITE) and b_yara: res = handle_indicator_ind( stream_md5, self.obj.source, IndicatorTypes.MD5, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, self.current_task.user, add_relationship=True, source_method=self.name, campaign=self.obj.campaign, description='Extracted by service ' + self.name) self._info('Add indicator md5:' + str(stream_md5) + ' -- id: ' + str(res)) #contains file if 'ContainedObjects' in result_extract and type( result_extract['ContainedObjects'] ) is list and result_extract['ContainedObjects']: for item_v in result_extract['ContainedObjects']: if item_v['FileMD5'] and item_v['FileType'] and item_v[ 'FileSize']: #search if file exist id_ = Sample.objects( md5=str(item_v['FileMD5'])).only('id').first() sample_exist = False ioc_exist = False if id_: sample_exist = True id_ = Indicator.objects( value=str(item_v['FileMD5'])).only('id').first() if id_: ioc_exist = True self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Contains md5 files', item_v['FileMD5'], { 'type': str(item_v['FileType']), 'size': str(item_v['FileSize']), 'Exists Sample': str(sample_exist), 'Exists IOC md5': str(ioc_exist) }) for item_v in result_extract['ContainedObjects']: #re do loop for best display result parse_result(self, item_v, acl_write, stream_md5)
def create_indicator_from_object(rel_type, rel_id, ind_type, value, analyst, request): """ Create an indicator out of this object. :param rel_type: The top-level object type this object is for. :type rel_type: str :param rel_id: The ObjectId of the top-level object. :param ind_type: The indicator type to use. :type ind_type: str :param value: The indicator value. :type value: str :param analyst: The user creating this indicator. :type analyst: str :param request: The Django request. :type request: :class:`django.http.HttpRequest` :returns: dict with keys "success" (bool) and "message" (str) """ result = None me = class_from_id(rel_type, rel_id) if not me: result = {'success': False, 'message': "Could not find %s" % rel_type} elif value == None or value.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty value field" } elif ind_type == None or ind_type.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty type field" } else: value = value.lower().strip() ind_type = ind_type.strip() create_indicator_result = {} ind_tlist = ind_type.split(" - ") if ind_tlist[0] == ind_tlist[1]: ind_type = ind_tlist[0] from crits.indicators.handlers import handle_indicator_ind if hasattr(me, 'source'): create_indicator_result = handle_indicator_ind(value, me.source, '', ind_type, analyst=analyst, add_domain=True) else: # In case the top level item doesn't have sources (such as campaign)... # then just default to the user's organization create_indicator_result = handle_indicator_ind( value, get_user_organization(analyst), '', ind_type, analyst=analyst, add_domain=True) # Check if an error occurred, if it did then return the error result if create_indicator_result.get('success', True) == False: return result indicator = Indicator.objects(ind_type=ind_type, value=value).first() if not indicator: result = { 'success': False, 'message': "Could not create indicator" } else: results = me.add_relationship(rel_item=indicator, rel_type="Related_To", analyst=analyst, get_rels=True) if results['success']: me.save(username=analyst) indicator.save(username=analyst) relationship = {'type': rel_type, 'value': rel_id} message = render_to_string( 'relationships_listing_widget.html', { 'relationship': relationship, 'nohide': True, 'relationships': results['message'] }, RequestContext(request)) result = {'success': True, 'message': message} else: message = "Indicator created. Could not create relationship" result = {'success': False, 'message': message} return result
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]: return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "} is_new_indicator = False dmain = None ip = None rank = { 'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4, } indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True if 'campaign' in ind: if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0: confidence = ind.get('campaign_confidence', 'low') ind['campaign'] = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if isinstance(ind['campaign'], EmbeddedCampaign): indicator.add_campaign(ind['campaign']) elif isinstance(ind['campaign'], list): for campaign in ind['campaign']: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value url_contains_ip = False if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain_or_ip = urlparse.urlparse(ind_value).hostname elif ind_type == "URI - Domain Name": domain_or_ip = ind_value (sdomain, fqdn) = get_domain(domain_or_ip) if sdomain == "no_tld_found_error" and ind_type == "URI - URL": try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass if not url_contains_ip: success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success['object'] if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = 'Address - ipv4-addr' except DjangoValidationError: ind_type = 'Address - ipv6-addr' success = None if add_domain: success = ip_add_update(ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] indicator.save(username=analyst) if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {'success': True, 'objectid': str(indicator.id), 'is_new_indicator': is_new_indicator, 'object': indicator}
def parse_result(self, result_extract, response_dict, config, md5_parent): stream_md5 = None user = self.current_task.user self.config = config acl_write = user.has_access_to(SampleACL.WRITE) if type(result_extract) is dict: #make reccursion extract each file embbed if 'FileMD5' in result_extract and result_extract['FileMD5']: tmp_dict = {} b_yara = False b_ioc = False #extract info no_info = [ 'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile', 'FileMD5', 'RootFileType', 'TempDirExtract', 'GlobalIOC' ] for key, value in result_extract.iteritems(): if not key in no_info: self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Info', key, {'value': str(value)}) #add download info if result_extract['FileMD5'] in response_dict: self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Info', 'Download embed file', {'value': sfa_api + vx}) #GLOBAL IOC if 'GlobalIOC' in result_extract and result_extract['GlobalIOC']: for key, value in result_extract['GlobalIOC'].iteritems(): self._add_result('Global IOC by categories', key, {'value': str(value)}) #extract yara match if result_extract['Yara']: for item_v in result_extract['Yara']: #self._info("Dict:"+str(item_v)) for key, value in item_v.iteritems(): data = { 'description': '', 'ioc': '', 'tags': '', 'score': '0' } for kx, vx in value.iteritems(): data[kx] = str(vx) self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Signatures yara matched', key, data) score_conf = re.sub("\D", "", config['import_yara_score']) if acl_write and config[ 'import_file_yara'] and 'score' in value and int( value['score']) >= int(score_conf): id_ = Sample.objects(md5=result_extract['FileMD5'] ).only('id').first() if id_: self._info( 'Add relationship with sample existed:' + str(stream_md5)) #make relationship id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes. CONTAINED_WITHIN, rel_date=datetime.now(), analyst=self.current_task.user.username) elif result_extract['FileMD5'] in response_dict: content_tmp = download( self, response_dict[result_extract['FileMD5']]) if content_tmp: name = str(result_extract['FileMD5']) if 'CDBNAME' in result_extract: name = str(result_extract['CDBNAME']) obj_parent = None if md5_parent: obj_parent = Sample.objects( md5=md5_parent).only('id').first() if not obj_parent: sample = handle_file( name, content_tmp, self.obj.source, related_id=str(self.obj.id), related_type=str( self.obj._meta['crits_type']), campaign=self.obj.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) else: sample = handle_file( name, content_tmp, obj_parent.source, related_id=str(obj_parent.id), related_type=str( obj_parent._meta['crits_type'] ), campaign=obj_parent.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) self._info('Add sample ' + str(name) + ' - MD5:' + str(result_extract['FileMD5'])) #extract IOC if result_extract['ExtractInfo']: for item_v in result_extract['ExtractInfo']: for key, value in item_v.iteritems(): self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Extract potential IOC', key, {'value': str(value)}) if acl_write and config['import_file_ioc']: id_ = Sample.objects(md5=result_extract['FileMD5'] ).only('id').first() if id_: self._info( 'Add relationship with sample existed:' + str(stream_md5)) #make relationship id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes. CONTAINED_WITHIN, rel_date=datetime.now(), analyst=self.current_task.user.username) elif result_extract['FileMD5'] in response_dict: content_tmp = download( self, response_dict[result_extract['FileMD5']]) if content_tmp: name = str(result_extract['FileMD5']) if 'CDBNAME' in result_extract: name = str(result_extract['CDBNAME']) obj_parent = None if md5_parent: obj_parent = Sample.objects( md5=md5_parent).only('id').first() if not obj_parent: sample = handle_file( name, content_tmp, self.obj.source, related_id=str(self.obj.id), related_type=str( self.obj._meta['crits_type']), campaign=self.obj.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) else: sample = handle_file( name, content_tmp, obj_parent.source, related_id=str(obj_parent.id), related_type=str( obj_parent._meta['crits_type'] ), campaign=obj_parent.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) self._info('Add sample ' + str(name) + ' - MD5:' + str(result_extract['FileMD5'])) #contains file if 'ContainedObjects' in result_extract and type( result_extract['ContainedObjects'] ) is list and result_extract['ContainedObjects']: for item_v in result_extract['ContainedObjects']: if item_v['FileMD5'] and item_v['FileType'] and item_v[ 'FileSize']: #search if file exist id_ = Sample.objects( md5=str(item_v['FileMD5'])).only('id').first() sample_exist = False ioc_exist = False if id_: sample_exist = True id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes.RELATED_TO, #rel_date=datetime.now(), analyst=self.current_task.user.username) id_ = Indicator.objects( value=str(item_v['FileMD5'])).only('id').first() if id_: ioc_exist = True id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes.RELATED_TO, #rel_date=datetime.now(), analyst=self.current_task.user.username) self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Contains md5 files', item_v['FileMD5'], { 'type': str(item_v['FileType']), 'size': str(item_v['FileSize']), 'Exists Sample': str(sample_exist), 'Exists IOC md5': str(ioc_exist) }) for item_v in result_extract['ContainedObjects']: #re do loop for best display result parse_result(self, item_v, response_dict, config, stream_md5)
def check_indicators_ip(self,obj, config): if settings.HTTP_PROXY: proxies = {'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY} else: proxies = {} indicators = [] url = config['av_url'] api = config['av_api'] ''' Detect if IP address if IPv4 or IPv6 ''' try: ''' Check if ipv4 ''' if socket.inet_aton(str(obj.ip)): self._info("IPv4 Address : "+str(obj.ip)) request_url = url+'indicators/IPv4/'+str(obj.ip)+'/malware' request_url_general = url + 'indicators/IPv4/' + str(obj.ip) + '/general' request_url_reputation = url + 'indicators/IPv4/' + str(obj.ip) + '/reputation' request_url_list = url + 'indicators/IPv4/' + str(obj.ip) + '/url_list' headers = {'X-OTX-API-KEY' : api} r = requests.get(request_url, headers=headers, verify=False, proxies=proxies) r_g = requests.get(request_url_general, headers=headers, verify=False, proxies=proxies) r_rp = requests.get(request_url_reputation, headers=headers, verify=False, proxies=proxies) r_url_list = requests.get(request_url_list, headers=headers, verify=False, proxies=proxies) if r.status_code !=200: self._error("Response not OK") return results = r.json() results_general = r_g.json() results_reputation = r_rp.json() results_url_list = r_url_list.json() geolocation = { 'Indicator': results_general.get('indicator'), 'Country': results_general.get('country_name'), 'whois': results_general.get('whois') } ''' Simple Geolocation Data ''' if geolocation is not None: self._add_result("General Information", results_general.get('indicator'), geolocation) ''' Related malware hashes. ''' for i in results.get('data'): self._add_result("Related Malicious Hash",i.get('hash')) ''' Pulse Information ''' if results_general.get('pulse_info') is not None: for m,n in results_general.get('pulse_info').iteritems(): if 'pulses' in m and bool(n): pulses = n self._add_result("Pulses Found",str(obj.ip), pulses) ''' Get reputational data ''' activities = results_reputation['reputation']['activities'] for active in activities: self._add_result("IP Reputation-Activities",results_reputation['reputation']['address'], active) domains = results_reputation['reputation']['domains'] for domain in domains: indicators.append([domain, IndicatorTypes.DOMAIN]) # Enable user to add unique indicators for this sample added = [] for item in indicators: if item[0]: indicator = item[0].lower() if indicator not in added: added.append(indicator) tdict = {} if item[1] in (IndicatorTypes.IPV4_ADDRESS, IndicatorTypes.DOMAIN): tdict = {'Type': item[1]} id_ = Indicator.objects(value=indicator).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('add_alienvault_indicators', indicator, tdict) ''' Get URL List :URLs analyzed by AlienVault Labs which point to or are somehow associated with this IP address. ''' for lst in results_url_list['url_list']: self._add_result("URL List","List", lst) else: self._info("IPv6 Address : " + str(obj.ip)) request_url = url + 'indicators/IPv6/' + str(obj.ip) + '/malware' request_url_general = url + 'indicators/IPv6/' + str(obj.ip) + '/general' request_url_reputation = url + 'indicators/IPv6/' + str(obj.ip) + '/reputation' request_url_list = url + 'indicators/IPv6/' + str(obj.ip) + '/url_list' headers = {'X-OTX-API-KEY': api} r = requests.get(request_url, headers=headers, verify=False, proxies=proxies) r_g = requests.get(request_url_general, headers=headers, verify=False, proxies=proxies) r_rp = requests.get(request_url_reputation, headers=headers, verify=False, proxies=proxies) r_url_list = requests.get(request_url_list, headers=headers, verify=False, proxies=proxies) if r.status_code != 200: self._error("Response not OK") return results = r.json() results_general = r_g.json() results_reputation = r_rp.json() results_url_list = r_url_list.json() geolocation = { 'Indicator': results_general.get('indicator'), 'Country': results_general.get('country_name'), 'whois': results_general.get('whois') } ''' Simple Geolocation Data ''' if geolocation is not None: self._add_result("General Information", results_general.get('indicator'), geolocation) ''' Related malicious hashes. ''' for i in results.get('data'): self._add_result("Related Malicious Hash", i.get('hash')) ''' Pulse Information ''' if results_general.get('pulse_info') is not None: for m, n in results_general.get('pulse_info').iteritems(): if 'pulses' in m and bool(n): pulses = n self._add_result("Pulses Found", str(obj.ip), pulses) ''' Get reputational data ''' activities = results_reputation['reputation']['activities'] for active in activities: self._add_result("IP Reputation-Activities", results_reputation['reputation']['address'], active) domains = results_reputation['reputation']['domains'] for domain in domains: indicators.append([domain, IndicatorTypes.DOMAIN]) # Enable user to add unique indicators for this sample added = [] for item in indicators: if item[0]: indicator = item[0].lower() if indicator not in added: added.append(indicator) tdict = {} if item[1] in (IndicatorTypes.IPV4_ADDRESS, IndicatorTypes.DOMAIN): tdict = {'Type': item[1]} id_ = Indicator.objects(value=indicator).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('add_alienvault_indicators', indicator, tdict) ''' Get URL List :URLs analyzed by AlienVault Labs which point to or are somehow associated with this IP address. ''' for lst in results_url_list['url_list']: self._add_result("URL List", "List", lst) except socket.error: self._error("Couldn't establish connections or invalid IP address")
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ is_new_indicator = False rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4} indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True ec = None if 'campaign' in ind: confidence = 'low' if 'campaign_confidence' in ind: confidence = ind['campaign_confidence'] ec = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get( indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get( indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if ec: indicator.add_campaign(ec) indicator.save(username=analyst) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain = ind_value.split("/")[2] elif ind_type == "URI - Domain Name": domain = ind_value #try: (sdomain, fqdn) = get_domain(domain) success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain).first() else: dmain = success['object'] if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) indicator.save(username=analyst) elif ind_type.startswith( "Address - ip") or ind_type == "Address - cidr": success = None if add_domain: success = ip_add_update(indicator.value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(None, indicator, analyst) return { 'success': True, 'objectid': indicator.id, 'is_new_indicator': is_new_indicator, 'object': indicator }
def get_indicator_details(indicator_id, analyst): """ Generate the data to render the Indicator details template. :param indicator_id: The ObjectId of the Indicator to get details for. :type indicator_id: str :param analyst: The user requesting this information. :type analyst: str :returns: template (str), arguments (dict) """ template = None users_sources = user_sources(analyst) indicator = Indicator.objects(id=indicator_id, source__name__in=users_sources).first() if not indicator: error = ("Either this indicator does not exist or you do " "not have permission to view it.") template = "error.html" args = {'error': error} return template, args forms = {} forms['new_action'] = IndicatorActionsForm(initial={ 'analyst': analyst, 'active': "off", 'date': datetime.datetime.now() }) forms['new_activity'] = IndicatorActivityForm( initial={ 'analyst': analyst, 'date': datetime.datetime.now() }) forms['new_campaign'] = CampaignForm() #'date': datetime.datetime.now(), forms['new_source'] = SourceForm(analyst, initial={'date': datetime.datetime.now()}) forms['download_form'] = DownloadFileForm(initial={ "obj_type": 'Indicator', "obj_id": indicator_id }) indicator.sanitize("%s" % analyst) # remove pending notifications for user remove_user_from_notification("%s" % analyst, indicator_id, 'Indicator') # subscription subscription = { 'type': 'Indicator', 'id': indicator_id, 'subscribed': is_user_subscribed("%s" % analyst, 'Indicator', indicator_id), } # relationship relationship = { 'type': 'Indicator', 'value': indicator_id, } #objects objects = indicator.sort_objects() #relationships relationships = indicator.sort_relationships("%s" % analyst, meta=True) #comments comments = {'comments': indicator.get_comments(), 'url_key': indicator_id} # favorites favorite = is_user_favorite("%s" % analyst, 'Indicator', indicator.id) # services manager = crits.service_env.manager service_list = manager.get_supported_services('Indicator', True) args = { 'objects': objects, 'relationships': relationships, 'comments': comments, 'relationship': relationship, 'subscription': subscription, "indicator": indicator, "forms": forms, "indicator_id": indicator_id, 'service_list': service_list, 'favorite': favorite, 'rt_url': settings.RT_URL } return template, args
def create_indicator_and_ip(type_, id_, ip, analyst): """ Add indicators for an IP address. :param type_: The CRITs top-level object we are getting this IP from. :type type_: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param id_: The ObjectId of the top-level object to search for. :type id_: str :param ip: The IP address to generate an indicator out of. :type ip: str :param analyst: The user adding this indicator. :type analyst: str :returns: dict with keys: "success" (boolean), "message" (str), "value" (str) """ obj_class = class_from_id(type_, id_) if obj_class: ip_class = IP.objects(ip=ip).first() ind_type = "Address - ipv4-addr" ind_class = Indicator.objects(ind_type=ind_type, value=ip).first() # setup IP if ip_class: ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) else: ip_class = IP() ip_class.ip = ip ip_class.source = obj_class.source ip_class.save(username=analyst) ip_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) # setup Indicator message = "" if ind_class: message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) else: ind_class = Indicator() ind_class.source = obj_class.source ind_class.ind_type = ind_type ind_class.value = ip ind_class.save(username=analyst) message = ind_class.add_relationship(rel_item=obj_class, rel_type="Related_To", analyst=analyst) ind_class.add_relationship(rel_item=ip_class, rel_type="Related_To", analyst=analyst) # save try: obj_class.save(username=analyst) ip_class.save(username=analyst) ind_class.save(username=analyst) if message['success']: rels = obj_class.sort_relationships("%s" % analyst, meta=True) return { 'success': True, 'message': rels, 'value': obj_class.id } else: return {'success': False, 'message': message['message']} except Exception, e: return {'success': False, 'message': e}
def create_indicator_from_object(rel_type, rel_id, ind_type, value, source_name, method, reference, analyst, request): """ Create an indicator out of this object. :param rel_type: The top-level object type this object is for. :type rel_type: str :param rel_id: The ObjectId of the top-level object. :param ind_type: The indicator type to use. :type ind_type: str :param value: The indicator value. :type value: str :param source_name: The source name for the indicator. :type source_name: str :param method: The source method for the indicator. :type method: str :param reference: The source reference for the indicator. :type reference: str :param analyst: The user creating this indicator. :type analyst: str :param request: The Django request. :type request: :class:`django.http.HttpRequest` :returns: dict with keys "success" (bool) and "message" (str) """ result = None me = class_from_id(rel_type, rel_id) if not me: result = {'success': False, 'message': "Could not find %s" % rel_type} elif value == None or value.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty value field"} elif ind_type == None or ind_type.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty type field"} elif source_name == None or source_name.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty source field"} else: value = value.lower().strip() ind_type = ind_type.strip() source_name = source_name.strip() create_indicator_result = {} ind_tlist = ind_type.split(" - ") if ind_tlist[0] == ind_tlist[1]: ind_type = ind_tlist[0] from crits.indicators.handlers import handle_indicator_ind campaign = me.campaign if hasattr(me, 'campaign') else None create_indicator_result = handle_indicator_ind(value, source_name, reference, ind_type, analyst, method, add_domain=True, campaign=campaign) # Check if an error occurred, if it did then return the error result if create_indicator_result.get('success', True) == False: return create_indicator_result indicator = Indicator.objects(ind_type=ind_type, value=value).first() if not indicator: result = {'success': False, 'message': "Could not create indicator"} else: results = me.add_relationship(rel_item=indicator, rel_type="Related_To", analyst=analyst, get_rels=True) if results['success']: me.save(username=analyst) indicator.save(username=analyst) relationship= {'type': rel_type, 'value': rel_id} message = render_to_string('relationships_listing_widget.html', {'relationship': relationship, 'nohide': True, 'relationships': results['message']}, RequestContext(request)) result = {'success': True, 'message': message} else: message = "Indicator created. Could not create relationship" result = {'success': False, 'message': message} return result
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature from crits.targets.target import Target # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in [ 'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator', 'Screenshot' ] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Actor': return Actor.objects(name=value).first() if type_ == 'Backdoor': return Backdoor.objects(id=value).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(name=value).first() elif type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(message_id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Exploit': return Exploit.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Signature': return Signature.objects(md5=value).first() elif type_ == 'Target': target = Target.objects(email_address=value).first() if target: return target else: return Target.objects(email_address__iexact=value).first() else: return None