def add_object(type_, id_, object_type, source, method, reference, tlp, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_objects: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_sort_relationships: Return all relationships and meta, sorted :type is_sort_relationships: bool :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ # if object_type is a validated indicator type, then validate value if value: from crits.indicators.handlers import validate_indicator_value (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no TLO provided return {"success": True} if not tlo: if type_ and id_: tlo = class_from_id(type_, id_) if not tlo: return {'success': False, 'message': "Failed to find TLO"} try: if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename ret = tlo.add_object(object_type, value, source, method, reference, user) if not ret['success']: msg = '%s! [Type: "%s"][Value: "%s"]' return {"success": False, "message": msg % (ret['message'], object_type, value)} else: results = {'success': True} if not is_validate_only: # save the object tlo.update(add_to_set__obj=ret['object']) results['message'] = "Object added successfully" if file_: # do we have a pcap? if detect_pcap(data): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and not is_validate_only: campaign = tlo.campaign if hasattr(tlo, 'campaign') else None from crits.indicators.handlers import handle_indicator_ind ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: forge_relationship(class_=tlo, right_class=ind_res['object'], rel_type=RelationshipTypes.RELATED_TO, user=user) else: msg = "Object added, but failed to add Indicator.<br>Error: %s" results['message'] = msg % ind_res.get('message') if is_sort_relationships == True: results['relationships'] = tlo.sort_relationships(user, meta=True) if get_objects: results['objects'] = tlo.sort_objects() results['id'] = str(tlo.id) return results except ValidationError as e: return {'success': False, 'message': str(e)}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results if name == "URL" and "://" not in value.split('.')[0]: return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"} elif object_type == "Address": if "ipv4" in name: try: validate_ipv4_address(value) except DjangoValidationError: return {"success" : False, "message" : "Invalid IPv4 address. "} elif "ipv6" in name: try: validate_ipv6_address(value) except DjangoValidationError: return {"success" : False, "message" : "Invalid IPv6 address. "} elif "cidr" in name: try: if '/' not in value: raise ValidationError("") cidr_parts = value.split('/') if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128: raise ValidationError("") if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32: raise ValidationError("") validate_ipv46_address(cidr_parts[0]) except (ValidationError, ValueError) as cidr_error: return {"success" : False, "message" : "Invalid CIDR address. "} try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def _add_grid(self): print "[+] adding to gridfs" put_file(self.test_md5, self.test_data)
def add_object(type_, id_, object_type, source, method, reference, tlp, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_objects: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_sort_relationships: Return all relationships and meta, sorted :type is_sort_relationships: bool :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ # if object_type is a validated indicator type, then validate value if value: from crits.indicators.handlers import validate_indicator_value (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no TLO provided return {"success": True} if not tlo: if type_ and id_: tlo = class_from_id(type_, id_) if not tlo: return {'success': False, 'message': "Failed to find TLO"} try: if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename ret = tlo.add_object(object_type, value, source, method, reference, user) if not ret['success']: msg = '%s! [Type: "%s"][Value: "%s"]' return { "success": False, "message": msg % (ret['message'], object_type, value) } else: results = {'success': True} if not is_validate_only: # save the object tlo.update(add_to_set__obj=ret['object']) results['message'] = "Object added successfully" if file_: # do we have a pcap? if detect_pcap(data): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and not is_validate_only: campaign = tlo.campaign if hasattr(tlo, 'campaign') else None from crits.indicators.handlers import handle_indicator_ind ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: forge_relationship(class_=tlo, right_class=ind_res['object'], rel_type=RelationshipTypes.RELATED_TO, user=user) else: msg = "Object added, but failed to add Indicator.<br>Error: %s" results['message'] = msg % ind_res.get('message') if is_sort_relationships == True: results['relationships'] = tlo.sort_relationships(user, meta=True) if get_objects: results['objects'] = tlo.sort_objects() results['id'] = str(tlo.id) return results except ValidationError as e: return {'success': False, 'message': str(e)}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, indicator_campaign=None, indicator_campaign_confidence=None, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, parent_id=oid, parent_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method=method, add_domain=True, campaign=indicator_campaign, campaign_confidence=indicator_campaign_confidence, cache=cache) if ind_res['success']: ind = ind_res['object'] # Inherit campaigns from top level item when creating # an indicator from an object if no campaigns were specified if indicator_campaign == None and ind != None: for campaign in obj.campaign: ec = EmbeddedCampaign(name=campaign.name, confidence=campaign.confidence, description="", analyst=analyst, date=datetime.datetime.now()) ind.add_campaign(ec) ind.save(username=analyst) forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() return results except ValidationError, e: return {'success': False, 'message': e}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, indicator_campaign=None, indicator_campaign_confidence=None, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) ind_res = handle_indicator_ind( value, source, reference, object_type, analyst, method=method, add_domain=True, campaign=indicator_campaign, campaign_confidence=indicator_campaign_confidence, cache=cache) if ind_res['success']: ind = ind_res['object'] # Inherit campaigns from top level item when creating # an indicator from an object if no campaigns were specified if indicator_campaign == None and ind != None: for campaign in obj.campaign: ec = EmbeddedCampaign( name=campaign.name, confidence=campaign.confidence, description="", analyst=analyst, date=datetime.datetime.now()) ind.add_campaign(ec) ind.save(username=analyst) forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() return results except ValidationError, e: return {'success': False, 'message': e}
def add_object(type_, id_, object_type, source, method, reference, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} obj = tlo if id_ == None: id_ = "" if obj == None: obj = class_from_id(type_, id_) from crits.indicators.handlers import validate_indicator_value if value is not None: (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no obj provided results['success'] = True return results if not obj: results['message'] = "TLO could not be found" results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, value, source, method, reference, user) if is_validate_only == False: obj.save(username=user) new_len = len(obj.obj) if new_len > cur_len: if not is_validate_only: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, method, reference, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(class_=obj, right_class=ind, rel_type=RelationshipTypes.RELATED_TO, user=user, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( user, meta=True) else: results['relationships'] = obj.sort_relationships( user, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results if name == "URL" and "://" not in value.split('.')[0]: return { "success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)" } elif object_type == "Address": if "ipv4" in name: try: validate_ipv4_address(value) except DjangoValidationError: return {"success": False, "message": "Invalid IPv4 address. "} elif "ipv6" in name: try: validate_ipv6_address(value) except DjangoValidationError: return {"success": False, "message": "Invalid IPv6 address. "} elif "cidr" in name: try: if '/' not in value: raise ValidationError("") cidr_parts = value.split('/') if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128: raise ValidationError("") if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32: raise ValidationError("") validate_ipv46_address(cidr_parts[0]) except (ValidationError, ValueError) as cidr_error: return {"success": False, "message": "Invalid CIDR address. "} try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def add_object(type_, id_, object_type, source, method, reference, user, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if id_ == None: id_ = "" if obj == None: obj = class_from_id(type_, id_) from crits.indicators.handlers import validate_indicator_value if value is not None: (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no obj provided results['success'] = True return results if not obj: results['message'] = "TLO could not be found" results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, value, source, method, reference, user) if is_validate_only == False: obj.save(username=user) new_len = len(obj.obj) if new_len > cur_len: if not is_validate_only: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, method, reference, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(class_=obj, right_class=ind, rel_type=RelationshipTypes.RELATED_TO, user=user, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(user, meta=True) else: results['relationships'] = obj.sort_relationships(user, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
if opts.gridfs: print "Copying GridFS data matching md5s to new collection..." filename = opts.gridfs elif opts.griddelete: print "Deleting matching md5s from GridFS..." filename = opts.griddelete count = 0 no_data = 0 try: with open(filename) as o: for line in o: md5 = line.strip() data = get_file(md5, opts.collection) if data: if opts.gridfs: put_file(md5, data) s = Sample.objects(md5=md5).first() if s: s.discover_binary() try: s.save() except: error_count += 1 err.write("Error saving sample for discover binary: %s" % md5) elif opts.griddelete: delete_file(md5, opts.collection) count += 1 else: no_data += 1 if opts.gridfs: print "Copied %s md5s to new collection." % count
if opts.gridfs: print "Copying GridFS data matching md5s to new collection..." filename = opts.gridfs elif opts.griddelete: print "Deleting matching md5s from GridFS..." filename = opts.griddelete count = 0 no_data = 0 try: with open(filename) as o: for line in o: md5 = line.strip() data = get_file(md5, opts.collection) if data: if opts.gridfs: put_file(md5, data) s = Sample.objects(md5=md5).first() if s: s.discover_binary() try: s.save() except: error_count += 1 err.write( "Error saving sample for discover binary: %s" % md5) elif opts.griddelete: delete_file(md5, opts.collection) count += 1 else: no_data += 1