def add_object(type_, id_, object_type, source, method, reference, tlp, user,
value=None, file_=None, add_indicator=False, get_objects=True,
tlo=None, is_sort_relationships=False, is_validate_only=False,
is_validate_locally=False, cache={}, **kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value,
source, method, reference, user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {"success": False,
"message": msg % (ret['message'], object_type, value)}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
obj = tlo
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, value, source, method, reference, user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
tlp,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value, source, method, reference,
user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {
"success": False,
"message": msg % (ret['message'], object_type, value)
}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def add_object(type_, id_, object_type, source, method,
reference, user, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
value,
source,
method,
reference,
user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}