def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method='', reference='', tlp='',
copy_rels=False, bucket_list=None, ticket=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param tlp: TLP for the source.
:type tlp: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if isinstance(data, unicode):
data=data.encode('utf-8')
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
# raw_data.source = [source_name]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
raw_data.add_source(source)
else:
return {"success":False,
"message": "User does not have permission to add object using source %s." % source_name}
# this will handle adding a new source, or an instance automatically
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
raw_data.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
raw_data.save(username=user.username)
if related_obj and relationship_type and raw_data:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
raw_data.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
raw_data.save(username=user.username)
raw_data.reload()
# save raw_data
raw_data.save(username=user.username)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method=None, copy_rels=False,
bucket_list=None, ticket=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
data = data.encode('utf-8')
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# create source
source = create_embedded_source(source_name,
date=timestamp,
reference='',
method=method,
analyst=user)
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if raw_data:
raw_data.add_source(source)
else:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
raw_data.source = [source]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
raw_data.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type=rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
# save raw_data
raw_data.save(username=user)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
}
return status