def generate_raw_data_versions(_id):
"""
Generate a list of available versions for this RawData.
:param _id: The ObjectId of the RawData to generate versions for.
:type _id: str
:returns: list
"""
raw_data = RawData.objects(id=_id).only('link_id').first()
if not raw_data:
return []
else:
versions = []
rvs = RawData.objects(link_id=raw_data.link_id).only('id',
'title',
'version',
'data')
for rv in rvs:
link = reverse('crits-raw_data-views-raw_data_details',
args=(rv.id,))
versions.append({'title': rv.title,
'version': rv.version,
'data': rv.data,
'link': link})
return versions
def update_raw_data_highlight_comment(_id, comment, line, analyst):
"""
Update a highlight comment.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param comment: The comment to add.
:type comment: str
:param line: The line this comment is associated with.
:type line: str, int
:param analyst: The user updating the comment.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
raw_data = RawData.objects(id=_id).first()
if not raw_data:
return None
else:
i = 0
for highlight in raw_data.highlights:
if highlight.line == int(line):
raw_data.highlights[i].comment = comment
try:
raw_data.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': str(e)}
i += 1
return {'success': False, 'message': 'Could not find highlight.'}
def update_raw_data_highlight_date(_id, date, line, analyst):
"""
Update a highlight date.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param date: The date to set.
:type date: str
:param line: The line this date is associated with.
:type line: str, int
:param analyst: The user updating the date.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
raw_data = RawData.objects(id=_id).first()
if not raw_data:
return None
else:
for highlight in raw_data.highlights:
if highlight.line == int(line):
highlight.line_date = parse(date, fuzzy=True)
try:
raw_data.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': str(e)}
return {'success': False, 'message': 'Could not find highlight.'}
def new_highlight(_id, line_num, line_data, analyst):
"""
Add a new highlight.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param line_num: The line to highlight.
:type line_num: str, int
:param line_data: The data on this line.
:type line_data: str
:param analyst: The user highlighting this line.
:type analyst: str
:returns: dict with keys "success" (boolean) and "html" (str)
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.add_highlight(line_num, line_data, analyst)
try:
raw_data.save(username=analyst)
html = render_to_string('raw_data_highlights.html',
{'raw_data': {'id': _id,
'highlights': raw_data.highlights}})
return {'success': True,
'html': html,
}
except ValidationError, e:
return e
def delete_highlight(_id, line_num, analyst):
"""
Delete a highlight from RawData.
:param _id: The ObjectId of the RawData to update.
:type _id: str
:param line_num: Line number of the highlight to delete.
:type line_num: str, int
:param analyst: The user deleting this highlight.
:type analyst: str
:returns: dict with keys "success" (boolean) and "html" (str)
"""
raw_data = RawData.objects(id=_id).first()
highlights = len(raw_data.highlights)
raw_data.remove_highlight(line_num, analyst)
if len(raw_data.highlights) < highlights:
try:
raw_data.save(username=analyst)
html = render_to_string('raw_data_highlights.html',
{'raw_data': {'id': _id,
'highlights': raw_data.highlights}})
return {'success': True,
'html': html,
}
except ValidationError, e:
return e
def create_indicator_from_raw(type_, id_, value, analyst):
"""
Add indicators from raw data.
:param type_: The indicator type to add.
:type type_: str
:param id_: The ObjectId of the RawData object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
raw_data = RawData.objects(id=id_).first()
if not raw_data:
return {'success': False,
'message': 'Could not find raw data'}
source = raw_data.source
bucket_list = raw_data.bucket_list
campaign = None
campaign_confidence = None
if len(raw_data.campaign) > 0:
campaign = raw_data.campaign[0].name
campaign_confidence = raw_data.campaign[0].confidence
result = handle_indicator_ind(value, source, reference=None, ctype=type_,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
raw_data.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
raw_data.save(username=analyst)
for rel in raw_data.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
raw_data.reload()
rels = raw_data.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def get_id_from_link_and_version(link, version):
"""
Get the ObjectId from a link_id and version number.
:param link: The link_id of the RawData.
:type link: str
:param version: The version number of the RawData.
:type version: int
:returns: None, ObjectId
"""
raw_data = RawData.objects(link_id=link, version=version).only('id').first()
if not raw_data:
return None
else:
return raw_data.id
def get_id_from_link_and_version(link, version):
"""
Get the ObjectId from a link_id and version number.
:param link: The link_id of the RawData.
:type link: str
:param version: The version number of the RawData.
:type version: int
:returns: None, ObjectId
"""
raw_data = RawData.objects(link_id=link,
version=version).only('id').first()
if not raw_data:
return None
else:
return raw_data.id
def delete_raw_data(_id, username=None):
"""
Delete RawData from CRITs.
:param _id: The ObjectId of the RawData to delete.
:type _id: str
:param username: The user deleting this RawData.
:type username: str
:returns: bool
"""
raw_data = RawData.objects(id=_id).first()
if raw_data:
raw_data.delete(username=username)
return True
else:
return False
def delete_raw_data(_id, username=None):
"""
Delete RawData from CRITs.
:param _id: The ObjectId of the RawData to delete.
:type _id: str
:param username: The user deleting this RawData.
:type username: str
:returns: bool
"""
raw_data = RawData.objects(id=_id).first()
if raw_data:
raw_data.delete(username=username)
return True
else:
return False
def new_inline_comment(_id, comment, line_num, analyst):
"""
Add a new inline comment.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param comment: The comment to add.
:type comment: str
:param line_num: The line this comment is associated with.
:type line_num: str, int
:param analyst: The user adding this comment.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"line" (int),
"html" (str)
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.add_inline_comment(comment, line_num, analyst)
try:
raw_data.save(username=analyst)
html = render_to_string(
'inline_comment.html', {
'username': analyst,
'comment': comment,
'date': datetime.datetime.now(),
'line': line_num,
'raw_data': {
'id': _id
}
})
return {
'success': True,
'message': 'Comment for line %s added successfully!' % line_num,
'inline': True,
'line': line_num,
'html': html,
}
except ValidationError, e:
return e
def update_raw_data_description(_id, description, analyst):
"""
Update the RawData description.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param description: The description to set.
:type description: str
:param analyst: The user updating the description.
:type analyst: str
:returns: None
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.description = description
try:
raw_data.save(username=analyst)
return None
except ValidationError, e:
return e
def update_raw_data_tool_details(_id, details, analyst):
"""
Update the RawData tool details.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param details: The detail to set.
:type detail: str
:param analyst: The user updating the details.
:type analyst: str
:returns: None
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.tool.details = details
try:
raw_data.save(username=analyst)
return None
except ValidationError, e:
return e
def update_raw_data_tool_name(_id, name, analyst):
"""
Update the RawData tool name.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param name: The name to set.
:type name: str
:param analyst: The user updating the name.
:type analyst: str
:returns: None
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.tool.name = name
try:
raw_data.save(username=analyst)
return None
except ValidationError, e:
return e
def new_inline_comment(_id, comment, line_num, analyst):
"""
Add a new inline comment.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param comment: The comment to add.
:type comment: str
:param line_num: The line this comment is associated with.
:type line_num: str, int
:param analyst: The user adding this comment.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"line" (int),
"html" (str)
:raises: ValidationError
"""
raw_data = RawData.objects(id=_id).first()
raw_data.add_inline_comment(comment, line_num, analyst)
try:
raw_data.save(username=analyst)
html = render_to_string('inline_comment.html',
{'username': analyst,
'comment': comment,
'date': datetime.datetime.now(),
'line': line_num,
'raw_data': {'id': _id}})
return {'success': True,
'message': 'Comment for line %s added successfully!' % line_num,
'inline': True,
'line': line_num,
'html': html,
}
except ValidationError, e:
return e
def generate_inline_comments(_id):
"""
Generate the inline comments for RawData.
:param _id: The ObjectId of the RawData to generate inline comments for.
:type _id: str
:returns: list
"""
raw_data = RawData.objects(id=_id).first()
if not raw_data:
return []
else:
inlines = []
for i in raw_data.inlines:
html = render_to_string('inline_comment.html',
{'username': i.analyst,
'comment': i.comment,
'date': i.date,
'line': i.line,
'raw_data': {'id': _id}})
inlines.append({'line': i.line, 'html': html})
return inlines
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def _scan(self, context):
if isinstance(context, RawDataContext):
raw_data = RawData.objects(id=context.identifier).first()
if not raw_data:
self._debug("Could not find raw data to parse.")
return
data = raw_data.data
elif isinstance(context, SampleContext):
data = make_ascii_strings(md5=context.identifier)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {"Type": "IP Address"}
id_ = Indicator.objects(value=ip).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential IP Address", ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {"Type": "Domain"}
id_ = Indicator.objects(value=domain).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential Domains", domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {"Type": "Email"}
id_ = Indicator.objects(value=email).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential Emails", email, tdict)
def update_raw_data_type(_id, data_type, analyst):
"""
Update the RawData data type.
:param _id: ObjectId of the RawData to update.
:type _id: str
:param data_type: The data type to set.
:type data_type: str
:param analyst: The user updating the data type.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
raw_data = RawData.objects(id=_id).first()
data_type = RawDataType.objects(name=data_type).first()
if not data_type:
return None
else:
raw_data.data_type = data_type.name
try:
raw_data.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': str(e)}
def _scan(self, context):
if isinstance(context, RawDataContext):
raw_data = RawData.objects(id=context.identifier).first()
if not raw_data:
self._debug("Could not find raw data to parse.")
return
data = raw_data.data
elif isinstance(context, SampleContext):
data = make_ascii_strings(md5=context.identifier)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': "IP Address"}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': "Domain"}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': "Email"}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
def get_raw_data_details(_id, user):
"""
Generate the data to render the RawData details template.
:param _id: The ObjectId of the RawData to get details for.
:type _id: str
:param user: The user requesting this information.
:type user: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(user)
if not _id:
raw_data = None
else:
raw_data = RawData.objects(id=_id, source__name__in=sources).first()
if not user.check_source_tlp(raw_data):
raw_data = None
if not raw_data:
template = "error.html"
args = {'error': 'raw_data not yet available or you do not have access to view it.'}
else:
raw_data.sanitize("%s" % user)
# remove pending notifications for user
remove_user_from_notification("%s" % user, raw_data.id, 'RawData')
# subscription
subscription = {
'type': 'RawData',
'id': raw_data.id,
'subscribed': is_user_subscribed("%s" % user,
'RawData', raw_data.id),
}
#objects
objects = raw_data.sort_objects()
#relationships
relationships = raw_data.sort_relationships("%s" % user, meta=True)
# relationship
relationship = {
'type': 'RawData',
'value': raw_data.id
}
versions = len(RawData.objects(link_id=raw_data.link_id).only('id'))
#comments
comments = {'comments': raw_data.get_comments(),
'url_key': _id}
#screenshots
screenshots = raw_data.get_screenshots(user)
# favorites
favorite = is_user_favorite("%s" % user, 'RawData', raw_data.id)
# services
service_list = get_supported_services('RawData')
# analysis results
service_results = raw_data.get_analysis_results()
args = {'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
"versions": versions,
"service_results": service_results,
"raw_data": raw_data,
"RawDataACL": RawDataACL}
return template, args
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method=None, copy_rels=False,
bucket_list=None, ticket=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
data = data.encode('utf-8')
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# create source
source = create_embedded_source(source_name,
date=timestamp,
reference='',
method=method,
analyst=user)
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if raw_data:
raw_data.add_source(source)
else:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
raw_data.source = [source]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
raw_data.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type=rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
# save raw_data
raw_data.save(username=user)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
}
return status
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method='', reference='', tlp='',
copy_rels=False, bucket_list=None, ticket=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param tlp: TLP for the source.
:type tlp: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if isinstance(data, unicode):
data=data.encode('utf-8')
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
# raw_data.source = [source_name]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
raw_data.add_source(source)
else:
return {"success":False,
"message": "User does not have permission to add object using source %s." % source_name}
# this will handle adding a new source, or an instance automatically
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
raw_data.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
raw_data.save(username=user.username)
if related_obj and relationship_type and raw_data:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
raw_data.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
raw_data.save(username=user.username)
raw_data.reload()
# save raw_data
raw_data.save(username=user.username)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None