def get_objects_type(name=None):
"""
Get the type of an ObjectType.
:param name: The name of the ObjectType.
:type name: str
:returns: str
"""
from crits.objects.object_type import ObjectType
if not name:
return None
obj = ObjectType.objects(name=name).first()
if obj:
return obj.object_type
return None
def get_objects_type(name=None):
"""
Get the type of an ObjectType.
:param name: The name of the ObjectType.
:type name: str
:returns: str
"""
from crits.objects.object_type import ObjectType
if not name:
return None
obj = ObjectType.objects(name=name).first()
if obj:
return obj.object_type
return None
def get_objects_datatype(name, otype):
"""
Get the datatype of an ObjectType.
:param name: The name of the ObjectType.
:type name: str
:param otype: The type of the ObjectType
:returns: str
"""
from crits.objects.object_type import ObjectType
obj = ObjectType.objects(name=name, object_type=otype).first()
if obj:
key = obj.datatype.keys()[0]
return key
else:
return None
def get_objects_datatype(name, otype):
"""
Get the datatype of an ObjectType.
:param name: The name of the ObjectType.
:type name: str
:param otype: The type of the ObjectType
:returns: str
"""
from crits.objects.object_type import ObjectType
obj = ObjectType.objects(name=name,
object_type=otype).first()
if obj:
key = obj.datatype.keys()[0]
return key
else:
return None
def handle_indicator_csv(csv_data, source, method, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower().replace(' - ', '-')] = c['name']
valid_actions = {}
for a in IndicatorAction.objects(active='on'):
valid_actions[a['name'].lower().replace(' - ', '-')] = a['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower().replace(' - ', '-')] = name
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, valid_ind_types, 'Type')
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value "
if not ind['type']:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, 'Campaign')
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, valid_campaign_confidence,
'Campaign Confidence',
default='low')
actions = d.get('Action', '')
if actions:
actions = get_verified_field(actions.split(','), valid_actions)
if not actions:
result['success'] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind['confidence'] = get_verified_field(d, valid_ratings, 'Confidence',
default='unknown')
ind['impact'] = get_verified_field(d, valid_ratings, 'Impact',
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
response = handle_indicator_insert(ind, source, reference, analyst=username,
method=method, add_domain=add_domain)
except Exception, e:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response['success']:
if actions:
action = {'active': 'on',
'analyst': username,
'begin_date': '',
'end_date': '',
'performed_date': '',
'reason': '',
'date': datetime.datetime.now()}
for action_type in actions:
action['action_type'] = action_type
action_add(response.get('objectid'), action)
else:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response['message'])
continue
added += 1
def handle_indicator_csv(csv_data,
source,
reference,
ctype,
username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = "Indicators added successfully!"
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'
}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'
}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False,
datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower()] = name
# Start line-by-line import
processed = 0
for d in data:
processed += 1
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, 'Type', valid_ind_types)
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value. "
if not ind['type']:
i += "No valid Indicator type. "
result_message += "Cannot process row: %s. %s<br />" % (processed,
i)
continue
campaign = get_verified_field(d, 'Campaign', valid_campaigns)
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(
d,
'Campaign Confidence',
valid_campaign_confidence,
default='low')
ind['confidence'] = get_verified_field(d,
'Confidence',
valid_ratings,
default='unknown')
ind['impact'] = get_verified_field(d,
'Impact',
valid_ratings,
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(
form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(
form_consts.Common.TICKET, '')
try:
handle_indicator_insert(ind,
source,
reference,
analyst=username,
add_domain=add_domain)
except Exception, e:
result['success'] = False
result['message'] = str(e)
return result
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatType, ActorMotivation
from crits.actors.actor import ActorSophistication, ActorIntendedEffect
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'ActorThreatType':
return ActorThreatType.objects(id=_id).first()
elif type_ == 'ActorMotivation':
return ActorMotivation.objects(id=_id).first()
elif type_ == 'ActorSophistication':
return ActorSophistication.objects(id=_id).first()
elif type_ == 'ActorIntendedEffect':
return ActorIntendedEffect.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def handle_indicator_csv(csv_data, source, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = "Indicators added successfully!"
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower()] = name
# Start line-by-line import
processed = 0
for d in data:
processed += 1
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, 'Type', valid_ind_types)
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value. "
if not ind['type']:
i += "No valid Indicator type. "
result_message += "Cannot process row: %s. %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, 'Campaign', valid_campaigns)
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, 'Campaign Confidence',
valid_campaign_confidence,
default='low')
ind['confidence'] = get_verified_field(d, 'Confidence', valid_ratings,
default='unknown')
ind['impact'] = get_verified_field(d, 'Impact', valid_ratings,
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
handle_indicator_insert(ind, source, reference, analyst=username,
add_domain=add_domain)
except Exception, e:
result['success'] = False
result['message'] = str(e)
return result
def handle_indicator_csv(csv_data, source, method, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower().replace(' - ', '-')] = c['name']
valid_actions = {}
for a in IndicatorAction.objects(active='on'):
valid_actions[a['name'].lower().replace(' - ', '-')] = a['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower().replace(' - ', '-')] = name
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, valid_ind_types, 'Type')
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value "
if not ind['type']:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, 'Campaign')
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, valid_campaign_confidence,
'Campaign Confidence',
default='low')
actions = d.get('Action', '')
if actions:
actions = get_verified_field(actions.split(','), valid_actions)
if not actions:
result['success'] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind['confidence'] = get_verified_field(d, valid_ratings, 'Confidence',
default='unknown')
ind['impact'] = get_verified_field(d, valid_ratings, 'Impact',
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
response = handle_indicator_insert(ind, source, reference, analyst=username,
method=method, add_domain=add_domain)
except Exception, e:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response['success']:
if actions:
action = {'active': 'on',
'analyst': username,
'begin_date': '',
'end_date': '',
'performed_date': '',
'reason': '',
'date': datetime.datetime.now()}
for action_type in actions:
action['action_type'] = action_type
action_add(response.get('objectid'), action)
else:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response['message'])
continue
added += 1
