def run(self, obj, config):
if isinstance(obj, RawData):
data = obj.data
elif isinstance(obj, Sample):
samp_data = obj.filedata.read()
data = make_ascii_strings(data=samp_data)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': "IP Address"}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': "Domain"}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': "Email"}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
def set_indicator_type(indicator_id, itype, username):
"""
Set the Indicator type.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param itype: The new indicator type.
:type itype: str
:param username: The user updating the indicator.
:type username: str
:returns: dict with key "success" (boolean)
"""
# check to ensure we're not duping an existing indicator
indicator = Indicator.objects(id=indicator_id).first()
value = indicator.value
ind_check = Indicator.objects(ind_type=itype, value=value).first()
if ind_check:
# we found a dupe
return {'success': False}
else:
try:
indicator.ind_type = itype
indicator.save(username=username)
return {'success': True}
except ValidationError:
return {'success': False}
def run(self, obj, config):
if isinstance(obj, RawData):
data = obj.data
elif isinstance(obj, Sample):
samp_data = obj.filedata.read()
data = make_ascii_strings(data=samp_data)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': "IP Address"}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': "Domain"}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': "Email"}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
def set_indicator_attack_type(id_, attack_type, user, **kwargs):
"""
Set the Indicator attack type.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param attack_type: The new indicator attack type.
:type attack_type: str
:param user: The user updating the indicator.
:type user: str
:returns: dict with key "success" (boolean)
"""
# check to ensure we're not duping an existing indicator
indicator = Indicator.objects(id=id_).first()
value = indicator.value
ind_check = Indicator.objects(attack_type=attack_type, value=value).first()
if ind_check:
# we found a dupe
return {"success": False, "message": "Duplicate would exist making this change."}
elif attack_type not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Attack Type."}
else:
try:
indicator.attack_type = attack_type
indicator.save(username=user)
return {"success": True}
except ValidationError:
return {"success": False}
def set_indicator_type(indicator_id, itype, username):
"""
Set the Indicator type.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param itype: The new indicator type.
:type itype: str
:param username: The user updating the indicator.
:type username: str
:returns: dict with key "success" (boolean)
"""
# check to ensure we're not duping an existing indicator
indicator = Indicator.objects(id=indicator_id).first()
value = indicator.value
ind_check = Indicator.objects(ind_type=itype, value=value).first()
if ind_check:
# we found a dupe
return {'success': False}
else:
try:
indicator.ind_type = itype
indicator.save(username=username)
return {'success': True}
except ValidationError:
return {'success': False}
def run(self, obj, config):
if isinstance(obj, Event):
data = obj.description
elif isinstance(obj, RawData):
data = obj.data
elif isinstance(obj, Sample):
samp_data = obj.filedata.read()
data = make_ascii_strings(data=samp_data)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': IndicatorTypes.IPV4_ADDRESS}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': IndicatorTypes.DOMAIN}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': IndicatorTypes.EMAIL_ADDRESS}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
hashes = extract_hashes(data)
for hash_ in hashes:
type_ = hash_[0]
val = hash_[1]
tdict = {'Type': type_}
if type_ == IndicatorTypes.MD5:
id_ = Sample.objects(md5=val).only('id').first()
elif type_ == IndicatorTypes.SHA1:
id_ = Sample.objects(sha1=val).only('id').first()
elif type_ == IndicatorTypes.SHA256:
id_ = Sample.objects(sha256=val).only('id').first()
elif type_ == IndicatorTypes.SSDEEP:
id_ = Sample.objects(ssdeep=val).only('id').first()
else:
id_ = None
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Samples', val, tdict)
def ci_update(indicator_id, ci_type, value, analyst):
"""
Update confidence or impact for an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param ci_type: What we are updating.
:type ci_type: str ("confidence" or "impact")
:param value: The value to set.
:type value: str ("unknown", "benign", "low", "medium", "high")
:param analyst: The user updating this indicator.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
indicator = Indicator.objects(id=indicator_id).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
if ci_type == "confidence" or ci_type == "impact":
try:
if ci_type == "confidence":
indicator.set_confidence(analyst, value)
else:
indicator.set_impact(analyst, value)
indicator.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, "message": e}
def activity_update(id_, activity, user=None, **kwargs):
"""
Update activity for an Indicator.
:param id_: The ObjectId of the indicator to update.
:type id_: str
:param activity: The activity information.
:type activity: dict
:param user: The user updating the activity.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(user)
indicator = Indicator.objects(id=id_, source__name__in=sources).first()
if not indicator:
return {"success": False, "message": "Could not find Indicator"}
try:
activity = datetime_parser(activity)
activity["analyst"] = user
indicator.edit_activity(
activity["analyst"], activity["start_date"], activity["end_date"], activity["description"], activity["date"]
)
indicator.save(username=user)
return {"success": True, "object": activity}
except ValidationError, e:
return {"success": False, "message": e}
def ci_search(itype, confidence, impact, actions):
"""
Find indicators based on type, confidence, impact, and/or actions.
:param itype: The indicator type to search for.
:type itype: str
:param confidence: The confidence level(s) to search for.
:type confidence: str
:param impact: The impact level(s) to search for.
:type impact: str
:param actions: The action(s) to search for.
:type actions: str
:returns: :class:`crits.core.crits_mongoengine.CritsQuerySet`
"""
query = {}
if confidence:
item_list = confidence.replace(' ', '').split(',')
query["confidence.rating"] = {"$in": item_list}
if impact:
item_list = impact.replace(' ', '').split(',')
query["impact.rating"] = {"$in": item_list}
if actions:
item_list = actions.split(',')
query["actions.action_type"] = {"$in": item_list}
query["type"] = "%s" % itype.strip()
result_filter = ('type', 'value', 'confidence', 'impact', 'actions')
results = Indicator.objects(__raw__=query).only(*result_filter)
return results
def action_add(indicator_id, action):
"""
Add an action to an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param action: The information about the action.
:type action: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(action['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
try:
indicator.add_action(action['action_type'],
action['active'],
action['analyst'],
action['begin_date'],
action['end_date'],
action['performed_date'],
action['reason'],
action['date'])
indicator.save(username=action['analyst'])
return {'success': True, 'object': action}
except ValidationError, e:
return {'success': False, 'message': e}
def activity_update(indicator_id, activity):
"""
Update activity for an Indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param activity: The activity information.
:type activity: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(activity['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False, 'message': 'Could not find Indicator'}
try:
indicator.edit_activity(activity['analyst'], activity['start_date'],
activity['end_date'], activity['description'],
activity['date'])
indicator.save(username=activity['analyst'])
return {'success': True, 'object': activity}
except ValidationError, e:
return {'success': False, 'message': e}
def activity_update(indicator_id, activity):
"""
Update activity for an Indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param activity: The activity information.
:type activity: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(activity['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
try:
indicator.edit_activity(activity['analyst'],
activity['start_date'],
activity['end_date'],
activity['description'],
activity['date'])
indicator.save(username=activity['analyst'])
return {'success': True, 'object': activity}
except ValidationError, e:
return {'success': False, 'message': e}
def ci_search(itype, confidence, impact, actions):
"""
Find indicators based on type, confidence, impact, and/or actions.
:param itype: The indicator type to search for.
:type itype: str
:param confidence: The confidence level(s) to search for.
:type confidence: str
:param impact: The impact level(s) to search for.
:type impact: str
:param actions: The action(s) to search for.
:type actions: str
:returns: :class:`crits.core.crits_mongoengine.CritsQuerySet`
"""
query = {}
if confidence:
item_list = confidence.replace(' ', '').split(',')
query["confidence.rating"] = {"$in": item_list}
if impact:
item_list = impact.replace(' ', '').split(',')
query["impact.rating"] = {"$in": item_list}
if actions:
item_list = actions.split(',')
query["actions.action_type"] = {"$in": item_list}
query["type"] = "%s" % itype.strip()
result_filter = ('type', 'value', 'confidence', 'impact', 'actions')
results = Indicator.objects(__raw__=query).only(*result_filter)
return results
def ci_update(indicator_id, ci_type, value, analyst):
"""
Update confidence or impact for an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param ci_type: What we are updating.
:type ci_type: str ("confidence" or "impact")
:param value: The value to set.
:type value: str ("unknown", "benign", "low", "medium", "high")
:param analyst: The user updating this indicator.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
indicator = Indicator.objects(id=indicator_id).first()
if not indicator:
return {'success': False, 'message': 'Could not find Indicator'}
if ci_type == "confidence" or ci_type == "impact":
try:
if ci_type == "confidence":
indicator.set_confidence(analyst, value)
else:
indicator.set_impact(analyst, value)
indicator.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, "message": e}
def action_add(indicator_id, action):
"""
Add an action to an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param action: The information about the action.
:type action: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(action['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False, 'message': 'Could not find Indicator'}
try:
indicator.add_action(action['action_type'], action['active'],
action['analyst'], action['begin_date'],
action['end_date'], action['performed_date'],
action['reason'], action['date'])
indicator.save(username=action['analyst'])
return {'success': True, 'object': action}
except ValidationError, e:
return {'success': False, 'message': e}
def create_indicator_from_obj(ind_type, obj_type, id_, value, analyst):
"""
Add indicators from CRITs object.
:param ind_type: The indicator type to add.
:type ind_type: str
:param obj_type: The CRITs type of the parent object.
:type obj_type: str
:param id_: The ObjectId of the parent object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj = class_from_id(obj_type, id_)
if not obj:
return {'success': False, 'message': 'Could not find object.'}
source = obj.source
bucket_list = obj.bucket_list
campaign = None
campaign_confidence = None
if len(obj.campaign) > 0:
campaign = obj.campaign[0].name
campaign_confidence = obj.campaign[0].confidence
result = handle_indicator_ind(value,
source,
reference=None,
ctype=ind_type,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
obj.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
obj.save(username=analyst)
for rel in obj.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
obj.reload()
rels = obj.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def create_indicator_from_raw(type_, id_, value, analyst):
"""
Add indicators from raw data.
:param type_: The indicator type to add.
:type type_: str
:param id_: The ObjectId of the RawData object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
raw_data = RawData.objects(id=id_).first()
if not raw_data:
return {'success': False, 'message': 'Could not find raw data'}
source = raw_data.source
bucket_list = raw_data.bucket_list
campaign = None
campaign_confidence = None
if len(raw_data.campaign) > 0:
campaign = raw_data.campaign[0].name
campaign_confidence = raw_data.campaign[0].confidence
result = handle_indicator_ind(value,
source,
reference=None,
ctype=type_,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
raw_data.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
raw_data.save(username=analyst)
for rel in raw_data.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
raw_data.reload()
rels = raw_data.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def create_indicator_from_obj(ind_type, obj_type, id_, value, analyst):
"""
Add indicators from CRITs object.
:param ind_type: The indicator type to add.
:type ind_type: str
:param obj_type: The CRITs type of the parent object.
:type obj_type: str
:param id_: The ObjectId of the parent object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj = class_from_id(obj_type, id_)
if not obj:
return {'success': False, 'message': 'Could not find object.'}
source = obj.source
bucket_list = obj.bucket_list
campaign = None
campaign_confidence = None
if len(obj.campaign) > 0:
campaign = obj.campaign[0].name
campaign_confidence = obj.campaign[0].confidence
result = handle_indicator_ind(value, source, reference=None, ctype=ind_type,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
obj.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
obj.save(username=analyst)
for rel in obj.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
obj.reload()
rels = obj.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def create_indicator_from_raw(type_, id_, value, analyst):
"""
Add indicators from raw data.
:param type_: The indicator type to add.
:type type_: str
:param id_: The ObjectId of the RawData object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
raw_data = RawData.objects(id=id_).first()
if not raw_data:
return {'success': False,
'message': 'Could not find raw data'}
source = raw_data.source
bucket_list = raw_data.bucket_list
campaign = None
campaign_confidence = None
if len(raw_data.campaign) > 0:
campaign = raw_data.campaign[0].name
campaign_confidence = raw_data.campaign[0].confidence
result = handle_indicator_ind(value, source, reference=None, ctype=type_,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
raw_data.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
raw_data.save(username=analyst)
for rel in raw_data.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
raw_data.reload()
rels = raw_data.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
def _scan(self, context):
if isinstance(context, RawDataContext):
raw_data = RawData.objects(id=context.identifier).first()
if not raw_data:
self._debug("Could not find raw data to parse.")
return
data = raw_data.data
elif isinstance(context, SampleContext):
data = make_ascii_strings(md5=context.identifier)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': "IP Address"}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': "Domain"}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': "Email"}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
def _scan(self, context):
if isinstance(context, RawDataContext):
raw_data = RawData.objects(id=context.identifier).first()
if not raw_data:
self._debug("Could not find raw data to parse.")
return
data = raw_data.data
elif isinstance(context, SampleContext):
data = make_ascii_strings(md5=context.identifier)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {"Type": "IP Address"}
id_ = Indicator.objects(value=ip).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential IP Address", ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {"Type": "Domain"}
id_ = Indicator.objects(value=domain).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential Domains", domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {"Type": "Email"}
id_ = Indicator.objects(value=email).only("id").first()
if id_:
tdict["exists"] = str(id_.id)
self._add_result("Potential Emails", email, tdict)
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
for indicator in indicators: # for each STIX indicator
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = Indicator.from_cybox(item, [self.source])
obj.save(username=self.source_instance.analyst)
self.imported.append((Indicator._meta['crits_type'], obj))
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def indicator_remove(_id, username):
"""
Remove an Indicator from CRITs.
:param _id: The ObjectId of the indicator to remove.
:type _id: str
:param username: The user removing the indicator.
:type username: str
:returns: dict with keys "success" (boolean) and "message" (list) if failed.
"""
if is_admin(username):
indicator = Indicator.objects(id=_id).first()
if indicator:
indicator.delete(username=username)
return {'success': True}
else:
return {'success': False, 'message': ['Cannot find Indicator']}
else:
return {'success': False, 'message': ['Must be an admin to delete']}
def indicator_remove(_id, username):
"""
Remove an Indicator from CRITs.
:param _id: The ObjectId of the indicator to remove.
:type _id: str
:param username: The user removing the indicator.
:type username: str
:returns: dict with keys "success" (boolean) and "message" (list) if failed.
"""
if is_admin(username):
indicator = Indicator.objects(id=_id).first()
if indicator:
indicator.delete(username=username)
return {'success': True}
else:
return {'success': False, 'message': ['Cannot find Indicator']}
else:
return {'success': False, 'message': ['Must be an admin to delete']}
def action_remove(indicator_id, date, analyst):
"""
Remove an action from an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param date: The date of the action to remove.
:type date: datetime.datetime
:param analyst: The user removing the action.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
indicator = Indicator.objects(id=indicator_id).first()
if not indicator:
return {'success': False, 'message': 'Could not find Indicator'}
try:
indicator.delete_action(date)
indicator.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def action_remove(indicator_id, date, analyst):
"""
Remove an action from an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param date: The date of the action to remove.
:type date: datetime.datetime
:param analyst: The user removing the action.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
indicator = Indicator.objects(id=indicator_id).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
try:
indicator.delete_action(date)
indicator.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def activity_remove(id_, date, user, **kwargs):
"""
Remove activity from an Indicator.
:param id_: The ObjectId of the indicator to update.
:type id_: str
:param date: The date of the activity to remove.
:type date: datetime.datetime
:param user: The user removing this activity.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
indicator = Indicator.objects(id=id_).first()
if not indicator:
return {"success": False, "message": "Could not find Indicator"}
try:
date = datetime_parser(date)
indicator.delete_activity(date)
indicator.save(username=user)
return {"success": True}
except ValidationError, e:
return {"success": False, "message": e}
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def parse_result(self, result_extract, acl_write, md5_parent):
stream_md5 = None
if type(result_extract) is dict:
#make reccursion extract each file embbed
if 'FileMD5' in result_extract and result_extract['FileMD5']:
tmp_dict = {}
b_yara = False
b_ioc = False
#extract info
no_info = [
'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile',
'FileMD5', 'RootFileType', 'TempDirExtract'
]
for key, value in result_extract.iteritems():
if not key in no_info:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info', key,
{'value': str(value)})
#extract yara match
if result_extract['Yara']:
for item_v in result_extract['Yara']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Signatures yara matched', key, value)
b_yara = True
#extract IOC
if result_extract['ExtractInfo']:
for item_v in result_extract['ExtractInfo']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Extract potential IOC', key,
{'value': str(value)})
b_ioc = True
#add_sample
if 'PathFile' in result_extract and type(
result_extract['PathFile']) is list and len(
result_extract['PathFile']) > 0:
if os.path.isfile(str(result_extract['PathFile'][0])):
with open(str(result_extract['PathFile'][0]),
'r') as content_file_tmp:
content_tmp = content_file_tmp.read()
stream_md5 = hashlib.md5(content_tmp).hexdigest()
name = str(stream_md5).decode('ascii', errors='ignore')
id_ = Sample.objects(md5=stream_md5).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
else:
if acl_write and (
self.config['import_file'] or
(self.config['import_file_yara'] and b_yara) or
(self.config['import_file_ioc'] and b_ioc)):
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(stream_md5))
else:
#add IOC if not add sample
if self.current_task.user.has_access_to(
IndicatorACL.WRITE) and b_yara:
res = handle_indicator_ind(
stream_md5,
self.obj.source,
IndicatorTypes.MD5,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
self.current_task.user,
add_relationship=True,
source_method=self.name,
campaign=self.obj.campaign,
description='Extracted by service ' +
self.name)
self._info('Add indicator md5:' +
str(stream_md5) + ' -- id: ' +
str(res))
#contains file
if 'ContainedObjects' in result_extract and type(
result_extract['ContainedObjects']
) is list and result_extract['ContainedObjects']:
for item_v in result_extract['ContainedObjects']:
if item_v['FileMD5'] and item_v['FileType'] and item_v[
'FileSize']:
#search if file exist
id_ = Sample.objects(
md5=str(item_v['FileMD5'])).only('id').first()
sample_exist = False
ioc_exist = False
if id_:
sample_exist = True
id_ = Indicator.objects(
value=str(item_v['FileMD5'])).only('id').first()
if id_:
ioc_exist = True
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Contains md5 files', item_v['FileMD5'], {
'type': str(item_v['FileType']),
'size': str(item_v['FileSize']),
'Exists Sample': str(sample_exist),
'Exists IOC md5': str(ioc_exist)
})
for item_v in result_extract['ContainedObjects']:
#re do loop for best display result
parse_result(self, item_v, acl_write, stream_md5)
def get_indicator_details(indicator_id, analyst):
"""
Generate the data to render the Indicator details template.
:param indicator_id: The ObjectId of the Indicator to get details for.
:type indicator_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
users_sources = user_sources(analyst)
indicator = Indicator.objects(id=indicator_id,
source__name__in=users_sources).first()
if not indicator:
error = ("Either this indicator does not exist or you do "
"not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
forms['new_action'] = IndicatorActionsForm(initial={'analyst': analyst,
'active': "off",
'date': datetime.datetime.now()})
forms['new_activity'] = IndicatorActivityForm(initial={'analyst': analyst,
'date': datetime.datetime.now()})
forms['new_campaign'] = CampaignForm()#'date': datetime.datetime.now(),
forms['new_source'] = SourceForm(analyst, initial={'date': datetime.datetime.now()})
forms['download_form'] = DownloadFileForm(initial={"obj_type": 'Indicator',
"obj_id": indicator_id})
indicator.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, indicator_id, 'Indicator')
# subscription
subscription = {
'type': 'Indicator',
'id': indicator_id,
'subscribed': is_user_subscribed("%s" % analyst,
'Indicator',
indicator_id),
}
# relationship
relationship = {
'type': 'Indicator',
'value': indicator_id,
}
#objects
objects = indicator.sort_objects()
#relationships
relationships = indicator.sort_relationships("%s" % analyst, meta=True)
#comments
comments = {'comments': indicator.get_comments(),
'url_key': indicator_id}
#screenshots
screenshots = indicator.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Indicator', indicator.id)
# services
service_list = get_supported_services('Indicator')
# analysis results
service_results = indicator.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
'comments': comments,
'relationship': relationship,
'subscription': subscription,
"indicator": indicator,
"forms": forms,
"indicator_id": indicator_id,
'screenshots': screenshots,
'service_list': service_list,
'service_results': service_results,
'favorite': favorite,
'rt_url': settings.RT_URL}
return template, args
def create_indicator_from_object(rel_type, rel_id, ind_type, value, analyst,
request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False, 'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty value field"
}
elif ind_type == None or ind_type.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty type field"
}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
if hasattr(me, 'source'):
create_indicator_result = handle_indicator_ind(value,
me.source,
'',
ind_type,
analyst=analyst,
add_domain=True)
else:
# In case the top level item doesn't have sources (such as campaign)...
# then just default to the user's organization
create_indicator_result = handle_indicator_ind(
value,
get_user_organization(analyst),
'',
ind_type,
analyst=analyst,
add_domain=True)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return result
indicator = Indicator.objects(ind_type=ind_type, value=value).first()
if not indicator:
result = {
'success': False,
'message': "Could not create indicator"
}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship = {'type': rel_type, 'value': rel_id}
message = render_to_string(
'relationships_listing_widget.html', {
'relationship': relationship,
'nohide': True,
'relationships': results['message']
}, RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False, 'message': message}
return result
def create_indicator_and_ip(type_, id_, ip, analyst):
"""
Add indicators for an IP address.
:param type_: The CRITs top-level object we are getting this IP from.
:type type_: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param id_: The ObjectId of the top-level object to search for.
:type id_: str
:param ip: The IP address to generate an indicator out of.
:type ip: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj_class = class_from_id(type_, id_)
if obj_class:
ip_class = IP.objects(ip=ip).first()
ind_type = "Address - ipv4-addr"
ind_class = Indicator.objects(ind_type=ind_type, value=ip).first()
# setup IP
if ip_class:
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
else:
ip_class = IP()
ip_class.ip = ip
ip_class.source = obj_class.source
ip_class.save(username=analyst)
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
# setup Indicator
message = ""
if ind_class:
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
else:
ind_class = Indicator()
ind_class.source = obj_class.source
ind_class.ind_type = ind_type
ind_class.value = ip
ind_class.save(username=analyst)
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
# save
try:
obj_class.save(username=analyst)
ip_class.save(username=analyst)
ind_class.save(username=analyst)
if message['success']:
rels = obj_class.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': obj_class.id}
else:
return {'success': False, 'message': message['message']}
except Exception, e:
return {'success': False, 'message': e}
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain = ind_value.split("/")[2]
elif ind_type == "URI - Domain Name":
domain = ind_value
#try:
(sdomain, fqdn) = get_domain(domain)
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
elif ind_type.startswith(
"Address - ip") or ind_type == "Address - cidr":
success = None
if add_domain:
success = ip_add_update(indicator.value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {
'success': True,
'objectid': indicator.id,
'is_new_indicator': is_new_indicator,
'object': indicator
}
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, tlp, analyst,
request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False, 'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty value field"
}
elif ind_type == None or ind_type.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty type field"
}
elif source_name == None or source_name.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty source field"
}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(
value,
source_name,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type, value=value).first()
if not indicator:
result = {
'success': False,
'message': "Could not create indicator"
}
else:
results = me.add_relationship(indicator,
RelationshipTypes.RELATED_TO,
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
relationship = {'type': rel_type, 'value': rel_id}
message = render_to_string(
'relationships_listing_widget.html', {
'relationship': relationship,
'nohide': True,
'relationships': results['message']
}, RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False, 'message': message}
return result
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(
Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def create_indicator_and_ip(type_, id_, ip, analyst):
"""
Add indicators for an IP address.
:param type_: The CRITs top-level object we are getting this IP from.
:type type_: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param id_: The ObjectId of the top-level object to search for.
:type id_: str
:param ip: The IP address to generate an indicator out of.
:type ip: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj_class = class_from_id(type_, id_)
if obj_class:
ip_class = IP.objects(ip=ip).first()
ind_type = "Address - ipv4-addr"
ind_class = Indicator.objects(ind_type=ind_type, value=ip).first()
# setup IP
if ip_class:
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
else:
ip_class = IP()
ip_class.ip = ip
ip_class.source = obj_class.source
ip_class.save(username=analyst)
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
# setup Indicator
message = ""
if ind_class:
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
else:
ind_class = Indicator()
ind_class.source = obj_class.source
ind_class.ind_type = ind_type
ind_class.value = ip
ind_class.save(username=analyst)
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
# save
try:
obj_class.save(username=analyst)
ip_class.save(username=analyst)
ind_class.save(username=analyst)
if message['success']:
rels = obj_class.sort_relationships("%s" % analyst, meta=True)
return {
'success': True,
'message': rels,
'value': obj_class.id
}
else:
return {'success': False, 'message': message['message']}
except Exception, e:
return {'success': False, 'message': e}
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def sample_network(self, tg_id):
"""
Get Sample Network indicators for a given ThreatGRID id
"""
indicators = []
url = '/api/v2/samples/' + tg_id + '/analysis/network_streams'
response = self.api_request(url, {}, 'get')
if response:
if response.get('data'):
# DNS
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == 'DNS':
# Process DNS lookups
dns_objects = item.get('decoded')
for obj in dns_objects:
result = {
'dns_query': dns_objects[obj].get('query', {}).get('query_data'),
'dns_type': dns_objects[obj].get('query', {}).get('query_type'),
}
dns_qid = dns_objects[obj].get('query', {}).get('query_id')
# Find the answer for each DNS query by id, type
for answer in dns_objects[obj].get('answers', []):
if answer.get('answer_id', 0) == dns_qid:
if answer.get('answer_type', '') == result['dns_type']:
result['dns_answer'] = answer.get('answer_data')
break
indicators.append([result.get('dns_query'), IndicatorTypes.DOMAIN])
indicators.append([result.get('dns_answer'), IndicatorTypes.IPV4_ADDRESS])
self._add_result('threatgrid_dns'.format(tg_id), result.pop('dns_query'), result)
self._notify()
# HTTP
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == 'HTTP':
for decode in item.get('decoded'):
for entry in decode:
# Only show HTTP requests
if entry.get('type') == 'request':
result = {
'host': entry.get('host'),
'method': entry.get('method'),
'url': entry.get('url'),
'ua': entry.get('headers', {}).get('user-agent'),
'referer': entry.get('headers', {}).get('referer'),
'dst': item.get('dst'),
'dst_port': item.get('dst_port'),
}
indicators.append([result.get('host'), IndicatorTypes.DOMAIN])
indicators.append([result.get('dst'), IndicatorTypes.IPV4_ADDRESS])
self._add_result('threatgrid_http'.format(tg_id), result.pop('host'), result)
self._notify()
# IP/Other
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == None:
result = {
'transport': item.get('transport'),
'src': item.get('src'),
'src_port': item.get('src_port'),
'dst': item.get('dst'),
'dst_port': item.get('dst_port'),
'bytes': item.get('bytes'),
'packets': item.get('packets'),
}
indicators.append([result.get('dst'), IndicatorTypes.IPV4_ADDRESS])
self._add_result('threatgrid_ip'.format(tg_id), result.pop('transport'), result)
self._notify()
# Enable user to add unique indicators for this sample
added = []
for item in indicators:
if item[0]:
indicator = item[0].lower()
if indicator not in added:
added.append(indicator)
tdict = {}
if item[1] in (IndicatorTypes.IPV4_ADDRESS, IndicatorTypes.DOMAIN):
tdict = {'Type': item[1]}
id_ = Indicator.objects(value=indicator).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('add_threatgrid_indicators', indicator, tdict)
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def to_stix(self, username=None):
"""
Converts a CRITs event to a STIX document.
The resulting document includes all related emails, samples, and
indicators converted to CybOX Observable objects.
Returns the STIX document and releasability constraints.
(NOTE: the following statement is untrue until the
releasability checking is finished, which includes setting
releasability on all CRITs objects.)
Raises UnreleasableEventError if the releasability on the
relationships and the event do not share any common releasability
sources.
"""
from crits.emails.email import Email
from crits.samples.sample import Sample
from crits.indicators.indicator import Indicator
from cybox.common import Time, ToolInformationList, ToolInformation
from cybox.core import Observables
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
from stix.common import InformationSource
from stix.common.identity import Identity
stix_indicators = []
stix_observables = []
final_objects = []
# create a list of sources to send as part of the results.
# list should be limited to the sources this user is allowed to use.
# this list should be used along with the list of objects to set the
# appropriate source's 'released' key to True for each object.
final_sources = []
user_source_list = user_sources(username)
for f in self.releasability:
if f.name in user_source_list:
final_sources.append(f.name)
final_sources = set(final_sources)
# TODO: eventually we can use class_from_id instead of the if block
# but only once we support all CRITs types.
for r in self.relationships:
obj = None
if r.rel_type == Email._meta['crits_type']:
obj = Email.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
stix_observables.append(ind[0])
elif r.rel_type == Sample._meta['crits_type']:
obj = Sample.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
for i in ind:
stix_observables.append(i)
elif r.rel_type == Indicator._meta['crits_type']:
#NOTE: Currently this will raise an exception if there
# are multiple indicators with the same value.
# Should be fixed automatically once we transition
# indicators to be related based on ObjectId rather
# than value.
obj = Indicator.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_stix_indicator()
stix_indicators.append(ind)
else:
continue
#Create a releasability list that is the intersection of
# each related item's releasability with the event's
# releasability. If the resulting set is empty, raise exception
#TODO: Set releasability on all objects so that we actually
# get results here instead of always raising an exception.
if obj:
releas_sources = set([rel.name for rel in releas])
final_sources = final_sources.intersection(releas_sources)
#TODO: uncomment the following lines when objects have
# releasability set.
#if not final_sources:
# raise UnreleasableEventError(r.value)
# add to the final_objects list to send as part of the results
final_objects.append(obj)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(
time=Time(produced_time= datetime.datetime.now()),
identity = Identity(name=settings.COMPANY_NAME),
tools = tool_list
)
description = StructuredText(value=self.description)
header = STIXHeader(information_source=i_s,
description=description,
package_intent=self.event_type,
title=self.title)
return (STIXPackage(indicators=stix_indicators,
observables=Observables(stix_observables),
stix_header=header,
id_=self.event_id),
final_sources,
final_objects)
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def parse_result(self, result_extract, response_dict, config, md5_parent):
stream_md5 = None
user = self.current_task.user
self.config = config
acl_write = user.has_access_to(SampleACL.WRITE)
if type(result_extract) is dict:
#make reccursion extract each file embbed
if 'FileMD5' in result_extract and result_extract['FileMD5']:
tmp_dict = {}
b_yara = False
b_ioc = False
#extract info
no_info = [
'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile',
'FileMD5', 'RootFileType', 'TempDirExtract', 'GlobalIOC'
]
for key, value in result_extract.iteritems():
if not key in no_info:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info', key,
{'value': str(value)})
#add download info
if result_extract['FileMD5'] in response_dict:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info',
'Download embed file', {'value': sfa_api + vx})
#GLOBAL IOC
if 'GlobalIOC' in result_extract and result_extract['GlobalIOC']:
for key, value in result_extract['GlobalIOC'].iteritems():
self._add_result('Global IOC by categories', key,
{'value': str(value)})
#extract yara match
if result_extract['Yara']:
for item_v in result_extract['Yara']:
#self._info("Dict:"+str(item_v))
for key, value in item_v.iteritems():
data = {
'description': '',
'ioc': '',
'tags': '',
'score': '0'
}
for kx, vx in value.iteritems():
data[kx] = str(vx)
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Signatures yara matched', key, data)
score_conf = re.sub("\D", "",
config['import_yara_score'])
if acl_write and config[
'import_file_yara'] and 'score' in value and int(
value['score']) >= int(score_conf):
id_ = Sample.objects(md5=result_extract['FileMD5']
).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.
CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
elif result_extract['FileMD5'] in response_dict:
content_tmp = download(
self,
response_dict[result_extract['FileMD5']])
if content_tmp:
name = str(result_extract['FileMD5'])
if 'CDBNAME' in result_extract:
name = str(result_extract['CDBNAME'])
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']
),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(name) +
' - MD5:' +
str(result_extract['FileMD5']))
#extract IOC
if result_extract['ExtractInfo']:
for item_v in result_extract['ExtractInfo']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Extract potential IOC', key,
{'value': str(value)})
if acl_write and config['import_file_ioc']:
id_ = Sample.objects(md5=result_extract['FileMD5']
).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.
CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
elif result_extract['FileMD5'] in response_dict:
content_tmp = download(
self,
response_dict[result_extract['FileMD5']])
if content_tmp:
name = str(result_extract['FileMD5'])
if 'CDBNAME' in result_extract:
name = str(result_extract['CDBNAME'])
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']
),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(name) +
' - MD5:' +
str(result_extract['FileMD5']))
#contains file
if 'ContainedObjects' in result_extract and type(
result_extract['ContainedObjects']
) is list and result_extract['ContainedObjects']:
for item_v in result_extract['ContainedObjects']:
if item_v['FileMD5'] and item_v['FileType'] and item_v[
'FileSize']:
#search if file exist
id_ = Sample.objects(
md5=str(item_v['FileMD5'])).only('id').first()
sample_exist = False
ioc_exist = False
if id_:
sample_exist = True
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.RELATED_TO,
#rel_date=datetime.now(),
analyst=self.current_task.user.username)
id_ = Indicator.objects(
value=str(item_v['FileMD5'])).only('id').first()
if id_:
ioc_exist = True
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.RELATED_TO,
#rel_date=datetime.now(),
analyst=self.current_task.user.username)
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Contains md5 files', item_v['FileMD5'], {
'type': str(item_v['FileType']),
'size': str(item_v['FileSize']),
'Exists Sample': str(sample_exist),
'Exists IOC md5': str(ioc_exist)
})
for item_v in result_extract['ContainedObjects']:
#re do loop for best display result
parse_result(self, item_v, response_dict, config,
stream_md5)
def get_indicator_details(indicator_id, analyst):
"""
Generate the data to render the Indicator details template.
:param indicator_id: The ObjectId of the Indicator to get details for.
:type indicator_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
users_sources = user_sources(analyst)
indicator = Indicator.objects(id=indicator_id, source__name__in=users_sources).first()
if not indicator:
error = "Either this indicator does not exist or you do " "not have permission to view it."
template = "error.html"
args = {"error": error}
return template, args
forms = {}
forms["new_activity"] = IndicatorActivityForm(initial={"analyst": analyst, "date": datetime.datetime.now()})
forms["new_campaign"] = CampaignForm() #'date': datetime.datetime.now(),
forms["new_source"] = SourceForm(analyst, initial={"date": datetime.datetime.now()})
forms["download_form"] = DownloadFileForm(initial={"obj_type": "Indicator", "obj_id": indicator_id})
indicator.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, indicator_id, "Indicator")
# subscription
subscription = {
"type": "Indicator",
"id": indicator_id,
"subscribed": is_user_subscribed("%s" % analyst, "Indicator", indicator_id),
}
# relationship
relationship = {"type": "Indicator", "value": indicator_id}
# objects
objects = indicator.sort_objects()
# relationships
relationships = indicator.sort_relationships("%s" % analyst, meta=True)
# comments
comments = {"comments": indicator.get_comments(), "url_key": indicator_id}
# screenshots
screenshots = indicator.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, "Indicator", indicator.id)
# services
service_list = get_supported_services("Indicator")
# analysis results
service_results = indicator.get_analysis_results()
args = {
"objects": objects,
"relationships": relationships,
"comments": comments,
"relationship": relationship,
"subscription": subscription,
"indicator": indicator,
"forms": forms,
"indicator_id": indicator_id,
"screenshots": screenshots,
"service_list": service_list,
"service_results": service_results,
"favorite": favorite,
"rt_url": settings.RT_URL,
}
return template, args
def get_indicator_details(indicator_id, analyst):
"""
Generate the data to render the Indicator details template.
:param indicator_id: The ObjectId of the Indicator to get details for.
:type indicator_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
users_sources = user_sources(analyst)
indicator = Indicator.objects(id=indicator_id,
source__name__in=users_sources).first()
if not indicator:
error = ("Either this indicator does not exist or you do "
"not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
forms['new_action'] = IndicatorActionsForm(initial={
'analyst': analyst,
'active': "off",
'date': datetime.datetime.now()
})
forms['new_activity'] = IndicatorActivityForm(
initial={
'analyst': analyst,
'date': datetime.datetime.now()
})
forms['new_campaign'] = CampaignForm() #'date': datetime.datetime.now(),
forms['new_source'] = SourceForm(analyst,
initial={'date': datetime.datetime.now()})
forms['download_form'] = DownloadFileForm(initial={
"obj_type": 'Indicator',
"obj_id": indicator_id
})
indicator.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, indicator_id, 'Indicator')
# subscription
subscription = {
'type':
'Indicator',
'id':
indicator_id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Indicator', indicator_id),
}
# relationship
relationship = {
'type': 'Indicator',
'value': indicator_id,
}
#objects
objects = indicator.sort_objects()
#relationships
relationships = indicator.sort_relationships("%s" % analyst, meta=True)
#comments
comments = {'comments': indicator.get_comments(), 'url_key': indicator_id}
# favorites
favorite = is_user_favorite("%s" % analyst, 'Indicator', indicator.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('Indicator', True)
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'relationship': relationship,
'subscription': subscription,
"indicator": indicator,
"forms": forms,
"indicator_id": indicator_id,
'service_list': service_list,
'favorite': favorite,
'rt_url': settings.RT_URL
}
return template, args
def check_indicators_ip(self,obj, config):
if settings.HTTP_PROXY:
proxies = {'http': settings.HTTP_PROXY,
'https': settings.HTTP_PROXY}
else:
proxies = {}
indicators = []
url = config['av_url']
api = config['av_api']
'''
Detect if IP address if IPv4 or IPv6
'''
try:
'''
Check if ipv4
'''
if socket.inet_aton(str(obj.ip)):
self._info("IPv4 Address : "+str(obj.ip))
request_url = url+'indicators/IPv4/'+str(obj.ip)+'/malware'
request_url_general = url + 'indicators/IPv4/' + str(obj.ip) + '/general'
request_url_reputation = url + 'indicators/IPv4/' + str(obj.ip) + '/reputation'
request_url_list = url + 'indicators/IPv4/' + str(obj.ip) + '/url_list'
headers = {'X-OTX-API-KEY' : api}
r = requests.get(request_url, headers=headers, verify=False, proxies=proxies)
r_g = requests.get(request_url_general, headers=headers, verify=False, proxies=proxies)
r_rp = requests.get(request_url_reputation, headers=headers, verify=False, proxies=proxies)
r_url_list = requests.get(request_url_list, headers=headers, verify=False, proxies=proxies)
if r.status_code !=200:
self._error("Response not OK")
return
results = r.json()
results_general = r_g.json()
results_reputation = r_rp.json()
results_url_list = r_url_list.json()
geolocation = {
'Indicator': results_general.get('indicator'),
'Country': results_general.get('country_name'),
'whois': results_general.get('whois')
}
'''
Simple Geolocation Data
'''
if geolocation is not None:
self._add_result("General Information", results_general.get('indicator'), geolocation)
'''
Related malware hashes.
'''
for i in results.get('data'):
self._add_result("Related Malicious Hash",i.get('hash'))
'''
Pulse Information
'''
if results_general.get('pulse_info') is not None:
for m,n in results_general.get('pulse_info').iteritems():
if 'pulses' in m and bool(n):
pulses = n
self._add_result("Pulses Found",str(obj.ip), pulses)
'''
Get reputational data
'''
activities = results_reputation['reputation']['activities']
for active in activities:
self._add_result("IP Reputation-Activities",results_reputation['reputation']['address'], active)
domains = results_reputation['reputation']['domains']
for domain in domains:
indicators.append([domain, IndicatorTypes.DOMAIN])
# Enable user to add unique indicators for this sample
added = []
for item in indicators:
if item[0]:
indicator = item[0].lower()
if indicator not in added:
added.append(indicator)
tdict = {}
if item[1] in (IndicatorTypes.IPV4_ADDRESS, IndicatorTypes.DOMAIN):
tdict = {'Type': item[1]}
id_ = Indicator.objects(value=indicator).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('add_alienvault_indicators', indicator, tdict)
'''
Get URL List :URLs analyzed by AlienVault Labs which point to or are somehow associated with this IP address.
'''
for lst in results_url_list['url_list']:
self._add_result("URL List","List", lst)
else:
self._info("IPv6 Address : " + str(obj.ip))
request_url = url + 'indicators/IPv6/' + str(obj.ip) + '/malware'
request_url_general = url + 'indicators/IPv6/' + str(obj.ip) + '/general'
request_url_reputation = url + 'indicators/IPv6/' + str(obj.ip) + '/reputation'
request_url_list = url + 'indicators/IPv6/' + str(obj.ip) + '/url_list'
headers = {'X-OTX-API-KEY': api}
r = requests.get(request_url, headers=headers, verify=False, proxies=proxies)
r_g = requests.get(request_url_general, headers=headers, verify=False, proxies=proxies)
r_rp = requests.get(request_url_reputation, headers=headers, verify=False, proxies=proxies)
r_url_list = requests.get(request_url_list, headers=headers, verify=False, proxies=proxies)
if r.status_code != 200:
self._error("Response not OK")
return
results = r.json()
results_general = r_g.json()
results_reputation = r_rp.json()
results_url_list = r_url_list.json()
geolocation = {
'Indicator': results_general.get('indicator'),
'Country': results_general.get('country_name'),
'whois': results_general.get('whois')
}
'''
Simple Geolocation Data
'''
if geolocation is not None:
self._add_result("General Information", results_general.get('indicator'), geolocation)
'''
Related malicious hashes.
'''
for i in results.get('data'):
self._add_result("Related Malicious Hash", i.get('hash'))
'''
Pulse Information
'''
if results_general.get('pulse_info') is not None:
for m, n in results_general.get('pulse_info').iteritems():
if 'pulses' in m and bool(n):
pulses = n
self._add_result("Pulses Found", str(obj.ip), pulses)
'''
Get reputational data
'''
activities = results_reputation['reputation']['activities']
for active in activities:
self._add_result("IP Reputation-Activities", results_reputation['reputation']['address'], active)
domains = results_reputation['reputation']['domains']
for domain in domains:
indicators.append([domain, IndicatorTypes.DOMAIN])
# Enable user to add unique indicators for this sample
added = []
for item in indicators:
if item[0]:
indicator = item[0].lower()
if indicator not in added:
added.append(indicator)
tdict = {}
if item[1] in (IndicatorTypes.IPV4_ADDRESS, IndicatorTypes.DOMAIN):
tdict = {'Type': item[1]}
id_ = Indicator.objects(value=indicator).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('add_alienvault_indicators', indicator, tdict)
'''
Get URL List :URLs analyzed by AlienVault Labs which point to or are somehow associated with this IP address.
'''
for lst in results_url_list['url_list']:
self._add_result("URL List", "List", lst)
except socket.error:
self._error("Couldn't establish connections or invalid IP address")
def sample_network(self, tg_id):
"""
Get Sample Network indicators for a given ThreatGRID id
"""
indicators = []
url = '/api/v2/samples/' + tg_id + '/analysis/network_streams'
response = self.api_request(url, {}, 'get')
if response:
if response.get('data'):
# DNS
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == 'DNS':
# Process DNS lookups
dns_objects = item.get('decoded')
for obj in dns_objects:
result = {
'dns_query':
dns_objects[obj].get('query',
{}).get('query_data'),
'dns_type':
dns_objects[obj].get('query',
{}).get('query_type'),
}
dns_qid = dns_objects[obj].get('query',
{}).get('query_id')
# Find the answer for each DNS query by id, type
for answer in dns_objects[obj].get('answers', []):
if answer.get('answer_id', 0) == dns_qid:
if answer.get('answer_type',
'') == result['dns_type']:
result['dns_answer'] = answer.get(
'answer_data')
break
indicators.append(
[result.get('dns_query'), 'Domain'])
indicators.append(
[result.get('dns_answer'), 'IP Address'])
self._add_result('threatgrid_dns'.format(tg_id),
result.pop('dns_query'), result)
self._notify()
# HTTP
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == 'HTTP':
for decode in item.get('decoded'):
for entry in decode:
# Only show HTTP requests
if entry.get('type') == 'request':
result = {
'host':
entry.get('host'),
'method':
entry.get('method'),
'url':
entry.get('url'),
'ua':
entry.get('headers',
{}).get('user-agent'),
'referer':
entry.get('headers',
{}).get('referer'),
'dst':
item.get('dst'),
'dst_port':
item.get('dst_port'),
}
indicators.append(
[result.get('host'), 'Domain'])
indicators.append(
[result.get('dst'), 'IP Address'])
self._add_result(
'threatgrid_http'.format(tg_id),
result.pop('host'), result)
self._notify()
# IP/Other
for num in response.get('data', {}).get('items'):
item = response['data']['items'][num]
if item.get('protocol') == None:
result = {
'transport': item.get('transport'),
'src': item.get('src'),
'src_port': item.get('src_port'),
'dst': item.get('dst'),
'dst_port': item.get('dst_port'),
'bytes': item.get('bytes'),
'packets': item.get('packets'),
}
indicators.append([result.get('dst'), 'IP Address'])
self._add_result('threatgrid_ip'.format(tg_id),
result.pop('transport'), result)
self._notify()
# Enable user to add unique indicators for this sample
added = []
for item in indicators:
if item[0]:
indicator = item[0].lower()
if indicator not in added:
added.append(indicator)
tdict = {}
if item[1] == 'Domain':
tdict = {'Type': 'Domain'}
id_ = Indicator.objects(
value=indicator).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
elif item[1] == 'IP Address':
tdict = {'Type': "IP Address"}
id_ = Indicator.objects(
value=indicator).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('add_threatgrid_indicators',
indicator, tdict)
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, analyst, request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False,
'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty value field"}
elif ind_type == None or ind_type.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty type field"}
elif source_name == None or source_name.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty source field"}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(value,
source_name,
reference,
ind_type,
analyst,
method,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type,
value=value).first()
if not indicator:
result = {'success': False,
'message': "Could not create indicator"}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship= {'type': rel_type, 'value': rel_id}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['message']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False,
'message': message}
return result
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
