from django import forms
from django.forms.utils import ErrorList
from crits.campaigns.campaign import Campaign
from crits.core import form_consts
from crits.core.forms import add_bucketlist_to_form, add_ticket_to_form
from crits.core.widgets import CalWidget
from crits.core.handlers import get_source_names, get_item_names
from crits.core.user_tools import get_user_organization
from crits.vocabulary.ips import IPTypes
from crits.vocabulary.relationships import RelationshipTypes
ip_choices = [(c,c) for c in IPTypes.values(sort=True)]
relationship_choices = [(c, c) for c in RelationshipTypes.values(sort=True)]
class TLDUpdateForm(forms.Form):
"""
Django form for updating TLD entries.
"""
error_css_class = 'error'
required_css_class = 'required'
filedata = forms.FileField()
class AddDomainForm(forms.Form):
"""
Django form for adding a domain.
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype, [self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {
'success': False,
'reason': 'No IP Type'
}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and
(isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val), [self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact)
and rel_obj.properties.type_
== Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get(
'date', 'None'), data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data, analyst, "STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(
data[key], self.source, analyst,
description, title, d_type, "STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj, res,
ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(
cbx_obj.id_, []).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to',
'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(
data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append(
(oid, RelationshipTypes.RELATED_TO, oid2,
"High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append(
(oid, RelationshipTypes.CONTAINS, rel_id,
"High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, "%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
from django.conf import settings
from django import forms
from django.forms.utils import ErrorList
from crits.campaigns.campaign import Campaign
from crits.core import form_consts
from crits.core.forms import add_bucketlist_to_form, add_ticket_to_form
from crits.core.widgets import CalWidget
from crits.core.handlers import get_source_names, get_item_names
from crits.core.user_tools import get_user_organization
from crits.vocabulary.ips import IPTypes
ip_choices = [(c, c) for c in IPTypes.values(sort=True)]
class TLDUpdateForm(forms.Form):
"""
Django form for updating TLD entries.
"""
error_css_class = 'error'
required_css_class = 'required'
filedata = forms.FileField()
class AddDomainForm(forms.Form):
"""
Django form for adding a domain.
"""
def make_cybox_object(type_, value=None):
"""
Converts type_, name, and value to a CybOX object instance.
:param type_: The object type.
:type type_: str
:param value: The object value.
:type value: str
:returns: CybOX object
"""
if type_ == IndicatorTypes.USER_ID:
acct = Account()
acct.description = value
return acct
elif type_ in IPTypes.values():
if type_ == IPTypes.IPV4_ADDRESS:
name = 'ipv4-addr'
elif type_ == IPTypes.IPV6_ADDRESS:
name = 'ipv6-addr'
elif type_ == IPTypes.IPV4_SUBNET:
name = 'ipv4-net'
elif type_ == IPTypes.IPV6_SUBNET:
name = 'ipv6-net'
return Address(category=name, address_value=value)
elif type_ == IndicatorTypes.API_KEY:
api = API()
api.description = value
return api
elif type_ == IndicatorTypes.DOMAIN:
obj = DomainName()
obj.value = value
return obj
elif type_ == IndicatorTypes.USER_AGENT:
obj = HTTPRequestHeaderFields()
obj.user_agent = value
return obj
elif type_ == IndicatorTypes.MUTEX:
m = Mutex()
m.named = True
m.name = String(value)
return m
elif type_ in (IndicatorTypes.SOURCE_PORT,
IndicatorTypes.DEST_PORT):
p = Port()
try:
p.port_value = PositiveInteger(value)
except ValueError: # XXX: Raise a better exception...
raise UnsupportedCybOXObjectTypeError(type_, name)
return p
elif type_ == IndicatorTypes.PROCESS_NAME:
p = Process()
p.name = String(value)
return p
elif type_ == IndicatorTypes.URI:
r = URI()
r.type_ = 'URL'
r.value = value
return r
elif type_ in (IndicatorTypes.REGISTRY_KEY,
IndicatorTypes.REG_KEY_CREATED,
IndicatorTypes.REG_KEY_DELETED,
IndicatorTypes.REG_KEY_ENUMERATED,
IndicatorTypes.REG_KEY_MONITORED,
IndicatorTypes.REG_KEY_OPENED):
obj = WinRegistryKey()
obj.key = value
return obj
"""
The following are types that are listed in the 'Indicator Type' box of
the 'New Indicator' dialog in CRITs. These types, unlike those handled
above, cannot be written to or read from CybOX at this point.
The reason for the type being omitted is written as a comment inline.
This can (and should) be revisited as new versions of CybOX are released.
NOTE: You will have to update the corresponding make_crits_object function
with handling for the reverse direction.
In the mean time, these types will raise unsupported errors.
"""
#elif type_ == "Device": # No CybOX API
#elif type_ == "DNS Cache": # No CybOX API
#elif type_ == "GUI": # revisit when CRITs supports width & height specification
#elif type_ == "HTTP Session": # No good mapping between CybOX/CRITs
#elif type_ == "Linux Package": # No CybOX API
#elif type_ == "Network Packet": # No good mapping between CybOX/CRITs
#elif type_ == "Network Route Entry": # No CybOX API
#elif type_ == "Network Route": # No CybOX API
#elif type_ == "Network Subnet": # No CybOX API
#elif type_ == "Semaphore": # No CybOX API
#elif type_ == "Socket": # No good mapping between CybOX/CRITs
#elif type_ == "UNIX File": # No CybOX API
#elif type_ == "UNIX Network Route Entry": # No CybOX API
#elif type_ == "UNIX Pipe": # No CybOX API
#elif type_ == "UNIX Process": # No CybOX API
#elif type_ == "UNIX User Account": # No CybOX API
#elif type_ == "UNIX Volume": # No CybOX API
#elif type_ == "User Session": # No CybOX API
#elif type_ == "Whois": # No good mapping between CybOX/CRITs
#elif type_ == "Win Computer Account": # No CybOX API
#elif type_ == "Win Critical Section": # No CybOX API
#elif type_ == "Win Executable File": # No good mapping between CybOX/CRITs
#elif type_ == "Win File": # No good mapping between CybOX/CRITs
#elif type_ == "Win Kernel": # No CybOX API
#elif type_ == "Win Mutex": # No good mapping between CybOX/CRITs
#elif type_ == "Win Network Route Entry": # No CybOX API
#elif type_ == "Win Pipe": # No good mapping between CybOX/CRITs
#elif type_ == "Win Prefetch": # No CybOX API
#elif type_ == "Win Semaphore": # No CybOX API
#elif type_ == "Win System Restore": # No CybOX API
#elif type_ == "Win Thread": # No good mapping between CybOX/CRITs
#elif type_ == "Win Waitable Timer": # No CybOX API
raise UnsupportedCybOXObjectTypeError(type_)
def make_cybox_object(type_, value=None):
"""
Converts type_, name, and value to a CybOX object instance.
:param type_: The object type.
:type type_: str
:param value: The object value.
:type value: str
:returns: CybOX object
"""
if type_ == IndicatorTypes.USER_ID:
acct = Account()
acct.description = value
return acct
elif type_ in IPTypes.values():
if type_ == IPTypes.IPV4_ADDRESS:
name = 'ipv4-addr'
elif type_ == IPTypes.IPV6_ADDRESS:
name = 'ipv6-addr'
elif type_ == IPTypes.IPV4_SUBNET:
name = 'ipv4-net'
elif type_ == IPTypes.IPV6_SUBNET:
name = 'ipv6-net'
return Address(category=name, address_value=value)
elif type_ == IndicatorTypes.API_KEY:
api = API()
api.description = value
return api
elif type_ == IndicatorTypes.DOMAIN:
obj = DomainName()
obj.value = value
elif type_ == IndicatorTypes.USER_AGENT:
obj = HTTPRequestHeaderFields()
obj.user_agent = value
return obj
elif type_ == IndicatorTypes.MUTEX:
m = Mutex()
m.named = True
m.name = String(value)
return m
elif type_ in (IndicatorTypes.SOURCE_PORT,
IndicatorTypes.DEST_PORT):
p = Port()
try:
p.port_value = PositiveInteger(value)
except ValueError: # XXX: Raise a better exception...
raise UnsupportedCybOXObjectTypeError(type_, name)
return p
elif type_ == IndicatorTypes.PROCESS_NAME:
p = Process()
p.name = String(value)
return p
elif type_ == IndicatorTypes.URI:
r = URI()
r.type_ = 'URL'
r.value = value
return r
elif type_ in (IndicatorTypes.REGISTRY_KEY,
IndicatorTypes.REG_KEY_CREATED,
IndicatorTypes.REG_KEY_DELETED,
IndicatorTypes.REG_KEY_ENUMERATED,
IndicatorTypes.REG_KEY_MONITORED,
IndicatorTypes.REG_KEY_OPENED):
obj = WinRegistryKey()
obj.key = value
return obj
"""
The following are types that are listed in the 'Indicator Type' box of
the 'New Indicator' dialog in CRITs. These types, unlike those handled
above, cannot be written to or read from CybOX at this point.
The reason for the type being omitted is written as a comment inline.
This can (and should) be revisited as new versions of CybOX are released.
NOTE: You will have to update the corresponding make_crits_object function
with handling for the reverse direction.
In the mean time, these types will raise unsupported errors.
"""
#elif type_ == "Device": # No CybOX API
#elif type_ == "DNS Cache": # No CybOX API
#elif type_ == "GUI": # revisit when CRITs supports width & height specification
#elif type_ == "HTTP Session": # No good mapping between CybOX/CRITs
#elif type_ == "Linux Package": # No CybOX API
#elif type_ == "Network Packet": # No good mapping between CybOX/CRITs
#elif type_ == "Network Route Entry": # No CybOX API
#elif type_ == "Network Route": # No CybOX API
#elif type_ == "Network Subnet": # No CybOX API
#elif type_ == "Semaphore": # No CybOX API
#elif type_ == "Socket": # No good mapping between CybOX/CRITs
#elif type_ == "UNIX File": # No CybOX API
#elif type_ == "UNIX Network Route Entry": # No CybOX API
#elif type_ == "UNIX Pipe": # No CybOX API
#elif type_ == "UNIX Process": # No CybOX API
#elif type_ == "UNIX User Account": # No CybOX API
#elif type_ == "UNIX Volume": # No CybOX API
#elif type_ == "User Session": # No CybOX API
#elif type_ == "Whois": # No good mapping between CybOX/CRITs
#elif type_ == "Win Computer Account": # No CybOX API
#elif type_ == "Win Critical Section": # No CybOX API
#elif type_ == "Win Executable File": # No good mapping between CybOX/CRITs
#elif type_ == "Win File": # No good mapping between CybOX/CRITs
#elif type_ == "Win Kernel": # No CybOX API
#elif type_ == "Win Mutex": # No good mapping between CybOX/CRITs
#elif type_ == "Win Network Route Entry": # No CybOX API
#elif type_ == "Win Pipe": # No good mapping between CybOX/CRITs
#elif type_ == "Win Prefetch": # No CybOX API
#elif type_ == "Win Semaphore": # No CybOX API
#elif type_ == "Win System Restore": # No CybOX API
#elif type_ == "Win Thread": # No good mapping between CybOX/CRITs
#elif type_ == "Win Waitable Timer": # No CybOX API
raise UnsupportedCybOXObjectTypeError(type_, name)
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def validate_indicator_value(value, ind_type):
"""
Check that a given value is valid for a particular Indicator type.
:param value: The value to be validated
:type value: str
:param ind_type: The indicator type to validate against
:type ind_type: str
:returns: tuple: (Valid value, Error message)
"""
value = value.strip()
domain = ""
# URL
if ind_type == IndicatorTypes.URI:
if "://" not in value.split(".")[0]:
return ("", "URI must contain protocol " "prefix (e.g. http://, https://, ftp://) ")
domain_or_ip = urlparse.urlparse(value).hostname
try:
validate_ipv46_address(domain_or_ip)
return (value, "")
except DjangoValidationError:
domain = domain_or_ip
# Email address
if ind_type in (
IndicatorTypes.EMAIL_ADDRESS,
IndicatorTypes.EMAIL_FROM,
IndicatorTypes.EMAIL_REPLY_TO,
IndicatorTypes.EMAIL_SENDER,
):
if "@" not in value:
return ("", "Email address must contain an '@'")
domain_or_ip = value.split("@")[-1]
if domain_or_ip[0] == "[" and domain_or_ip[-1] == "]":
try:
validate_ipv46_address(domain_or_ip[1:-1])
return (value, "")
except DjangoValidationError:
return ("", "Email address does not contain a valid IP")
else:
domain = domain_or_ip
# IPs
if ind_type in IPTypes.values():
(ip_address, error) = validate_and_normalize_ip(value, ind_type)
if error:
return ("", error)
else:
return (ip_address, "")
# Domains
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI) or domain:
(root, domain, error) = get_valid_root_domain(domain or value)
if error:
return ("", error)
else:
return (value, "")
return (value, "")
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {'success': False, 'reason': 'No IP Type'}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and (isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val),
[self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data,
analyst,
"STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(data[key],
self.source,
analyst,
description,
title,
d_type,
"STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj,
res, ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
x, y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to', 'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append((oid,
RelationshipTypes.RELATED_TO,
oid2, "High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append((oid,
RelationshipTypes.CONTAINS,
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
"%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI