def __get_inspected_objs(self, source_file):
email1 = EmailMessage()
email1.from_ = '*****@*****.**'
email1.to = '*****@*****.**'
email1.subject = 'Subject1'
email1.date = DateTime('21-06-2015T19:52:00')
email1.add_related(source_file, "Extracted_From", inline=False)
return [email1]
def to_cybox(self, exclude=None):
"""
Convert an email to a CybOX Observables.
Pass parameter exclude to specify fields that should not be
included in the returned object.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
if exclude == None:
exclude = []
observables = []
obj = EmailMessage()
# Assume there is going to be at least one header
obj.header = EmailHeader()
if 'message_id' not in exclude:
obj.header.message_id = String(self.message_id)
if 'subject' not in exclude:
obj.header.subject = String(self.subject)
if 'sender' not in exclude:
obj.header.sender = Address(self.reply_to, Address.CAT_EMAIL)
if 'reply_to' not in exclude:
obj.header.reply_to = Address(self.reply_to, Address.CAT_EMAIL)
if 'x_originating_ip' not in exclude:
obj.header.x_originating_ip = Address(self.x_originating_ip,
Address.CAT_IPV4)
if 'raw_body' not in exclude:
obj.raw_body = self.raw_body
if 'raw_header' not in exclude:
obj.raw_header = self.raw_header
#copy fields where the names differ between objects
if 'helo' not in exclude and 'email_server' not in exclude:
obj.email_server = String(self.helo)
if ('from_' not in exclude and 'from' not in exclude
and 'from_address' not in exclude):
obj.header.from_ = EmailAddress(self.from_address)
if 'date' not in exclude and 'isodate' not in exclude:
obj.header.date = DateTime(self.isodate)
observables.append(Observable(obj))
return (observables, self.releasability)
def __create_cybox_headers(self, msg):
""" Returns a CybOX EmailHeaderType object """
if self.__verbose_output:
sys.stderr.write("** parsing headers\n")
headers = EmailHeader()
if 'received' in self.headers:
headers.received_lines = self._parse_received_headers(msg)
if 'to' in self.headers:
headers.to = _get_email_recipients(msg['to'])
if 'cc' in self.headers:
headers.cc = _get_email_recipients(msg['cc'])
if 'bcc' in self.headers:
headers.bcc = _get_email_recipients(msg['bcc'])
if 'from' in self.headers:
headers.from_ = _get_single_email_address(msg['from'])
if 'sender' in self.headers:
headers.sender = _get_single_email_address(msg['sender'])
if 'reply-to' in self.headers:
headers.reply_to = _get_single_email_address(msg['reply-to'])
if 'subject' in self.headers:
headers.subject = String(msg['subject'])
if 'in-reply-to' in self.headers:
headers.in_reply_to = String(msg['in-reply-to'])
if 'errors-to' in self.headers:
headers.errors_to = String(msg['errors-to'])
if 'date' in self.headers:
headers.date = DateTime(msg['date'])
if 'message-id' in self.headers:
headers.message_id = String(msg['message-id'])
if 'boundary' in self.headers:
headers.boundary = String(msg['boundary'])
if 'content-type' in self.headers:
headers.content_type = String(msg['content-type'])
if 'mime-version' in self.headers:
headers.mime_version = String(msg['mime-version'])
if 'precedence' in self.headers:
headers.precedence = String(msg['precedence'])
if 'user-agent' in self.headers:
headers.user_agent = String(msg['user-agent'])
if 'x-mailer' in self.headers:
headers.x_mailer = String(msg['x-mailer'])
if 'x-originating-ip' in self.headers:
headers.x_originating_ip = Address(msg['x-originating-ip'],
Address.CAT_IPV4)
if 'x-priority' in self.headers:
headers.x_priority = String(msg['x-priority'])
return headers
def to_cybox_observable(obj, exclude=None, bin_fmt="raw"):
"""
Convert a CRITs TLO to a CybOX Observable.
:param obj: The TLO to convert.
:type obj: :class:`crits.core.crits_mongoengine.CRITsBaseAttributes`
:param exclude: Attributes to exclude.
:type exclude: list
:param bin_fmt: The format for the binary (if applicable).
:type bin_fmt: str
"""
type_ = obj._meta['crits_type']
if type_ == 'Certificate':
custom_prop = Property(
) # make a custom property so CRITs import can identify Certificate exports
custom_prop.name = "crits_type"
custom_prop.description = "Indicates the CRITs type of the object this CybOX object represents"
custom_prop._value = "Certificate"
obje = File() # represent cert information as file
obje.md5 = obj.md5
obje.file_name = obj.filename
obje.file_format = obj.filetype
obje.size_in_bytes = obj.size
obje.custom_properties = CustomProperties()
obje.custom_properties.append(custom_prop)
obs = Observable(obje)
obs.description = obj.description
data = obj.filedata.read()
if data: # if cert data available
a = Artifact(data, Artifact.TYPE_FILE) # create artifact w/data
a.packaging.append(Base64Encoding())
obje.add_related(a, "Child_Of") # relate artifact to file
return ([obs], obj.releasability)
elif type_ == 'Domain':
obje = DomainName()
obje.value = obj.domain
obje.type_ = obj.record_type
return ([Observable(obje)], obj.releasability)
elif type_ == 'Email':
if exclude == None:
exclude = []
observables = []
obje = EmailMessage()
# Assume there is going to be at least one header
obje.header = EmailHeader()
if 'message_id' not in exclude:
obje.header.message_id = String(obj.message_id)
if 'subject' not in exclude:
obje.header.subject = String(obj.subject)
if 'sender' not in exclude:
obje.header.sender = Address(obj.sender, Address.CAT_EMAIL)
if 'reply_to' not in exclude:
obje.header.reply_to = Address(obj.reply_to, Address.CAT_EMAIL)
if 'x_originating_ip' not in exclude:
obje.header.x_originating_ip = Address(obj.x_originating_ip,
Address.CAT_IPV4)
if 'x_mailer' not in exclude:
obje.header.x_mailer = String(obj.x_mailer)
if 'boundary' not in exclude:
obje.header.boundary = String(obj.boundary)
if 'raw_body' not in exclude:
obje.raw_body = obj.raw_body
if 'raw_header' not in exclude:
obje.raw_header = obj.raw_header
#copy fields where the names differ between objects
if 'helo' not in exclude and 'email_server' not in exclude:
obje.email_server = String(obj.helo)
if ('from_' not in exclude and 'from' not in exclude
and 'from_address' not in exclude):
obje.header.from_ = EmailAddress(obj.from_address)
if 'date' not in exclude and 'isodate' not in exclude:
obje.header.date = DateTime(obj.isodate)
obje.attachments = Attachments()
observables.append(Observable(obje))
return (observables, obj.releasability)
elif type_ == 'Indicator':
observables = []
obje = make_cybox_object(obj.ind_type, obj.value)
observables.append(Observable(obje))
return (observables, obj.releasability)
elif type_ == 'IP':
obje = Address()
obje.address_value = obj.ip
if obj.ip_type == IPTypes.IPv4_ADDRESS:
obje.category = "ipv4-addr"
elif obj.ip_type == IPTypes.IPv6_ADDRESS:
obje.category = "ipv6-addr"
elif obj.ip_type == IPTypes.IPv4_SUBNET:
obje.category = "ipv4-net"
elif obj.ip_type == IPTypes.IPv6_SUBNET:
obje.category = "ipv6-subnet"
return ([Observable(obje)], obj.releasability)
elif type_ == 'PCAP':
obje = File()
obje.md5 = obj.md5
obje.file_name = obj.filename
obje.file_format = obj.contentType
obje.size_in_bytes = obj.length
obs = Observable(obje)
obs.description = obj.description
art = Artifact(obj.filedata.read(), Artifact.TYPE_NETWORK)
art.packaging.append(Base64Encoding())
obje.add_related(art, "Child_Of") # relate artifact to file
return ([obs], obj.releasability)
elif type_ == 'RawData':
obje = Artifact(obj.data.encode('utf-8'), Artifact.TYPE_FILE)
obje.packaging.append(Base64Encoding())
obs = Observable(obje)
obs.description = obj.description
return ([obs], obj.releasability)
elif type_ == 'Sample':
if exclude == None:
exclude = []
observables = []
f = File()
for attr in ['md5', 'sha1', 'sha256']:
if attr not in exclude:
val = getattr(obj, attr, None)
if val:
setattr(f, attr, val)
if obj.ssdeep and 'ssdeep' not in exclude:
f.add_hash(Hash(obj.ssdeep, Hash.TYPE_SSDEEP))
if 'size' not in exclude and 'size_in_bytes' not in exclude:
f.size_in_bytes = UnsignedLong(obj.size)
if 'filename' not in exclude and 'file_name' not in exclude:
f.file_name = obj.filename
# create an Artifact object for the binary if it exists
if 'filedata' not in exclude and bin_fmt:
data = obj.filedata.read()
if data: # if sample data available
a = Artifact(data,
Artifact.TYPE_FILE) # create artifact w/data
if bin_fmt == "zlib":
a.packaging.append(ZlibCompression())
a.packaging.append(Base64Encoding())
elif bin_fmt == "base64":
a.packaging.append(Base64Encoding())
f.add_related(a, "Child_Of") # relate artifact to file
if 'filetype' not in exclude and 'file_format' not in exclude:
#NOTE: this doesn't work because the CybOX File object does not
# have any support built in for setting the filetype to a
# CybOX-binding friendly object (e.g., calling .to_dict() on
# the resulting CybOX object fails on this field.
f.file_format = obj.filetype
observables.append(Observable(f))
return (observables, obj.releasability)
else:
return (None, None)
def __create_whois_object(self, domain):
""" Creates a CybOX WHOISObjectType object """
if not domain:
return None
if(self.__verbose_output):
sys.stderr.write("** creating Whois object for: %s\n" % domain)
if self.http_whois:
record = self.__get_whois_record_http(domain)
else:
record = self.__get_whois_record(domain)
if not record:
return None
whois = WhoisEntry()
record['status'] = ['OK' if status == 'ACTIVE' else status for status in record['status']]
#Only build registrar info objects if we have the relevant info
if (record['registrar'] or record['whois_server'] or
record['registrar_address'] or record['referral_url'] or
record['registrar_contacts']):
registrar = WhoisRegistrar()
registrar.name = String(record.get('registrar'))
registrar.address = String(record.get('registrar_address'))
registrar.whois_server = URI(record.get('whois_server'))
registrar.referral_url = URI(record.get('referral_url'))
contacts = WhoisContacts()
for email in record['registrar_contacts']:
contact = WhoisContact()
contact.contact_type = 'ADMIN'
contact.name = String(record.get('registrar'))
contact.email_address = EmailAddress(email)
contacts.append(contact)
registrar.contacts = contacts
whois.registrar_info = registrar
whois.domain_name = self.__create_domain_name_object(record.get('domain_name'))
nservers = WhoisNameservers()
for url in record.get('name_servers', []):
nservers.append(self.__create_url_object(url))
if nservers:
whois.nameservers = nservers
status = WhoisStatuses()
for s in record.get('status', []):
status.append(WhoisStatus(s))
if status:
whois.status = status
whois.updated_date = DateTime(record.get('updated_date'))
whois.creation_date = DateTime(record.get('creation_date'))
whois.expiration_date = DateTime(record.get('expiration_date'))
return whois
def __create_cybox_headers(self, msg):
""" Returns a CybOX EmailHeaderType object """
if self.__verbose_output:
sys.stderr.write("** parsing headers\n")
headers = EmailHeader()
if 'received' in self.headers:
lines = self._parse_received_headers(msg)
if lines:
headers.received_lines = lines
if 'to' in self.headers:
headers.to = _get_email_recipients(msg['to'])
if msg['delivered-to'] and not headers.to:
headers.to = _get_email_recipients(msg['delivered-to'])
if 'cc' in self.headers:
headers.cc = _get_email_recipients(msg['cc'])
if 'bcc' in self.headers:
headers.bcc = _get_email_recipients(msg['bcc'])
if 'from' in self.headers:
headers.from_ = _get_single_email_address(msg['from'])
if 'sender' in self.headers:
headers.sender = _get_single_email_address(msg['sender'])
if 'reply-to' in self.headers:
headers.reply_to = _get_single_email_address(msg['reply-to'])
if 'subject' in self.headers and 'subject' in msg:
headers.subject = String(msg['subject'])
if 'in-reply-to' in self.headers and 'in-reply-to' in msg:
headers.in_reply_to = String(msg['in-reply-to'])
if 'errors-to' in self.headers and 'errors-to' in msg:
headers.errors_to = String(msg['errors-to'])
if 'date' in self.headers and 'date' in msg:
headers.date = DateTime(msg['date'])
if 'message-id' in self.headers and 'message-id' in msg:
headers.message_id = String(msg['message-id'])
if 'boundary' in self.headers and 'boundary' in msg:
headers.boundary = String(msg['boundary'])
if 'content-type' in self.headers and 'content-type' in msg:
headers.content_type = String(msg['content-type'])
if 'mime-version' in self.headers and 'mime-version' in msg:
headers.mime_version = String(msg['mime-version'])
if 'precedence' in self.headers and 'precedence' in msg:
headers.precedence = String(msg['precedence'])
if 'user-agent' in self.headers and 'user-agent' in msg:
headers.user_agent = String(msg['user-agent'])
if 'x-mailer' in self.headers and 'x-mailer' in msg:
headers.x_mailer = String(msg['x-mailer'])
if 'x-originating-ip' in self.headers and msg['x-originating-ip']:
headers.x_originating_ip = Address(msg['x-originating-ip'],
Address.CAT_IPV4)
if 'x-priority' in self.headers and 'x-priority' in msg:
#Must be a digit - pull one out of anything that could be a string such as 3 (Normal)
import re
priority = ''
for p in re.findall(r'\d+',msg['x-priority']):
if p.isdigit():
priority = p
if priority:
headers.x_priority = String(priority)
return headers
def test_list_dates(self):
dt = DateTime([self.dt, self.dt, self.dt])
self.assertEqual(3, len(dt.value))
expected = "{0}{1}{0}{1}{0}".format(self.dt.isoformat(), DEFAULT_DELIM)
actual = normalize_to_xml(dt.serialized_value, DEFAULT_DELIM)
self.assertEqual(expected, actual)
def test_parse_date_string(self):
cybox_dt2 = DateTime(self.dt_str)
self.assertEqual(self.dt, cybox_dt2.value)
self.assertEqual(self.dt.isoformat(), cybox_dt2.serialized_value)
self.assertEqual(self.dt.isoformat(), six.text_type(cybox_dt2))
def test_parse_datetime(self):
cybox_dt = DateTime(self.dt)
self.assertEqual(self.dt, cybox_dt.value)
self.assertEqual(self.dt.isoformat(), six.text_type(cybox_dt))
def test_isodate(self):
now = datetime.datetime.now()
dt = DateTime(now)
self.assertEqual(now.isoformat(), dt.serialized_value)
self.assertEqual(now.isoformat(), dt.to_obj().valueOf_)
build_information.compiler_version = String('')
build_information.linker_name = String('')
build_information.linker_version = String('')
# https://cybox.readthedocs.org/en/stable/_modules/cybox/common/digitalsignature.html#DigitalSignature
digital_signature = DigitalSignature()
digital_signature.certificate_issuer = String('')
digital_signature.certificate_subject = String('')
digital_signature.signature_description = String('')
digital_signature.signature_exists = None
digital_signature.signature_verified = None
# https://cybox.readthedocs.org/en/stable/_modules/cybox/objects/win_executable_file_object.html#PEExports
exports = PEExports()
exports.exported_functions = PEExportedFunctions()
exports.exports_time_stamp = DateTime()
exports.name = String()
exports.number_of_addresses = Long()
exports.number_of_functions = Integer()
exports.number_of_names = Long()
# The Extraneous_Bytes field specifies the number of extraneous bytes contained in the PE binary.
extraneous_bytes = Integer()
# https://cybox.readthedocs.org/en/stable/_modules/cybox/objects/win_executable_file_object.html#PEHeaders
headers = PEHeaders()
headers.dos_header = DOSHeader()
headers.entropy = Entropy()
headers.file_header = PEFileHeader()
headers.hashes = HashList()
headers.optional_header = PEOptionalHeader()
