def json2observable(config, src, dest, endpoint, json_, crits_id):
# TODO split into smaller functions
'''transform crits observables into cybox'''
try:
set_id_method(IDGenerator.METHOD_UUID)
xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
if endpoint == 'ips':
crits_types = {
'Address - cidr': 'cidr',
'Address - ipv4-addr': 'ipv4-addr',
'Address - ipv4-net': 'ipv4-net',
'Address - ipv4-net-mask': 'ipv4-netmask',
'Address - ipv6-addr': 'ipv6-addr',
'Address - ipv6-net': 'ipv6-net',
'Address - ipv6-net-mask': 'ipv6-netmask'
}
addr = Address(address_value=json_['ip'],
category=crits_types[json_['type']])
addr.condition = 'Equals'
observable_ = Observable(addr)
elif endpoint == 'domains':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = json_['domain']
domain.condition = 'Equals'
observable_ = Observable(domain)
elif endpoint == 'samples':
crits_types = {
'md5': 'MD5',
'sha1': 'SHA1',
'sha224': 'SHA224',
'sha256': 'SHA256',
'sha384': 'SHA384',
'sha512': 'SHA512',
'ssdeep': 'SSDEEP'
}
file_object = File()
file_object.file_name = json_['filename']
for hash in crits_types.keys():
if hash in json_:
file_object.add_hash(
Hash(json_[hash], type_=crits_types[hash]))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
observable_ = Observable(file_object)
elif endpoint == 'emails':
crits_types = {
'subject': 'subject',
'to': 'to',
'cc': 'cc',
'from_address': 'from_',
'sender': 'sender',
'date': 'date',
'message_id': 'message_id',
'reply_to': 'reply_to',
'boundary': 'boundary',
'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'
}
email = EmailMessage()
email.header = EmailHeader()
for k in crits_types.keys():
val = json_.get(k, None)
if val:
email.header.__setattr__(crits_types[k], val)
email.header.__getattribute__(crits_types[k]).condition = \
'Equals'
observable_ = Observable(email)
else:
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='crits', obj_type=endpoint, id_=crits_id))
return (None)
observable_.id = xmlns_name + ':observable-' + crits_id
observable_.id_ = observable_.id
return (observable_)
except:
e = sys.exc_info()[0]
config['logger'].error(log.log_messages['obj_convert_error'].format(
src_type='crits',
src_obj='observable',
id_=crits_id,
dest_type='cybox',
dest_obj='observable'))
config['logger'].exception(e)
return (None)
def json2observable(config, src, dest, endpoint, json_, crits_id):
# TODO split into smaller functions
'''transform crits observables into cybox'''
try:
set_id_method(IDGenerator.METHOD_UUID)
xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
if endpoint == 'ips':
crits_types = {'Address - cidr': 'cidr',
'Address - ipv4-addr': 'ipv4-addr',
'Address - ipv4-net': 'ipv4-net',
'Address - ipv4-net-mask': 'ipv4-netmask',
'Address - ipv6-addr': 'ipv6-addr',
'Address - ipv6-net': 'ipv6-net',
'Address - ipv6-net-mask': 'ipv6-netmask'}
addr = Address(address_value=json_['ip'],
category=crits_types[json_['type']])
addr.condition = 'Equals'
observable_ = Observable(addr)
elif endpoint == 'domains':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = json_['domain']
domain.condition = 'Equals'
observable_ = Observable(domain)
elif endpoint == 'samples':
crits_types = {'md5': 'MD5',
'sha1': 'SHA1',
'sha224': 'SHA224',
'sha256': 'SHA256',
'sha384': 'SHA384',
'sha512': 'SHA512',
'ssdeep': 'SSDEEP'}
file_object = File()
file_object.file_name = json_['filename']
for hash in crits_types.keys():
if hash in json_:
file_object.add_hash(Hash(json_[hash],
type_=crits_types[hash]))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
observable_ = Observable(file_object)
elif endpoint == 'emails':
crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
'from_address': 'from_', 'sender': 'sender',
'date': 'date', 'message_id': 'message_id',
'reply_to': 'reply_to', 'boundary': 'boundary',
'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'}
email = EmailMessage()
email.header = EmailHeader()
for k in crits_types.keys():
val = json_.get(k, None)
if val:
email.header.__setattr__(crits_types[k], val)
email.header.__getattribute__(crits_types[k]).condition = \
'Equals'
observable_ = Observable(email)
else:
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='crits', obj_type=endpoint, id_=crits_id))
return(None)
observable_.id = xmlns_name + ':observable-' + crits_id
observable_.id_ = observable_.id
return(observable_)
except:
e = sys.exc_info()[0]
config['logger'].error(
log.log_messages['obj_convert_error'].format(
src_type='crits', src_obj='observable', id_=crits_id,
dest_type='cybox', dest_obj='observable'))
config['logger'].exception(e)
return(None)
def genStixDoc(
outputDir_,
targetFileSha1_,
targetFileSha256_,
targetFileSha512_,
targetFileSsdeep_,
targetFileMd5_,
targetFileSize_,
targetFileName_,
ipv4Addresses_,
hostNames_):
"""
Generate Stix document from the input values. The doc structure is the file
object along with the related network items: addresses, domain names. Output
is written to files, which are then wrapped with taxii and uploaded using a
separate script.
"""
parsedTargetFileName = reFileName(targetFileName_)[1]
parsedTargetFilePrefix = reFileName(targetFileName_)[0]
stix.utils.set_id_namespace({"http://www.equifax.com/cuckoo2Stix" : "cuckoo2Stix"})
NS = cybox.utils.Namespace("http://www.equifax.com/cuckoo2Stix", "cuckoo2Stix")
cybox.utils.set_id_namespace(NS)
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_header.description = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_package.stix_header = stix_header
# Create the ttp
malware_instance = MalwareInstance()
malware_instance.add_name(parsedTargetFileName)
malware_instance.description = targetFileSha1_
ttp = TTP(title='TTP: ' + parsedTargetFileName)
ttp.behavior = Behavior()
ttp.behavior.add_malware_instance(malware_instance)
stix_package.add_ttp(ttp)
# Create the indicator for the ipv4 addresses
ipv4Object = Address(ipv4Addresses_, Address.CAT_IPV4)
ipv4Object.condition = 'Equals'
ipv4Indicator = Indicator()
ipv4Indicator.title = parsedTargetFileName + ': ipv4 addresses'
ipv4Indicator.add_indicator_type('IP Watchlist')
ipv4Indicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
ipv4Indicator.observable = ipv4Object
ipv4Indicator.confidence = 'Low'
# Create the indicator for the domain names
domainNameObject = DomainName()
domainNameObject.value = hostNames_
domainNameObject.condition = 'Equals'
domainNameIndicator = Indicator()
domainNameIndicator.title = parsedTargetFileName + ': domain names'
domainNameIndicator.add_indicator_type('Domain Watchlist')
domainNameIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
domainNameIndicator.observable = domainNameObject
domainNameIndicator.confidence = 'Low'
# Create the indicator for the file
fileObject = File()
fileObject.file_name = parsedTargetFileName
fileObject.file_name.condition = 'Equals'
fileObject.size_in_bytes = targetFileSize_
fileObject.size_in_bytes.condition = 'Equals'
fileObject.add_hash(Hash(targetFileSha1_, type_='SHA1', exact=True))
fileObject.add_hash(Hash(targetFileSha256_, type_='SHA256', exact=True))
fileObject.add_hash(Hash(targetFileSha512_, type_='SHA512', exact=True))
fileObject.add_hash(Hash(targetFileSsdeep_, type_='SSDEEP', exact=True))
fileObject.add_hash(Hash(targetFileMd5_, type_='MD5', exact=True))
fileIndicator = Indicator()
fileIndicator.title = parsedTargetFileName + ': hashes'
fileIndicator.description = parsedTargetFilePrefix
fileIndicator.add_indicator_type('File Hash Watchlist')
fileIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship="Indicates Malware"))
fileIndicator.observable = fileObject
fileIndicator.confidence = 'Low'
stix_package.indicators = [fileIndicator, ipv4Indicator, domainNameIndicator]
stagedStixDoc = stix_package.to_xml()
stagedStixDoc = fixAddressObject(stagedStixDoc)
stagedStixDoc = fixDomainObject(stagedStixDoc)
today = datetime.datetime.now()
now = today.strftime('%Y-%m-%d_%H%M%S')
if not os.path.exists(outputDir_):
os.makedirs(outputDir_)
with open (outputDir_ + '/' + now + '-' + targetFileSha1_ + '.stix.xml', 'a') as myfile:
myfile.write(stagedStixDoc)
_l.debug('Wrote file: ' + now + '-' + targetFileSha1_ + '.stix.xml')
return
def gen_stix_observable_sample(config, target=None, datatype=None,
title='random test data',
description='random test data',
package_intents='Indicators - Watchlist',
tlp_color='WHITE'):
'''generate sample stix data comprised of indicator_count
indicators of type datatype'''
# setup the xmlns...
xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name']
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = '../../../../descendant-or-self::node()'
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == 'ip':
addr = Address(address_value=datagen.generate_random_ip_address(),
category='ipv4-addr')
addr.condition = 'Equals'
stix_package.add_observable(Observable(addr))
elif datatype == 'domain':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = datagen.generate_random_domain(config)
domain.condition = 'Equals'
stix_package.add_observable(Observable(domain))
elif datatype == 'filehash':
file_object = File()
file_object.file_name = str(uuid.uuid4()) + '.exe'
hashes = datagen.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == 'email':
try:
msg = datagen.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {'Subject': 'subject', 'To': 'to', 'Cc':
'cc', 'Bcc': 'bcc', 'From': 'from_',
'Sender': 'sender', 'Date': 'date',
'Message-ID': 'message_id', 'Reply-To':
'reply_to', 'In-Reply-To': 'in_reply_to',
'Content-Type': 'content_type', 'Errors-To':
'errors_to', 'Precedence': 'precedence',
'Boundary': 'boundary', 'MIME-Version':
'mime_version', 'X-Mailer': 'x_mailer',
'User-Agent': 'user_agent',
'X-Originating-IP': 'x_originating_ip',
'X-Priority': 'x_priority'}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = \
'Equals'
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return(None)
observable_id = stix_package.observables.observables[0].id_
return(observable_id, stix_package)
def gen_stix_observable_sample(config,
target=None,
datatype=None,
title='random test data',
description='random test data',
package_intents='Indicators - Watchlist',
tlp_color='WHITE'):
'''generate sample stix data comprised of indicator_count
indicators of type datatype'''
# setup the xmlns...
xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name']
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = '../../../../descendant-or-self::node()'
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == 'ip':
addr = Address(address_value=datagen.generate_random_ip_address(),
category='ipv4-addr')
addr.condition = 'Equals'
stix_package.add_observable(Observable(addr))
elif datatype == 'domain':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = datagen.generate_random_domain(config)
domain.condition = 'Equals'
stix_package.add_observable(Observable(domain))
elif datatype == 'filehash':
file_object = File()
file_object.file_name = str(uuid.uuid4()) + '.exe'
hashes = datagen.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == 'email':
try:
msg = datagen.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {
'Subject': 'subject',
'To': 'to',
'Cc': 'cc',
'Bcc': 'bcc',
'From': 'from_',
'Sender': 'sender',
'Date': 'date',
'Message-ID': 'message_id',
'Reply-To': 'reply_to',
'In-Reply-To': 'in_reply_to',
'Content-Type': 'content_type',
'Errors-To': 'errors_to',
'Precedence': 'precedence',
'Boundary': 'boundary',
'MIME-Version': 'mime_version',
'X-Mailer': 'x_mailer',
'User-Agent': 'user_agent',
'X-Originating-IP': 'x_originating_ip',
'X-Priority': 'x_priority'
}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = \
'Equals'
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return (None)
observable_id = stix_package.observables.observables[0].id_
return (observable_id, stix_package)
def genStixDoc(
outputDir_,
targetFileSha1_,
targetFileSha256_,
targetFileSha512_,
targetFileSsdeep_,
targetFileMd5_,
targetFileSize_,
targetFileName_,
ipv4Addresses_,
hostNames_):
"""
Generate Stix document from the input values. The doc structure is the file
object along with the related network items: addresses, domain names. Output
is written to files, which are then wrapped with taxii and uploaded using a
separate script.
"""
parsedTargetFileName = reFileName(targetFileName_)[1]
parsedTargetFilePrefix = reFileName(targetFileName_)[0]
stix.utils.set_id_namespace({"http://www.equifax.com/cuckoo2Stix" : "cuckoo2Stix"})
NS = cybox.utils.Namespace("http://www.equifax.com/cuckoo2Stix", "cuckoo2Stix")
cybox.utils.set_id_namespace(NS)
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_header.description = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_package.stix_header = stix_header
# Create the ttp
malware_instance = MalwareInstance()
malware_instance.add_name(parsedTargetFileName)
malware_instance.description = targetFileSha1_
ttp = TTP(title='TTP: ' + parsedTargetFileName)
ttp.behavior = Behavior()
ttp.behavior.add_malware_instance(malware_instance)
stix_package.add_ttp(ttp)
# Create the indicator for the ipv4 addresses
ipv4Object = Address(ipv4Addresses_, Address.CAT_IPV4)
ipv4Object.condition = 'Equals'
ipv4Indicator = Indicator()
ipv4Indicator.title = parsedTargetFileName + ': ipv4 addresses'
ipv4Indicator.add_indicator_type('IP Watchlist')
ipv4Indicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
ipv4Indicator.observable = ipv4Object
ipv4Indicator.confidence = 'Low'
# Create the indicator for the domain names
domainNameObject = DomainName()
domainNameObject.value = hostNames_
domainNameObject.condition = 'Equals'
domainNameIndicator = Indicator()
domainNameIndicator.title = parsedTargetFileName + ': domain names'
domainNameIndicator.add_indicator_type('Domain Watchlist')
domainNameIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
domainNameIndicator.observable = domainNameObject
domainNameIndicator.confidence = 'Low'
# Create the indicator for the file
fileObject = File()
fileObject.file_name = parsedTargetFileName
fileObject.file_name.condition = 'Equals'
fileObject.size_in_bytes = targetFileSize_
fileObject.size_in_bytes.condition = 'Equals'
fileObject.add_hash(Hash(targetFileSha1_, type_='SHA1', exact=True))
fileObject.add_hash(Hash(targetFileSha256_, type_='SHA256', exact=True))
fileObject.add_hash(Hash(targetFileSha512_, type_='SHA512', exact=True))
fileObject.add_hash(Hash(targetFileSsdeep_, type_='SSDEEP', exact=True))
fileObject.add_hash(Hash(targetFileMd5_, type_='MD5', exact=True))
fileIndicator = Indicator()
fileIndicator.title = parsedTargetFileName + ': hashes'
fileIndicator.description = parsedTargetFilePrefix
fileIndicator.add_indicator_type('File Hash Watchlist')
fileIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship="Indicates Malware"))
fileIndicator.observable = fileObject
fileIndicator.confidence = 'Low'
stix_package.indicators = [fileIndicator, ipv4Indicator, domainNameIndicator]
stagedStixDoc = stix_package.to_xml()
stagedStixDoc = fixAddressObject(stagedStixDoc)
stagedStixDoc = fixDomainObject(stagedStixDoc)
today = datetime.datetime.now()
now = today.strftime('%Y-%m-%d_%H%M%S')
if not os.path.exists(outputDir_):
os.makedirs(outputDir_)
with open (outputDir_ + '/' + now + '-' + targetFileSha1_ + '.stix.xml', 'a') as myfile:
myfile.write(stagedStixDoc)
_l.debug('Wrote file: ' + now + '-' + targetFileSha1_ + '.stix.xml')
return
def gen_stix_observable_sample(
config,
target=None,
datatype=None,
title="random test data",
description="random test data",
package_intents="Indicators - Watchlist",
tlp_color="WHITE",
):
"""generate sample stix data comprised of indicator_count
indicators of type datatype"""
# setup the xmlns...
xmlns_url = config["edge"]["sites"][target]["stix"]["xmlns_url"]
xmlns_name = config["edge"]["sites"][target]["stix"]["xmlns_name"]
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = "../../../../descendant-or-self::node()"
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == "ip":
addr = Address(address_value=datagen_.generate_random_ip_address(), category="ipv4-addr")
addr.condition = "Equals"
stix_package.add_observable(Observable(addr))
elif datatype == "domain":
domain = DomainName()
domain.type_ = "FQDN"
domain.value = datagen_.generate_random_domain(config)
domain.condition = "Equals"
stix_package.add_observable(Observable(domain))
elif datatype == "filehash":
file_object = File()
file_object.file_name = str(uuid.uuid4()) + ".exe"
hashes = datagen_.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == "email":
try:
msg = datagen_.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {
"Subject": "subject",
"To": "to",
"Cc": "cc",
"Bcc": "bcc",
"From": "from_",
"Sender": "sender",
"Date": "date",
"Message-ID": "message_id",
"Reply-To": "reply_to",
"In-Reply-To": "in_reply_to",
"Content-Type": "content_type",
"Errors-To": "errors_to",
"Precedence": "precedence",
"Boundary": "boundary",
"MIME-Version": "mime_version",
"X-Mailer": "x_mailer",
"User-Agent": "user_agent",
"X-Originating-IP": "x_originating_ip",
"X-Priority": "x_priority",
}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = "Equals"
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return None
observable_id = stix_package.observables.observables[0].id_
return (observable_id, stix_package)
