def generateDomainIPObservable(indicator, attribute):
indicator.add_indicator_type("Domain Watchlist")
domain = attribute["value"].split('|')[0]
ip = attribute["value"].split('|')[1]
address_object = resolveIPType(ip, attribute["type"])
address_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":AddressObject-" + attribute[
"uuid"]
address_observable = Observable(address_object)
address_observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":Address-" + attribute[
"uuid"]
domain_object = DomainName()
domain_object.value = domain
domain_object.value.condition = "Equals"
domain_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":DomainNameObject-" + attribute[
"uuid"]
domain_observable = Observable(domain_object)
domain_observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":DomainName-" + attribute[
"uuid"]
compositeObject = ObservableComposition(
observables=[address_observable, domain_observable])
compositeObject.operator = "AND"
observable = Observable(
id_=cybox.utils.idgen.__generator.namespace.prefix +
":ObservableComposition-" + attribute["uuid"])
observable.observable_composition = compositeObject
return observable
def gen_fqdns(self, count):
'''Generate list of FQDNs'''
domains = []
for i in range(0, count):
domain = DomainName()
domain.value = self.fake.domain_name()
domains.append(domain)
return domains
def domainlist_indicator(domains=[]):
domainlist = Indicator()
domainlist.add_indicator_type("Domain Watchlist")
for d in domains:
domain = DomainName()
domain.value = d
domainlist.add_observable(domain)
return domainlist
def generateDomainIPObservable(indicator, attribute):
indicator.add_indicator_type("Domain Watchlist")
compositeObject = ObservableComposition()
compositeObject.operator = "AND"
domain = attribute["value"].split('|')[0]
ip = attribute["value"].split('|')[1]
address_object = resolveIPType(ip, attribute["type"])
domain_object = DomainName()
domain_object.value = domain
compositeObject.add(address_object)
compositeObject.add(domain_object)
return compositeObject
def to_cybox_observable(self):
"""
Convert a Domain to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
obj = DomainName()
obj.value = self.domain
obj.type_ = self.record_type
return ([Observable(obj)], self.releasability)
def to_cybox_observable(self):
"""
Convert a Domain to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
obj = DomainName()
obj.value = self.domain
obj.type_ = self.record_type
return ([Observable(obj)], self.releasability)
def domainNameobj(domain):
# cybox stuff
d = DomainName()
d.value = domain
# stix stuff
indicator1 = Indicator()
indicator1.title = "Domain name"
indicator1.description = (
"An indicator containing a suspicious domain name")
indicator1.set_produced_time(utils.dates.now())
indicator1.add_object(d)
return indicator1
def generate_domain_ip_observable(self, indicator, attribute):
indicator.add_indicator_type("Domain Watchlist")
domain, ip = attribute.value.split('|')
address_object = self.resolve_ip_type(attribute.type, ip)
address_object.parent.id_ = "{}:AddressObject-{}".format(self.namespace_prefix, attribute.uuid)
address_observable = Observable(address_object)
address_observable.id_ = "{}:Address-{}".format(self.namespace_prefix, attribute.uuid)
domain_object = DomainName()
domain_object.value = domain
domain_object.value.condition = "Equals"
domain_object.parent.id_ = "{}:DomainNameObject-{}".format(self.namespace_prefix, attribute.uuid)
domain_observable = Observable(domain_object)
domain_observable.id_ = "{}:DomainName-{}".format(self.namespace_prefix, attribute.uuid)
composite_object = ObservableComposition(observables=[address_observable, domain_observable])
composite_object.operator = "AND"
observable = Observable(id_="{}:ObservableComposition-{}".format(self.namespace_prefix, attribute.uuid))
observable.observable_composition = composite_object
return observable
def add_obs_to_pkg(obs, pkg):
if "ip" in obs:
for i in obs["ip"]:
address = Address()
address.address_value = i
pkg.add_observable(address)
if "host" in obs:
for h in obs["host"]:
hostname = Hostname()
hostname.hostname_value = h
pkg.add_observable(hostname)
if "domain" in obs:
for d in obs["domain"]:
domain = DomainName()
domain.value = d
pkg.add_observable(domain)
return pkg
def add_raw_indicator(self , orig_indicator, ts=None):
indicator_value = orig_indicator
if not self._is_ascii(indicator_value):
return False
indicator_type, _ = guess_type(indicator_value)
# Create a CyboX File Object
if indicator_type == StixItemType.IPADDR:
title = "Malicious IPv4 - %s" % indicator_value
descr = "Malicious IPv4 involved with %s" % self._pkg.stix_header.title
cybox = Address(indicator_value , Address.CAT_IPV4)
elif indicator_type == StixItemType.DOMAIN:
title = "Malicious domain - %s" % indicator_value
descr = "Malicious domain involved with %s" % self._pkg.stix_header.title
cybox = DomainName()
cybox.value = indicator_value
elif indicator_type == StixItemType.MD5:
title = "Malicious MD5 - %s" % indicator_value
descr = "Malicious MD5 involved with %s" % self._pkg.stix_header.title
cybox = File()
cybox.add_hash(indicator_value )
elif indicator_type == StixItemType.SHA256:
title = "Malicious SHA256 - %s" % indicator_value
descr = "Malicious SHA256 involved with %s" % self._pkg.stix_header.title
cybox = File()
cybox.add_hash(indicator_value )
elif indicator_type == StixItemType.SHA1:
title = "Malicious SHA1 - %s" % indicator_value
descr = "Malicious SHA1 involved with %s" % self._pkg.stix_header.title
cybox = File()
cybox.add_hash(indicator_value )
elif indicator_type == StixItemType.URL:
title = "Malicious URL - %s" % indicator_value
descr = "Malicious URL involved with %s" % self._pkg.stix_header.title
cybox = URI()
cybox.value = indicator_value
cybox.type_ = URI.TYPE_URL
if indicator_type == StixItemType.UNKNOWN:
return False
indicator = Indicator()
indicator.title = title
indicator.description = descr
indicator.add_object(cybox)
indicator.set_producer_identity(self.__author)
if ts:
indicator.set_produced_time(ts)
else:
indicator.set_produced_time(utils.dates.now())
self._add(indicator)
return True
def get_observable(self):
if self.type_ == 'md5':
o_ = File()
o_.md5 = self.value
elif self.type_ == 'sha1':
o_ = File()
o_.sha1 = self.value
elif self.type_ == 'sha256':
o_ = File()
o_.sha256 = self.value
elif self.type_ == 'sha512':
o_ = File()
o_.sha256 = self.value
elif self.type_ == 'url':
o_ = URI(value=self.value, type_=URI.TYPE_URL)
elif self.type_ == 'hostname':
o_ = DomainName()
o_.value = self.value
o_.type_ = 'FQDN'
elif self.type_ == 'domain':
o_ = DomainName()
o_.value = self.value
o_.type_ = 'domain'
elif self.type_ == 'ip-dst':
o_ = Address(address_value=self.value, category=Address.CAT_IPV4)
o_.is_destination = True
o_.is_Source = False
elif self.type_ == 'email-src':
o_ = Address(address_value=self.value, category=Address.CAT_EMAIL)
elif self.type_ == 'email-subject':
o_ = EmailMessage()
o_.subject = self.value
else:
# print 'skip>>>>: type_: ' + str(self.type_)
o_ = None
return o_
def to_cybox_observable(obj, exclude=None, bin_fmt="raw"):
"""
Convert a CRITs TLO to a CybOX Observable.
:param obj: The TLO to convert.
:type obj: :class:`crits.core.crits_mongoengine.CRITsBaseAttributes`
:param exclude: Attributes to exclude.
:type exclude: list
:param bin_fmt: The format for the binary (if applicable).
:type bin_fmt: str
"""
type_ = obj._meta['crits_type']
if type_ == 'Certificate':
custom_prop = Property(
) # make a custom property so CRITs import can identify Certificate exports
custom_prop.name = "crits_type"
custom_prop.description = "Indicates the CRITs type of the object this CybOX object represents"
custom_prop._value = "Certificate"
obje = File() # represent cert information as file
obje.md5 = obj.md5
obje.file_name = obj.filename
obje.file_format = obj.filetype
obje.size_in_bytes = obj.size
obje.custom_properties = CustomProperties()
obje.custom_properties.append(custom_prop)
obs = Observable(obje)
obs.description = obj.description
data = obj.filedata.read()
if data: # if cert data available
a = Artifact(data, Artifact.TYPE_FILE) # create artifact w/data
a.packaging.append(Base64Encoding())
obje.add_related(a, "Child_Of") # relate artifact to file
return ([obs], obj.releasability)
elif type_ == 'Domain':
obje = DomainName()
obje.value = obj.domain
obje.type_ = obj.record_type
return ([Observable(obje)], obj.releasability)
elif type_ == 'Email':
if exclude == None:
exclude = []
observables = []
obje = EmailMessage()
# Assume there is going to be at least one header
obje.header = EmailHeader()
if 'message_id' not in exclude:
obje.header.message_id = String(obj.message_id)
if 'subject' not in exclude:
obje.header.subject = String(obj.subject)
if 'sender' not in exclude:
obje.header.sender = Address(obj.sender, Address.CAT_EMAIL)
if 'reply_to' not in exclude:
obje.header.reply_to = Address(obj.reply_to, Address.CAT_EMAIL)
if 'x_originating_ip' not in exclude:
obje.header.x_originating_ip = Address(obj.x_originating_ip,
Address.CAT_IPV4)
if 'x_mailer' not in exclude:
obje.header.x_mailer = String(obj.x_mailer)
if 'boundary' not in exclude:
obje.header.boundary = String(obj.boundary)
if 'raw_body' not in exclude:
obje.raw_body = obj.raw_body
if 'raw_header' not in exclude:
obje.raw_header = obj.raw_header
#copy fields where the names differ between objects
if 'helo' not in exclude and 'email_server' not in exclude:
obje.email_server = String(obj.helo)
if ('from_' not in exclude and 'from' not in exclude
and 'from_address' not in exclude):
obje.header.from_ = EmailAddress(obj.from_address)
if 'date' not in exclude and 'isodate' not in exclude:
obje.header.date = DateTime(obj.isodate)
obje.attachments = Attachments()
observables.append(Observable(obje))
return (observables, obj.releasability)
elif type_ == 'Indicator':
observables = []
obje = make_cybox_object(obj.ind_type, obj.value)
observables.append(Observable(obje))
return (observables, obj.releasability)
elif type_ == 'IP':
obje = Address()
obje.address_value = obj.ip
if obj.ip_type == IPTypes.IPv4_ADDRESS:
obje.category = "ipv4-addr"
elif obj.ip_type == IPTypes.IPv6_ADDRESS:
obje.category = "ipv6-addr"
elif obj.ip_type == IPTypes.IPv4_SUBNET:
obje.category = "ipv4-net"
elif obj.ip_type == IPTypes.IPv6_SUBNET:
obje.category = "ipv6-subnet"
return ([Observable(obje)], obj.releasability)
elif type_ == 'PCAP':
obje = File()
obje.md5 = obj.md5
obje.file_name = obj.filename
obje.file_format = obj.contentType
obje.size_in_bytes = obj.length
obs = Observable(obje)
obs.description = obj.description
art = Artifact(obj.filedata.read(), Artifact.TYPE_NETWORK)
art.packaging.append(Base64Encoding())
obje.add_related(art, "Child_Of") # relate artifact to file
return ([obs], obj.releasability)
elif type_ == 'RawData':
obje = Artifact(obj.data.encode('utf-8'), Artifact.TYPE_FILE)
obje.packaging.append(Base64Encoding())
obs = Observable(obje)
obs.description = obj.description
return ([obs], obj.releasability)
elif type_ == 'Sample':
if exclude == None:
exclude = []
observables = []
f = File()
for attr in ['md5', 'sha1', 'sha256']:
if attr not in exclude:
val = getattr(obj, attr, None)
if val:
setattr(f, attr, val)
if obj.ssdeep and 'ssdeep' not in exclude:
f.add_hash(Hash(obj.ssdeep, Hash.TYPE_SSDEEP))
if 'size' not in exclude and 'size_in_bytes' not in exclude:
f.size_in_bytes = UnsignedLong(obj.size)
if 'filename' not in exclude and 'file_name' not in exclude:
f.file_name = obj.filename
# create an Artifact object for the binary if it exists
if 'filedata' not in exclude and bin_fmt:
data = obj.filedata.read()
if data: # if sample data available
a = Artifact(data,
Artifact.TYPE_FILE) # create artifact w/data
if bin_fmt == "zlib":
a.packaging.append(ZlibCompression())
a.packaging.append(Base64Encoding())
elif bin_fmt == "base64":
a.packaging.append(Base64Encoding())
f.add_related(a, "Child_Of") # relate artifact to file
if 'filetype' not in exclude and 'file_format' not in exclude:
#NOTE: this doesn't work because the CybOX File object does not
# have any support built in for setting the filetype to a
# CybOX-binding friendly object (e.g., calling .to_dict() on
# the resulting CybOX object fails on this field.
f.file_format = obj.filetype
observables.append(Observable(f))
return (observables, obj.releasability)
else:
return (None, None)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
### Add Generated observable to Indicator
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
infoSrc = InformationSource(identity=Identity(name=srcObj.Domain))
infoSrc.add_contributing_source(data[sKey]['attrib']['ref'])
if len(srcObj.Domain) > 0:
objIndicator.producer = infoSrc
if data[sKey]['attrib']['lstDateVF']:
objIndicator.set_produced_time(
data[sKey]['attrib']['lstDateVF'][0])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Generate Indicator Title based on availbe data
lstContainng = []
lstIs = []
sTitle = 'This domain ' + data[sKey]['attrib'][
'domain'] + ' has been identified as malicious'
if len(data[sKey]['attrib']['ref']):
sTitle += ' by ' + data[sKey]['attrib']['ref']
if len(data[sKey]['attrib']['type']) > 0:
sTitle += ', via this vector [' + data[sKey]['attrib'][
'type'] + '].'
else:
sTitle += '.'
objIndicator.title = sTitle
### Generate Indicator Description based on availbe data
sDscrpt = 'Lehigh.edu site has added this domain ' + data[sKey][
'attrib']['domain']
sDscrpt += ' to recommend block list.'
sDscrpt += ' This site has been identified as malicious'
sDscrpt += ' by ' + data[sKey]['attrib']['ref']
sDscrpt += ' and was still attive on the following dates ' + str(
data[sKey]['attrib']['lstDateVF']) + "."
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_type("Remote Access Trojan")
ttpTitle = data[sKey]['attrib']['type']
objTTP = TTP(title=ttpTitle)
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def genStixDoc(
outputDir_,
targetFileSha1_,
targetFileSha256_,
targetFileSha512_,
targetFileSsdeep_,
targetFileMd5_,
targetFileSize_,
targetFileName_,
ipv4Addresses_,
hostNames_):
"""
Generate Stix document from the input values. The doc structure is the file
object along with the related network items: addresses, domain names. Output
is written to files, which are then wrapped with taxii and uploaded using a
separate script.
"""
parsedTargetFileName = reFileName(targetFileName_)[1]
parsedTargetFilePrefix = reFileName(targetFileName_)[0]
stix.utils.set_id_namespace({"http://www.equifax.com/cuckoo2Stix" : "cuckoo2Stix"})
NS = cybox.utils.Namespace("http://www.equifax.com/cuckoo2Stix", "cuckoo2Stix")
cybox.utils.set_id_namespace(NS)
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_header.description = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_package.stix_header = stix_header
# Create the ttp
malware_instance = MalwareInstance()
malware_instance.add_name(parsedTargetFileName)
malware_instance.description = targetFileSha1_
ttp = TTP(title='TTP: ' + parsedTargetFileName)
ttp.behavior = Behavior()
ttp.behavior.add_malware_instance(malware_instance)
stix_package.add_ttp(ttp)
# Create the indicator for the ipv4 addresses
ipv4Object = Address(ipv4Addresses_, Address.CAT_IPV4)
ipv4Object.condition = 'Equals'
ipv4Indicator = Indicator()
ipv4Indicator.title = parsedTargetFileName + ': ipv4 addresses'
ipv4Indicator.add_indicator_type('IP Watchlist')
ipv4Indicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
ipv4Indicator.observable = ipv4Object
ipv4Indicator.confidence = 'Low'
# Create the indicator for the domain names
domainNameObject = DomainName()
domainNameObject.value = hostNames_
domainNameObject.condition = 'Equals'
domainNameIndicator = Indicator()
domainNameIndicator.title = parsedTargetFileName + ': domain names'
domainNameIndicator.add_indicator_type('Domain Watchlist')
domainNameIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
domainNameIndicator.observable = domainNameObject
domainNameIndicator.confidence = 'Low'
# Create the indicator for the file
fileObject = File()
fileObject.file_name = parsedTargetFileName
fileObject.file_name.condition = 'Equals'
fileObject.size_in_bytes = targetFileSize_
fileObject.size_in_bytes.condition = 'Equals'
fileObject.add_hash(Hash(targetFileSha1_, type_='SHA1', exact=True))
fileObject.add_hash(Hash(targetFileSha256_, type_='SHA256', exact=True))
fileObject.add_hash(Hash(targetFileSha512_, type_='SHA512', exact=True))
fileObject.add_hash(Hash(targetFileSsdeep_, type_='SSDEEP', exact=True))
fileObject.add_hash(Hash(targetFileMd5_, type_='MD5', exact=True))
fileIndicator = Indicator()
fileIndicator.title = parsedTargetFileName + ': hashes'
fileIndicator.description = parsedTargetFilePrefix
fileIndicator.add_indicator_type('File Hash Watchlist')
fileIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship="Indicates Malware"))
fileIndicator.observable = fileObject
fileIndicator.confidence = 'Low'
stix_package.indicators = [fileIndicator, ipv4Indicator, domainNameIndicator]
stagedStixDoc = stix_package.to_xml()
stagedStixDoc = fixAddressObject(stagedStixDoc)
stagedStixDoc = fixDomainObject(stagedStixDoc)
today = datetime.datetime.now()
now = today.strftime('%Y-%m-%d_%H%M%S')
if not os.path.exists(outputDir_):
os.makedirs(outputDir_)
with open (outputDir_ + '/' + now + '-' + targetFileSha1_ + '.stix.xml', 'a') as myfile:
myfile.write(stagedStixDoc)
_l.debug('Wrote file: ' + now + '-' + targetFileSha1_ + '.stix.xml')
return
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_destination = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = objAddr.CAT_IPV4
elif isIPv6(sAddr):
objAddr.category = objAddr.CAT_IPV6
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isDestination: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['title']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
obsURI.title = 'URI: ' + sURI
sDscrpt = 'URI: ' + sURI + " | "
sDscrpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Generate Indicator Title based on availbe data
sTitle = 'C2C Site: ' + sURI
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if len(sAddr) > 0:
sAddLine = "This IP address " + sAddr
if len(sDomain) > 0:
sAddLine = "This domain " + sDomain
if len(sAddr) > 0 and len(sDomain) > 0:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt += sAddLine
sDscrpt = sDscrpt + " has been identified as a command and control site for " + data[
sKey]['attrib']['dscrpt'] + ' malware'
if len(srcObj.Domain) > 0:
sDscrpt = sDscrpt + " by " + srcObj.Domain
else:
sDscrpt = sDscrpt + "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + data[
sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
pass
#Parse TTP
sType = data[sKey]['attrib']['dscrpt']
if len(sType) > 0:
objMalware = MalwareInstance()
objMalware.add_name(sType)
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = ""
objMalware.description = ""
objTTP = TTP(title=sType)
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
# if 'indicator' in data[sKey]['meta']['IDs']:
# objIndicator.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({objIndicator.id_:'indicator'})
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['IP Address']
if sAddr:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
# if 'address"' in data[sKey]['meta']['IDs']:
# obsAddr.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({objIndicator.id_:'address'})
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['Hostname']
if sDomain:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
# if 'domain' in data[sKey]['meta']['IDs']:
# obsDomain.id_ = data[sKey]['meta']['IDs'].key
# else:
# data[sKey]['meta']['IDs'].update({obsDomain.id_:'domain'})
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
### Parsing Port Number
sPortList = data[sKey]['attrib']['Ports']
for item in sPortList:
if sPortList[item]:
objPort = Port()
sPort = sPortList[item]
objPort.port_value = int(sPort)
objPort.port_value.condition = 'Equals'
objPort.layer4_protocol = 'TCP'
obsPort = Observable(objPort)
objPort = None
obsPort.sighting_count = 1
obsPort.title = 'Port: ' + str(sPort)
sDscrpt = 'PortNumber: ' + str(sPort) + " | "
sDscrpt += "Protocol: TCP | "
obsPort.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsPort)
### Add Generated observable to Indicator
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
infoSrc = InformationSource(identity=Identity(name=srcObj.Domain))
#infoSrc.add_contributing_source(data[sKey]['attrib']['ref'])
objIndicator.producer = infoSrc
# if data[sKey]['attrib']['lstDateVF']:
# objIndicator.set_produced_time(data[sKey]['attrib']['lstDateVF'][0]);
objIndicator.set_received_time(data[sKey]['meta']['dateDL'])
### Generate Indicator Title based on availbe data
lstContainng = []
lstIs = []
sTitle = ' This'
if data[sKey]['attrib']['Hostname']:
sTitle += ' domain ' + data[sKey]['attrib']['Hostname']
else:
sTitle += ' ipAddress ' + sKey
sTitle += ' has been identified as a TOR network "Exit Point" router'
objIndicator.title = sTitle
### Generate Indicator Description based on availbe data
sDscrpt = ' torstatus.blutmagie.de has identified this'
if data[sKey]['attrib']['Hostname']:
sDscrpt += ' domain ' + data[sKey]['attrib']['Hostname']
else:
sDscrpt += ' ipAddress ' + sKey
# sDscrpt += ' with a router name of "' + data[sKey]['attrib']['Router Name'] + '"'
# if data[sKey]['attrib']['Ports']['ORPort']:
# sDscrpt += ' using ORPort: ' + str(data[sKey]['attrib']['Ports']['ORPort'])
# if data[sKey]['attrib']['Ports']['DirPort']:
# sDscrpt += ' and DirPort: ' + str(data[sKey]['attrib']['Ports']['DirPort'])
sDscrpt += ' as a TOR network "Exit Point" router'
if data[sKey]['attrib']['Country Code']:
sCntry_code = data[sKey]['attrib']['Country Code']
if sCntry_code in dictCC2CN:
sCntry_name = dictCC2CN[sCntry_code]
sDscrpt += ', which appears to be located in ' + sCntry_name
sDscrpt += '. \n\n RawData: ' + str(data[sKey]['attrib'])
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
# objMalware = MalwareInstance()
# objMalware.add_type("Remote Access Trojan")
# ttpTitle = data[sKey]['attrib']['type']
# objTTP = TTP(title=ttpTitle)
# objTTP.behavior = Behavior()
# objTTP.behavior.add_malware_instance(objMalware)
# objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
# locDataFile = 'db_' + srcObj.fileName.split('.')[0] + '.json'
# sndFile_Dict2JSON(data,locDataFile);
# data = None
return (stix_package)
def get_indicator_from_json(indicator_json, user_timezone):
type_ = indicator_json['type']
v = indicator_json['value']
title = indicator_json['title']
o_ = None
# ipv4か?
if type_ == JSON_OBJECT_TYPE_IPV4:
o_ = Address()
o_.address_value = v.replace('[', '').replace(']', '')
# urlか?
if type_ == JSON_OBJECT_TYPE_URI:
o_ = URI()
o_.value = v
# md5か?
if type_ == JSON_OBJECT_TYPE_MD5:
o_ = File()
o_.md5 = v
# sha1か?
if type_ == JSON_OBJECT_TYPE_SHA1:
o_ = File()
o_.sha1 = v
# sha256か?
if type_ == JSON_OBJECT_TYPE_SHA256:
o_ = File()
o_.sha256 = v
# sha512か?
if type_ == JSON_OBJECT_TYPE_SHA512:
o_ = File()
o_.sha512 = v
# email-addressか?
if type_ == JSON_OBJECT_TYPE_EMAIL_ADDRESS:
o_ = EmailAddress()
o_.address_value = v
# domainか?
if type_ == JSON_OBJECT_TYPE_DOMAIN:
o_ = DomainName()
o_.value = v.replace('[', '').replace(']', '')
# file名か?
if type_ == JSON_OBJECT_TYPE_FILE_NAME:
o_ = File()
o_.file_name = v
# なにも該当していないので None
if o_ is None:
print('何も該当なし:' + str(type_) + ':' + str(v))
return None
# indicator 作って返却
indicator_title = '%s (%s)' % (v, title)
ind = CommonExtractor.get_indicator_from_object(
o_, indicator_title, user_timezone)
return ind
def gen_stix_observable_sample(
config,
target=None,
datatype=None,
title="random test data",
description="random test data",
package_intents="Indicators - Watchlist",
tlp_color="WHITE",
):
"""generate sample stix data comprised of indicator_count
indicators of type datatype"""
# setup the xmlns...
xmlns_url = config["edge"]["sites"][target]["stix"]["xmlns_url"]
xmlns_name = config["edge"]["sites"][target]["stix"]["xmlns_name"]
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = "../../../../descendant-or-self::node()"
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == "ip":
addr = Address(address_value=datagen_.generate_random_ip_address(), category="ipv4-addr")
addr.condition = "Equals"
stix_package.add_observable(Observable(addr))
elif datatype == "domain":
domain = DomainName()
domain.type_ = "FQDN"
domain.value = datagen_.generate_random_domain(config)
domain.condition = "Equals"
stix_package.add_observable(Observable(domain))
elif datatype == "filehash":
file_object = File()
file_object.file_name = str(uuid.uuid4()) + ".exe"
hashes = datagen_.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == "email":
try:
msg = datagen_.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {
"Subject": "subject",
"To": "to",
"Cc": "cc",
"Bcc": "bcc",
"From": "from_",
"Sender": "sender",
"Date": "date",
"Message-ID": "message_id",
"Reply-To": "reply_to",
"In-Reply-To": "in_reply_to",
"Content-Type": "content_type",
"Errors-To": "errors_to",
"Precedence": "precedence",
"Boundary": "boundary",
"MIME-Version": "mime_version",
"X-Mailer": "x_mailer",
"User-Agent": "user_agent",
"X-Originating-IP": "x_originating_ip",
"X-Priority": "x_priority",
}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = "Equals"
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return None
observable_id = stix_package.observables.observables[0].id_
return (observable_id, stix_package)
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
### Input Check
if srcObj is None or data is None:
# TODO: Needs error msg: Missing srcData Object
return False
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if sAddr:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if sDomain:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
objIndicator.add_indicator_type("Domain Watchlist")
# Parser File Hash
# sHash = data[sKey]['attrib']['hash'];
# if len(sHash) > 0:
# objFile = File()
# sFileName = data[sKey]['attrib']['fileName']
# if len(sFileName) > 0:
# objFile.file_name = sFileName
# objFile.file_format = sFileName.split('.')[1]
# objFile.add_hash(Hash(sHash, exact=True))
# obsFile = Observable(objFile)
# objFile = None;
# obsFile.sighting_count = 1
# obsFile.title = 'File: ' + sFileName
# sDscrpt = 'FileName: ' + sFileName + " | "
# sDscrpt += "FileHash: " + sHash + " | "
# obsFile.description = "<![CDATA[" + sDscrpt + "]]>"
# listOBS.append(obsFile)
# obsFile = None;
# objIndicator.add_indicator_type("File Hash Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(srcObj.Domain) > 0:
objIndicator.set_producer_identity(srcObj.Domain)
if data[sKey]['attrib']['dateVF']:
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Old Title / Description Generator
#objIndicator.title = data[sKey]['attrib']['title'];
#objIndicator.description = "<![CDATA[" + data[sKey]['attrib']['dscrpt'] + "]]>";
### Generate Indicator Title based on availbe data
sTitle = 'Feodo Tracker: '
if sAddr:
sAddLine = "This IP address has been identified as malicious"
if sDomain:
sAddLine = "This domain has been identified as malicious"
if len(sAddLine) > 0:
sTitle += " | " + sAddLine
if len(srcObj.Domain) > 0:
sTitle += " by " + srcObj.Domain
else:
sTitle += "."
if len(sTitle) > 0:
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if sAddr:
sAddLine = "This IP address " + sAddr
if sDomain:
sAddLine = "This domain " + sDomain
if sAddr and sDomain:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt += sAddLine
sDscrpt += " has been identified as malicious"
if len(srcObj.Domain) > 0:
sDscrpt += " by " + srcObj.Domain
else:
sDscrpt += "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + \
data[sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_name("Cridex")
objMalware.add_name("Bugat")
objMalware.add_name("Dridex")
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = "Feodo (also known as Cridex or Bugat) is a Trojan used to commit ebanking fraud and steal sensitive information from the victims computer, such as credit card details or credentials"
sDscrpt = "Feodo (also known as Cridex or Bugat) is a Trojan used to commit ebanking fraud and steal sensitive information from the victims computer, such as credit card details or credentials. At the moment, Feodo Tracker is tracking four versions of Feodo, and they are labeled by Feodo Tracker as version A, version B, version C and version D:\n"
sDscrpt += "\n"
sDscrpt += " Version A: Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.\n"
sDscrpt += " Version B: Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.\n"
sDscrpt += " Version C: Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.\n"
sDscrpt += " Version D: Successor of Cridex. This version is also known as Dridex\n"
objMalware.description = "<![CDATA[" + sDscrpt + "]]>"
objTTP = TTP(title="Feodo")
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = srcObj.Domain + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return stix_package
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
oObsSrcData = genObsSrcData(srcObj, data[sKey])
### Parsing IP Address
sAddr = data[sKey]['attrib']['ip']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_destination = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.type = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.type = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
oObsSrcData.sighting_count = 1
obsAddr.observable_source.append(oObsSrcData)
sTitle = 'IP: ' + sAddr
obsAddr.title = sTitle
sDscpt = 'ipv4-addr' + ': ' + sAddr + " | "
sDscpt += "is_destination: True | "
if data[sKey]['attrib']['first']:
sDscpt += "firstSeen: " + data[sKey]['attrib']['first'] + " | "
if data[sKey]['attrib']['last']:
sDscpt += "lastSeen: " + data[sKey]['attrib']['last'] + " | "
obsAddr.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
### Parsing Network Address
sAddr = data[sKey]['attrib']['inetnum']
if sAddr:
objAddr = Address()
objAddr.is_destination = True
sAddrNet = sAddr.split("-")
objAddr.address_value = sAddrNet[0].strip(
) + "##comma##" + sAddrNet[1].strip()
objAddr.address_value.condition = 'InclusiveBetween'
objAddr.category = 'ipv4-net'
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
oObsSrcData.sighting_count = 1
obsAddr.observable_source.append(oObsSrcData)
sTitle = 'NETWORK_range: ' + sAddr
obsAddr.title = sTitle
sDscpt = 'ipv4-net' + ': ' + sAddr + " | "
if data[sKey]['attrib']['netname']:
sDscpt += 'netName' + ': ' + data[sKey]['attrib'][
'netname'] + " | "
obsAddr.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
### Parsing Email Address
sEMAIL = data[sKey]['attrib']['email']
if sEMAIL:
objEmail = EmailAddress()
#objEmail.is_source = True
objEmail.address_value = sEMAIL
objEmail.address_value.condition = 'Equals'
objEmail.category = 'e-mail'
obsEmail = Observable(objEmail)
objEmail = None
obsEmail.sighting_count = 1
oObsSrcData.sighting_count = 1
if len(data[sKey]['attrib']['source']) > 0:
oObsSrcData.name = data[sKey]['attrib']['source']
obsEmail.observable_source.append(oObsSrcData)
sTitle = 'REGISTRAR_email: ' + sEMAIL
obsEmail.title = sTitle
sDscrpt = 'REGISTRAR_email: ' + sEMAIL
if data[sKey]['attrib']['descr']:
sDscrpt += " | REGISTRAR_name: " + data[sKey]['attrib'][
'descr'] + " | "
obsEmail.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsEmail)
obsEmail = None
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
objDomain.is_destination = True
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
oObsSrcData.sighting_count = 1
obsDomain.observable_source.append(oObsSrcData)
obsDomain.title = 'Domain: ' + sDomain
sDscpt = 'Domain: ' + sDomain + " | "
sDscpt += "isDestination: True | "
if data[sKey]['attrib']['first']:
sDscpt += "firstSeen: " + data[sKey]['attrib']['first'] + " | "
if data[sKey]['attrib']['last']:
sDscpt += "lastSeen: " + data[sKey]['attrib']['last'] + " | "
#if
obsDomain.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['url']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
oObsSrcData.sighting_count = 1
obsURI.observable_source.append(oObsSrcData)
obsURI.title = 'URI: ' + sURI
sDscpt = 'URI: ' + sURI + " | "
sDscpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
sDscrpt = None
sCntry_code = None
sCntry_name = None
sRgstra_email = None
sRgstra_name = None
# add Phishing Email Target
# add Phishing email Details phishtank_ID
if data[sKey]['attrib']['country']:
sCntry_code = data[sKey]['attrib']['country']
if sCntry_code in dictCC2CN:
sCntry_name = dictCC2CN[sCntry_code]
if data[sKey]['attrib']['email'] > 0:
sRgstra_email = data[sKey]['attrib']['email']
if data[sKey]['attrib']['descr']:
sRgstra_name = data[sKey]['attrib']['descr']
sDscrpt = " clean-mx.de has identified this "
if isIPv4(data[sKey]['attrib']['domain']):
sDscrpt += "ip address " + data[sKey]['attrib']['domain'] + " "
else:
sDscrpt += "domain " + data[sKey]['attrib']['domain'] + " "
sDscrpt += "as malicious "
if data[sKey]['attrib']['target']:
sDscrpt += "and uses phishing email(s) targeting " + data[sKey][
'attrib']['target'] + " users with "
else:
sDscrpt += "and sent out "
sDscrpt += "email containg this url <-Do Not Connect-> {" + data[sKey][
'attrib']['url'] + "} <-Do Not Connect-> link. "
if data[sKey]['attrib']['phishtank']:
sDscrpt += "For more detail on the specific phisihing email use this phishtank ID [" + data[
sKey]['attrib']['phishtank'] + "]. "
if sCntry_code:
sDscrpt += " This url appears to originated in " + sCntry_code
if sCntry_name:
sDscrpt += " (" + sCntry_name + ")"
if sCntry_code and (sRgstra_email or sRgstra_name):
sDscrpt += " and is "
if sRgstra_email:
sDscrpt += "register to " + sRgstra_email
if sRgstra_email and sRgstra_name:
sDscrpt += " of " + sRgstra_name
elif sRgstra_name:
sDscrpt += "register to " + sRgstra_name
sDscrpt += "."
if sCntry_code or sRgstra_email or sRgstra_name:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
sTitle = 'Phishing ID:' + sKey + " "
if data[sKey]['attrib']["target"]:
sTitle += "Target: " + data[sKey]['attrib']["target"] + " "
if data[sKey]['attrib']["url"]:
sTitle += "URL: " + data[sKey]['attrib']["url"] + " "
objIndicator.title = sTitle
### Add Generated observable to Indicator
objIndicator.add_indicator_type("IP Watchlist")
objIndicator.observable_composition_operator = 'OR'
objIndicator.observables = listOBS
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['first'])
objIndicator.set_received_time(data[sKey]['dateDL'])
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
#sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + srcObj.srcTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
coa.efficacy = HighMediumLow('High')
coa.stage = COAStage('Response')
coa.type_ = CourseOfActionType('Perimeter Blocking')
# Related Observable (by id)
addr = Address(address_value=ip, category=Address.CAT_IPV4)
observable = Observable(addr)
coa.parameter_observables = Observables(observables=Observable(
idref=observable.id_))
# Related CoA (basic; by id)
coa2 = CourseOfAction(title='Block domain traffic to {}'.format(domain))
related_coa = RelatedCOA(CourseOfAction(idref=coa2.id_))
coa.related_coas.append(related_coa)
# Indicator
indicator = Indicator(title='Indicator')
domain_name = DomainName()
domain_name.value = domain
indicator.add_observable(domain_name)
indicator.suggested_coas.append(CourseOfAction(idref=coa2.id_))
# Generate STIX Package
stix_package = STIXPackage()
stix_package.add_observable(observable)
stix_package.add_course_of_action(coa)
stix_package.add_course_of_action(coa2)
stix_package.add_indicator(indicator)
print(stix_package.to_xml().decode())
def main():
# define constants
TI_REQUEST_URL = "https://api.intelgraph.idefense.com/rest/threatindicator/v0"
# iDefense API Key
# To avoid hard-coding creds, I'm using environment variables
if os.environ.get('IDEF_TOKEN') is None:
print(
"error: please store your iDefense IntelGraph API key in the IDEF_TOKEN environment"
)
sys.exit(1)
API_KEY = os.environ.get('IDEF_TOKEN')
API_SECRET = ''
# TODO: use command-line parameter
timestr = datetime.datetime.utcnow() - datetime.timedelta(days=1)
LAST_IMPORT = timestr.strftime("%Y-%m-%dT%H:%M:%S") + ".000Z"
HEADERS = {
"Content-Type": "application/json",
"auth-token": API_KEY,
"X-Api-Key-Proof": API_SECRET
}
print(HEADERS)
page = 1
more_data = True
count = 0
# Set namespace
NAMESPACE = Namespace("https://intelgraph.idefense.com", "idefense")
set_id_namespace(NAMESPACE)
# Create STIX Package
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.description = "iDefense Threat Indicators Feed"
stix_package.stix_header = stix_header
ttps = {}
malware = {}
try:
while more_data:
request_payload = {
"start_date": LAST_IMPORT,
"page_size": 200,
"page": page
}
r = requests.post(TI_REQUEST_URL,
headers=HEADERS,
data=json.dumps(request_payload))
print(r)
response = []
if r.status_code == requests.codes.ok:
try:
# Read in response as json
response = r.json()
except (ValueError, KeyError):
print("Response couldn't be decoded :(")
more_data = False
continue
more_data = response.get('more', 'False')
print("Page %d ==> %s (%s)" %
(page, response['more'], response['total_size']))
page += 1
# Iterate the response
for indicatorD in response['results']:
count += 1
# Indicator value such as the value of the IP/Domain/URL
indicator = indicatorD.get('key')
print(indicator, indicatorD.get('type'))
if indicatorD.get('last_seen_as') is None:
last_seen_as = 'UNKNOWN'
else:
last_seen_as = ''.join(indicatorD.get('last_seen_as'))
# Identify TTP
if last_seen_as not in ttps:
ttps[last_seen_as] = TTP(title=last_seen_as)
stix_package.add_ttp(ttps[last_seen_as])
# Identify malware source
if 'files' in indicatorD:
for hashD in indicatorD['files']:
md5 = hashD.get('key')
# Malware Family classification of the hash if available
if hashD.get('malware_family') is None:
malware_family = "Unknown"
else:
malware_family = ''.join(
hashD.get('malware_family'))
if md5 not in malware:
malware[md5] = add_malware(
md5, malware_family, hashD.get('uuid'))
if indicatorD.get('type') == "url":
# Create indicator
indicator = Indicator(
id_="indicator-{0}".format(indicatorD.get('uuid')),
title=''.join(indicatorD.get('malware_family')),
timestamp=indicatorD.get('last_seen'))
indicator.add_indicator_type("URL Watchlist")
# Populate URL
url = URI()
url.value = indicatorD.get('key')
url.type_ = URI.TYPE_URL
url.value.condition = "Equals"
indicator.add_observable(url)
elif indicatorD.get('type') == "domain":
# Populate domain name
indicator = Indicator(
id_="indicator-{0}".format(indicatorD.get('uuid')),
title=''.join(indicatorD.get('malware_family')),
timestamp=indicatorD.get('last_seen'))
indicator.add_indicator_type("Domain Watchlist")
domain = DomainName()
domain.value = indicatorD.get('key')
domain.value.condition = "Equals"
indicator.add_observable(domain)
elif indicatorD.get('type') == "ip":
# Create indicator
indicator = Indicator(
id_="indicator-{0}".format(indicatorD.get('uuid')),
title=indicatorD.get('malware_family'),
timestamp=indicatorD.get('last_seen'))
indicator.add_indicator_type("IP Watchlist")
# Populate IP address
addr = Address(address_value=indicatorD.get('key'),
category=Address.CAT_IPV4)
addr.condition = "Equals"
indicator.add_observable(addr)
# Link TTP
indicator.add_indicated_ttp(
TTP(idref=ttps[last_seen_as].id_))
# Indicate confidence score
indicator.confidence = Confidence(
value=VocabString(indicatorD.get('confidence')))
# Add related indicator to malware
indicator.add_related_indicator(malware[md5])
# Add to package
stix_package.add_indicator(indicator)
else:
print("API request couldn't be fulfilled due status code: %d" %
r.status_code)
more_data = False
except requests.exceptions.ConnectionError as e:
print("Check your network connection\n %s" % str(e))
except requests.exceptions.HTTPError as e:
print("Bad HTTP response\n %s" % str(e))
except Exception as e:
print("Uncaught exception\n %s" % str(e))
# Output to XML
with open('stix-1.2.1.xml', 'wb') as f:
f.write(stix_package.to_xml())
def build(self):
self.stix_header.title = self.pulse["name"]
self.stix_header.description = self.pulse["description"]
self.stix_header.short_description = "%spulse/%s" % (PULSE_SERVER_BASE, str(self.pulse["id"]))
self.stix_header.package_intents.append(PackageIntent.TERM_INDICATORS)
self.stix_header.information_source = InformationSource()
self.stix_header.information_source.time = Time()
self.stix_header.information_source.description = "Alienvault OTX - https://otx.alienvault.com/"
self.stix_header.information_source.time.produced_time = self.pulse["modified"]
self.stix_header.information_source.identity = Identity()
self.stix_header.information_source.identity.name = "Alienvault OTX"
self.stix_package.stix_header = self.stix_header
hashes = []
addresses = []
domains = []
urls = []
mails = []
for p_indicator in self.pulse["indicators"]:
if p_indicator["type"] in self.hash_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (p_indicator["indicator"], PULSE_SERVER_BASE, str(self.pulse["id"]))
file_ = File()
hash_ = Hash(p_indicator["indicator"], self.hash_translation[p_indicator["type"]])
file_.add_hash(hash_)
observable_ = Observable(file_)
elif p_indicator["type"] in self.address_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (p_indicator["indicator"], PULSE_SERVER_BASE, str(self.pulse["id"]))
ipv4_ = Address.from_dict({'address_value': p_indicator["indicator"],
'category': self.address_translation[p_indicator["type"]]})
observable_ = Observable(ipv4_)
elif p_indicator["type"] in self.name_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (p_indicator["indicator"], PULSE_SERVER_BASE, str(self.pulse["id"]))
domain_ = DomainName.from_dict({'value': p_indicator["indicator"], 'type':'FQDN'})
observable_ = Observable(domain_)
elif p_indicator["type"] == "URL":
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (p_indicator["indicator"], PULSE_SERVER_BASE, str(self.pulse["id"]))
url_ = URI.from_dict({'value': p_indicator["indicator"], 'type': URI.TYPE_URL})
observable_ = Observable(url_)
elif p_indicator["type"] == "email":
email_ = Address.from_dict({'address_value': p_indicator["indicator"], 'category': Address.CAT_EMAIL})
observable_ = Observable(email_)
#elif p_indicator["type"] == "CVE":
# vuln_ = Vulnerability()
# vuln_.cveid = p_indicator["indicator"].upper()
# observable_ = Observable(vuln_)
elif p_indicator["type"] == "Mutex":
mutex_ = Mutex.from_dict({'named': True, 'name': p_indicator["indicator"]})
observable_ = Observable(mutex_)
elif p_indicator["type"] == "CIDR":
nrange = IP(p_indicator["indicator"])
nrange_values = nrange.strNormal(3).replace("-", ",")
ipv4_ = Address.from_dict({'address_value': nrange_values, 'category': Address.CAT_IPV4})
ipv4_.address_value.condition = "InclusiveBetween"
observable_ = Observable(ipv4_)
else:
continue
mind = Indicator()
mind.description = p_indicator["description"]
mind.title = "%s from %spulse/%s" % (p_indicator["indicator"], PULSE_SERVER_BASE, str(self.pulse["id"]))
observable_.title = "%s - %s" % (p_indicator["type"], p_indicator["indicator"])
mind.add_observable(observable_)
self.stix_package.add_indicator(mind)
def gen_stix_observable_sample(config, target=None, datatype=None,
title='random test data',
description='random test data',
package_intents='Indicators - Watchlist',
tlp_color='WHITE'):
'''generate sample stix data comprised of indicator_count
indicators of type datatype'''
# setup the xmlns...
xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name']
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = '../../../../descendant-or-self::node()'
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == 'ip':
addr = Address(address_value=datagen.generate_random_ip_address(),
category='ipv4-addr')
addr.condition = 'Equals'
stix_package.add_observable(Observable(addr))
elif datatype == 'domain':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = datagen.generate_random_domain(config)
domain.condition = 'Equals'
stix_package.add_observable(Observable(domain))
elif datatype == 'filehash':
file_object = File()
file_object.file_name = str(uuid.uuid4()) + '.exe'
hashes = datagen.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == 'email':
try:
msg = datagen.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {'Subject': 'subject', 'To': 'to', 'Cc':
'cc', 'Bcc': 'bcc', 'From': 'from_',
'Sender': 'sender', 'Date': 'date',
'Message-ID': 'message_id', 'Reply-To':
'reply_to', 'In-Reply-To': 'in_reply_to',
'Content-Type': 'content_type', 'Errors-To':
'errors_to', 'Precedence': 'precedence',
'Boundary': 'boundary', 'MIME-Version':
'mime_version', 'X-Mailer': 'x_mailer',
'User-Agent': 'user_agent',
'X-Originating-IP': 'x_originating_ip',
'X-Priority': 'x_priority'}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = \
'Equals'
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return(None)
observable_id = stix_package.observables.observables[0].id_
return(observable_id, stix_package)
def build(self):
"""Define the STIX report."""
self.stix_header.title = self.pulse["name"]
self.stix_header.description = self.pulse["description"]
self.stix_header.package_intents = "Indicators"
self.stix_header.information_source = InformationSource()
self.stix_header.information_source.time = Time()
self.stix_header.information_source.time.received_time = self.pulse[
"modified"]
self.stix_header.information_source.time.produced_time = self.stix_package.timestamp
self.stix_header.information_source.identity = Identity()
self.stix_header.information_source.identity.name = IDENTITY_NAME
# self.stix_package.stix_header = self.stix_header
# self.stix_package.stix_header.handling = self._marking()
# self.report = Report()
# self.report.header = Header()
# self.report.header.title = self.pulse["name"]
# self.report.header.descriptions = self.pulse["description"]
# self.report.header.intents = "Indicators"
# self.report.header.short_description = "%spulse/%s" % (
# PULSE_SERVER_BASE, str(self.pulse["id"]))
# self.report.header.information_source = InformationSource()
# self.report.header.information_source.time = Time()
# self.report.header.information_source.time.received_time = self.pulse[
# "modified"]
# self.report.header.information_source.time.produced_time = self.report.timestamp
# self.report.header.information_source.identity = Identity()
# self.report.header.information_source.identity.name = IDENTITY_NAME
hashes = False
addresses = False
emails = False
domains = False
urls = False
mutex = False
hash_indicator = Indicator()
hash_indicator.set_producer_identity(IDENTITY_NAME)
hash_indicator.set_produced_time(hash_indicator.timestamp)
hash_indicator.set_received_time(self.pulse["modified"])
hash_indicator.title = "[OTX] [Files] " + self.pulse["name"]
hash_indicator.add_indicator_type("File Hash Watchlist")
hash_indicator.confidence = "Low"
address_indicator = Indicator()
address_indicator.set_producer_identity(IDENTITY_NAME)
address_indicator.set_produced_time(address_indicator.timestamp)
address_indicator.set_received_time(self.pulse["modified"])
address_indicator.title = "[OTX] [IP] " + self.pulse["name"]
address_indicator.add_indicator_type("IP Watchlist")
address_indicator.confidence = "Low"
domain_indicator = Indicator()
domain_indicator.set_producer_identity(IDENTITY_NAME)
domain_indicator.set_produced_time(domain_indicator.timestamp)
domain_indicator.set_received_time(self.pulse["modified"])
domain_indicator.title = "[OTX] [Domain] " + self.pulse["name"]
domain_indicator.add_indicator_type("Domain Watchlist")
domain_indicator.confidence = "Low"
url_indicator = Indicator()
url_indicator.set_producer_identity(IDENTITY_NAME)
url_indicator.set_produced_time(url_indicator.timestamp)
url_indicator.set_received_time(self.pulse["modified"])
url_indicator.title = "[OTX] [URL] " + self.pulse["name"]
url_indicator.add_indicator_type("URL Watchlist")
url_indicator.confidence = "Low"
email_indicator = Indicator()
email_indicator.set_producer_identity(IDENTITY_NAME)
email_indicator.set_produced_time(email_indicator.timestamp)
email_indicator.set_received_time(self.pulse["modified"])
email_indicator.title = "[OTX] [Email] " + self.pulse["name"]
email_indicator.add_indicator_type("Malicious E-mail")
email_indicator.confidence = "Low"
mutex_indicator = Indicator()
mutex_indicator.set_producer_identity(IDENTITY_NAME)
mutex_indicator.set_produced_time(mutex_indicator.timestamp)
mutex_indicator.set_received_time(self.pulse["modified"])
mutex_indicator.title = "[OTX] [Mutex] " + self.pulse["name"]
mutex_indicator.add_indicator_type("Malware Artifacts")
mutex_indicator.confidence = "Low"
for p_indicator in self.pulse["indicators"]:
if p_indicator["type"] in self.hash_translation:
file_object = File()
file_object.add_hash(Hash(p_indicator["indicator"]))
file_object.hashes[0].simple_hash_value.condition = "Equals"
file_object.hashes[0].type_.condition = "Equals"
file_obs = Observable(file_object)
file_obs.title = "File: " + \
str(file_object.hashes[0].type_) + \
" - " + p_indicator["indicator"]
hash_indicator.add_observable(file_obs)
try:
hash_indicator.description = p_indicator["description"]
except KeyError:
hash_indicator.description = ""
hashes = True
elif p_indicator["type"] in self.address_translation:
ip = Address()
ip.address_value = p_indicator["indicator"]
ip.category = self.address_translation[p_indicator["type"]]
ip.address_value.condition = "Equals"
ip_obs = Observable(ip)
ip_obs.title = "Address: " + str(ip.address_value)
address_indicator.add_observable(ip_obs)
try:
address_indicator.description = p_indicator["description"]
except KeyError:
address_indicator.description = ""
addresses = True
elif p_indicator["type"] in self.name_translation:
domain = DomainName()
domain.value = p_indicator["indicator"]
domain.type_ = "FQDN"
domain.value.condition = "Equals"
domain_obs = Observable(domain)
domain_obs.title = "Domain: " + str(domain.value)
domain_indicator.add_observable(domain_obs)
try:
domain_indicator.description = p_indicator["description"]
except KeyError:
domain_indicator.description = ""
domains = True
elif p_indicator["type"] == "URL":
url = URI()
url.value = p_indicator["indicator"]
url.type_ = URI.TYPE_URL
url.value.condition = "Equals"
url_obs = Observable(url)
url_obs.title = "URI: " + str(url.value)
url_indicator.add_observable(url_obs)
try:
url_indicator.description = p_indicator["description"]
except KeyError:
url_indicator.description = ""
urls = True
elif p_indicator["type"] == "email":
email = Address()
email.address_value = p_indicator["indicator"]
email.category = "e-mail"
email.address_value.condition = "Equals"
email_obs = Observable(email)
email_obs.title = "Address: " + str(email.address_value)
email_indicator.add_observable(email_obs)
try:
email_indicator.description = p_indicator["indicator"]
except KeyError:
email_indicator.description = ""
emails = True
elif p_indicator["type"] == "Mutex":
mutex = Mutex()
mutex.name = p_indicator["indicator"]
mutex.named = True
mutex_obs = Observable(mutex)
mutex_obs.title = "Mutex: " + str(mutex.name)
mutex_indicator.add_observable(mutex_obs)
try:
mutex_indicator.description = p_indicator["indicator"]
except KeyError:
mutex_indicator.description = ""
mutex = True
else:
continue
if hashes:
self.stix_package.add_indicator(hash_indicator)
if addresses:
self.stix_package.add_indicator(address_indicator)
if domains:
self.stix_package.add_indicator(domain_indicator)
if urls:
self.stix_package.add_indicator(url_indicator)
if emails:
self.stix_package.add_indicator(email_indicator)
if mutex:
self.stix_package.add_indicator(mutex_indicator)
def gen_stix_observable_sample(config,
target=None,
datatype=None,
title='random test data',
description='random test data',
package_intents='Indicators - Watchlist',
tlp_color='WHITE'):
'''generate sample stix data comprised of indicator_count
indicators of type datatype'''
# setup the xmlns...
xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name']
set_stix_id_namespace({xmlns_url: xmlns_name})
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
# construct a stix package...
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = title
stix_header.description = description
stix_header.package_intents = package_intents
marking = MarkingSpecification()
marking.controlled_structure = '../../../../descendant-or-self::node()'
tlp_marking = TLPMarkingStructure()
tlp_marking.color = tlp_color
marking.marking_structures.append(tlp_marking)
stix_package.stix_header = stix_header
stix_package.stix_header.handling = Marking()
stix_package.stix_header.handling.add_marking(marking)
# ...and stuff it full of random sample data :-)
if datatype == 'ip':
addr = Address(address_value=datagen.generate_random_ip_address(),
category='ipv4-addr')
addr.condition = 'Equals'
stix_package.add_observable(Observable(addr))
elif datatype == 'domain':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = datagen.generate_random_domain(config)
domain.condition = 'Equals'
stix_package.add_observable(Observable(domain))
elif datatype == 'filehash':
file_object = File()
file_object.file_name = str(uuid.uuid4()) + '.exe'
hashes = datagen.generate_random_hashes()
for hash in hashes.keys():
file_object.add_hash(Hash(hashes[hash], type_=hash.upper()))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
stix_package.add_observable(Observable(file_object))
elif datatype == 'email':
try:
msg = datagen.get_random_spam_msg(config)
email = EmailMessage()
email.header = EmailHeader()
header_map = {
'Subject': 'subject',
'To': 'to',
'Cc': 'cc',
'Bcc': 'bcc',
'From': 'from_',
'Sender': 'sender',
'Date': 'date',
'Message-ID': 'message_id',
'Reply-To': 'reply_to',
'In-Reply-To': 'in_reply_to',
'Content-Type': 'content_type',
'Errors-To': 'errors_to',
'Precedence': 'precedence',
'Boundary': 'boundary',
'MIME-Version': 'mime_version',
'X-Mailer': 'x_mailer',
'User-Agent': 'user_agent',
'X-Originating-IP': 'x_originating_ip',
'X-Priority': 'x_priority'
}
# TODO handle received_lines
for key in header_map.keys():
val = msg.get(key, None)
if val:
email.header.__setattr__(header_map[key], val)
email.header.__getattribute__(header_map[key]).condition = \
'Equals'
# TODO handle email bodies (it's mostly all there except for
# handling weird text encoding problems that were making
# libcybox stacktrace)
# body = get_email_payload(random_spam_msg)
# if body:
# email.raw_body = body
stix_package.add_observable(Observable(email))
except:
return (None)
observable_id = stix_package.observables.observables[0].id_
return (observable_id, stix_package)
def build(self):
self.stix_header.title = self.pulse["name"]
self.stix_header.description = self.pulse["description"]
self.stix_header.short_description = "%spulse/%s" % (
PULSE_SERVER_BASE, str(self.pulse["id"]))
self.stix_header.package_intents.append(PackageIntent.TERM_INDICATORS)
self.stix_header.information_source = InformationSource()
self.stix_header.information_source.time = Time()
self.stix_header.information_source.description = "Alienvault OTX - https://otx.alienvault.com/"
self.stix_header.information_source.time.produced_time = self.pulse[
"modified"]
self.stix_header.information_source.identity = Identity()
self.stix_header.information_source.identity.name = "Alienvault OTX"
self.stix_package.stix_header = self.stix_header
hashes = []
addresses = []
domains = []
urls = []
mails = []
for p_indicator in self.pulse["indicators"]:
if p_indicator["type"] in self.hash_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (
p_indicator["indicator"], PULSE_SERVER_BASE,
str(self.pulse["id"]))
file_ = File()
hash_ = Hash(p_indicator["indicator"],
self.hash_translation[p_indicator["type"]])
file_.add_hash(hash_)
observable_ = Observable(file_)
elif p_indicator["type"] in self.address_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (
p_indicator["indicator"], PULSE_SERVER_BASE,
str(self.pulse["id"]))
ipv4_ = Address.from_dict({
'address_value':
p_indicator["indicator"],
'category':
self.address_translation[p_indicator["type"]]
})
observable_ = Observable(ipv4_)
elif p_indicator["type"] in self.name_translation:
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (
p_indicator["indicator"], PULSE_SERVER_BASE,
str(self.pulse["id"]))
domain_ = DomainName.from_dict({
'value':
p_indicator["indicator"],
'type':
'FQDN'
})
observable_ = Observable(domain_)
elif p_indicator["type"] == "URL":
new_ind = Indicator()
new_ind.description = p_indicator["description"]
new_ind.title = "%s from %spulse/%s" % (
p_indicator["indicator"], PULSE_SERVER_BASE,
str(self.pulse["id"]))
url_ = URI.from_dict({
'value': p_indicator["indicator"],
'type': URI.TYPE_URL
})
observable_ = Observable(url_)
elif p_indicator["type"] == "email":
email_ = Address.from_dict({
'address_value':
p_indicator["indicator"],
'category':
Address.CAT_EMAIL
})
observable_ = Observable(email_)
#elif p_indicator["type"] == "CVE":
# vuln_ = Vulnerability()
# vuln_.cveid = p_indicator["indicator"].upper()
# observable_ = Observable(vuln_)
elif p_indicator["type"] == "Mutex":
mutex_ = Mutex.from_dict({
'named': True,
'name': p_indicator["indicator"]
})
observable_ = Observable(mutex_)
elif p_indicator["type"] == "CIDR":
nrange = IP(p_indicator["indicator"])
nrange_values = nrange.strNormal(3).replace("-", ",")
ipv4_ = Address.from_dict({
'address_value': nrange_values,
'category': Address.CAT_IPV4
})
ipv4_.address_value.condition = "InclusiveBetween"
observable_ = Observable(ipv4_)
else:
continue
mind = Indicator()
mind.description = p_indicator["description"]
mind.title = "%s from %spulse/%s" % (p_indicator["indicator"],
PULSE_SERVER_BASE,
str(self.pulse["id"]))
observable_.title = "%s - %s" % (p_indicator["type"],
p_indicator["indicator"])
mind.add_observable(observable_)
self.stix_package.add_indicator(mind)
def csv2stix(outFormat,inFile):
#=============
# Build package metadata
#=============
stix_package = STIXPackage()
stix_package.stix_header = STIXHeader()
stix_package.stix_header.title = "TG3390"
stix_package.stix_header.description = "Dell SecureWorks Counter Threat Unit(TM) (CTU) researchers investigated activities associated with Threat Group-3390[1] (TG-3390) - http://www.secureworks.com/cyber-threat-intelligence/threats/threat-group-3390-targets-organizations-for-cyberespionage/"
marking_specification = MarkingSpecification()
marking_specification.controlled_structure = "../../../../descendant-or-self::node()"
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
handling = Marking()
handling.add_marking(marking_specification)
stix_package.stix_header.handling = handling
#=============
# Build package structure
#=============
ta_tg3390 = ThreatActor(title="TG3390")
ta_tg3390.identity = Identity(name="TG3390")
attack_pattern = AttackPattern()
attack_pattern.description = ("Infrastructure Building")
ttp_infrastructure = TTP(title="Infrastructure Building")
ttp_infrastructure.behavior = Behavior()
ttp_infrastructure.behavior.add_attack_pattern(attack_pattern)
ttp_infrastructure.add_intended_effect("Unauthorized Access")
infra_domainInd = Indicator(title="Domains associated with TG3390 Infrastructure")
infra_domainInd.add_indicator_type("Domain Watchlist")
infra_domainInd.confidence = "High"
infra_domainInd.add_indicated_ttp(TTP(idref=ttp_infrastructure.id_))
infra_IPInd = Indicator(title="[H] IP Addresses associated with TG3390 Infrastructure")
infra_IPInd.add_indicator_type("IP Watchlist")
infra_IPInd.confidence = "High"
infra_IPInd.add_indicated_ttp(TTP(idref=ttp_infrastructure.id_))
infra_IPInd_M = Indicator(title="[M] IP Addresses associated with TG3390 Infrastructure")
infra_IPInd_M.add_indicator_type("IP Watchlist")
infra_IPInd_M.confidence = "Medium"
infra_IPInd_M.add_indicated_ttp(TTP(idref=ttp_infrastructure.id_))
httpBrowserObj = MalwareInstance()
httpBrowserObj.add_name("HTTP Browser")
ttp_httpB = TTP(title="HTTP Browser")
ttp_httpB.behavior = Behavior()
ttp_httpB.behavior.add_malware_instance(httpBrowserObj)
ttp_httpB.add_intended_effect("Theft - Intellectual Property")
httpB_hashInd = Indicator(title="File hashes for HTTP Browser")
httpB_hashInd.add_indicator_type("File Hash Watchlist")
httpB_hashInd.confidence = "High"
httpB_hashInd.add_indicated_ttp(TTP(idref=ttp_httpB.id_))
httpBrowserDropperObj = MalwareInstance()
httpBrowserDropperObj.add_name("HTTP Browser Dropper")
ttp_httpBDpr = TTP(title="HTTP Browser Dropper")
ttp_httpBDpr.behavior = Behavior()
ttp_httpBDpr.behavior.add_malware_instance(httpBrowserDropperObj)
ttp_httpBDpr.add_intended_effect("Theft - Intellectual Property")
httpBDpr_hashInd = Indicator(title="File hashes for HTTP Browser Dropper")
httpBDpr_hashInd.add_indicator_type("File Hash Watchlist")
httpBDpr_hashInd.confidence = "High"
httpBDpr_hashInd.add_indicated_ttp(TTP(idref=ttp_httpBDpr.id_))
plugXObj = MalwareInstance()
plugXObj.add_name("PlugX Dropper")
ttp_plugX = TTP(title="PlugX Dropper")
ttp_plugX.behavior = Behavior()
ttp_plugX.behavior.add_malware_instance(plugXObj)
ttp_plugX.add_intended_effect("Theft - Intellectual Property")
plugX_hashInd = Indicator(title="File hashes for PlugX Dropper")
plugX_hashInd.add_indicator_type("File Hash Watchlist")
plugX_hashInd.confidence = "High"
plugX_hashInd.add_indicated_ttp(TTP(idref=ttp_plugX.id_))
#=============
# Process content in to structure
#=============
ip_rules = []
ip_rules_M = []
domain_rules = []
with open(inFile, 'rb') as f:
reader = csv.reader(f)
for row in reader:
obs = row[0]
obsType = row[1]
description = row[2]
confidence = row[3]
#print obs,obsType,description,confidence
if description == 'TG-3390 infrastructure':
if obsType == 'Domain name':
domain_obj = DomainName()
domain_obj.value = obs
#domain_obj.title = description
infra_domainInd.add_object(domain_obj)
domain_rule = generate_snort([obs], 'DomainName', str(infra_domainInd.id_).split(':',1)[1].split('-',1)[1])
domain_rules.append(domain_rule)
elif obsType == 'IP address':
ipv4_obj = Address()
ipv4_obj.category = "ipv4-addr"
ipv4_obj.address_value = obs
ipv4_obj.title = description
ind_ref = str(infra_IPInd.id_).split(':',1)[1].split('-',1)[1]
if confidence == "High":
infra_IPInd.add_object(ipv4_obj)
ip_rules.append(obs)
else:
infra_IPInd_M.add_object(ipv4_obj)
ip_rules_M.append(obs)
else:
print "TTP Infra: obsType is wrong"
elif description == 'HttpBrowser RAT dropper':
file_obj = File()
file_obj.add_hash(Hash(obs))
file_obj.title = description
httpBDpr_hashInd.add_observable(file_obj)
elif description == 'HttpBrowser RAT':
file_obj = File()
file_obj.add_hash(Hash(obs))
file_obj.title = description
httpB_hashInd.add_observable(file_obj)
elif description == 'PlugX RAT dropper':
file_obj = File()
file_obj.add_hash(Hash(obs))
file_obj.title = description
plugX_hashInd.add_observable(file_obj)
else:
print "TTP not found"
#print ip_rules
ip_rule = generate_snort(ip_rules, 'Address', str(infra_IPInd.id_).split(':',1)[1].split('-',1)[1])
ip_rule_M = generate_snort(ip_rules_M, 'Address', str(infra_IPInd_M.id_).split(':',1)[1].split('-',1)[1])
ip_tm = SnortTestMechanism()
ip_tm.rules = [ip_rule]
ip_tm.efficacy = "High"
ip_tm.producer = InformationSource(identity=Identity(name="Auto"))
infra_IPInd.test_mechanisms = [ip_tm]
ip_M_tm = SnortTestMechanism()
ip_M_tm.rules = [ip_rule_M]
ip_M_tm.efficacy = "Medium"
ip_M_tm.producer = InformationSource(identity=Identity(name="Auto"))
infra_IPInd_M.test_mechanisms = [ip_M_tm]
domain_tm = SnortTestMechanism()
domain_tm.rules = domain_rules
domain_tm.efficacy = "High"
domain_tm.producer = InformationSource(identity=Identity(name="Auto"))
infra_domainInd.test_mechanisms = [domain_tm]
#=============
# Add the composed content to the structure
#=============
stix_package.add_indicator(infra_domainInd)
stix_package.add_indicator(infra_IPInd)
stix_package.add_indicator(infra_IPInd_M)
stix_package.add_indicator(httpBDpr_hashInd)
stix_package.add_indicator(httpB_hashInd)
stix_package.add_indicator(plugX_hashInd)
stix_package.add_ttp(ttp_infrastructure)
stix_package.add_ttp(ttp_httpB)
stix_package.add_ttp(ttp_httpBDpr)
stix_package.add_ttp(ttp_plugX)
"""
if outFormat =='dict':
print stix_package.to_dict()
if outFormat =='json':
parsed = stix_package.to_json()
jsonpkg = json.loads(parsed)
pprint.pprint(jsonpkg)
if outFormat =='stix':
print stix_package.to_xml()
"""
#if verbose:
#print stix_package.to_xml()
#pprint(stix_package.to_json())
return stix_package
def main(iocs=iocs):
stix_header = STIXHeader(title=iocs['title'],
description=iocs['desc'],
package_intents=["Indicators - Watchlist"])
stix_package = STIXPackage(stix_header=stix_header)
# add indicator - file hash
if iocs.get('hash'):
indicator_file_hash = Indicator(title="Malicious File")
indicator_file_hash.add_indicator_type("File Hash Watchlist")
for file_hash in iocs['hash']:
file_object = File()
file_object.add_hash(Hash(file_hash))
file_object.hashes[0].simple_hash_value.condition = "Equals"
file_object.hashes[0].type_.condition = "Equals"
indicator_file_hash.add_observable(file_object)
stix_package.add_indicator(indicator_file_hash)
# add indicator - file name
if iocs.get('fname'):
indicator_filename = Indicator(title="Malicious File Name")
for file in iocs['fname']:
file_object = File()
file_object.file_name = file
indicator_filename.add_observable(file_object)
stix_package.add_indicator(indicator_filename)
# add indicator - ip address
if iocs.get('ips'):
indicator_ip = Indicator(title="Malicious IP Address")
indicator_ip.add_indicator_type("IP Watchlist")
for ip in iocs['ips']:
addr = Address(address_value=ip, category=Address.CAT_IPV4)
addr.condition = "Equals"
indicator_ip.add_observable(addr)
stix_package.add_indicator(indicator_ip)
# add indicator - domains
if iocs.get('domains'):
indicator_domains = Indicator(title="Malicious Domains")
indicator_domains.add_indicator_type("Domain Watchlist")
for domain in iocs['domains']:
domain_name = DomainName()
domain_name.value = domain
indicator_domains.add_observable(domain_name)
stix_package.add_indicator(indicator_domains)
# add indicator - url
if iocs.get('urls'):
indicator_url = Indicator(title='Malicious URL')
indicator_url.add_indicator_type("URL Watchlist")
for _url in iocs['urls']:
url = URI()
url.value = _url
url.type_ = URI.TYPE_URL
url.value.condition = "Equals"
# url.value.condition = "Contains"
indicator_url.add_observable(url)
stix_package.add_indicator(indicator_url)
# add indicator - email subject
if iocs.get('subject'):
indicator_email_subject = Indicator(title='Malicious E-mail Subject')
indicator_email_subject.add_indicator_type("Malicious E-mail")
for subject in iocs['subject']:
email_subject_object = EmailMessage()
email_subject_object.header = EmailHeader()
email_subject_object.header.subject = subject
email_subject_object.header.subject.condition = "StartsWith"
indicator_email_subject.add_observable(email_subject_object)
stix_package.add_indicator(indicator_email_subject)
# add indicator - email sender
if iocs.get('senders'):
indicator_email_sender = Indicator(title='Malicious E-mail Sender')
indicator_email_sender.add_indicator_type("Malicious E-mail")
for sender in iocs['senders']:
email_sender_object = EmailMessage()
email_sender_object.header = EmailHeader()
email_sender_object.header.sender = sender
email_sender_object.header.sender.condition = "Equals"
indicator_email_sender.add_observable(email_sender_object)
stix_package.add_indicator(indicator_email_sender)
# print(stix_package.to_xml(encoding=None))
# print(type(stix_package.to_xml(encoding=None)))
return stix_package.to_xml(encoding=None)
def genStixDoc(
outputDir_,
targetFileSha1_,
targetFileSha256_,
targetFileSha512_,
targetFileSsdeep_,
targetFileMd5_,
targetFileSize_,
targetFileName_,
ipv4Addresses_,
hostNames_):
"""
Generate Stix document from the input values. The doc structure is the file
object along with the related network items: addresses, domain names. Output
is written to files, which are then wrapped with taxii and uploaded using a
separate script.
"""
parsedTargetFileName = reFileName(targetFileName_)[1]
parsedTargetFilePrefix = reFileName(targetFileName_)[0]
stix.utils.set_id_namespace({"http://www.equifax.com/cuckoo2Stix" : "cuckoo2Stix"})
NS = cybox.utils.Namespace("http://www.equifax.com/cuckoo2Stix", "cuckoo2Stix")
cybox.utils.set_id_namespace(NS)
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_header.description = 'File: ' + parsedTargetFileName + ' with the associated hashes, network indicators'
stix_package.stix_header = stix_header
# Create the ttp
malware_instance = MalwareInstance()
malware_instance.add_name(parsedTargetFileName)
malware_instance.description = targetFileSha1_
ttp = TTP(title='TTP: ' + parsedTargetFileName)
ttp.behavior = Behavior()
ttp.behavior.add_malware_instance(malware_instance)
stix_package.add_ttp(ttp)
# Create the indicator for the ipv4 addresses
ipv4Object = Address(ipv4Addresses_, Address.CAT_IPV4)
ipv4Object.condition = 'Equals'
ipv4Indicator = Indicator()
ipv4Indicator.title = parsedTargetFileName + ': ipv4 addresses'
ipv4Indicator.add_indicator_type('IP Watchlist')
ipv4Indicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
ipv4Indicator.observable = ipv4Object
ipv4Indicator.confidence = 'Low'
# Create the indicator for the domain names
domainNameObject = DomainName()
domainNameObject.value = hostNames_
domainNameObject.condition = 'Equals'
domainNameIndicator = Indicator()
domainNameIndicator.title = parsedTargetFileName + ': domain names'
domainNameIndicator.add_indicator_type('Domain Watchlist')
domainNameIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship='Indicates Malware'))
domainNameIndicator.observable = domainNameObject
domainNameIndicator.confidence = 'Low'
# Create the indicator for the file
fileObject = File()
fileObject.file_name = parsedTargetFileName
fileObject.file_name.condition = 'Equals'
fileObject.size_in_bytes = targetFileSize_
fileObject.size_in_bytes.condition = 'Equals'
fileObject.add_hash(Hash(targetFileSha1_, type_='SHA1', exact=True))
fileObject.add_hash(Hash(targetFileSha256_, type_='SHA256', exact=True))
fileObject.add_hash(Hash(targetFileSha512_, type_='SHA512', exact=True))
fileObject.add_hash(Hash(targetFileSsdeep_, type_='SSDEEP', exact=True))
fileObject.add_hash(Hash(targetFileMd5_, type_='MD5', exact=True))
fileIndicator = Indicator()
fileIndicator.title = parsedTargetFileName + ': hashes'
fileIndicator.description = parsedTargetFilePrefix
fileIndicator.add_indicator_type('File Hash Watchlist')
fileIndicator.add_indicated_ttp(RelatedTTP(TTP(idref=ttp.id_), relationship="Indicates Malware"))
fileIndicator.observable = fileObject
fileIndicator.confidence = 'Low'
stix_package.indicators = [fileIndicator, ipv4Indicator, domainNameIndicator]
stagedStixDoc = stix_package.to_xml()
stagedStixDoc = fixAddressObject(stagedStixDoc)
stagedStixDoc = fixDomainObject(stagedStixDoc)
today = datetime.datetime.now()
now = today.strftime('%Y-%m-%d_%H%M%S')
if not os.path.exists(outputDir_):
os.makedirs(outputDir_)
with open (outputDir_ + '/' + now + '-' + targetFileSha1_ + '.stix.xml', 'a') as myfile:
myfile.write(stagedStixDoc)
_l.debug('Wrote file: ' + now + '-' + targetFileSha1_ + '.stix.xml')
return
def json2observable(config, src, dest, endpoint, json_, crits_id):
# TODO split into smaller functions
'''transform crits observables into cybox'''
try:
set_id_method(IDGenerator.METHOD_UUID)
xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
if endpoint == 'ips':
crits_types = {'Address - cidr': 'cidr',
'Address - ipv4-addr': 'ipv4-addr',
'Address - ipv4-net': 'ipv4-net',
'Address - ipv4-net-mask': 'ipv4-netmask',
'Address - ipv6-addr': 'ipv6-addr',
'Address - ipv6-net': 'ipv6-net',
'Address - ipv6-net-mask': 'ipv6-netmask'}
addr = Address(address_value=json_['ip'],
category=crits_types[json_['type']])
addr.condition = 'Equals'
observable_ = Observable(addr)
elif endpoint == 'domains':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = json_['domain']
domain.condition = 'Equals'
observable_ = Observable(domain)
elif endpoint == 'samples':
crits_types = {'md5': 'MD5',
'sha1': 'SHA1',
'sha224': 'SHA224',
'sha256': 'SHA256',
'sha384': 'SHA384',
'sha512': 'SHA512',
'ssdeep': 'SSDEEP'}
file_object = File()
file_object.file_name = json_['filename']
for hash in crits_types.keys():
if hash in json_:
file_object.add_hash(Hash(json_[hash],
type_=crits_types[hash]))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
observable_ = Observable(file_object)
elif endpoint == 'emails':
crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
'from_address': 'from_', 'sender': 'sender',
'date': 'date', 'message_id': 'message_id',
'reply_to': 'reply_to', 'boundary': 'boundary',
'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'}
email = EmailMessage()
email.header = EmailHeader()
for k in crits_types.keys():
val = json_.get(k, None)
if val:
email.header.__setattr__(crits_types[k], val)
email.header.__getattribute__(crits_types[k]).condition = \
'Equals'
observable_ = Observable(email)
else:
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='crits', obj_type=endpoint, id_=crits_id))
return(None)
observable_.id = xmlns_name + ':observable-' + crits_id
observable_.id_ = observable_.id
return(observable_)
except:
e = sys.exc_info()[0]
config['logger'].error(
log.log_messages['obj_convert_error'].format(
src_type='crits', src_obj='observable', id_=crits_id,
dest_type='cybox', dest_obj='observable'))
config['logger'].exception(e)
return(None)
def generate_stix_file(input_file, list_type, delimiter, list_name, tc_name, tmp_dir, validate, verbose):
# observable limit per generated stix file
OBSERVABLES_PER_STIX_FILE = 3000
if verbose:
logging.info("=====================")
logging.info("== GENERATING STIX ==")
logging.info("=====================")
# download or open input file
if validators.url(input_file):
res = requests.get(input_file)
items = res.text.split(delimiter)
else:
# exit if input file doesn't exist
if not os.path.isfile(input_file):
logging.error("Supplied input file '{}' doesn't exist".format(input_file))
sys.exit("Error: Supplied input file '{}' doesn't exist".format(input_file))
else:
with open(input_file, 'r') as f:
items = f.read().split(delimiter)
logging.info("Successfully parsed input file at {}".format(input_file))
# slice input into batches
for batch_num, index in enumerate(range(0, len(items), OBSERVABLES_PER_STIX_FILE), 1):
# slice handles out of bounds indices gracefully
batch_items = items[index:index + OBSERVABLES_PER_STIX_FILE]
# create the STIX Package
package = STIXPackage()
# create the STIX Header and add a description
header = STIXHeader()
package.stix_header = header
reporttime = datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M:%S %Z')
# create indicator for each item in the batch
for item in batch_items:
item = item.strip()
# basic filtering of empty items and comments
if not item or item.startswith(('#', '//', '--')):
continue
if list_type == 'ip':
indicator_obj = Address()
# attempt to parse as an ip address
try:
parsed_ip = ipaddress.ip_address(item)
if parsed_ip.version == 4:
indicator_obj.category = Address.CAT_IPV4
elif parsed_ip.version == 6:
indicator_obj.category = Address.CAT_IPV6
else:
logging.warning("Unknown IP Address version type: {} - skipping".format(parsed_ip.version))
continue
except ValueError:
# if ip address parsing fails then attempt to parse as an ip network
try:
parsed_ip = ipaddress.ip_network(item, strict=False)
indicator_obj.category = Address.CAT_CIDR
except ValueError:
logging.warning("IP Address {} is neither an IPv4, IPv6, nor CIDR - skipping".format(item))
continue
indicator_obj.address_value = str(parsed_ip)
indicator_obj.condition = "Equals"
indicator_type = "IP Watchlist"
# customizable components below
indicator_title = "IP: {}"
indicator_description = "IP {} reported from {}"
elif list_type == 'domain':
# validate domain
if validate and not validators.domain(item):
logging.warning("Invalid domain: {} - skipping".format(item))
continue
indicator_obj = DomainName()
indicator_obj.value = item
indicator_type = "Domain Watchlist"
# customizable components below
indicator_title = "Domain: {}"
indicator_description = "Domain {} reported from {}"
elif list_type == 'url':
# validate url
if validate and not validators.url(item):
logging.warning("Invalid url: {} - skipping".format(item))
continue
indicator_obj = URI()
indicator_obj.value = item
indicator_obj.type_ = URI.TYPE_URL
indicator_obj.condition = "Equals"
indicator_type = "URL Watchlist"
# customizable components below
indicator_title = "URL: {}"
indicator_description = "URL {} reported from {}"
else:
# invalid input type
logging.error("invalid input type encountered")
raise Exception('Error: invalid input type encountered')
# create a basic Indicator object from the item
indicator = Indicator()
indicator.title = indicator_title.format(str(item))
indicator.description = indicator_description.format(str(item), list_name)
indicator.add_indicator_type(indicator_type)
indicator.set_producer_identity(list_name)
indicator.set_produced_time(str(reporttime))
indicator.add_observable(indicator_obj)
# add the indicator to the stix package
package.add_indicator(indicator)
# save each batch in a separate stix file with the filename ending ..._part_N.stix
collection_filename = "{}_part_{}.stix".format(strip_non_alphanum(tc_name), batch_num)
with open(os.path.join(tmp_dir, collection_filename), 'wb') as f:
f.write(package.to_xml())
logging.info("Successfully created stix file {}".format(collection_filename))
# clear cybox cache to prevent an Out of Memory error
# https://cybox.readthedocs.io/en/stable/api/cybox/core/object.html#cybox.core.object.Object
cache_clear()
return
def index2stix(local_index, orig_stix):
#=============
# Build package metadata
#=============
new_stix = STIXPackage()
new_stix.stix_header = STIXHeader()
new_stix.stix_header.title = "TG3390 - Enrichment"
new_stix.stix_header.description = "Enrichment stix file to the Dell SecureWorks Counter Threat Unit(TM) (CTU) researchers investigated activities associated with Threat Group-3390[1] (TG-3390) - http://www.secureworks.com/cyber-threat-intelligence/threats/threat-group-3390-targets-organizations-for-cyberespionage/"
marking_specification = MarkingSpecification()
marking_specification.controlled_structure = "../../../../descendant-or-self::node()"
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
handling = Marking()
handling.add_marking(marking_specification)
new_stix.stix_header.handling = handling
enrich_IPs = Indicator(title="Suspected TG3390 IP Addresses obtained through automated enrichment")
enrich_IPs.add_indicator_type("IP Watchlist")
enrich_IPs.confidence = "Low"
related_IPs = Indicator(title="Related indicator wrapper for source of enrichment")
related_IPs.add_indicator_type("IP Watchlist")
related_IPs.confidence = "Medium"
enrich_Domains = Indicator(title="Suspected TG3390 Domains obtained through automated enrichment")
enrich_Domains.add_indicator_type("Domain Watchlist")
enrich_Domains.confidence = "Low"
related_Domains = Indicator(title="Related indicator wrapper for source of enrichment")
related_Domains.add_indicator_type("Domain Watchlist")
related_Domains.confidence = "Medium"
# START with the ones that already have ids:
#if verbose:
#print_chain(local_index)
new_ref_created = True
while new_ref_created:
new_ref_created = False
for ind_type in local_index:
for obs in local_index[ind_type]:
id_tobe_referenced = local_index[ind_type][obs][0]
#print id_tobe_referenced[:10]
if id_tobe_referenced[:10] != '{{no_ref}}':
ref_obs = Observable()
ref_obs.id_ = id_tobe_referenced.replace("{{no_ref}}","")
ref_obs.description = 'Source of enrichment for: '
create_ref_obs = False
for entry in local_index[ind_type][obs]:
if type(entry) is list:
if len(entry)>0:
for item in entry:
ref, child_ind_type = get_ref_from_obs(item, local_index)
#print item
if ref == '{{no_ref}}' or ref == '':
create_ref_obs = True
new_ref_created = True
#print 'Create new, complete, observable for ' + item
#print child_ind_type
#Create the new observable for item and add as object to appropriate Indicator
if child_ind_type == 'DomainName':
append_ind = enrich_Domains
related_ind = related_Domains
new_obj = DomainName()
new_obj.value = item
#enrich_Domains.add_object(domain_obj)
elif child_ind_type == 'Address':
append_ind = enrich_IPs
related_ind = related_IPs
new_obj = Address()
new_obj.category = "ipv4-addr"
new_obj.address_value = item
#enrich_IPs.add_object(ipv4_obj)
else:
print 'Unsupported indicator type: ' + child_ind_type
new_obs = Observable(new_obj)
new_obs_ref = new_obs.id_
append_ind.add_observable(new_obs)
ref = new_obs_ref
#local_index[item][0] = ref
set_obs_ref(item, new_obs_ref, local_index)
#print 'Adding ref to: ' + ref_obs.id_ + ' of ' + ref
ref_obs.description = str(ref_obs.description) + ref.replace("{{no_ref}}","") + ', '
if create_ref_obs:
#Add the new ref obs to Related Indicators
related_ind.add_observable(ref_obs)
#print related_ind.to_xml()
create_ref_obs = False
related_ind1 = RelatedIndicator(related_IPs, relationship='Source of enrichment for IPs')
related_ind2 = RelatedIndicator(related_Domains, relationship='Source of enrichment for Domains')
enrich_IPs.related_indicators.append(related_ind1)
enrich_Domains.related_indicators.append(related_ind2)
new_stix.add_indicator(enrich_IPs)
new_stix.add_indicator(enrich_Domains)
#new_stix_json = json.loads(new_stix.to_json())
#new_stix_xml = new_stix.to_xml()
#if verbose:
#print new_stix.to_xml()
return new_stix
def make_cybox_object(type_, value=None):
"""
Converts type_, name, and value to a CybOX object instance.
:param type_: The object type.
:type type_: str
:param value: The object value.
:type value: str
:returns: CybOX object
"""
if type_ == IndicatorTypes.USER_ID:
acct = Account()
acct.description = value
return acct
elif type_ in IPTypes.values():
if type_ == IPTypes.IPV4_ADDRESS:
name = 'ipv4-addr'
elif type_ == IPTypes.IPV6_ADDRESS:
name = 'ipv6-addr'
elif type_ == IPTypes.IPV4_SUBNET:
name = 'ipv4-net'
elif type_ == IPTypes.IPV6_SUBNET:
name = 'ipv6-net'
return Address(category=name, address_value=value)
elif type_ == IndicatorTypes.API_KEY:
api = API()
api.description = value
return api
elif type_ == IndicatorTypes.DOMAIN:
obj = DomainName()
obj.value = value
elif type_ == IndicatorTypes.USER_AGENT:
obj = HTTPRequestHeaderFields()
obj.user_agent = value
return obj
elif type_ == IndicatorTypes.MUTEX:
m = Mutex()
m.named = True
m.name = String(value)
return m
elif type_ in (IndicatorTypes.SOURCE_PORT,
IndicatorTypes.DEST_PORT):
p = Port()
try:
p.port_value = PositiveInteger(value)
except ValueError: # XXX: Raise a better exception...
raise UnsupportedCybOXObjectTypeError(type_, name)
return p
elif type_ == IndicatorTypes.PROCESS_NAME:
p = Process()
p.name = String(value)
return p
elif type_ == IndicatorTypes.URI:
r = URI()
r.type_ = 'URL'
r.value = value
return r
elif type_ in (IndicatorTypes.REGISTRY_KEY,
IndicatorTypes.REG_KEY_CREATED,
IndicatorTypes.REG_KEY_DELETED,
IndicatorTypes.REG_KEY_ENUMERATED,
IndicatorTypes.REG_KEY_MONITORED,
IndicatorTypes.REG_KEY_OPENED):
obj = WinRegistryKey()
obj.key = value
return obj
"""
The following are types that are listed in the 'Indicator Type' box of
the 'New Indicator' dialog in CRITs. These types, unlike those handled
above, cannot be written to or read from CybOX at this point.
The reason for the type being omitted is written as a comment inline.
This can (and should) be revisited as new versions of CybOX are released.
NOTE: You will have to update the corresponding make_crits_object function
with handling for the reverse direction.
In the mean time, these types will raise unsupported errors.
"""
#elif type_ == "Device": # No CybOX API
#elif type_ == "DNS Cache": # No CybOX API
#elif type_ == "GUI": # revisit when CRITs supports width & height specification
#elif type_ == "HTTP Session": # No good mapping between CybOX/CRITs
#elif type_ == "Linux Package": # No CybOX API
#elif type_ == "Network Packet": # No good mapping between CybOX/CRITs
#elif type_ == "Network Route Entry": # No CybOX API
#elif type_ == "Network Route": # No CybOX API
#elif type_ == "Network Subnet": # No CybOX API
#elif type_ == "Semaphore": # No CybOX API
#elif type_ == "Socket": # No good mapping between CybOX/CRITs
#elif type_ == "UNIX File": # No CybOX API
#elif type_ == "UNIX Network Route Entry": # No CybOX API
#elif type_ == "UNIX Pipe": # No CybOX API
#elif type_ == "UNIX Process": # No CybOX API
#elif type_ == "UNIX User Account": # No CybOX API
#elif type_ == "UNIX Volume": # No CybOX API
#elif type_ == "User Session": # No CybOX API
#elif type_ == "Whois": # No good mapping between CybOX/CRITs
#elif type_ == "Win Computer Account": # No CybOX API
#elif type_ == "Win Critical Section": # No CybOX API
#elif type_ == "Win Executable File": # No good mapping between CybOX/CRITs
#elif type_ == "Win File": # No good mapping between CybOX/CRITs
#elif type_ == "Win Kernel": # No CybOX API
#elif type_ == "Win Mutex": # No good mapping between CybOX/CRITs
#elif type_ == "Win Network Route Entry": # No CybOX API
#elif type_ == "Win Pipe": # No good mapping between CybOX/CRITs
#elif type_ == "Win Prefetch": # No CybOX API
#elif type_ == "Win Semaphore": # No CybOX API
#elif type_ == "Win System Restore": # No CybOX API
#elif type_ == "Win Thread": # No good mapping between CybOX/CRITs
#elif type_ == "Win Waitable Timer": # No CybOX API
raise UnsupportedCybOXObjectTypeError(type_, name)
def json2observable(config, src, dest, endpoint, json_, crits_id):
# TODO split into smaller functions
'''transform crits observables into cybox'''
try:
set_id_method(IDGenerator.METHOD_UUID)
xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url']
xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name))
if endpoint == 'ips':
crits_types = {
'Address - cidr': 'cidr',
'Address - ipv4-addr': 'ipv4-addr',
'Address - ipv4-net': 'ipv4-net',
'Address - ipv4-net-mask': 'ipv4-netmask',
'Address - ipv6-addr': 'ipv6-addr',
'Address - ipv6-net': 'ipv6-net',
'Address - ipv6-net-mask': 'ipv6-netmask'
}
addr = Address(address_value=json_['ip'],
category=crits_types[json_['type']])
addr.condition = 'Equals'
observable_ = Observable(addr)
elif endpoint == 'domains':
domain = DomainName()
domain.type_ = 'FQDN'
domain.value = json_['domain']
domain.condition = 'Equals'
observable_ = Observable(domain)
elif endpoint == 'samples':
crits_types = {
'md5': 'MD5',
'sha1': 'SHA1',
'sha224': 'SHA224',
'sha256': 'SHA256',
'sha384': 'SHA384',
'sha512': 'SHA512',
'ssdeep': 'SSDEEP'
}
file_object = File()
file_object.file_name = json_['filename']
for hash in crits_types.keys():
if hash in json_:
file_object.add_hash(
Hash(json_[hash], type_=crits_types[hash]))
for i in file_object.hashes:
i.simple_hash_value.condition = "Equals"
observable_ = Observable(file_object)
elif endpoint == 'emails':
crits_types = {
'subject': 'subject',
'to': 'to',
'cc': 'cc',
'from_address': 'from_',
'sender': 'sender',
'date': 'date',
'message_id': 'message_id',
'reply_to': 'reply_to',
'boundary': 'boundary',
'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'
}
email = EmailMessage()
email.header = EmailHeader()
for k in crits_types.keys():
val = json_.get(k, None)
if val:
email.header.__setattr__(crits_types[k], val)
email.header.__getattribute__(crits_types[k]).condition = \
'Equals'
observable_ = Observable(email)
else:
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='crits', obj_type=endpoint, id_=crits_id))
return (None)
observable_.id = xmlns_name + ':observable-' + crits_id
observable_.id_ = observable_.id
return (observable_)
except:
e = sys.exc_info()[0]
config['logger'].error(log.log_messages['obj_convert_error'].format(
src_type='crits',
src_obj='observable',
id_=crits_id,
dest_type='cybox',
dest_obj='observable'))
config['logger'].exception(e)
return (None)
kc_phases = KillChainPhasesReference()
kc_phases.append(KillChainPhaseReference(name=phase.name))
ttp.kill_chain_phases = kc_phases
# TTP - Resource (Tool, Infrastructure, Personas)
resource = Resource()
tool = ToolInformation(title='malware.exe')
tool.type_ = AttackerToolType('Malware')
tool.description = 'Tool Description'
tool.short_description = 'Tool Short Description'
infrastructure = Infrastructure(title='Leveraged Domains')
infrastructure.types = AttackerInfrastructureType('Domain Registration')
infrastructure.description = 'Infrastructure Description'
infrastructure.short_description = 'Infrastructure Short Description'
domain = DomainName()
domain.value = 'totally-not-malware.biz'
observable = Observable(domain)
infrastructure.observable_characterization = Observables(
Observable(idref=observable.id_))
personas = Personas()
personas.append(Identity(name='Stephen Golub'))
resource = Resource(tools=Tools(tool),
infrastructure=infrastructure,
personas=personas)
ttp.resources = resource
related_ttp = RelatedTTP(TTP(idref=ttp.id_))
def adptr_dict2STIX(srcObj, data):
sTxt = "Called... "
sndMSG(sTxt, 'INFO', 'adptr_dict2STIX()')
stixObj = None
### Input Check
if srcObj == None or data == None:
#TODO: Needs error msg: Missing srcData Object
return (False)
### Generate NameSpace id tags
STIX_NAMESPACE = {"http://hailataxii.com": "opensource"}
OBS_NAMESPACE = Namespace("http://hailataxii.com", "opensource")
stix_set_id_namespace(STIX_NAMESPACE)
obs_set_id_namespace(OBS_NAMESPACE)
### Building STIX Wrapper
stix_package = STIXPackage()
objIndicator = Indicator()
### Bulid Object Data
for sKey in data:
objIndicator = Indicator()
listOBS = []
### Parsing IP Address
sAddr = data[sKey]['attrib']['ipAddr']
if len(sAddr) > 0:
objAddr = Address()
objAddr.is_source = True
objAddr.address_value = sAddr
objAddr.address_value.condition = 'Equals'
if isIPv4(sAddr):
objAddr.category = 'ipv4-addr'
elif isIPv6(sAddr):
objAddr.category = 'ipv6-addr'
else:
continue
obsAddr = Observable(objAddr)
objAddr = None
obsAddr.sighting_count = 1
obsAddr.title = 'IP: ' + sAddr
sDscrpt = 'IPv4' + ': ' + sAddr + " | "
sDscrpt += "isSource: True | "
obsAddr.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsAddr)
obsAddr = None
objIndicator.add_indicator_type("IP Watchlist")
### Parsing Domain
sDomain = data[sKey]['attrib']['domain']
if len(sDomain) > 0:
objDomain = DomainName()
objDomain.value = sDomain
objDomain.value.condition = 'Equals'
if isFQDN(sDomain):
objDomain.type = 'FQDN'
elif isTLD(sDomain):
objDomain.type = 'TLD'
else:
continue
obsDomain = Observable(objDomain)
objDomain = None
obsDomain.sighting_count = 1
obsDomain.title = 'Domain: ' + sDomain
sDscrpt = 'Domain: ' + sDomain + " | "
sDscrpt += "isFQDN: True | "
obsDomain.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsDomain)
obsDomain = None
objIndicator.add_indicator_type("Domain Watchlist")
#Parser URI
sURI = data[sKey]['attrib']['URI']
if len(sURI) > 0:
objURI = URI()
objURI.value = sURI
objURI.value.condition = 'Equals'
objURI.type_ = URI.TYPE_URL
obsURI = Observable(objURI)
objURI = None
obsURI.sighting_count = 1
obsURI.title = 'URI: ' + sURI
sDscrpt = 'URI: ' + sURI + " | "
sDscrpt += "Type: URL | "
obsURI.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsURI)
obsURI = None
objIndicator.add_indicator_type("URL Watchlist")
#Parser File Hash
sHash = data[sKey]['attrib']['hash']
if len(sHash) > 0:
objFile = File()
sFileName = data[sKey]['attrib']['fileName']
if len(sFileName) > 0:
objFile.file_name = sFileName
objFile.file_format = sFileName.split('.')[1]
objFile.add_hash(Hash(sHash, exact=True))
obsFile = Observable(objFile)
objFile = None
obsFile.sighting_count = 1
obsFile.title = 'File: ' + sFileName
sDscrpt = 'FileName: ' + sFileName + " | "
sDscrpt += "FileHash: " + sHash + " | "
obsFile.description = "<![CDATA[" + sDscrpt + "]]>"
listOBS.append(obsFile)
obsFile = None
objIndicator.add_indicator_type("File Hash Watchlist")
### Add Generated observable to Indicator
objIndicator.observables = listOBS
objIndicator.observable_composition_operator = 'OR'
#Parsing Producer
sProducer = srcObj.Domain
if len(sProducer) > 0:
objIndicator.set_producer_identity(sProducer)
objIndicator.set_produced_time(data[sKey]['attrib']['dateVF'])
objIndicator.set_received_time(data[sKey]['dateDL'])
### Old Title / Description Generator
#objIndicator.title = data[sKey]['attrib']['title'];
#objIndicator.description = "<![CDATA[" + data[sKey]['attrib']['dscrpt'] + "]]>";
### Generate Indicator Title based on availbe data
sTitle = 'ZeuS Tracker (' + data[sKey]['attrib'][
'status'] + ')| ' + data[sKey]['attrib']['title']
if len(sAddr) > 0:
sAddLine = "This IP address has been identified as malicious"
if len(sDomain) > 0:
sAddLine = "This domain has been identified as malicious"
if len(sAddLine) > 0:
sTitle = sTitle + " | " + sAddLine
if len(srcObj.Domain) > 0:
sTitle = sTitle + " by " + srcObj.Domain
else:
sTitle = sTitle + "."
if len(sTitle) > 0:
objIndicator.title = sTitle
#Generate Indicator Description based on availbe data
sDscrpt = ""
if len(sAddr) > 0:
sAddLine = "This IP address " + sAddr
if len(sDomain) > 0:
sAddLine = "This domain " + sDomain
if len(sAddr) > 0 and len(sDomain) > 0:
sAddLine = "This domain " + sDomain + " (" + sAddr + ")"
if len(sAddLine) > 0:
sDscrpt = sDscrpt + sAddLine
sDscrpt = sDscrpt + " has been identified as malicious"
if len(srcObj.Domain) > 0:
sDscrpt = sDscrpt + " by " + srcObj.Domain
else:
sDscrpt = sDscrpt + "."
sDscrpt = sDscrpt + ". For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [" + data[
sKey]['attrib']['link'] + "]."
if len(sDscrpt) > 0:
objIndicator.description = "<![CDATA[" + sDscrpt + "]]>"
#Parse TTP
objMalware = MalwareInstance()
objMalware.add_name("ZeuS")
objMalware.add_name("Zbot")
objMalware.add_name("Zeus")
objMalware.add_type("Remote Access Trojan")
objMalware.short_description = "Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system"
objMalware.description = "Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))"
objTTP = TTP(title="ZeuS")
objTTP.behavior = Behavior()
objTTP.behavior.add_malware_instance(objMalware)
objIndicator.add_indicated_ttp(objTTP)
#objIndicator.add_indicated_ttp(TTP(idref=objTTP.id_))
#stix_package.add_ttp(objTTP)
stix_package.add_indicator(objIndicator)
objIndicator = None
### STIX Package Meta Data
stix_header = STIXHeader()
stix_header.title = srcObj.pkgTitle
stix_header.description = "<![CDATA[" + srcObj.pkgDscrpt + "]]>"
### Understanding markings http://stixproject.github.io/idioms/features/data-markings/
marking_specification = MarkingSpecification()
classLevel = SimpleMarkingStructure()
classLevel.statement = "Unclassified (Public)"
marking_specification.marking_structures.append(classLevel)
objTOU = TermsOfUseMarkingStructure()
sTOU = open('tou.txt').read()
objTOU.terms_of_use = sProducer + " | " + sTOU
marking_specification.marking_structures.append(objTOU)
tlp = TLPMarkingStructure()
tlp.color = "WHITE"
marking_specification.marking_structures.append(tlp)
marking_specification.controlled_structure = "//node()"
handling = Marking()
handling.add_marking(marking_specification)
stix_header.handling = handling
stix_package.stix_header = stix_header
stix_header = None
### Generate STIX XML File
locSTIXFile = 'STIX_' + srcObj.fileName.split('.')[0] + '.xml'
sndFile(stix_package.to_xml(), locSTIXFile)
return (stix_package)
def create_object(self, ce1sus_object, event_permissions, user):
definition_name = ce1sus_object.definition.name
obj = Object()
identifier = 'ce1sus:Object-{0}'.format(ce1sus_object.uuid)
obj.id_ = identifier
# try to find automatic the object container
try:
clazz = get_class(
'cybox.objects.{0}_object'.format(definition_name.lower()),
definition_name)
instance = clazz()
if definition_name == 'Disk':
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
except ImportError:
if definition_name == 'WinEventLog':
instance = WinEventLog()
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
elif definition_name == 'UserAccount':
instance = UserAccount()
elif definition_name == 'WinService':
instance = WinService()
elif definition_name == 'WindowsRegistryKey':
instance = WinRegistryKey()
elif definition_name == 'NetworkConnection':
instance = NetworkConnection()
elif definition_name == 'WinVolume':
instance = WinVolume()
# TODO: check why this must be set stix bug?
setattr(instance, 'drive_type', None)
elif definition_name == 'WinKernelHook':
instance = WinKernelHook()
elif definition_name == 'WinDriver':
instance = WinDriver()
elif definition_name == 'DomainName':
instance = DomainName()
# TODO: try to map it manually
elif definition_name == 'email':
instance = EmailMessage()
else:
raise Ce1susStixMapperException(
'Required to map manually {0}'.format(definition_name))
obj.properties = instance
attributes = ce1sus_object.get_attributes_for_permissions(
event_permissions, user)
for attribute in attributes:
self.map_attribtue(instance, attribute)
rel_objects = ce1sus_object.get_related_objects_for_permissions(
event_permissions, user)
for rel_object in rel_objects:
ob = self.create_object(rel_object.object, event_permissions, user)
if ob:
rel_obj = self.create_related_object(rel_object,
event_permissions, user)
# cybox_rel_object = RelatedObject(properties=ob.properties, relationship=rel_object.relation)
obj.related_objects.append(rel_obj)
return obj
def make_cybox_object(type_, value=None):
"""
Converts type_, name, and value to a CybOX object instance.
:param type_: The object type.
:type type_: str
:param value: The object value.
:type value: str
:returns: CybOX object
"""
if type_ == IndicatorTypes.USER_ID:
acct = Account()
acct.description = value
return acct
elif type_ in IPTypes.values():
if type_ == IPTypes.IPV4_ADDRESS:
name = 'ipv4-addr'
elif type_ == IPTypes.IPV6_ADDRESS:
name = 'ipv6-addr'
elif type_ == IPTypes.IPV4_SUBNET:
name = 'ipv4-net'
elif type_ == IPTypes.IPV6_SUBNET:
name = 'ipv6-net'
return Address(category=name, address_value=value)
elif type_ == IndicatorTypes.API_KEY:
api = API()
api.description = value
return api
elif type_ == IndicatorTypes.DOMAIN:
obj = DomainName()
obj.value = value
return obj
elif type_ == IndicatorTypes.USER_AGENT:
obj = HTTPRequestHeaderFields()
obj.user_agent = value
return obj
elif type_ == IndicatorTypes.MUTEX:
m = Mutex()
m.named = True
m.name = String(value)
return m
elif type_ in (IndicatorTypes.SOURCE_PORT,
IndicatorTypes.DEST_PORT):
p = Port()
try:
p.port_value = PositiveInteger(value)
except ValueError: # XXX: Raise a better exception...
raise UnsupportedCybOXObjectTypeError(type_, name)
return p
elif type_ == IndicatorTypes.PROCESS_NAME:
p = Process()
p.name = String(value)
return p
elif type_ == IndicatorTypes.URI:
r = URI()
r.type_ = 'URL'
r.value = value
return r
elif type_ in (IndicatorTypes.REGISTRY_KEY,
IndicatorTypes.REG_KEY_CREATED,
IndicatorTypes.REG_KEY_DELETED,
IndicatorTypes.REG_KEY_ENUMERATED,
IndicatorTypes.REG_KEY_MONITORED,
IndicatorTypes.REG_KEY_OPENED):
obj = WinRegistryKey()
obj.key = value
return obj
"""
The following are types that are listed in the 'Indicator Type' box of
the 'New Indicator' dialog in CRITs. These types, unlike those handled
above, cannot be written to or read from CybOX at this point.
The reason for the type being omitted is written as a comment inline.
This can (and should) be revisited as new versions of CybOX are released.
NOTE: You will have to update the corresponding make_crits_object function
with handling for the reverse direction.
In the mean time, these types will raise unsupported errors.
"""
#elif type_ == "Device": # No CybOX API
#elif type_ == "DNS Cache": # No CybOX API
#elif type_ == "GUI": # revisit when CRITs supports width & height specification
#elif type_ == "HTTP Session": # No good mapping between CybOX/CRITs
#elif type_ == "Linux Package": # No CybOX API
#elif type_ == "Network Packet": # No good mapping between CybOX/CRITs
#elif type_ == "Network Route Entry": # No CybOX API
#elif type_ == "Network Route": # No CybOX API
#elif type_ == "Network Subnet": # No CybOX API
#elif type_ == "Semaphore": # No CybOX API
#elif type_ == "Socket": # No good mapping between CybOX/CRITs
#elif type_ == "UNIX File": # No CybOX API
#elif type_ == "UNIX Network Route Entry": # No CybOX API
#elif type_ == "UNIX Pipe": # No CybOX API
#elif type_ == "UNIX Process": # No CybOX API
#elif type_ == "UNIX User Account": # No CybOX API
#elif type_ == "UNIX Volume": # No CybOX API
#elif type_ == "User Session": # No CybOX API
#elif type_ == "Whois": # No good mapping between CybOX/CRITs
#elif type_ == "Win Computer Account": # No CybOX API
#elif type_ == "Win Critical Section": # No CybOX API
#elif type_ == "Win Executable File": # No good mapping between CybOX/CRITs
#elif type_ == "Win File": # No good mapping between CybOX/CRITs
#elif type_ == "Win Kernel": # No CybOX API
#elif type_ == "Win Mutex": # No good mapping between CybOX/CRITs
#elif type_ == "Win Network Route Entry": # No CybOX API
#elif type_ == "Win Pipe": # No good mapping between CybOX/CRITs
#elif type_ == "Win Prefetch": # No CybOX API
#elif type_ == "Win Semaphore": # No CybOX API
#elif type_ == "Win System Restore": # No CybOX API
#elif type_ == "Win Thread": # No good mapping between CybOX/CRITs
#elif type_ == "Win Waitable Timer": # No CybOX API
raise UnsupportedCybOXObjectTypeError(type_)
def main():
# get args
parser = argparse.ArgumentParser ( description = "Parse a given CSV from Shadowserver and output STIX XML to stdout"
, formatter_class=argparse.ArgumentDefaultsHelpFormatter )
parser.add_argument("--infile","-f", help="input CSV with bot data", default = "bots.csv")
args = parser.parse_args()
# setup stix document
stix_package = STIXPackage()
stix_header = STIXHeader()
stix_header.title = "Bot Server IP addresses"
stix_header.description = "IP addresses connecting to bot control servers at a given port"
stix_header.add_package_intent ("Indicators - Watchlist")
# add marking
mark = Marking()
markspec = MarkingSpecification()
markstruct = SimpleMarkingStructure()
markstruct.statement = "Usage of this information, including integration into security mechanisms implies agreement with the Shadowserver Terms of Service available at https://www.shadowserver.org/wiki/pmwiki.php/Shadowserver/TermsOfService"
markspec.marking_structures.append(markstruct)
mark.add_marking(markspec)
stix_header.handling = mark
# include author info
stix_header.information_source = InformationSource()
stix_header.information_source.time = Time()
stix_header.information_source.time.produced_time =datetime.now(tzutc())
stix_header.information_source.tools = ToolInformationList()
stix_header.information_source.tools.append("ShadowBotnetIP-STIXParser")
stix_header.information_source.identity = Identity()
stix_header.information_source.identity.name = "MITRE STIX Team"
stix_header.information_source.add_role(VocabString("Format Transformer"))
src = InformationSource()
src.description = "https://www.shadowserver.org/wiki/pmwiki.php/Services/Botnet-CCIP"
srcident = Identity()
srcident.name = "shadowserver.org"
src.identity = srcident
src.add_role(VocabString("Originating Publisher"))
stix_header.information_source.add_contributing_source(src)
stix_package.stix_header = stix_header
# add TTP for overall indicators
bot_ttp = TTP()
bot_ttp.title = 'Botnet C2'
bot_ttp.resources = Resource()
bot_ttp.resources.infrastructure = Infrastructure()
bot_ttp.resources.infrastructure.title = 'Botnet C2'
stix_package.add_ttp(bot_ttp)
# read input data
fd = open (args.infile, "rb")
infile = csv.DictReader(fd)
for row in infile:
# split indicators out, may be 1..n with positional storage, same port and channel, inconsistent delims
domain = row['Domain'].split()
country = row['Country'].split()
region = row['Region'].split('|')
state = row['State'].split('|')
asn = row['ASN'].split()
asname = row['AS Name'].split()
asdesc = row['AS Description'].split('|')
index = 0
for ip in row['IP Address'].split():
indicator = Indicator()
indicator.title = "IP indicator for " + row['Channel']
indicator.description = "Bot connecting to control server"
# point to overall TTP
indicator.add_indicated_ttp(TTP(idref=bot_ttp.id_))
# add our IP and port
sock = SocketAddress()
sock.ip_address = ip
# add sighting
sight = Sighting()
sight.timestamp = ""
obs = Observable(item=sock.ip_address)
obsref = Observable(idref=obs.id_)
sight.related_observables.append(obsref)
indicator.sightings.append(sight)
stix_package.add_observable(obs)
# add pattern for indicator
sock_pattern = SocketAddress()
sock_pattern.ip_address = ip
port = Port()
port.port_value = row['Port']
sock_pattern.port = port
sock_pattern.ip_address.condition= "Equals"
sock_pattern.port.port_value.condition= "Equals"
indicator.add_object(sock_pattern)
stix_package.add_indicator(indicator)
# add domain
domain_obj = DomainName()
domain_obj.value = domain[index]
domain_obj.add_related(sock.ip_address,"Resolved_To", inline=False)
stix_package.add_observable(domain_obj)
# add whois obs
whois_obj = WhoisEntry()
registrar = WhoisRegistrar()
registrar.name = asname[index]
registrar.address = state[index] + region[index] + country[index]
whois_obj.registrar_info = registrar
whois_obj.add_related(sock.ip_address,"Characterizes", inline=False)
stix_package.add_observable(whois_obj)
# add ASN obj
asn_obj = AutonomousSystem()
asn_obj.name = asname[index]
asn_obj.number = asn[index]
asn_obj.handle = "AS" + str(asn[index])
asn_obj.add_related(sock.ip_address,"Contains", inline=False)
stix_package.add_observable(asn_obj)
# iterate
index = index + 1
print stix_package.to_xml()
