def file_to_stix(file_):
'''transform files into stix packages'''
try:
stix_package = STIXPackage.from_xml(file_)
except UnsupportedVersionError as ex:
updated = ramrod.update(file_)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
return stix_package
def file_to_stix(file_):
'''transform files into stix packages'''
try:
stix_package = STIXPackage.from_xml(file_)
except UnsupportedVersionError as ex:
updated = ramrod.update(file_)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
return stix_package
def __init__(self, source_item):
self.source_item = source_item
try:
self.stix_package = STIXPackage.from_xml(self.io())
except UnsupportedVersionError:
updated = ramrod.update(self.io(), to_='1.1.1')
document = updated.document.as_stringio()
self.stix_package = STIXPackage.from_xml(document)
except Exception:
logging.error('error parsing STIX package (%s)', self.file_name())
self.stix_package = None
def taxii_content_block_to_stix(content_block):
'''transform taxii content blocks into stix packages'''
xml = StringIO(content_block.content)
try:
stix_package = STIXPackage.from_xml(xml)
except UnsupportedVersionError as ex:
updated = ramrod.update(xml)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
xml.close()
return stix_package
def taxii_content_block_to_stix(content_block):
'''transform taxii content blocks into stix packages'''
xml = StringIO(content_block.content)
try:
stix_package = STIXPackage.from_xml(xml)
except UnsupportedVersionError as ex:
updated = ramrod.update(xml)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
xml.close()
return stix_package
def load_stix_package(self, stix_file):
"""Helper for loading and updating (if required) a STIX package."""
try:
package = STIXPackage.from_xml(stix_file)
except UnsupportedVersionError:
updated = ramrod.update(stix_file, to_='1.1.1')
document = updated.document.as_stringio()
try:
package = STIXPackage.from_xml(document)
except Exception:
package = None
except Exception:
package = None
return package
def __init__(self, source_item):
self.source_item = source_item
with warnings.catch_warnings():
warnings.filterwarnings("ignore","The use of this field has been deprecated",UserWarning)
try:
self.stix_package = STIXPackage.from_xml(self.io())
except UnsupportedVersionError:
updated = ramrod.update(self.io(), to_=LATEST_STIX_VERSION)
document = updated.document.as_stringio()
self.stix_package = STIXPackage.from_xml(document)
except Exception:
logging.error('error parsing STIX package (%s)', self.file_name())
self.stix_package = None
self.stix_version = self.stix_package.version
def get_domain_from_XML(XMLData, hostURL):
#Conevert XML to STIX Object -> dictionary
stixPackage = STIXPackage.from_xml(etree.fromstring(XMLData))
stixDict = stixPackage.to_dict()
domains = []
#Check type of XML
acceptableDataType = ['URIObjectType', 'DomainNameObjectType']
#Loop through every Indicator
for indicator in stixDict['indicators']:
dataType = indicator['observable']['object']['properties'][
'xsi:type']
#Check data can be converted to domain or not?
if (dataType in acceptableDataType):
rawDomain = indicator['observable']['object']['properties'][
'value']
# eti.eset.com has domain nested in another tag
if (hostURL == 'eti.eset.com'):
rawDomain = rawDomain['value']
domain = Ultility.get_converted_domain(rawDomain)
if domain is not None:
domains.append(domain)
#Remove Duplicated Domains
domains = list(dict.fromkeys(domains))
return domains
def process(self):
defaults = self._defaults()
try:
feed = STIXPackage.from_xml(self.rule.remote)
except Exception as e:
self.logger.error('Error parsing feed: {}'.format(e))
self.logger.error(defaults['remote'])
raise e
d = feed.to_dict()
header = d['stix_header']
for e in d.get('indicators'):
if not e['observable']:
continue
i = copy.deepcopy(defaults)
i['description'] = e['title'].lower()
i['lasttime'] = e['timestamp']
i['indicator'] = e['observable']['object']['properties']['value'][
'value'].lower()
i['tlp'] = header['handling'][0]['marking_structures'][1][
'color'].lower()
if not i.get('indicator'):
continue
if self.itype:
if resolve_itype(i['indicator']) != self.itype:
continue
yield i
def main():
stix_package = STIXPackage.from_xml('sample-incidents.xml')
data = {
'incidents': {
}
}
ttps = {}
for ttp in stix_package.ttps:
ttps[ttp.id_] = ttp
data['incidents'][ttp.title] = []
observables = {}
for observable in stix_package.observables.observables:
observables[observable.id_] = observable
for incident in stix_package.incidents:
ip = observables[incident.related_observables[0].item.idref].object_.properties.address_value.value
ttp = ttps[incident.leveraged_ttps[0].item.idref]
time = incident.time.first_malicious_action.value.isoformat()
data['incidents'][ttp.title].append({
'ip': ip,
'time': time
})
print data
def main():
stix_package = STIXPackage.from_xml("file-hash-reputation.xml")
for indicator in stix_package.indicators:
print "Hash: " + indicator.observable.object_.properties.hashes[0].simple_hash_value.value
print "Reputation: " + indicator.confidence.value.value
print "TTP: " + indicator.indicated_ttps[0].item.title
def strip_observables(pkg_path):
'''Strips observable from a package, support multiple structures'''
result = Observables()
pkg = STIXPackage.from_xml(pkg_path)
processed = []
for ind in pkg.indicators:
if ind.composite_indicator_expression:
"""The indicator is a compsite structure, this references other indicators, which reference the observables..."""
cyboxobject = ObservableComposition()
cyboxobject.operator = str(ind.observable_composition_operator)
for x in ind.composite_indicator_expression:
"""For every indicator in the composite list, get referenced indicator"""
ref_ind = getindicator_by_id(pkg, str(x._idref))
if ref_ind.observables:
for y in ref_ind.observables:
"""For every referenced observable, get the object"""
ref_obs = getobservable_by_id(pkg, y._idref)
if ref_obs:
cyboxobject.add(ref_obs)
processed.append(ref_obs.id_)
result.add(cyboxobject)
if ind.observables:
for x in ind.observables:
if x is not None:
if x.id_ not in processed:
result.add(x)
processed.append(x.id_)
if pkg.observables:
for x in pkg.observables:
if x is not None:
if x.id_ not in processed:
result.add(x)
scanfile = open(os.path.join(iocname,"scan.json"),'w')
scanfile.write(json.dumps(walkobservables(result).to_dict(), indent=4))
scanfile.close()
def main():
poll_response = 'file-hash-rep-poll-response.xml'
f = open(poll_response, 'r')
msg = tm11.get_message_from_xml(f.read())
requested_hash = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
requested_hash_type = 'MD5'
#Iterate over the content blocks
for content_block in msg.content_blocks:
if content_block.content_binding.binding_id != CB_STIX_XML_111:
raise ValueError('Something other than STIX 1.1.1 was attempted!')
# Deserialize the STIX_Package
stix_package = STIXPackage.from_xml(StringIO(content_block.content))
indicator = get_first_matching_indicator(stix_package, requested_hash,
requested_hash_type)
indicated_ttp = get_first_parseable_indicated_ttp(indicator)
confidence = indicated_ttp.confidence.value
ttp = get_indicated_ttp(stix_package, indicated_ttp.id_)
if ttp.title != 'Malicious File':
raise ValueError('Don\'t know how to handle that TTP')
if confidence in ('High', 'Medium'):
print "DO NOT OPEN THE FILE"
elif confidence in ('Low', 'Unknown'):
print "THINK TWICE ABOUT OPENING THE FILE"
elif confidence in ('None', ):
print "Go ahead!"
else:
raise ValueError("Unknown confidence: %s!" % confidence)
def main():
stix_package = STIXPackage.from_xml('sample-combined.xml')
data = {
'incidents': {
}
}
ttps = {}
for ttp in stix_package.ttps:
ttps[ttp.id_] = ttp
data['incidents'][ttp.title] = []
observables = {}
for observable in stix_package.observables.observables:
observables[observable.id_] = observable
indicators = {}
for indicator in stix_package.indicators:
indicators[indicator.id_] = indicator
for incident in stix_package.incidents:
ip = observables[incident.related_observables[0].item.idref].object_.properties.address_value.value
ttp = ttps[incident.leveraged_ttps[0].item.idref]
time = incident.time.first_malicious_action.value.isoformat()
address_value = indicators[incident.related_indicators[0].item.idref].observable.object_.properties.address_value
data['incidents'][ttp.title].append({
'ip': ip,
'time': time,
'pattern': "IP %(condition)s(%(ip)s)" % {'condition': address_value.condition, 'ip': address_value.value}
})
print(data)
def main():
stix_package = STIXPackage.from_xml('snort-test-mechanism.xml')
ttps = {}
for ttp in stix_package.ttps.ttps:
ttps[ttp.id_] = ttp
ets = {}
for et in stix_package.exploit_targets:
ets[et.id_] = et
for indicator in stix_package.indicators:
print "== INDICATOR =="
print "Title: " + indicator.title
print "Confidence: " + indicator.confidence.value.value
for indicated_ttp in indicator.indicated_ttps:
ttp = ttps[indicated_ttp.item.idref] # Resolve the TTP by idref
et = ets[ttp.exploit_targets[0].item.idref] # Resolve the ET by idref
print "Indicated TTP: " + ttp.title + " (" + et.vulnerabilities[0].cve_id + ")"
for tm in indicator.test_mechanisms:
print "Producer: " + tm.producer.identity.name
print "Efficacy: " + tm.efficacy.value.value
for rule in tm.rules:
print "Rule: " + rule.value
def pre_import_stix(file, cluster=None):
from stix.core import STIXPackage
pkg = STIXPackage()
pkg = pkg.from_xml(file)
reports = pkg.reports
header = None
timestamp = ""
try:
header = reports[0].header
timestamp = reports[0].timestamp
except:
header = pkg.header
# sc = header_to_subcluster(header)
sc = {"name": header.title, "description": header.description, "firstseen": timestamp}
"""
campaigns= pkg.campaigns
for campaign in campaigns:
s = campaign_to_subcluster(campaign)
if not s in sc:
sc.append(s)
"""
# ttp = pkg.ttps
obs = pkg.observables
if sc:
sc["node"] = []
sc = obs_to_node(obs, sc)
sc["cluster"] = cluster
return sc
def test_identity_from_xml(self):
obj = self.klass.from_dict(self._full_dict)
sp = STIXPackage()
sp.add(obj)
s = BytesIO(sp.to_xml())
pkg = STIXPackage.from_xml(s)
self.assertTrue("CIQIdentity3.0InstanceType" in text_type(pkg.to_xml()))
def main():
poll_response = 'file-hash-rep-poll-response.xml'
f = open(poll_response, 'r')
msg = tm11.get_message_from_xml(f.read())
requested_hash = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
requested_hash_type = 'MD5'
#Iterate over the content blocks
for content_block in msg.content_blocks:
if content_block.content_binding.binding_id != CB_STIX_XML_111:
raise ValueError('Something other than STIX 1.1.1 was attempted!')
# Deserialize the STIX_Package
stix_package = STIXPackage.from_xml(StringIO(content_block.content))
indicator = get_first_matching_indicator(stix_package, requested_hash, requested_hash_type)
indicated_ttp = get_first_parseable_indicated_ttp(indicator)
confidence = indicated_ttp.confidence.value
ttp = get_indicated_ttp(stix_package, indicated_ttp.id_)
if ttp.title != 'Malicious File':
raise ValueError('Don\'t know how to handle that TTP')
if confidence in ('High','Medium'):
print "DO NOT OPEN THE FILE"
elif confidence in ('Low', 'Unknown'):
print "THINK TWICE ABOUT OPENING THE FILE"
elif confidence in ('None', ):
print "Go ahead!"
else:
raise ValueError("Unknown confidence: %s!" % confidence)
def _parse_taxii_content(self, taxii_content):
indicators_to_add = {}
for stix_obj in taxii_content:
try:
stix_parsed = STIXPackage.from_xml(lxml_fromstring(stix_obj.content))
except Exception as e:
logger.error('Error parsing STIX object: {}'.format(e))
continue
try:
tmp = _parse_stix_package(stix_parsed)
except NotImplementedError as e:
if str(e).endswith("AISMarkingStructure'"):
import stix.extensions.marking.ais
tmp = _parse_stix_package(stix_parsed)
else:
raise
for obs_key, value in tmp.items():
if obs_key in indicators_to_add:
indicators_to_add[obs_key].update(value)
else:
indicators_to_add[obs_key] = value
for i_dict in indicators_to_add.values():
if i_dict.get('indicator'):
logger.debug('adding indicator {}'.format(i_dict['indicator']))
yield Indicator(**i_dict)
def test_duplicate_ns_prefix(self):
"""Test that duplicate namespace prefix mappings raise errors.
"""
p = STIXPackage()
bad = {'bad:ns': 'stix'} # 'stix' is already default ns prefix
self.assertRaises(
nsparser.DuplicatePrefixError,
p.to_xml,
ns_dict=bad
)
# Build a valid stix document that has a default namespace remapped
# to another namespace. We remap 'cybox' to a bogus ns here.
xml = (
"""<stix:STIX_Package
xmlns:cybox="THISISGONNABEPROBLEM"
xmlns:stix="http://stix.mitre.org/stix-1"
version="1.2"
timestamp="2015-04-09T14:22:25.620831+00:00"/>"""
)
sio = StringIO(xml)
p = STIXPackage.from_xml(sio)
# Exporting should raise an error.
self.assertRaises(
nsparser.DuplicatePrefixError,
p.to_xml
)
def pre_import_stix(file, cluster=None):
from stix.core import STIXPackage
pkg = STIXPackage()
pkg = pkg.from_xml(file)
reports = pkg.reports
header = None
timestamp = ""
try:
header = reports[0].header
timestamp = reports[0].timestamp
except:
header = pkg.header
#sc = header_to_subcluster(header)
sc = {
"name": header.title,
"description": header.description,
"firstseen": timestamp,
}
"""
campaigns= pkg.campaigns
for campaign in campaigns:
s = campaign_to_subcluster(campaign)
if not s in sc:
sc.append(s)
"""
#ttp = pkg.ttps
obs = pkg.observables
if sc:
sc["node"] = []
sc = obs_to_node(obs, sc)
sc["cluster"] = cluster
return sc
def test_parsed_namespaces(self):
"""Test that non-default namespaces make it through the parse-serialize
process.
"""
xml = ("""<stix:STIX_Package
xmlns:TEST="a:test"
xmlns:FOO="a:foo"
xmlns:BAR="a:bar"
xmlns:cybox="http://cybox.mitre.org/cybox-2"
xmlns:cyboxCommon="http://cybox.mitre.org/common-2"
xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2"
xmlns:example="http://example.com"
xmlns:stix="http://stix.mitre.org/stix-1"
xmlns:stixCommon="http://stix.mitre.org/common-1"
xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
id="example:Package-e2454ee8-e59c-43ac-a085-46ae4516fd6e"
version="1.2"
timestamp="2015-04-09T14:22:25.620831+00:00"/>""")
sio = StringIO(xml)
p = STIXPackage.from_xml(sio)
serialized = p.to_xml()
e = lxml.etree.XML(serialized)
self.assertEqual(e.nsmap.get('TEST'), 'a:test')
self.assertEqual(e.nsmap.get('FOO'), 'a:foo')
self.assertEqual(e.nsmap.get('BAR'), 'a:bar')
def test_duplicate_ns_prefix(self):
"""Test that duplicate namespace prefix mappings raise errors.
"""
p = STIXPackage()
bad = {'bad:ns': 'stix'} # 'stix' is already default ns prefix
self.assertRaises(mixbox.namespaces.DuplicatePrefixError,
p.to_xml,
ns_dict=bad)
# Build a valid stix document that has a default namespace remapped
# to another namespace. We remap 'stixCommon' to a bogus ns here.
xml = ("""<stix:STIX_Package
xmlns:stixCommon="THISISGONNABEPROBLEM"
xmlns:stix="http://stix.mitre.org/stix-1"
xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"
version="1.2"
timestamp="2015-04-09T14:22:25.620831+00:00">
<stix:STIX_Header>
<stix:Description>A unit test</stix:Description>
</stix:STIX_Header>
</stix:STIX_Package>""")
sio = StringIO(xml)
p = STIXPackage.from_xml(sio)
# Exporting should raise an error.
self.assertRaises(mixbox.namespaces.DuplicatePrefixError, p.to_xml)
def build_report(fname):
""" parse the provided STIX package and create a
CB Feed Report that includes all suitable observables
as CB IOCs """
# The python STIX libs are pedantic about document versions. See
# https://github.com/STIXProject/python-stix/issues/124
# parser = EntityParser()
# pkg = parser.parse_xml(fname, check_version=False)
pkg = STIXPackage.from_xml(fname)
iocs = {}
if pkg.observables:
iocs = parse_observables(pkg.observables.observables)
if pkg.indicators:
for indicator in pkg.indicators:
iocs = merge(iocs, parse_observables(indicator.observables))
ts = int(time.mktime(pkg.timestamp.timetuple())) if pkg.timestamp else int(time.mktime(time.gmtime()))
fields = {'iocs': iocs,
'score': 100, # does STIX have a severity field?
'timestamp': ts,
'link': 'http://stix.mitre.org',
'id': pkg.id_,
'title': pkg.stix_header.title,
}
if len(iocs.keys()) == 0 or all(len(iocs[k]) == 0 for k in iocs):
print("-> No suitable observables found in {0}; skipping.".format(fname))
return None
print("-> Including %s observables from {0}.".format(sum(len(iocs[k]) for k in iocs), fname))
return CbReport(**fields)
def loadEvent(self, args, pathname):
try:
filename = '{}/tmp/{}'.format(pathname, args[1])
event = STIXPackage.from_xml(filename)
if "CIRCL:Package" in event.id_ and "CIRCL MISP" in event.stix_header.title:
fromMISP = True
else:
fromMISP = False
if fromMISP:
self.event = event.related_packages.related_package[
0].item.incidents[0]
else:
self.event = event
self.fromMISP = fromMISP
self.filename = filename
self.load_mapping()
except:
print(
json.dumps({
'success':
0,
'message':
'The temporary STIX export file could not be read'
}))
sys.exit(0)
def loadEvent(args, pathname):
try:
filename = '{}/tmp/{}'.format(pathname, args[1])
tempFile = open(filename, 'r')
fromMISP = True
stixJson = True
try:
event = json.loads(tempFile.read())
isJson = True
except:
event = STIXPackage.from_xml(filename)
try:
event = json.loads(event.to_json())
except:
stixJson = False
if args[1].startswith('misp.'):
event = event['related_packages']['related_packages'][0]
else:
fromMISP = False
isJson = False
return event, isJson, fromMISP, stixJson
except:
print(
json.dumps({
'success':
0,
'message':
'The temporary STIX export file could not be read'
}))
sys.exit(0)
def poll_feed(settings,subscription):
""" polls a TAXII feed"""
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(True)
client.set_auth_credentials({'username': settings['username'], 'password': settings['password']})
msg_id=uuid.uuid4().hex
poll_request1 = tm11.PollRequest(message_id=msg_id,collection_name=settings['subscriptions'][subscription]['collection_name'],subscription_id=settings['subscriptions'][subscription]['subscription_id'])
poll_xml=poll_request1.to_xml()
http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/', VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(http_resp, poll_request1.message_id)
observables={}
indicators = json.loads(taxii_message.to_json())
if 'content_blocks' in indicators.keys():
for indicator in indicators['content_blocks']:
open('/tmp/indicator.xml','w').write(indicator['content'])
indi=STIXPackage.from_xml('/tmp/indicator.xml').to_dict()
if 'observables' in indi.keys():
for obs in indi['observables']['observables']:
if 'object' in obs.keys():
ot=obs['object']['properties']['xsi:type']
if ot in settings['supported_objects'].keys() and not ot in observables.keys():
observables[ot]=[]
if ot in settings['supported_objects'].keys() and settings['supported_objects'][ot] in obs['object']['properties'].keys():
# note, you will only be able to process one property per object type, but you also know there's only one property you can process
try:
observables[ot].append(obs['object']['properties'][settings['supported_objects'][ot]])
except:
print "[-] you're dumb"
print supported_objects[ot], "not in:", obs['object']
return observables
def test_parsed_namespaces(self):
"""Test that non-default namespaces make it through the parse-serialize
process.
"""
xml = (
"""<stix:STIX_Package
xmlns:TEST="a:test"
xmlns:FOO="a:foo"
xmlns:BAR="a:bar"
xmlns:cybox="http://cybox.mitre.org/cybox-2"
xmlns:cyboxCommon="http://cybox.mitre.org/common-2"
xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2"
xmlns:example="http://example.com"
xmlns:stix="http://stix.mitre.org/stix-1"
xmlns:stixCommon="http://stix.mitre.org/common-1"
xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
id="example:Package-e2454ee8-e59c-43ac-a085-46ae4516fd6e"
version="1.2"
timestamp="2015-04-09T14:22:25.620831+00:00"/>"""
)
sio = StringIO(xml)
p = STIXPackage.from_xml(sio)
serialized = p.to_xml()
e = lxml.etree.XML(serialized)
self.assertEqual(e.nsmap.get('TEST'), 'a:test')
self.assertEqual(e.nsmap.get('FOO'), 'a:foo')
self.assertEqual(e.nsmap.get('BAR'), 'a:bar')
def main():
stix_package = STIXPackage.from_xml('file-hash-reputation.xml')
for indicator in stix_package.indicators:
print "Hash: " + indicator.observable.object_.properties.hashes[
0].simple_hash_value.value
print "Reputation: " + indicator.confidence.value.value
print "TTP: " + indicator.indicated_ttps[0].item.title
def test():
# Process a XML file on disk
stix_package = STIXPackage.from_xml(sys.argv[1])
stixParser = FSISAC_STIX_Parser()
iocs = stixParser.process_stix_dict(stix_package.to_dict())
j = stixParser.convert_to_json(iocs)
print j
def main():
stix_package = STIXPackage.from_xml('indicator-for-c2-ip-address.xml')
for indicator in stix_package.indicators:
print "--INDICATOR--"
ip = indicator.observable.object_.properties.address_value.value
print "IP: " + ip
for ttp in stix_package.ttps:
print "TTP: " + ttp.title
def main():
stix_package = STIXPackage.from_xml('indicator-for-c2-ip-address.xml')
for indicator in stix_package.indicators:
print "--INDICATOR--"
ip = indicator.observable.object_.properties.address_value.value
print "IP: " + ip
for ttp in stix_package.ttps:
print "TTP: " + ttp.title
def _main(first=True):
if first:
begin = datetime.datetime.now(tzutc()) - timedelta(days=int(days))
else:
try:
begin = _readTimestamp()
except IOError:
print("[-] No timestamp file found have you 'first_run'?")
sys.exit(0)
end = datetime.datetime.now(tzutc())
poll_params1 = tm11.PollParameters(
allow_asynch=False,
response_type=RT_COUNT_ONLY,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)],
)
try:
poll_req3 = tm11.PollRequest(
message_id='PollReq03',
collection_name=collection,
poll_parameters=poll_params1,
exclusive_begin_timestamp_label=begin,
inclusive_end_timestamp_label=end,
)
except ValueError:
print("[-] Invalid timestamp file")
sys.exit(0)
except Exception:
print("[-] Error with PollRequest")
poll_xml = poll_req3.to_xml()
http_resp = client.call_taxii_service2(
server, path, VID_TAXII_XML_11,
poll_xml, port=port)
taxii_message = t.get_message_from_http_response(
http_resp, poll_req3.message_id)
if taxii_message.message_type == MSG_POLL_RESPONSE:
if taxii_message.content_blocks:
try:
for content in taxii_message.content_blocks:
package_io = StringIO(content.content)
pkg = STIXPackage.from_xml(package_io)
title = pkg.id_.split(':', 1)[-1]
with open(title + ".xml", "w") as text_file:
text_file.write(content.content)
print("[+] Successfully generated " + title)
except Exception:
print("[-] Error with TAXII response")
else:
print("[+] No content returned")
_saveTimestamp(str(end))
else:
print("[-] Error with TAXII response")
def parse_stix_file(self, filename):
stix_package = STIXPackage.from_xml(filename)
stixParser = FSISAC_STIX_Parser()
iocs = stixParser.process_stix_dict(stix_package.to_dict())
j = stixParser.convert_to_json(iocs)
return j
def main():
fn = 'ex_01.xml'
stix_package = STIXPackage.from_xml(fn)
stix_dict = stix_package.to_dict() # parse to dictionary
pprint(stix_dict)
stix_package_two = STIXPackage.from_dict(stix_dict) # create python-stix object from dictionary
xml = stix_package_two.to_xml() # generate xml from python-stix object
print(xml)
def main():
for fn in FILENAMES:
print("FILENAME:", fn)
package = STIXPackage.from_xml(fn)
print(package.to_xml())
divider()
pprint.pprint(package.to_dict())
divider()
def main():
fn = 'ex_01.xml'
stix_package = STIXPackage.from_xml(fn)
stix_dict = stix_package.to_dict() # parse to dictionary
pprint(stix_dict)
stix_package_two = STIXPackage.from_dict(
stix_dict) # create python-stix object from dictionary
xml = stix_package_two.to_xml() # generate xml from python-stix object
print(xml)
def load_stix(stix):
# Just save the pain and load it if the first character is a <
if sys.version_info < (3, 5):
json_exception = ValueError
else:
json_exception = json.JSONDecodeError
if isinstance(stix, STIXPackage):
# Oh cool we're ok
# Who tried to load this? Honestly.
return stix
elif hasattr(stix, 'read'):
# It's a file!
# But somehow, sometimes, reading it returns a bytes stream and the loader dies on python 3.4.
# Luckily, STIXPackage.from_json (which is mixbox.Entity.from_json) will happily load a string.
# So we're going to play dirty.
data = stix.read()
if isinstance(data, bytes):
data = data.decode()
try:
# Try loading from JSON
stix_package = STIXPackage.from_json(data)
except json_exception:
# Ok then try loading from XML
# Loop zoop
stix.seek(0)
try:
stix_package = STIXPackage.from_xml(stix)
except Exception as ex:
# No joy. Quit.
raise STIXLoadError("Could not load stix file. {}".format(ex))
return stix_package
elif isinstance(stix, str):
# It's text, we'll need to use a temporary file
# Create a temporary file to load from
# Y'know I should probably give it a max size before jumping to disk
# idk, 10MB? Sounds reasonable.
f = SpooledTemporaryFile(max_size=10 * 1024)
# O I have idea for sneak
# Will be very sneak
# Write the (probably) XML to file
f.write(stix.encode("utf-8"))
# Reset the file so we can read from it
f.seek(0)
# AHA SNEAK DIDN'T EXPECT RECURSION DID YOU
return load_stix(f)
def __check_info_source(self, datafile):
"""
Parses the STIX Header element looking for an Information source element matching the configured value.
Currently only matches to STIX Header, long-term could check every Information Source element.
:param datafile: The file sent to be parsed, expected to be STIX XML format.
:return Boolean: True if match is found, otherwise False.
"""
ret = False
try:
# TODO: Parse only header or look for all Information_Source elements?
# attempt to map as a STIX Package, if fails force an update with ramrod
try:
stix_package = STIXPackage.from_xml(datafile)
except Exception as e:
self._logger.warning("Unable to parse XML to STIX, attempting version change Exception={0}".format(e))
updated = ramrod.update(datafile, force=True)
stix_package = STIXPackage.from_xml(updated)
# Compare the information source from the stix_header
info_src = stix_package.stix_header.information_source
if info_src.identity:
if info_src.identity.name in self._sources:
return True
# If there are contributing sources, parse and compare to source list
if info_src.contributing_sources:
if info_src.contributing_sources._inner_name == 'sources':
for x in info_src.contributing_sources._inner:
if type(x) is information_source.InformationSource:
if x.identity.name in self._sources:
return True
except Exception as e:
self._logger.warning(
"Unable to force XML to STIX with ramrod. Unable to check source Exception={0}".format(e))
return ret
if ret is False:
self._logger.info("Did not identify the requested source in STIX Header.")
return ret
def main():
stix_package = STIXPackage.from_xml('yara-test-mechanism.xml')
for indicator in stix_package.indicators:
print("== INDICATOR ==")
print("Title: " + indicator.title)
print("Description: " + indicator.description.value)
for tm in indicator.test_mechanisms:
print("Producer: " + tm.producer.identity.name)
print("Rule: %s" % tm.rule)
def run_stix(file_name):
# with open(file_name) as fin:
# e = etree.fromstring(fin.read())
# print(type(e))
# gives error with stix 1.1.1.12 when file with 1 block is provided
stix_package = STIXPackage.from_xml(file_name)
dict_stix_package = stix_package.to_dict()
for k, v in dict_stix_package.items():
print('%s\n%s' % (k, json.dumps(v, indent=2)))
def upload_xml(self, *vpath, **params):
user = None
try:
user = self.user_controller.get_user_by_username(
self.get_user().username)
except ControllerException as error:
self.logger.error(error)
raise HTTPError(400, error.message)
input_json = self.get_json()
filename = input_json['name']
self.logger.info(
'Starting to import xml form file {0}'.format(filename))
data = input_json['data']['data']
complete = params.get('complete', False)
inflated = params.get('inflated', False)
open_ioc_xml_string = base64.b64decode(data)
base_dir, stix_file_path = self.__make_stix_xml_string(
filename, open_ioc_xml_string)
stix_package = STIXPackage.from_xml(stix_file_path)
if not self.dump:
rmtree(base_dir)
try:
event = self.stix_mapper.map_stix_package(stix_package, user)
self.event_controller.insert_event(user, event)
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(
event.to_dict(complete, inflated, event_permissions, user))
except (ControllerIntegrityException, BrokerException) as error:
# TODO: merge
event = self.stix_mapper.map_stix_package(stix_package, user,
False, True)
local_event = self.event_controller.get_event_by_uuid(event.uuid)
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
merged_event = self.merger.merge_event(local_event, event, user,
event_permissions)
self.event_controller.update_event(user, merged_event, True, True)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(
merged_event.to_dict(complete, inflated, event_permissions,
user))
except (ControllerException, BrokerException) as error:
self.logger.error(error)
raise HTTPError(400, '0'.format(error.message))
def main():
stix_package = STIXPackage.from_xml('yara-test-mechanism.xml')
for indicator in stix_package.indicators:
print "== INDICATOR =="
print "Title: " + indicator.title
print "Description: " + indicator.description.value
for tm in indicator.test_mechanisms:
print "Producer: " + tm.producer.identity.name
for rule in tm.rules:
print "Rule: " + rule.value
def load_event(self, filename):
try:
return STIXPackage.from_xml(filename)
except Exception as ns_error:
if ns_error.__str__().startswith('Namespace not found:'):
ns_value = ns_error.ns_uri
prefix = ns_value.split('/')[-1]
ns = mixbox_ns.Namespace(ns_value, prefix, '')
mixbox_ns.register_namespace(ns)
return self.load_event(filename)
else:
return None
def parse_stix(self, reference=None, make_event=False, source=''):
"""
Parse the document.
:param reference: The reference to the data.
:type reference: str
:param make_event: Whether or not to create an Event for this document.
:type make_event: bool
:param source: The source of this document.
:type source: str
:raises: :class:`crits.standards.parsers.STIXParserException`
Until we have a way to map source strings in a STIX document to
a source in CRITs, we are being safe and using the source provided
as the true source.
"""
f = StringIO(self.data)
self.package = STIXPackage.from_xml(f)
f.close()
if not self.package:
raise STIXParserException("STIX package failure")
stix_header = self.package.stix_header
if stix_header and stix_header.information_source and stix_header.information_source.identity:
self.information_source = stix_header.information_source.identity.name
if self.information_source:
info_src = "STIX Source: %s" % self.information_source
if not reference:
reference = ''
else:
reference += ", "
reference += info_src
if does_source_exist(source):
self.source.name = source
elif does_source_exist(self.information_source):
self.source.name = self.information_source
else:
raise STIXParserException("No source to attribute data to.")
self.source_instance.reference = reference
self.source.instances.append(self.source_instance)
if make_event:
event = Event.from_stix(stix_package=self.package,
source=[self.source])
try:
event.save(username=self.source_instance.analyst)
self.imported.append((Event._meta['crits_type'], event))
except Exception, e:
self.failed.append(
(e.message, type(event).__name__, event.id_))
def _get_stix_package(ioc_xml):
if ioc_xml is not None and len(ioc_xml) > 0:
ns = namespaces.Namespace("http://openioc.org/openioc", "openioc", "")
idgen.set_id_namespace(ns)
stix_obj = to_stix(BytesIO(ioc_xml))
stix_package = STIXPackage_v1.from_xml(etree.fromstring(stix_obj.to_xml()))
return stix_package
else:
raise RuntimeError('request body is empty.')
def parse_stix(self, reference=None, make_event=False, source=''):
"""
Parse the document.
:param reference: The reference to the data.
:type reference: str
:param make_event: Whether or not to create an Event for this document.
:type make_event: bool
:param source: The source of this document.
:type source: str
:raises: :class:`crits.standards.parsers.STIXParserException`
Until we have a way to map source strings in a STIX document to
a source in CRITs, we are being safe and using the source provided
as the true source.
"""
f = StringIO(self.data)
self.package = STIXPackage.from_xml(f)
f.close()
if not self.package:
raise STIXParserException("STIX package failure")
stix_header = self.package.stix_header
if stix_header and stix_header.information_source and stix_header.information_source.identity:
self.information_source = stix_header.information_source.identity.name
if self.information_source:
info_src = "STIX Source: %s" % self.information_source
if not reference:
reference = ''
else:
reference += ", "
reference += info_src
if does_source_exist(source):
self.source.name = source
elif does_source_exist(self.information_source):
self.source.name = self.information_source
else:
raise STIXParserException("No source to attribute data to.")
self.source_instance.reference = reference
self.source.instances.append(self.source_instance)
if make_event:
event = Event.from_stix(stix_package=self.package)
try:
event.add_source(self.source)
event.save(username=self.source_instance.analyst)
self.imported.append((Event._meta['crits_type'], event))
except Exception, e:
self.failed.append((e.message, type(event).__name__, event.id_))
def get_api_packages(self, packages):
"""Returns a list of python-stix api STIXPackage objects.
Keyword arguments:
packages -- a list of STIX_Package etree Element objects
"""
api_packages = []
for package in packages:
xml = etree.tostring(package)
bytesio = BytesIO(xml)
stix_package = STIXPackage.from_xml(bytesio)
api_packages.append(stix_package[0])
return api_packages
def test_utf16_roundtrip(self):
sh = STIXHeader()
sh.title = UNICODE_STR
sp = STIXPackage()
sp.stix_header = sh
# serialize as utf-16
xml16 = sp.to_xml(encoding="utf-16")
# deserialize as utf-16
sp2 = STIXPackage.from_xml(StringIO(xml16), encoding="utf-16")
sh2 = sp2.stix_header
# check that the titles align
self.assertEqual(sh.title, sh2.title)
def process(self):
defaults = self._defaults()
try:
feed = STIXPackage.from_xml(self.rule.remote)
except Exception as e:
self.logger.error('Error parsing feed: {}'.format(e))
self.logger.error(defaults['remote'])
raise e
d = feed.to_dict()
header = d['stix_header']
for e in d.get('indicators'):
if not e['observable']:
continue
i = copy.deepcopy(defaults)
i['description'] = e['title'].lower()
i['lasttime'] = e.get('timestamp')
try:
i['indicator'] = e['observable']['object']['properties']['value']['value']
except KeyError:
if e['observable']['object']['properties'].get('address_value'):
i['indicator'] = e['observable']['object']['properties']['address_value']['value']
if e['observable']['object']['properties'].get('hashes'):
i['indicator'] = e['observable']['object']['properties']['hashes'][0]['simple_hash_value']['value']
if e['observable']['object']['properties'].get('header'):
i['indicator'] = e['observable']['object']['properties']['header']['from']['address_value']['value']
try:
i['tlp'] = header['handling'][0]['marking_structures'][1]['color'].lower()
except KeyError:
i['tlp'] = header['handling'][0]['marking_structures'][0]['color']
i['indicator'] = i['indicator'].lower()
i['tlp'] = i['tlp'].lower()
if not i.get('indicator'):
continue
if self.itype:
if resolve_itype(i['indicator']) != self.itype:
continue
yield i
def process_taxii_content_blocks(content_block):
"""
Process taxii content blocks
"""
indicators = dict()
observables = dict()
xml = StringIO.StringIO(content_block.content)
stix_package = STIXPackage.from_xml(xml)
xml.close()
if stix_package.observables:
for o in stix_package.observables.observables:
observables[o.id_] = o
if stix_package.indicators:
for i in stix_package.indicators:
indicators[i.id_] = i
return indicators, observables
def main():
stix_package = STIXPackage.from_xml('openioc-test-mechanism.xml')
for indicator in stix_package.indicators:
print("== INDICATOR ==")
print("Title: " + indicator.title)
print("Description: " + indicator.description.value)
for indicated_ttp in indicator.indicated_ttps:
ttp = stix_package.find(indicated_ttp.item.idref)
print("Indicated TTP: " + ttp.title)
for tm in indicator.test_mechanisms:
print("Producer: " + tm.producer.identity.name)
print("== IOC ==")
print(etree.tostring(tm.ioc))
print("== ENDIOC ==")
def main():
fn = sys.argv[-1] if len(sys.argv) == 2 else 'ex_01.xml'
stix_package = STIXPackage.from_xml(fn)
indicator_count = 0
observable_count = 0
object_count = 0
for child in stix_package.walk():
if isinstance(child, Indicator):
indicator_count += 1
elif isinstance(child, Observable):
observable_count += 1
elif isinstance(child, Object):
object_count += 1
print "Indicators:", indicator_count
print "Observables:", observable_count
print "Objects:", object_count
def process_taxii_content_blocks(config, content_block):
'''process taxii content blocks'''
incidents = dict()
indicators = dict()
observables = dict()
xml = StringIO.StringIO(content_block.content)
stix_package = STIXPackage.from_xml(xml)
xml.close()
if stix_package.incidents:
for j in stix_package.incidents:
incidents[j.id_] = j
if stix_package.indicators:
for i in stix_package.indicators:
indicators[i.id_] = i
if stix_package.observables:
for o in stix_package.observables.observables:
observables[o.id_] = o
return(incidents, indicators, observables)
def main():
stix_package = STIXPackage.from_xml('sample-indicators.xml')
data = {
'indicators': {
}
}
ttps = {}
for ttp in stix_package.ttps:
ttps[ttp.id_] = ttp
data['indicators'][ttp.title] = []
for indicator in stix_package.indicators:
ip = indicator.observable.object_.properties.address_value.value
ttp = ttps[indicator.indicated_ttps[0].item.idref]
data['indicators'][ttp.title].append(ip)
print data
def test(files):
'''Parses each file in the list of files and performs a to_obj(), from_obj()
to_dict(), from_dict(), and to_xml() on each STIXPackage
'''
info("testing [%s] files" % (len(files)))
for fn in files:
with open(fn, 'rb') as f:
try:
sp = STIXPackage.from_xml(f)
o = sp.to_obj()
sp2 = STIXPackage.from_obj(o)
d = sp.to_dict()
sp3 = STIXPackage.from_dict(d)
xml = sp.to_xml()
print "[+] Sucessfully tested %s" % fn
except Exception as ex:
tb = traceback.format_exc()
print "[!] Error with %s : %s" % (fn, str(ex))
print tb
