def test_make_i_node(self):
for op in self.operators:
inode = ioc_api.make_indicator_node(op)
self.assertEqual(inode.get('operator'), op.upper())
self.assertNotEqual(inode.get('id'), self.nid)
inode = ioc_api.make_indicator_node(op, nid=self.nid)
self.assertEqual(inode.get('operator'), op.upper())
self.assertEqual(inode.get('id'), self.nid)
def test_make_i_node(self):
for op in self.operators:
inode = ioc_api.make_indicator_node(op)
self.assertEqual(inode.get('operator'), op.upper())
self.assertNotEqual(inode.get('id'), self.nid)
inode = ioc_api.make_indicator_node(op, nid=self.nid)
self.assertEqual(inode.get('operator'), op.upper())
self.assertEqual(inode.get('id'), self.nid)
def addStrings(xmldoc, parentnode, strings):
# This simply adds an AND block of the strings found
if len(strings) > 0:
stringsind = ioc_api.make_indicator_node("AND")
for string in strings:
stringsinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/StringList/string", content=string, content_type="string")
stringsind.append(stringsinditem)
parentnode.append(stringsind)
else:
return
def convert_branch(self, old_node, new_node, comment_dict={}):
"""
convert_branch
recursively walk a indicator logic tree, starting from a Indicator node.
converts OpenIOC 1.0 Indicator/IndicatorItems to Openioc 1.1 and preserves order.
input
old_node: old node, an Indicator node, which we walk down to convert
new_node: new node, an Indicator node, which we add new IndicatorItem and Indicator nodes too
comment_dict: maps ids to comment values. only applied to IndicatorItem nodes
return
returns True upon completiong
may raise ValueError
"""
expected_tag = 'Indicator'
if old_node.tag != expected_tag:
raise ValueError('old_node expected tag is [%s]' % expected_tag)
for node in old_node.getchildren():
node_id = node.get('id')
if node.tag == 'IndicatorItem':
condition = node.get('condition')
negation = False
if condition.endswith('not'):
negation = True
condition = condition[:-3]
document = node.xpath('Context/@document')[0]
search = node.xpath('Context/@search')[0]
content_type = node.xpath('Content/@type')[0]
content = node.findtext('Content')
context_type = node.xpath('Context/@type')[0]
new_ii_node = ioc_api.make_indicatoritem_node(condition=condition,
document=document,
search=search,
content_type=content_type,
content=content,
context_type=context_type,
negate=negation,
nid=node_id)
# set comment
comment = node.find('Comment')
if comment is not None:
comment_dict[node_id] = comment.text
new_node.append(new_ii_node)
elif node.tag == 'Indicator':
operator = node.get('operator')
if operator.upper() not in ['OR', 'AND']:
raise IOCParseError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator))
new_i_node = ioc_api.make_indicator_node(operator, node_id)
new_node.append(new_i_node)
self.convert_branch(node, new_i_node, comment_dict)
else:
# should never get here
raise IOCParseError('node is not a Indicator/IndicatorItem')
return True
def convert_branch(self, old_node, new_node, comment_dict=None):
"""
recursively walk a indicator logic tree, starting from a Indicator node.
converts OpenIOC 1.0 Indicator/IndicatorItems to Openioc 1.1 and preserves order.
:param old_node: Indicator node, which we walk down to convert
:param new_node: Indicator node, which we add new IndicatorItem and Indicator nodes too
:param comment_dict: maps ids to comment values. only applied to IndicatorItem nodes
:return: True upon completion
:raises: UpgradeError if there is a problem during the conversion.
"""
expected_tag = 'Indicator'
if old_node.tag != expected_tag:
raise UpgradeError('old_node expected tag is [%s]' % expected_tag)
if not comment_dict:
comment_dict = {}
for node in old_node.getchildren():
node_id = node.get('id')
if node.tag == 'IndicatorItem':
condition = node.get('condition')
negation = False
if condition.endswith('not'):
negation = True
condition = condition[:-3]
document = node.xpath('Context/@document')[0]
search = node.xpath('Context/@search')[0]
content_type = node.xpath('Content/@type')[0]
content = node.findtext('Content')
context_type = node.xpath('Context/@type')[0]
new_ii_node = ioc_api.make_indicatoritem_node(condition=condition,
document=document,
search=search,
content_type=content_type,
content=content,
context_type=context_type,
negate=negation,
nid=node_id)
# set comment
comment = node.find('Comment')
if comment is not None:
comment_dict[node_id] = comment.text
new_node.append(new_ii_node)
elif node.tag == 'Indicator':
operator = node.get('operator')
if operator.upper() not in ['OR', 'AND']:
raise UpgradeError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator))
new_i_node = ioc_api.make_indicator_node(operator, node_id)
new_node.append(new_i_node)
self.convert_branch(node, new_i_node, comment_dict)
else:
# should never get here
raise UpgradeError('node is not a Indicator/IndicatorItem')
return True
def create_ioc_object(ioc_name, items, and_or=True):
ioc = ioc_api.IOC(name=ioc_name)
top_level_or_node = ioc.top_level_indicator
# build the definition
if and_or:
second_level_and_node = ioc_api.make_indicator_node('AND')
top_level_or_node.append(second_level_and_node)
for item in items:
condition, document, search, content_type, content = tuple(item)
# print condition, document, search, content_type, content
ii_node = ioc_api.make_indicatoritem_node(condition, document, search, content_type, content)
if and_or:
second_level_and_node.append(ii_node)
else:
top_level_or_node.append(ii_node)
# update the last modified time
ioc.set_lastmodified_date()
return ioc
def create_ioc_object(ioc_name, items, and_or=True):
ioc = ioc_api.IOC(name=ioc_name)
top_level_or_node = ioc.top_level_indicator
# build the definition
if and_or:
second_level_and_node = ioc_api.make_indicator_node('AND')
top_level_or_node.append(second_level_and_node)
for item in items:
condition, document, search, content_type, content = tuple(item)
# print condition, document, search, content_type, content
ii_node = ioc_api.make_indicatoritem_node(condition, document, search,
content_type, content)
if and_or:
second_level_and_node.append(ii_node)
else:
top_level_or_node.append(ii_node)
# update the last modified time
ioc.set_lastmodified_date()
return ioc
def test_schema_validation_from_api_fail(self):
schema = et.XMLSchema(et.parse(OPENIOC_11_SCHEMA))
ioc_obj = ioc_api.IOC(name=self.name,
description=self.description,
author=self.author,
links=self.links,
keywords=self.keywords,
iocid=self.iocid)
i_node = ioc_api.make_indicator_node('AND')
ioc_obj.top_level_indicator.append(i_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content=self.content_text,
context_type='mir')
ioc_obj.top_level_indicator.append(ii_node)
s = ioc_obj.write_ioc_to_string()
ioc_tree = et.fromstring(s)
# Ensure our output is valid after reserializing
with self.assertRaises(et.DocumentInvalid) as cm:
schema.assertValid(ioc_tree)
self.assertIn('This element is not expected', str(cm.exception))
def createMetaData(xmldoc, parentnode, metadata):
# load in the file name. It won't always be the actual file name
# because dionaea renames it with the md5 hash...but it might
# be the actual name
#
# As well, the items here generally won't change, so they are being
# put in an AND block
and_item = ioc_api.make_indicator_node('AND')
if metadata['malfilename'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/FileName", content=str(metadata['malfilename']), content_type="string")
and_item.append(inditem)
# file size
if metadata['malfilesize'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/SizeInBytes", content=str(metadata['malfilesize']), content_type="int")
and_item.append(inditem)
# file md5
if metadata['malmd5'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Md5Sum", content=metadata['malmd5'], content_type="md5")
and_item.append(inditem)
# md54k (http://www.md54k.org)
# md54k is not part of Mandiant's list of indicators
# so we are using our iocaware custom list (context_type="iocaware")
# Please see the iocware.iocterms file
if metadata['malmd54k'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Md54ksum", content=metadata['malmd54k'], content_type="md5", context_type="iocaware")
and_item.append(inditem)
if metadata['malsha1'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha1sum", content=metadata['malsha1'], content_type="sha1")
and_item.append(inditem)
if metadata['malsha256'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha256sum", content=metadata['malsha256'], content_type="sha256")
and_item.append(inditem)
# sha512 is not included in the list of OpenIOC indicators
# so the context_type="iocware" - please see the iocaware.iocterms file
if metadata["malsha512"] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha512sum", content=metadata['malsha512'], content_type="sha512", context_type="iocaware")
and_item.append(inditem)
# SSDeep also isn't included in the list of OpenIOC indicators
if metadata["malssdeep"] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Ssdeep", content=metadata["malssdeep"], content_type="ssdeep", context_type="iocaware")
and_item.append(inditem)
if metadata['malfiletype'] != "":
inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Type", content=metadata['malfiletype'], content_type="string")
and_item.append(inditem)
parentnode.append(and_item)
peinfoind = ioc_api.make_indicator_node("OR")
if len(metadata['iocimports']) > 0:
for importfunc in metadata['iocimports']:
importinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/ImportedModules/Module/ImportedFunctions/string", content=importfunc, content_type="string")
peinfoind.append(importinditem)
if len(metadata['iocexports']) > 0:
for exportfunc in metadata['iocexports']:
exportinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Exports/ExportedFunctions/string", content=exportfunc, content_type="string")
peinfoind.append(exportinditem)
if len(metadata['badpesections']) > 0:
for section in metadata['badpesections']:
sectionind = ioc_api.make_indicator_node("AND")
sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/Name", content=section[0], content_type="string")
sectionind.append(sectioninditem)
sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/SizeInBytes", content=str(section[1]), content_type="int")
sectionind.append(sectioninditem)
sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/Entropy/CurveData/float", content=str(section[2]), content_type="float")
sectionind.append(sectioninditem)
peinfoind.append(sectionind)
# Include any PE Version Information
if len(metadata['versioninfo']) > 0:
infoind = ioc_api.make_indicator_node("AND")
for infoitem in metadata['versioninfo']:
if metadata['versioninfo'][infoitem] != "" and metadata['versioninfo'][infoitem] is not None:
if "Version" in infoitem:
itemvalue = str(metadata['versioninfo'][infoitem]).replace(", ", ".")
else:
itemvalue = str(metadata['versioninfo'][infoitem])
infoitemsearch = "FileItem/PEInfo/VersionInfoItem/" + infoitem
infoinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search=infoitemsearch, content=str(itemvalue), content_type="string")
infoind.append(infoinditem)
peinfoind.append(infoind)
parentnode.append(peinfoind)
def createDynamicIndicators(xmldoc, parentnode, dynamicindicators):
filescreated = False
processesstarted = False
regkeyscreated = False
mutexescreated = False
hostscontacted = False
hasdynamicindicators = False
# Here we are just testing to see if the report had any
# of the various dynamic indicator types so we know whether
# or not to process them at all
if len(dynamicindicators['droppedfiles']) > 0:
filescreated = True
hasdynamicindicators = True
if len(dynamicindicators['processes']) > 0:
processesstarted = True
hasdynamicindicators = True
if len(dynamicindicators['regkeys']) > 0:
regkeyscreated = True
hasdynamicindicators = True
if len(dynamicindicators['mutexes']) > 0:
mutexescreated = True
hasdynamicindicators = True
if len(dynamicindicators['hosts']) > 0:
hostscontacted = True
hasdynamicindicators = True
if not hasdynamicindicators:
return
ind = ioc_api.make_indicator_node("OR")
if filescreated:
createdfilesind = ioc_api.make_indicator_node("OR")
for createdfile in dynamicindicators['droppedfiles']:
createdfilesinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/FilenameCreated", content=createdfile[0], content_type="string")
createdfilesind.append(createdfilesinditem)
ind.append(createdfilesind)
if processesstarted:
processesind = ioc_api.make_indicator_node("OR")
for process in dynamicindicators['processes']:
startedprocessesind = ioc_api.make_indicator_node("AND")
# Process name
startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/name", content=process[0], content_type="string")
startedprocessesind.append(startedprocessesitem)
# Process pid
startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/pid", content=str(process[1]), content_type="int")
startedprocessesind.append(startedprocessesitem)
# Process parent pid
startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/parentpid", content=str(process[2]), content_type="int")
startedprocessesind.append(startedprocessesitem)
processesind.append(startedprocessesind)
ind.append(processesind)
if regkeyscreated:
regkeyind = ioc_api.make_indicator_node("AND")
for regkey in dynamicindicators['regkeys']:
createdregkeysind = ioc_api.make_indicatoritem_node(condition="is", document="RegistryItem", search="RegistryItem/KeyPath", content=regkey, content_type="string")
regkeyind.append(createdregkeysind)
ind.append(regkeyind)
if mutexescreated:
mutexkeyind = ioc_api.make_indicator_node("OR")
for mutex in dynamicindicators['mutexes']:
createdmutexesind = ioc_api.make_indicatoritem_node(condition="contains", document="ProcessItem", search="ProcessItem/HandList/Handl/Name", content=mutex, content_type="string")
mutexkeyind.append(createdmutexesind)
ind.append(mutexkeyind)
if hostscontacted:
hostsind = ioc_api.make_indicator_node("OR")
for host in dynamicindicators['hosts']:
hostsinditem = ioc_api.make_indicatoritem_node(condition="is", document="PortItem", search="PortItem/remoteIP", content=host, content_type="string")
hostsind.append(hostsinditem)
ind.append(hostsind)
parentnode.append(ind)
return
def convert_branch(self,
old_node,
new_node,
ids_to_skip,
comment_dict=None):
"""
Recursively walk a indicator logic tree, starting from a Indicator node.
Converts OpenIOC 1.1 Indicator/IndicatorItems to Openioc 1.0 and preserves order.
:param old_node: An Indicator node, which we walk down to convert
:param new_node: An Indicator node, which we add new IndicatorItem and Indicator nodes too
:param ids_to_skip: set of node @id values not to convert
:param comment_dict: maps ids to comment values. only applied to IndicatorItem nodes
:return: returns True upon completion.
:raises: DowngradeError if there is a problem during the conversion.
"""
expected_tag = 'Indicator'
if old_node.tag != expected_tag:
raise DowngradeError('old_node expected tag is [%s]' %
expected_tag)
if not comment_dict:
comment_dict = {}
for node in old_node.getchildren():
node_id = node.get('id')
if node_id in ids_to_skip:
continue
if node.tag == 'IndicatorItem':
negation = node.get('negate')
condition = node.get('condition')
if 'true' in negation.lower():
new_condition = condition + 'not'
else:
new_condition = condition
document = node.xpath('Context/@document')[0]
search = node.xpath('Context/@search')[0]
content_type = node.xpath('Content/@type')[0]
content = node.findtext('Content')
context_type = node.xpath('Context/@type')[0]
new_ii_node = ioc_api.make_indicatoritem_node(
condition=condition,
document=document,
search=search,
content_type=content_type,
content=content,
context_type=context_type,
nid=node_id)
# set condition
new_ii_node.attrib['condition'] = new_condition
# set comment
if node_id in comment_dict:
comment = comment_dict[node_id]
comment_node = et.Element('Comment')
comment_node.text = comment
new_ii_node.append(comment_node)
# remove preserver-case and negate
del new_ii_node.attrib['negate']
del new_ii_node.attrib['preserve-case']
new_node.append(new_ii_node)
elif node.tag == 'Indicator':
operator = node.get('operator')
if operator.upper() not in ['OR', 'AND']:
raise DowngradeError(
'Indicator@operator is not AND/OR. [%s] has [%s]' %
(node_id, operator))
new_i_node = ioc_api.make_indicator_node(operator, node_id)
new_node.append(new_i_node)
self.convert_branch(node, new_i_node, ids_to_skip,
comment_dict)
else:
# should never get here
raise DowngradeError('node is not a Indicator/IndicatorItem')
return True
def convert_branch(self, old_node, new_node, ids_to_skip, comment_dict=None):
"""
convert_branch
recursively walk a indicator logic tree, starting from a Indicator node.
converts OpenIOC 1.1 Indicator/IndicatorItems to Openioc 1.0 and preserves order.
input
old_node: old node, an Indicator node, which we walk down to convert
new_node: new node, an Indicator node, which we add new IndicatorItem and Indicator nodes too
ids_to_skip: set of ids not to convert
comment_dict: maps ids to comment values. only applied to IndicatorItem nodes
return
returns True upon completiong
may raise ValueError
"""
expected_tag = 'Indicator'
if old_node.tag != expected_tag:
raise ValueError('old_node expected tag is [%s]' % expected_tag)
for node in old_node.getchildren():
node_id = node.get('id')
if node_id in ids_to_skip:
continue
if node.tag == 'IndicatorItem':
negation = node.get('negate')
condition = node.get('condition')
if 'true' in negation.lower():
new_condition = condition + 'not'
else:
new_condition = condition
document = node.xpath('Context/@document')[0]
search = node.xpath('Context/@search')[0]
content_type = node.xpath('Content/@type')[0]
content = node.findtext('Content')
context_type = node.xpath('Context/@type')[0]
new_ii_node = ioc_api.make_indicatoritem_node(condition=condition,
document=document,
search=search,
content_type=content_type,
content=content,
context_type=context_type,
nid=node_id)
# set condition
new_ii_node.attrib['condition'] = new_condition
# set comment
if node_id in comment_dict:
comment = comment_dict[node_id]
comment_node = et.Element('Comment')
comment_node.text = comment
new_ii_node.append(comment_node)
# remove preserver-case and negate
del new_ii_node.attrib['negate']
del new_ii_node.attrib['preserve-case']
new_node.append(new_ii_node)
elif node.tag == 'Indicator':
operator = node.get('operator')
if operator.upper() not in ['OR', 'AND']:
raise IOCParseError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator))
new_i_node = ioc_api.make_indicator_node(operator, node_id)
new_node.append(new_i_node)
self.convert_branch(node, new_i_node, ids_to_skip, comment_dict)
else:
# should never get here
raise IOCParseError('node is not a Indicator/IndicatorItem')
return True
def test_make_i_node_bad(self):
op = 'FooBar'
with self.assertRaises(ValueError):
inode = ioc_api.make_indicator_node(op)
def test_schema_validation_from_api_fix_ordering_failure(self):
schema = et.XMLSchema(et.parse(OPENIOC_11_SCHEMA))
ioc_obj = ioc_api.IOC(name=self.name,
description=self.description,
author=self.author,
links=self.links,
keywords=self.keywords,
iocid=self.iocid)
empty_i_node = ioc_api.make_indicator_node('AND', nid='EmptyANDnode')
ioc_obj.top_level_indicator.append(empty_i_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content=self.content_text,
context_type='mir')
ioc_obj.top_level_indicator.append(ii_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
ioc_obj.top_level_indicator.append(ii_node)
and_node_1 = ioc_api.make_indicator_node('AND', nid='ANDnode1')
ioc_obj.top_level_indicator.append(and_node_1)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='and node 1 content 1',
context_type='mir')
and_node_1.append(ii_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='and node 1 content 2',
context_type='mir')
and_node_1.append(ii_node)
and_node_2 = ioc_api.make_indicator_node('AND', nid='ANDnode2')
ioc_obj.top_level_indicator.append(and_node_2)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
and_node_2.append(ii_node)
or_node_1 = ioc_api.make_indicator_node('OR', nid='ORnode1')
and_node_2.append(or_node_1)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
or_node_1.append(ii_node)
inner_i_node = ioc_api.make_indicator_node('AND', nid='InnderANDnode')
or_node_1.append(inner_i_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
inner_i_node.append(ii_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
inner_i_node.append(ii_node)
ii_node = ioc_api.make_indicatoritem_node(
condition='is',
document=self.context_document,
search=self.context_search,
content_type=self.content_type,
content='test text',
context_type='mir')
or_node_1.append(ii_node)
ioc_api.fix_schema_node_ordering(ioc_obj.top_level_indicator)
s = ioc_obj.write_ioc_to_string()
print(s)
ioc_tree = et.fromstring(s)
# Ensure our output is valid after reserializing
schema.assertValid(ioc_tree)
self.assertTrue(schema.validate(ioc_tree))
def test_make_i_node_bad(self):
op = 'FooBar'
with self.assertRaises(ValueError):
inode = ioc_api.make_indicator_node(op)
