def test_make_i_node(self): for op in self.operators: inode = ioc_api.make_indicator_node(op) self.assertEqual(inode.get('operator'), op.upper()) self.assertNotEqual(inode.get('id'), self.nid) inode = ioc_api.make_indicator_node(op, nid=self.nid) self.assertEqual(inode.get('operator'), op.upper()) self.assertEqual(inode.get('id'), self.nid)
def addStrings(xmldoc, parentnode, strings): # This simply adds an AND block of the strings found if len(strings) > 0: stringsind = ioc_api.make_indicator_node("AND") for string in strings: stringsinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/StringList/string", content=string, content_type="string") stringsind.append(stringsinditem) parentnode.append(stringsind) else: return
def convert_branch(self, old_node, new_node, comment_dict={}): """ convert_branch recursively walk a indicator logic tree, starting from a Indicator node. converts OpenIOC 1.0 Indicator/IndicatorItems to Openioc 1.1 and preserves order. input old_node: old node, an Indicator node, which we walk down to convert new_node: new node, an Indicator node, which we add new IndicatorItem and Indicator nodes too comment_dict: maps ids to comment values. only applied to IndicatorItem nodes return returns True upon completiong may raise ValueError """ expected_tag = 'Indicator' if old_node.tag != expected_tag: raise ValueError('old_node expected tag is [%s]' % expected_tag) for node in old_node.getchildren(): node_id = node.get('id') if node.tag == 'IndicatorItem': condition = node.get('condition') negation = False if condition.endswith('not'): negation = True condition = condition[:-3] document = node.xpath('Context/@document')[0] search = node.xpath('Context/@search')[0] content_type = node.xpath('Content/@type')[0] content = node.findtext('Content') context_type = node.xpath('Context/@type')[0] new_ii_node = ioc_api.make_indicatoritem_node(condition=condition, document=document, search=search, content_type=content_type, content=content, context_type=context_type, negate=negation, nid=node_id) # set comment comment = node.find('Comment') if comment is not None: comment_dict[node_id] = comment.text new_node.append(new_ii_node) elif node.tag == 'Indicator': operator = node.get('operator') if operator.upper() not in ['OR', 'AND']: raise IOCParseError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator)) new_i_node = ioc_api.make_indicator_node(operator, node_id) new_node.append(new_i_node) self.convert_branch(node, new_i_node, comment_dict) else: # should never get here raise IOCParseError('node is not a Indicator/IndicatorItem') return True
def convert_branch(self, old_node, new_node, comment_dict=None): """ recursively walk a indicator logic tree, starting from a Indicator node. converts OpenIOC 1.0 Indicator/IndicatorItems to Openioc 1.1 and preserves order. :param old_node: Indicator node, which we walk down to convert :param new_node: Indicator node, which we add new IndicatorItem and Indicator nodes too :param comment_dict: maps ids to comment values. only applied to IndicatorItem nodes :return: True upon completion :raises: UpgradeError if there is a problem during the conversion. """ expected_tag = 'Indicator' if old_node.tag != expected_tag: raise UpgradeError('old_node expected tag is [%s]' % expected_tag) if not comment_dict: comment_dict = {} for node in old_node.getchildren(): node_id = node.get('id') if node.tag == 'IndicatorItem': condition = node.get('condition') negation = False if condition.endswith('not'): negation = True condition = condition[:-3] document = node.xpath('Context/@document')[0] search = node.xpath('Context/@search')[0] content_type = node.xpath('Content/@type')[0] content = node.findtext('Content') context_type = node.xpath('Context/@type')[0] new_ii_node = ioc_api.make_indicatoritem_node(condition=condition, document=document, search=search, content_type=content_type, content=content, context_type=context_type, negate=negation, nid=node_id) # set comment comment = node.find('Comment') if comment is not None: comment_dict[node_id] = comment.text new_node.append(new_ii_node) elif node.tag == 'Indicator': operator = node.get('operator') if operator.upper() not in ['OR', 'AND']: raise UpgradeError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator)) new_i_node = ioc_api.make_indicator_node(operator, node_id) new_node.append(new_i_node) self.convert_branch(node, new_i_node, comment_dict) else: # should never get here raise UpgradeError('node is not a Indicator/IndicatorItem') return True
def create_ioc_object(ioc_name, items, and_or=True): ioc = ioc_api.IOC(name=ioc_name) top_level_or_node = ioc.top_level_indicator # build the definition if and_or: second_level_and_node = ioc_api.make_indicator_node('AND') top_level_or_node.append(second_level_and_node) for item in items: condition, document, search, content_type, content = tuple(item) # print condition, document, search, content_type, content ii_node = ioc_api.make_indicatoritem_node(condition, document, search, content_type, content) if and_or: second_level_and_node.append(ii_node) else: top_level_or_node.append(ii_node) # update the last modified time ioc.set_lastmodified_date() return ioc
def test_schema_validation_from_api_fail(self): schema = et.XMLSchema(et.parse(OPENIOC_11_SCHEMA)) ioc_obj = ioc_api.IOC(name=self.name, description=self.description, author=self.author, links=self.links, keywords=self.keywords, iocid=self.iocid) i_node = ioc_api.make_indicator_node('AND') ioc_obj.top_level_indicator.append(i_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content=self.content_text, context_type='mir') ioc_obj.top_level_indicator.append(ii_node) s = ioc_obj.write_ioc_to_string() ioc_tree = et.fromstring(s) # Ensure our output is valid after reserializing with self.assertRaises(et.DocumentInvalid) as cm: schema.assertValid(ioc_tree) self.assertIn('This element is not expected', str(cm.exception))
def createMetaData(xmldoc, parentnode, metadata): # load in the file name. It won't always be the actual file name # because dionaea renames it with the md5 hash...but it might # be the actual name # # As well, the items here generally won't change, so they are being # put in an AND block and_item = ioc_api.make_indicator_node('AND') if metadata['malfilename'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/FileName", content=str(metadata['malfilename']), content_type="string") and_item.append(inditem) # file size if metadata['malfilesize'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/SizeInBytes", content=str(metadata['malfilesize']), content_type="int") and_item.append(inditem) # file md5 if metadata['malmd5'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Md5Sum", content=metadata['malmd5'], content_type="md5") and_item.append(inditem) # md54k (http://www.md54k.org) # md54k is not part of Mandiant's list of indicators # so we are using our iocaware custom list (context_type="iocaware") # Please see the iocware.iocterms file if metadata['malmd54k'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Md54ksum", content=metadata['malmd54k'], content_type="md5", context_type="iocaware") and_item.append(inditem) if metadata['malsha1'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha1sum", content=metadata['malsha1'], content_type="sha1") and_item.append(inditem) if metadata['malsha256'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha256sum", content=metadata['malsha256'], content_type="sha256") and_item.append(inditem) # sha512 is not included in the list of OpenIOC indicators # so the context_type="iocware" - please see the iocaware.iocterms file if metadata["malsha512"] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Sha512sum", content=metadata['malsha512'], content_type="sha512", context_type="iocaware") and_item.append(inditem) # SSDeep also isn't included in the list of OpenIOC indicators if metadata["malssdeep"] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/Ssdeep", content=metadata["malssdeep"], content_type="ssdeep", context_type="iocaware") and_item.append(inditem) if metadata['malfiletype'] != "": inditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Type", content=metadata['malfiletype'], content_type="string") and_item.append(inditem) parentnode.append(and_item) peinfoind = ioc_api.make_indicator_node("OR") if len(metadata['iocimports']) > 0: for importfunc in metadata['iocimports']: importinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/ImportedModules/Module/ImportedFunctions/string", content=importfunc, content_type="string") peinfoind.append(importinditem) if len(metadata['iocexports']) > 0: for exportfunc in metadata['iocexports']: exportinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Exports/ExportedFunctions/string", content=exportfunc, content_type="string") peinfoind.append(exportinditem) if len(metadata['badpesections']) > 0: for section in metadata['badpesections']: sectionind = ioc_api.make_indicator_node("AND") sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/Name", content=section[0], content_type="string") sectionind.append(sectioninditem) sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/SizeInBytes", content=str(section[1]), content_type="int") sectionind.append(sectioninditem) sectioninditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/PEInfo/Sections/Section/Entropy/CurveData/float", content=str(section[2]), content_type="float") sectionind.append(sectioninditem) peinfoind.append(sectionind) # Include any PE Version Information if len(metadata['versioninfo']) > 0: infoind = ioc_api.make_indicator_node("AND") for infoitem in metadata['versioninfo']: if metadata['versioninfo'][infoitem] != "" and metadata['versioninfo'][infoitem] is not None: if "Version" in infoitem: itemvalue = str(metadata['versioninfo'][infoitem]).replace(", ", ".") else: itemvalue = str(metadata['versioninfo'][infoitem]) infoitemsearch = "FileItem/PEInfo/VersionInfoItem/" + infoitem infoinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search=infoitemsearch, content=str(itemvalue), content_type="string") infoind.append(infoinditem) peinfoind.append(infoind) parentnode.append(peinfoind)
def createDynamicIndicators(xmldoc, parentnode, dynamicindicators): filescreated = False processesstarted = False regkeyscreated = False mutexescreated = False hostscontacted = False hasdynamicindicators = False # Here we are just testing to see if the report had any # of the various dynamic indicator types so we know whether # or not to process them at all if len(dynamicindicators['droppedfiles']) > 0: filescreated = True hasdynamicindicators = True if len(dynamicindicators['processes']) > 0: processesstarted = True hasdynamicindicators = True if len(dynamicindicators['regkeys']) > 0: regkeyscreated = True hasdynamicindicators = True if len(dynamicindicators['mutexes']) > 0: mutexescreated = True hasdynamicindicators = True if len(dynamicindicators['hosts']) > 0: hostscontacted = True hasdynamicindicators = True if not hasdynamicindicators: return ind = ioc_api.make_indicator_node("OR") if filescreated: createdfilesind = ioc_api.make_indicator_node("OR") for createdfile in dynamicindicators['droppedfiles']: createdfilesinditem = ioc_api.make_indicatoritem_node(condition="is", document="FileItem", search="FileItem/FilenameCreated", content=createdfile[0], content_type="string") createdfilesind.append(createdfilesinditem) ind.append(createdfilesind) if processesstarted: processesind = ioc_api.make_indicator_node("OR") for process in dynamicindicators['processes']: startedprocessesind = ioc_api.make_indicator_node("AND") # Process name startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/name", content=process[0], content_type="string") startedprocessesind.append(startedprocessesitem) # Process pid startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/pid", content=str(process[1]), content_type="int") startedprocessesind.append(startedprocessesitem) # Process parent pid startedprocessesitem = ioc_api.make_indicatoritem_node(condition="is", document="ProcessItem", search="ProcessItem/parentpid", content=str(process[2]), content_type="int") startedprocessesind.append(startedprocessesitem) processesind.append(startedprocessesind) ind.append(processesind) if regkeyscreated: regkeyind = ioc_api.make_indicator_node("AND") for regkey in dynamicindicators['regkeys']: createdregkeysind = ioc_api.make_indicatoritem_node(condition="is", document="RegistryItem", search="RegistryItem/KeyPath", content=regkey, content_type="string") regkeyind.append(createdregkeysind) ind.append(regkeyind) if mutexescreated: mutexkeyind = ioc_api.make_indicator_node("OR") for mutex in dynamicindicators['mutexes']: createdmutexesind = ioc_api.make_indicatoritem_node(condition="contains", document="ProcessItem", search="ProcessItem/HandList/Handl/Name", content=mutex, content_type="string") mutexkeyind.append(createdmutexesind) ind.append(mutexkeyind) if hostscontacted: hostsind = ioc_api.make_indicator_node("OR") for host in dynamicindicators['hosts']: hostsinditem = ioc_api.make_indicatoritem_node(condition="is", document="PortItem", search="PortItem/remoteIP", content=host, content_type="string") hostsind.append(hostsinditem) ind.append(hostsind) parentnode.append(ind) return
def convert_branch(self, old_node, new_node, ids_to_skip, comment_dict=None): """ Recursively walk a indicator logic tree, starting from a Indicator node. Converts OpenIOC 1.1 Indicator/IndicatorItems to Openioc 1.0 and preserves order. :param old_node: An Indicator node, which we walk down to convert :param new_node: An Indicator node, which we add new IndicatorItem and Indicator nodes too :param ids_to_skip: set of node @id values not to convert :param comment_dict: maps ids to comment values. only applied to IndicatorItem nodes :return: returns True upon completion. :raises: DowngradeError if there is a problem during the conversion. """ expected_tag = 'Indicator' if old_node.tag != expected_tag: raise DowngradeError('old_node expected tag is [%s]' % expected_tag) if not comment_dict: comment_dict = {} for node in old_node.getchildren(): node_id = node.get('id') if node_id in ids_to_skip: continue if node.tag == 'IndicatorItem': negation = node.get('negate') condition = node.get('condition') if 'true' in negation.lower(): new_condition = condition + 'not' else: new_condition = condition document = node.xpath('Context/@document')[0] search = node.xpath('Context/@search')[0] content_type = node.xpath('Content/@type')[0] content = node.findtext('Content') context_type = node.xpath('Context/@type')[0] new_ii_node = ioc_api.make_indicatoritem_node( condition=condition, document=document, search=search, content_type=content_type, content=content, context_type=context_type, nid=node_id) # set condition new_ii_node.attrib['condition'] = new_condition # set comment if node_id in comment_dict: comment = comment_dict[node_id] comment_node = et.Element('Comment') comment_node.text = comment new_ii_node.append(comment_node) # remove preserver-case and negate del new_ii_node.attrib['negate'] del new_ii_node.attrib['preserve-case'] new_node.append(new_ii_node) elif node.tag == 'Indicator': operator = node.get('operator') if operator.upper() not in ['OR', 'AND']: raise DowngradeError( 'Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator)) new_i_node = ioc_api.make_indicator_node(operator, node_id) new_node.append(new_i_node) self.convert_branch(node, new_i_node, ids_to_skip, comment_dict) else: # should never get here raise DowngradeError('node is not a Indicator/IndicatorItem') return True
def convert_branch(self, old_node, new_node, ids_to_skip, comment_dict=None): """ convert_branch recursively walk a indicator logic tree, starting from a Indicator node. converts OpenIOC 1.1 Indicator/IndicatorItems to Openioc 1.0 and preserves order. input old_node: old node, an Indicator node, which we walk down to convert new_node: new node, an Indicator node, which we add new IndicatorItem and Indicator nodes too ids_to_skip: set of ids not to convert comment_dict: maps ids to comment values. only applied to IndicatorItem nodes return returns True upon completiong may raise ValueError """ expected_tag = 'Indicator' if old_node.tag != expected_tag: raise ValueError('old_node expected tag is [%s]' % expected_tag) for node in old_node.getchildren(): node_id = node.get('id') if node_id in ids_to_skip: continue if node.tag == 'IndicatorItem': negation = node.get('negate') condition = node.get('condition') if 'true' in negation.lower(): new_condition = condition + 'not' else: new_condition = condition document = node.xpath('Context/@document')[0] search = node.xpath('Context/@search')[0] content_type = node.xpath('Content/@type')[0] content = node.findtext('Content') context_type = node.xpath('Context/@type')[0] new_ii_node = ioc_api.make_indicatoritem_node(condition=condition, document=document, search=search, content_type=content_type, content=content, context_type=context_type, nid=node_id) # set condition new_ii_node.attrib['condition'] = new_condition # set comment if node_id in comment_dict: comment = comment_dict[node_id] comment_node = et.Element('Comment') comment_node.text = comment new_ii_node.append(comment_node) # remove preserver-case and negate del new_ii_node.attrib['negate'] del new_ii_node.attrib['preserve-case'] new_node.append(new_ii_node) elif node.tag == 'Indicator': operator = node.get('operator') if operator.upper() not in ['OR', 'AND']: raise IOCParseError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator)) new_i_node = ioc_api.make_indicator_node(operator, node_id) new_node.append(new_i_node) self.convert_branch(node, new_i_node, ids_to_skip, comment_dict) else: # should never get here raise IOCParseError('node is not a Indicator/IndicatorItem') return True
def test_make_i_node_bad(self): op = 'FooBar' with self.assertRaises(ValueError): inode = ioc_api.make_indicator_node(op)
def test_schema_validation_from_api_fix_ordering_failure(self): schema = et.XMLSchema(et.parse(OPENIOC_11_SCHEMA)) ioc_obj = ioc_api.IOC(name=self.name, description=self.description, author=self.author, links=self.links, keywords=self.keywords, iocid=self.iocid) empty_i_node = ioc_api.make_indicator_node('AND', nid='EmptyANDnode') ioc_obj.top_level_indicator.append(empty_i_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content=self.content_text, context_type='mir') ioc_obj.top_level_indicator.append(ii_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') ioc_obj.top_level_indicator.append(ii_node) and_node_1 = ioc_api.make_indicator_node('AND', nid='ANDnode1') ioc_obj.top_level_indicator.append(and_node_1) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='and node 1 content 1', context_type='mir') and_node_1.append(ii_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='and node 1 content 2', context_type='mir') and_node_1.append(ii_node) and_node_2 = ioc_api.make_indicator_node('AND', nid='ANDnode2') ioc_obj.top_level_indicator.append(and_node_2) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') and_node_2.append(ii_node) or_node_1 = ioc_api.make_indicator_node('OR', nid='ORnode1') and_node_2.append(or_node_1) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') or_node_1.append(ii_node) inner_i_node = ioc_api.make_indicator_node('AND', nid='InnderANDnode') or_node_1.append(inner_i_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') inner_i_node.append(ii_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') inner_i_node.append(ii_node) ii_node = ioc_api.make_indicatoritem_node( condition='is', document=self.context_document, search=self.context_search, content_type=self.content_type, content='test text', context_type='mir') or_node_1.append(ii_node) ioc_api.fix_schema_node_ordering(ioc_obj.top_level_indicator) s = ioc_obj.write_ioc_to_string() print(s) ioc_tree = et.fromstring(s) # Ensure our output is valid after reserializing schema.assertValid(ioc_tree) self.assertTrue(schema.validate(ioc_tree))