def main(): # define constants TI_REQUEST_URL = "https://api.intelgraph.idefense.com/rest/threatindicator/v0" # iDefense API Key # To avoid hard-coding creds, I'm using environment variables if os.environ.get('IDEF_TOKEN') is None: print( "error: please store your iDefense IntelGraph API key in the IDEF_TOKEN environment" ) sys.exit(1) API_KEY = os.environ.get('IDEF_TOKEN') API_SECRET = '' # TODO: use command-line parameter timestr = datetime.datetime.utcnow() - datetime.timedelta(days=1) LAST_IMPORT = timestr.strftime("%Y-%m-%dT%H:%M:%S") + ".000Z" HEADERS = { "Content-Type": "application/json", "auth-token": API_KEY, "X-Api-Key-Proof": API_SECRET } print(HEADERS) page = 1 more_data = True count = 0 # Set namespace NAMESPACE = Namespace("https://intelgraph.idefense.com", "idefense") set_id_namespace(NAMESPACE) # Create STIX Package stix_package = STIXPackage() stix_header = STIXHeader() stix_header.description = "iDefense Threat Indicators Feed" stix_package.stix_header = stix_header ttps = {} malware = {} try: while more_data: request_payload = { "start_date": LAST_IMPORT, "page_size": 200, "page": page } r = requests.post(TI_REQUEST_URL, headers=HEADERS, data=json.dumps(request_payload)) print(r) response = [] if r.status_code == requests.codes.ok: try: # Read in response as json response = r.json() except (ValueError, KeyError): print("Response couldn't be decoded :(") more_data = False continue more_data = response.get('more', 'False') print("Page %d ==> %s (%s)" % (page, response['more'], response['total_size'])) page += 1 # Iterate the response for indicatorD in response['results']: count += 1 # Indicator value such as the value of the IP/Domain/URL indicator = indicatorD.get('key') print(indicator, indicatorD.get('type')) if indicatorD.get('last_seen_as') is None: last_seen_as = 'UNKNOWN' else: last_seen_as = ''.join(indicatorD.get('last_seen_as')) # Identify TTP if last_seen_as not in ttps: ttps[last_seen_as] = TTP(title=last_seen_as) stix_package.add_ttp(ttps[last_seen_as]) # Identify malware source if 'files' in indicatorD: for hashD in indicatorD['files']: md5 = hashD.get('key') # Malware Family classification of the hash if available if hashD.get('malware_family') is None: malware_family = "Unknown" else: malware_family = ''.join( hashD.get('malware_family')) if md5 not in malware: malware[md5] = add_malware( md5, malware_family, hashD.get('uuid')) if indicatorD.get('type') == "url": # Create indicator indicator = Indicator( id_="indicator-{0}".format(indicatorD.get('uuid')), title=''.join(indicatorD.get('malware_family')), timestamp=indicatorD.get('last_seen')) indicator.add_indicator_type("URL Watchlist") # Populate URL url = URI() url.value = indicatorD.get('key') url.type_ = URI.TYPE_URL url.value.condition = "Equals" indicator.add_observable(url) elif indicatorD.get('type') == "domain": # Populate domain name indicator = Indicator( id_="indicator-{0}".format(indicatorD.get('uuid')), title=''.join(indicatorD.get('malware_family')), timestamp=indicatorD.get('last_seen')) indicator.add_indicator_type("Domain Watchlist") domain = DomainName() domain.value = indicatorD.get('key') domain.value.condition = "Equals" indicator.add_observable(domain) elif indicatorD.get('type') == "ip": # Create indicator indicator = Indicator( id_="indicator-{0}".format(indicatorD.get('uuid')), title=indicatorD.get('malware_family'), timestamp=indicatorD.get('last_seen')) indicator.add_indicator_type("IP Watchlist") # Populate IP address addr = Address(address_value=indicatorD.get('key'), category=Address.CAT_IPV4) addr.condition = "Equals" indicator.add_observable(addr) # Link TTP indicator.add_indicated_ttp( TTP(idref=ttps[last_seen_as].id_)) # Indicate confidence score indicator.confidence = Confidence( value=VocabString(indicatorD.get('confidence'))) # Add related indicator to malware indicator.add_related_indicator(malware[md5]) # Add to package stix_package.add_indicator(indicator) else: print("API request couldn't be fulfilled due status code: %d" % r.status_code) more_data = False except requests.exceptions.ConnectionError as e: print("Check your network connection\n %s" % str(e)) except requests.exceptions.HTTPError as e: print("Bad HTTP response\n %s" % str(e)) except Exception as e: print("Uncaught exception\n %s" % str(e)) # Output to XML with open('stix-1.2.1.xml', 'wb') as f: f.write(stix_package.to_xml())
def to_stix(obj, items_to_convert=[], loaded=False, bin_fmt="raw", ref_id=None): """ Converts a CRITs object to a STIX document. The resulting document includes standardized representations of all related objects noted within items_to_convert. :param items_to_convert: The list of items to convert to STIX/CybOX :type items_to_convert: Either a list of CRITs objects OR a list of {'_type': CRITS_TYPE, '_id': CRITS_ID} dicts :param loaded: Set to True if you've passed a list of CRITs objects as the value for items_to_convert, else leave False. :type loaded: bool :param bin_fmt: Specifies the format for Sample data encoding. Options: None (don't include binary data in STIX output), "raw" (include binary data as is), "base64" (base64 encode binary data) :returns: A dict indicating which items mapped to STIX indicators, ['stix_indicators'] which items mapped to STIX observables, ['stix_observables'] which items are included in the resulting STIX doc, ['final_objects'] and the STIX doc itself ['stix_obj']. """ from cybox.common import Time, ToolInformationList, ToolInformation from stix.common import StructuredText, InformationSource from stix.core import STIXPackage, STIXHeader from stix.common.identity import Identity import stix.common.kill_chains.lmco as lmco # These lists are used to determine which CRITs objects # go in which part of the STIX document. ind_list = ['Indicator'] obs_list = ['Domain', 'Email', 'IP', 'Sample'] camp_list = ['Campaign'] actor_list = ['Actor'] # Store message stix_msg = { 'stix_incidents': [], 'stix_indicators': [], 'stix_observables': [], 'stix_actors': [], 'final_objects': [] } if not loaded: # if we have a list of object metadata, load it before processing items_to_convert = [class_from_id(item['_type'], item['_id']) for item in items_to_convert] # add self to the list of items to STIXify if obj not in items_to_convert: items_to_convert.append(obj) # add any email attachments attachments = [] for obj in items_to_convert: if obj._meta['crits_type'] == 'Email': for rel in obj.relationships: if rel.relationship == RelationshipTypes.CONTAINS: atch = class_from_id('Sample', rel.object_id) if atch not in items_to_convert: attachments.append(atch) items_to_convert.extend(attachments) # grab ObjectId of items refObjs = {key.id: 0 for key in items_to_convert} relationships = {} stix = [] stx = False tlp = None from stix.indicator import Indicator as S_Ind for obj in items_to_convert: obj_type = obj._meta['crits_type'] if obj_type == class_from_type('Event')._meta['crits_type']: stx, release = to_stix_incident(obj) stix_msg['stix_incidents'].append(stx) elif obj_type in ind_list: # convert to STIX indicators stx, releas = to_stix_indicator(obj) stix_msg['stix_indicators'].append(stx) refObjs[obj.id] = S_Ind(idref=stx.id_) elif obj_type in camp_list: camp = to_stix_campaign(obj, False) comm = to_stix_comments(obj) tlp = to_stix_tlp(obj) rel = to_stix_relationship(obj) sight = to_stix_sightings(obj) kill = to_stix_kill_chains(obj) ttp = to_stix_ttps(obj) rfi = to_stix_rfi(obj) ind = S_Ind() ind.title = "MARTI Campaign" ind.sightings.append(sight) for each in camp: ind.add_related_campaign(each) for each in comm: ind.add_related_indicator(each) for each in rel: ind.add_related_indicator(each) for each in rfi: ind.add_related_indicator(each) for each in kill: ind.add_kill_chain_phase(each) for each in ttp: ind.add_indicated_ttp(each) ind.producer = to_stix_information_source(obj) ind.short_descriptions = obj.sectors ind.descriptions = obj.aliases stx = ind stix_msg['stix_indicators'].append(stx) refObjs[obj.id] = S_Ind(idref=stx.id_) elif obj_type in obs_list: # convert to CybOX observable camp = to_stix_campaign(obj) comm = to_stix_comments(obj) rel = to_stix_relationship(obj) sight = to_stix_sightings(obj) kill = to_stix_kill_chains(obj) tlp = to_stix_tlp(obj) rfi = to_stix_rfi(obj) if obj_type == class_from_type('Sample')._meta['crits_type']: stx, releas = to_cybox_observable(obj, bin_fmt=bin_fmt) else: stx, releas = to_cybox_observable(obj) # wrap in stix Indicator ind = S_Ind() for ob in stx: ind.add_observable(ob) ind.sightings.append(sight) for each in camp: ind.add_related_campaign(each) for each in comm: ind.add_related_indicator(each) for each in rel: ind.add_related_indicator(each) for each in rfi: ind.add_related_indicator(each) for each in kill: ind.add_kill_chain_phase(each) ind.title = "CRITs %s Top-Level Object" % obj_type ind.description = ("This is simply a CRITs %s top-level " "object, not actually an Indicator. " "The Observable is wrapped in an Indicator" " to facilitate documentation of the " "relationship." % obj_type) ind.confidence = 'None' ind.producer = to_stix_information_source(obj) ind.short_descriptions = obj.sectors stx = ind stix_msg['stix_indicators'].append(stx) refObjs[obj.id] = S_Ind(idref=stx.id_) elif obj_type in actor_list: # convert to STIX actor stx, releas = to_stix_actor(obj) stix_msg['stix_actors'].append(stx) # get relationships from CRITs objects for rel in obj.relationships: if rel.object_id in refObjs: relationships.setdefault(stx.id_, {}) relationships[stx.id_][rel.object_id] = (rel.relationship, rel.rel_confidence.capitalize(), rel.rel_type) stix_msg['final_objects'].append(obj) if stx: stix.append(stx) # set relationships on STIX objects for stix_obj in stix: for rel in relationships.get(stix_obj.id_, {}): if isinstance(refObjs.get(rel), S_Ind): # if is STIX Indicator stix_obj.related_indicators.append(refObjs[rel]) rel_meta = relationships.get(stix_obj.id_)[rel] stix_obj.related_indicators[-1].relationship = rel_meta[0] stix_obj.related_indicators[-1].confidence = rel_meta[1] # Add any Email Attachments to CybOX EmailMessage Objects if isinstance(stix_obj, S_Ind): if 'EmailMessage' in stix_obj.observable.object_.id_: if rel_meta[0] == 'Contains' and rel_meta[2] == 'Sample': email = stix_obj.observable.object_.properties email.attachments.append(refObjs[rel].idref) tool_list = ToolInformationList() tool = ToolInformation("CRITs", "MITRE") tool.version = settings.CRITS_VERSION tool_list.append(tool) i_s = InformationSource( time=Time(produced_time= datetime.now()), identity=Identity(name=settings.COMPANY_NAME), tools=tool_list) if obj._meta['crits_type'] == "Event": stix_desc = obj.description() stix_int = obj.event_type() stix_title = obj.title() else: stix_desc = "STIX from %s" % settings.COMPANY_NAME stix_int = "Collective Threat Intelligence" stix_title = "Threat Intelligence Sharing" header = STIXHeader(information_source=i_s, description=StructuredText(value=stix_desc), package_intents=[stix_int], title=stix_title, handling=tlp) if not ref_id: ref_id = uuid.uuid4() stix_msg['stix_obj'] = STIXPackage(incidents=stix_msg['stix_incidents'], indicators=stix_msg['stix_indicators'], threat_actors=stix_msg['stix_actors'], stix_header=header, campaigns=camp, id_=ref_id) #print stix_msg['stix_obj'].to_xml() return stix_msg