def create_request_message(self, args): try: if args.begin_ts: begin_ts = dateutil.parser.parse(args.begin_ts) if not begin_ts.tzinfo: raise ValueError else: begin_ts = None if args.end_ts: end_ts = dateutil.parser.parse(args.end_ts) if not end_ts.tzinfo: raise ValueError else: end_ts = None except ValueError: print("Unable to parse timestamp value. Timestamp should include both date and time " \ "information along with a timezone or UTC offset (e.g., YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm). " \ "Aborting poll.") sys.exit() create_kwargs = { 'message_id': tm11.generate_message_id(), 'collection_name': args.collection, 'exclusive_begin_timestamp_label': begin_ts, 'inclusive_end_timestamp_label': end_ts } if args.subscription_id: create_kwargs['subscription_id'] = args.subscription_id else: create_kwargs['poll_parameters'] = tm11.PollRequest.PollParameters( ) poll_req = tm11.PollRequest(**create_kwargs) return poll_req
def test_10(self): """ Test a query. Should match just the APT1 report. """ test = tdq.Test(capability_id=tdq.CM_CORE, relationship=R_EQUALS, parameters={ P_VALUE: 'Unit 61398', P_MATCH_TYPE: 'case_sensitive_string' }) criterion = tdq.Criterion( target= 'STIX_Package/Threat_Actors/Threat_Actor/Identity/Specification/PartyName/OrganisationName/SubDivisionName', test=test) criteria = tdq.Criteria(OP_AND, criterion=[criterion]) q = tdq.DefaultQuery(CB_STIX_XML_111, criteria) pp = tm11.PollParameters() pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', poll_parameters=pp) #msg = self.send_poll_request('/services/test_poll_1/', VID_TAXII_XML_11, pr) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_POLL_RESPONSE) if len(msg.content_blocks) != 1: raise ValueError('Got %s CBs' % len(msg.content_blocks))
def prepare_request(collection_name, version, count_only=False, bindings=[], subscription_id=None): if version == 11: content_bindings = [tm11.ContentBinding(b) for b in bindings] if subscription_id: poll_parameters = None else: poll_parameters = tm11.PollParameters( response_type=( RT_FULL if not count_only else RT_COUNT_ONLY), content_bindings=content_bindings, ) return tm11.PollRequest( message_id=MESSAGE_ID, collection_name=collection_name, subscription_id=subscription_id, poll_parameters=poll_parameters ) elif version == 10: content_bindings = bindings return tm10.PollRequest( message_id=MESSAGE_ID, feed_name=collection_name, content_bindings=content_bindings, subscription_id=subscription_id )
def main(): # Create the test portion of the query my_test = tdq.Test(capability_id=CM_CORE, relationship=R_EQUALS, parameters={P_VALUE: value, P_MATCH_TYPE: 'case_insensitive_string'} ) #Put the test into a Criterion my_criterion = tdq.Criterion(target=target, test=my_test) # Put the Criterion into a Criteria my_criteria = tdq.Criteria(operator=OP_AND, criterion=[my_criterion], criteria=None) # Create a query with the criteria my_query = tdq.DefaultQuery(CB_STIX_XML_111, my_criteria) # Create a Poll Parameters that indicates # Only STIX 1.1.1 is accepted in response # and with the query created previously params = tm11.PollParameters(content_bindings=[tm11.ContentBinding(CB_STIX_XML_111)], query=my_query) poll_request = tm11.PollRequest(message_id=generate_message_id(), collection_name='file_hash_reputation', poll_parameters=params) print poll_request.to_xml(pretty_print=True)
def poll_11(self): auth_credentials_dict = self._get_auth_credentials_dict() self._client.set_auth_credentials(auth_credentials_dict) try: poll_parameters = tm11.PollParameters() poll_request = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=self._collection_name, exclusive_begin_timestamp_label=self._start, inclusive_end_timestamp_label=self._end, poll_parameters=poll_parameters, ) last_requested = datetime.datetime.now(pytz.utc) http_resp = self._client.call_taxii_service2( self._address, self._path, const.VID_TAXII_XML_11, poll_request.to_xml(), port=self._port) taxii_message = libtaxii.get_message_from_http_response( http_resp, poll_request.message_id) try: if taxii_message.content_blocks is None: return 0 except AttributeError: return 0 count = 0 for cb in taxii_message.content_blocks: _, stix_file_path = tempfile.mkstemp(suffix='.xml') with open(stix_file_path, 'wb+') as fp: fp.write(cb.content) try: from ctirs.core.stix.regist import regist if self._community is not None: regist(stix_file_path, self._community, self._via) count += 1 except BaseException: traceback.print_exc() self._taxii.last_requested = last_requested self._taxii.save() return count finally: if 'cert_file' in auth_credentials_dict: try: os.remove(auth_credentials_dict['cert_file']) except Exception: pass if 'key_file' in auth_credentials_dict: try: os.remove(auth_credentials_dict['key_file']) except Exception: pass
def _main(first=True): if first: begin = datetime.datetime.now(tzutc()) - timedelta(days=int(days)) else: try: begin = _readTimestamp() except IOError: print("[-] No timestamp file found have you 'first_run'?") sys.exit(0) end = datetime.datetime.now(tzutc()) poll_params1 = tm11.PollParameters( allow_asynch=False, response_type=RT_COUNT_ONLY, content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)], ) try: poll_req3 = tm11.PollRequest( message_id='PollReq03', collection_name=collection, poll_parameters=poll_params1, exclusive_begin_timestamp_label=begin, inclusive_end_timestamp_label=end, ) except ValueError: print("[-] Invalid timestamp file") sys.exit(0) except Exception: print("[-] Error with PollRequest") poll_xml = poll_req3.to_xml() http_resp = client.call_taxii_service2( server, path, VID_TAXII_XML_11, poll_xml, port=port) taxii_message = t.get_message_from_http_response( http_resp, poll_req3.message_id) if taxii_message.message_type == MSG_POLL_RESPONSE: if taxii_message.content_blocks: try: for content in taxii_message.content_blocks: package_io = StringIO(content.content) pkg = STIXPackage.from_xml(package_io) title = pkg.id_.split(':', 1)[-1] with open(title + ".xml", "w") as text_file: text_file.write(content.content) print("[+] Successfully generated " + title) except Exception: print("[-] Error with TAXII response") else: print("[+] No content returned") _saveTimestamp(str(end)) else: print("[-] Error with TAXII response")
def poll_11(self): #request xml作成 poll_parameters = tm11.PollParameters() poll_request = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=self._collection_name, exclusive_begin_timestamp_label=self._start, inclusive_end_timestamp_label=self._end, poll_parameters=poll_parameters, ) last_requested = datetime.datetime.now(pytz.utc) #from xml.dom.minidom import parseString #print( parseString(poll_request.to_xml()).toprettyxml(encoding='utf-8')) #request http_resp = self._client.call_taxii_service2( self._address, self._path, const.VID_TAXII_XML_11, poll_request.to_xml(), port=self._port) #taxii_message抽出 taxii_message = libtaxii.get_message_from_http_response(http_resp, poll_request.message_id) #content_blocks が None or 存在しない場合は登録ししない try: if taxii_message.content_blocks is None: return 0 except AttributeError: return 0 #content_blocksごとに登録処理 count = 0 for cb in taxii_message.content_blocks: #stixファイルを一時ファイルに出力 _,stix_file_path = tempfile.mkstemp(suffix='.xml') with open(stix_file_path,'wb+') as fp: #cb.contentがstixの中身 fp.write(cb.content) #登録 try: from ctirs.core.stix.regist import regist if self._community is not None: regist(stix_file_path,self._community,self._via) count += 1 except: #taxiiの場合は途中で失敗しても処理を継続する traceback.print_exc() #last_requested更新 self._taxii.last_requested = last_requested self._taxii.save() return count
def test_03(self): """ Test an invalid subscription ID """ pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', subscription_id='jdslkajdlksajdlksajld') msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_STATUS_MESSAGE, st=ST_NOT_FOUND, sd_keys=[SD_ITEM])
def test_09(self): """ Tests that a single PollRequest succeeds. """ pp = tm11.PollParameters() pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', poll_parameters=pp) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_POLL_RESPONSE) #msg = self.send_poll_request('/services/test_poll_1/', VID_TAXII_XML_11, pr) if len(msg.content_blocks) != 5: raise ValueError('Got %s CBs' % len(msg.content_blocks))
def test_04(self): """ Test a Content Binding ID not supported by the server """ pp = tm11.PollParameters( content_bindings=[tm11.ContentBinding('some_random_binding')]) pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', poll_parameters=pp) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_STATUS_MESSAGE, st=ST_UNSUPPORTED_CONTENT_BINDING, sd_keys=[SD_SUPPORTED_CONTENT])
def poll_feed(settings, subscription): """ polls a TAXII feed""" client = tc.HttpClient() client.set_auth_type(tc.HttpClient.AUTH_BASIC) client.set_use_https(True) client.set_auth_credentials({ 'username': settings['username'], 'password': settings['password'] }) msg_id = uuid.uuid4().hex poll_request1 = tm11.PollRequest(message_id=msg_id, collection_name=settings['subscriptions'] [subscription]['collection_name'], subscription_id=settings['subscriptions'] [subscription]['subscription_id']) poll_xml = poll_request1.to_xml() http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/', VID_TAXII_XML_11, poll_xml) taxii_message = t.get_message_from_http_response(http_resp, poll_request1.message_id) observables = {} indicators = json.loads(taxii_message.to_json()) if 'content_blocks' in indicators.keys(): for indicator in indicators['content_blocks']: open('/tmp/indicator.xml', 'w').write(indicator['content']) indi = STIXPackage.from_xml('/tmp/indicator.xml').to_dict() if 'observables' in indi.keys(): for obs in indi['observables']['observables']: if 'object' in obs.keys(): ot = obs['object']['properties']['xsi:type'] if ot in settings['supported_objects'].keys( ) and not ot in observables.keys(): observables[ot] = [] if ot in settings['supported_objects'].keys( ) and settings['supported_objects'][ot] in obs[ 'object']['properties'].keys(): # note, you will only be able to process one property per object type, but you also know there's only one property you can process try: observables[ot].append( obs['object']['properties'][ settings['supported_objects'][ot]]) except: print "[-] you're dumb" print supported_objects[ot], "not in:", obs[ 'object'] return observables
def test_02(self): """ Test a begin TS later than an end TS. """ begin_ts = datetime.now(tzutc()) end_ts = begin_ts - timedelta(days=-7) pp = tm11.PollParameters() pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', poll_parameters=pp, exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), ST_FAILURE)
def test_05(self): """ Test a supported Content Binding ID with an unsupported subtype """ pp = tm11.PollParameters(content_bindings=[ tm11.ContentBinding(CB_STIX_XML_111, subtype_ids=['FakeSubtype']) ]) pr = tm11.PollRequest(message_id=generate_message_id(), collection_name='default', poll_parameters=pp) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_STATUS_MESSAGE, st=ST_UNSUPPORTED_CONTENT_BINDING, sd_keys=[SD_SUPPORTED_CONTENT])
def test_01(self): """ Test an invalid collection name """ pp = tm11.PollParameters() pr = tm11.PollRequest( message_id=generate_message_id(), collection_name='INVALID_COLLECTION_NAME_13820198320', poll_parameters=pp) msg = make_request('/services/test_poll_1/', pr.to_xml(), get_headers(VID_TAXII_SERVICES_11, False), MSG_STATUS_MESSAGE, st=ST_NOT_FOUND, sd_keys=[SD_ITEM])
def indicators(self, collection_name, starttime=(datetime.now(tzutc()) - timedelta(hours=1)), endtime=datetime.now(tzutc()), subscription_id=None): delivery_params = tm11.DeliveryParameters(VID_TAXII_HTTP_10, self.remote, VID_TAXII_XML_11) """ poll_params = tm11.PollParameters(response_type=RT_COUNT_ONLY, #content_bindings=[tm11.ContentBinding(CB_STIX_XML_11)], delivery_parameters=delivery_params ) """ poll_params = tm11.PollRequest.PollParameters() poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(), collection_name=collection_name, exclusive_begin_timestamp_label=starttime, inclusive_end_timestamp_label=endtime, poll_parameters=poll_params, subscription_id=subscription_id) logger.debug('TAXII collection poll request: {}'.format( poll_req.to_xml(pretty_print=True))) poll_req_xml = poll_req.to_xml() http_resp = self.client.call_taxii_service2(self.up.hostname, self.up.path, VID_TAXII_XML_11, poll_req_xml, self.up.port) taxii_message = t.get_message_from_http_response( http_resp, poll_req.message_id) logger.debug('TAXII collection poll response: {}'.format( taxii_message.to_xml(pretty_print=True))) if taxii_message.message_type == MSG_STATUS_MESSAGE: if taxii_message.status_type == ST_SUCCESS: logger.info( 'TAXII polled successfully but returned no results') return [] raise RuntimeError('TAXII polling failed: {} - {}'.format( taxii_message.status_type, taxii_message.message)) return self._parse_taxii_content(taxii_message.content_blocks)
def poll(s, path="/", collection="default", query=None, begin_ts=None, end_ts=None): if query != None: query=s.create_query(query) poll_params=tm11.PollRequest.PollParameters(query=query) else: poll_params=tm11.PollRequest.PollParameters() # Create poll request poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(), collection_name=collection, exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts, poll_parameters=poll_params) # Convert to XML for request body poll_req_xml = poll_req.to_xml(True) # Create HTTP client client = tc.HttpClient() client.setProxy('noproxy') # Call TAXII service, using the body resp = client.callTaxiiService2(s.host, path, t.VID_TAXII_XML_11, poll_req_xml, s.port) # Get response resp = t.get_message_from_http_response(resp, '0') pkgs = [] # Process each content block for cb in resp.content_blocks: content = cb.content # Parse the payload, should be a STIX document. package = STIXPackage.from_xml(StringIO.StringIO(content)) pkgs.append(package) return resp.inclusive_end_timestamp_label, pkgs
def _prepare_poll_request( self, collection_name, begin_date=None, end_date=None, subscription_id=None, inbox_service=None, content_bindings=None, count_only=False, ): data = dict( message_id=self._generate_id(), collection_name=collection_name, exclusive_begin_timestamp_label=begin_date, inclusive_end_timestamp_label=end_date, ) if subscription_id: data["subscription_id"] = subscription_id else: _bindings = pack_content_bindings(content_bindings, version=11) poll_params = {"content_bindings": _bindings} if inbox_service: message_bindings = (inbox_service.message_bindings[0] if inbox_service.message_bindings else []) poll_params["delivery_parameters"] = tm11.DeliveryParameters( inbox_protocol=inbox_service.protocol, inbox_address=inbox_service.address, delivery_message_binding=message_bindings, ) poll_params["allow_asynch"] = True if count_only: poll_params["response_type"] = const.RT_COUNT_ONLY else: poll_params["response_type"] = const.RT_FULL data["poll_parameters"] = tm11.PollRequest.PollParameters( **poll_params) return tm11.PollRequest(**data)
def process_fsisac_stix_for_today(self): today_str = datetime.datetime.today().strftime('%Y-%m-%d') print "[*] Downloading stix for today (%s)..." % (today_str) # Create a TAXII Client client = tc.HttpClient() client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC ) # Username/password plus client cert auth client.set_use_https(True) # Use HTTPS # Update with your CIR credentials client.auth_credentials['username'] = FSISAC_USERNAME client.auth_credentials['password'] = FSISAC_PASSWORD client.auth_credentials['key_file'] = FSISAC_KEY client.auth_credentials['cert_file'] = FSISAC_CERT taxii_server = 'analysis.fsisac.com' taxii_service = '/taxii-discovery-service/' feed = 'system.Default' # TAXII feed to be polled. Update to poll a custom TAXII feed. # TAXII poll Exclusive Start Date and Inclusive End Date, as python datetime tuples. toyear = datetime.datetime.today().year tomonth = datetime.datetime.today().month today = datetime.datetime.today().day yesterday = datetime.datetime.today() + datetime.timedelta(days=-1) yesterday = yesterday.day # print "=" * 100 # print "DEBUGGING" # print "=" * 100 # print toyear, tomonth, yesterday # debug # print toyear, tomonth, today # debug # print "=" * 100 if yesterday == 31: start = datetime.datetime(toyear, tomonth - 1, yesterday, tzinfo=pytz.UTC) else: start = datetime.datetime(toyear, tomonth, yesterday, tzinfo=pytz.UTC) end = datetime.datetime(toyear, tomonth, today, tzinfo=pytz.UTC) # start = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC) # end = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC) # A TAXII poll can return a lot of data. For performance reasons, if the polling period spans multiple days, # only poll for one day at a time within the polling period. inc_start = start inc_end = inc_start + datetime.timedelta(days=1) while inc_start <= end: params = tm11.PollParameters() #Create the TAXII poll request poll_request = tm11.PollRequest( tm11.generate_message_id(), collection_name=feed, poll_parameters=params, exclusive_begin_timestamp_label=inc_start, inclusive_end_timestamp_label=inc_end) poll_xml = poll_request.to_xml() # Get the TAXII poll response http_resp = client.call_taxii_service2(taxii_server, taxii_service, VID_TAXII_XML_11, poll_xml) taxii_message = t.get_message_from_http_response( http_resp, poll_request.message_id) # Write each content block from the TAXII poll response to the "path" directory. for cb in taxii_message.content_blocks: #filename = gen_filename(taxii_message.collection_name, 'FSISAC_STIX111_', cb.timestamp_label.isoformat(), '.xml') filename = gen_filename('FSISAC', '_STIX111_', cb.timestamp_label.isoformat(), '.xml') with open(STIX_DOWNLOADED_PATH + "/" + filename, 'w') as outfile: outfile.write(cb.content) print "Written to %s" % filename # Increment to the next day in the specified date range. inc_start = inc_start + datetime.timedelta(days=1) inc_end = inc_end + datetime.timedelta(days=1)
def main(): # Create XML parser that can strip namespaces xmlParser = EntityParser() stix_package = None argParser = get_parser() args = argParser.parse_args() if args[0].help: print_help(argParser) # Import from a TAXII server elif args[0].taxii: begin_ts = None end_ts = None try: if args[0].begin_ts: structTime = time.strptime(args[0].begin_ts, '%Y-%m-%d %H:%M:%S') begin_ts = datetime.datetime(*structTime[:6]) begin_ts = begin_ts.replace(tzinfo=pytz.UTC) else: begin_ts = None if args[0].end_ts: structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S') end_ts = datetime.datetime(*structTime[:6]) end_ts = end_ts.replace(tzinfo=pytz.UTC) else: end_ts = None except ValueError: print >> sys.stderr, "Could not parse either start or end time" raise poll_req = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=args[0].collection, exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts, poll_parameters=tm11.PollRequest.PollParameters()) poll_req_xml = poll_req.to_xml() client = tc.HttpClient() if args[0].taxii_ssl: client.setUseHttps(True) if args[0].taxii_username: client.setAuthType(1) if not args[0].taxii_password: args[0].taxii_password = getpass.getpass( "Enter your taxii password: "******"/poll/", t.VID_TAXII_XML_11, poll_req_xml, args[0].taxiiport) response_message = t.get_message_from_http_response(resp, '0') response_dict = response_message.to_dict() indicators = 0 if 'content_blocks' in response_dict: for content in response_dict["content_blocks"]: bindingId = content["content_binding"]["binding_id"] if bindingId and bindingId.startswith( "urn:stix.mitre.org:xml"): #if args[0].verbose: # print >> sys.stderr, "RECIEVED STIX DATA:\n" #print >> sys.stderr, content["content"] try: # Parse the information stix_pkg = STIXPackage.from_xml( io.BytesIO(content["content"])) stix_package = stix_pkg indicators += process_package_dict( args, stix_package.to_dict()) except ValueError: print >> sys.stderr, "Could not parse STIX document: " print >> sys.stderr, content["content"] raise print "Imported", indicators, "indicators into set" else: print >> sys.stderr, "Invalid reponse from TAXII server" pprint.pprint(response_dict, sys.stderr) exit(255) # Import from a XML file on disk elif args[0].file: stix_package = STIXPackage.from_xml(args[0].file) indicators = process_package_dict(args, stix_package.to_dict()) print "Imported", indicators, "indicators into set" else: print >> sys.stderr, "Invalid arguments. Type 'python stix_Arcsight.py --help' for usage.\n"
def execute_taxii_agent(hostname=None, https=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None): ret = { 'Certificate': [], 'Domain': [], 'Email': [], 'Event': [], 'Indicator': [], 'IP': [], 'PCAP': [], 'RawData': [], 'Sample': [], 'successes': 0, 'failures': [], 'status': False, 'reason': '' } sc = get_config('taxii_service') # XXX: Validate these! if not hostname: hostname = str(sc['hostname']) if not keyfile: keyfile = str(sc['keyfile']) if not certfile: certfile = str(sc['certfile']) if not feed: feed = str(sc['data_feed']) if https == None: https = sc['https'] create_events = sc['create_events'] # Last document's end time is our start time. if not start: last = taxii.Taxii.get_last() if last: start = pytz.utc.localize(last.end) # If start is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(start, str): start = pytz.utc.localize(parse(start, fuzzy=True)) # store the current time as the time of this request runtime = datetime.now(tzutc()) # End time is always now, unless specified. if not end: end = runtime # If end is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(end, str): end = pytz.utc.localize(parse(end, fuzzy=True)) # compare start and end to make sure: # 1) start time is before end time # 2) end time is not in the future if (start != None and start >= end) and end > runtime: ret['reason'] = "Bad timestamp(s)" return ret client = tc.HttpClient() if https: client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile}) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy) crits_taxii = taxii.Taxii() crits_taxii.runtime = runtime crits_taxii.end = end # Poll using 1.1 then 1.0 if that fails. poll_msg = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=feed, poll_parameters=tm11.PollRequest.PollParameters(), exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id) if response.getcode( ) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE: # Check if this is a TAXII 1.0 server and try again if response.info().getheader( 'X-TAXII-Content-Type') == t.VID_TAXII_XML_10: poll_msg = tm.PollRequest(message_id=tm.generate_message_id(), feed_name=feed, exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response( response, poll_msg.message_id) if response.getcode( ) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret else: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret valid = tm.validate_xml(taxii_msg.to_xml()) if valid != True: ret['reason'] = "Invalid XML: %s" % valid return ret if taxii_msg.message_type != tm.MSG_POLL_RESPONSE: ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type return ret ret['status'] = True if not taxii_msg.content_blocks: crits_taxii.save() return ret mid = taxii_msg.message_id for content_block in taxii_msg.content_blocks: data = parse_content_block(content_block, keyfile, certfile) if not data: ret['failures'].append(('No data found in content block', 'Data')) continue objs = import_standards_doc(data, analyst, method, ref=mid, make_event=create_events) for k in objs['imported']: ret['successes'] += 1 ret[k[0]].append(k[1]) for k in objs['failed']: ret['failures'].append(k) crits_taxii.save() return ret
def main(): # This is a work-around for the fact that the 1.0 indicator type was removed from the STIX python # library, even though it is the essentially the same as the 1.1 type. We want to still support 1.0 # indicators since they are out there, and there is no difference for the purposes of this script. # vocabs._VOCAB_MAP["stixVocabs:IndicatorTypeVocab-1.0"] = IndicatorType # Create XML parser that can strip namespaces xmlParser = EntityParser() stix_package = None argParser = get_parser() args = argParser.parse_args() if args[0].help: print_help(argParser) # Import from a TAXII server elif args[0].referenceset and args[0].taxii: begin_ts = None end_ts = None try: if args[0].begin_ts: structTime = time.strptime(args[0].begin_ts, '%Y-%m-%d %H:%M:%S') begin_ts = datetime.datetime(*structTime[:6]) begin_ts = begin_ts.replace(tzinfo=pytz.UTC) else: begin_ts = None if args[0].end_ts: structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S') end_ts = datetime.datetime(*structTime[:6]) end_ts = end_ts.replace(tzinfo=pytz.UTC) else: end_ts = None except ValueError: print >> sys.stderr, "Could not parse either start or end time" raise #Create Request Header poll_req = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=args[0].collection, exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts, poll_parameters=tm11.PollRequest.PollParameters()) poll_req_xml = poll_req.to_xml() #Create request client object client = tc.HttpClient() #HTTPS Setting if args[0].taxii_ssl: client.set_use_https(True) # Basic Auth if args[0].taxii_username: if not args[0].taxii_password: args[0].taxii_password = getpass.getpass( "Enter your taxii password: "******"content_blocks"]: bindingId = content["content_binding"]["binding_id"] if bindingId and bindingId.startswith( "urn:stix.mitre.org:xml"): if args[0].verbose: print >> sys.stderr, "RECIEVED STIX DATA:\n" print >> sys.stderr, content["content"] try: # This string replace is a workaround for some invalid documents in my test server, if you don't need it, remove it xmlString = content["content"].replace( 'low', 'Low').replace('medium', 'Medium').replace('high', 'High') stix_package = xmlParser.parse_xml( io.BytesIO(xmlString), False) indicators += process_package_dict( args, stix_package.to_dict()) except ValueError: print >> sys.stderr, "Could not parse STIX document: " print >> sys.stderr, content["content"] raise print "Imported", indicators, "indicators into reference set", args[ 0].referenceset else: print >> sys.stderr, "Invalid reponse from TAXII server" pprint.pprint(response_dict, sys.stderr) exit(255) # Import from a XML file on disk elif args[0].referenceset and args[0].file: stix_package = xmlParser.parse_xml(args[0].file, False) indicators = process_package_dict(args, stix_package.to_dict()) print "Imported", indicators, "indicators into reference set", args[ 0].referenceset else: print >> sys.stderr, "Invalid arguments. Type 'python stix_import.py --help' for usage.\n"
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None): '''poll cti via taxii''' client = tc.HttpClient() client.setUseHttps(use_ssl) client.setAuthType(client.AUTH_BASIC) client.setAuthCredentials({'username': user, 'password': passwd}) cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = dict() earliest = poll_start(time_range) latest = nowutc() poll_window = 43200 # 12 hour blocks seem reasonable total_windows = (latest - earliest) / poll_window if (latest - earliest) % poll_window: total_windows += 1 if not quiet: widgets = [ 'TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA() ] progress = ProgressBar(widgets=widgets, maxval=total_windows).start() window_latest = latest window_earliest = window_latest - poll_window for i in range(total_windows): window_latest -= poll_window if window_earliest - poll_window < earliest: window_earliest = earliest else: window_earliest -= poll_window poll_params = tm11.PollParameters( allow_asynch=False, response_type=RT_FULL, content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)]) poll_request = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=collection, exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp( window_earliest).replace(tzinfo=pytz.utc), inclusive_end_timestamp_label=datetime.datetime.fromtimestamp( window_latest).replace(tzinfo=pytz.utc), poll_parameters=(poll_params)) http_response = client.callTaxiiService2(host, endpoint, t.VID_TAXII_XML_11, poll_request.to_xml(), port=port) taxii_message = t.get_message_from_http_response( http_response, poll_request.message_id) if isinstance(taxii_message, tm11.StatusMessage): print("TAXII connection error! %s" % (taxii_message.message)) elif isinstance(taxii_message, tm11.PollResponse): for content_block in taxii_message.content_blocks: try: stix_package = taxii_content_block_to_stix(content_block) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): if not k in cooked_cybox_objs.keys(): cooked_cybox_objs[k] = set() cooked_cybox_objs[k].update(raw_cybox_objs[k]) except: next if not quiet: progress.update(i) if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)
args_sts = date_N_hours_ago.strftime("%Y-%m-%d %H:%M:%S") args_ets = date_now.strftime("%Y-%m-%d %H:%M:%S") structTime = time.strptime(args_sts,'%Y-%m-%d %H:%M:%S') begin_ts = datetime(*structTime[:7]) begin_ts = begin_ts.replace(tzinfo=pytz.UTC) eTime = time.strptime(args_ets,'%Y-%m-%d %H:%M:%S') end_ts = datetime(*eTime[:7]) end_ts = end_ts.replace(tzinfo=pytz.UTC) ### CREATE THE POLL ## SET Collection Name poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(), collection_name='system.Default', exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts, poll_parameters=tm11.PollRequest.PollParameters()) poll_req_xml = poll_req.to_xml() ### SET SOME HEADERS headrs = [ "Content-Type: application/xml", "Content-Length: " + str(len(poll_req_xml)), "User-Agent: TAXII Client Application", "Accept: application/xml", "X-TAXII-Accept: urn:taxii.mitre.org:message:xml:1.1", "X-TAXII-Content-Type: urn:taxii.mitre.org:message:xml:1.1", "X-TAXII-Protocol: urn:taxii.mitre.org:protocol:https:1.0", ]
def main(): stix_package = None argParser = get_parser() args = argParser.parse_args() if args[0].help: print_help(argParser) # Import from a TAXII server elif args[0].taxii: begin_ts = None end_ts = None try: if args[0].begin_ts: structTime = time.strptime(args[0].begin_ts, '%Y-%m-%d %H:%M:%S') begin_ts = datetime.datetime(*structTime[:6]) begin_ts = begin_ts.replace(tzinfo=pytz.UTC) else: begin_ts = None if args[0].end_ts: structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S') end_ts = datetime.datetime(*structTime[:6]) end_ts = end_ts.replace(tzinfo=pytz.UTC) else: end_ts = None except ValueError: print('Could not parse either start or end time', file=sys.stderr) raise poll_req = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=args[0].collection, exclusive_begin_timestamp_label=begin_ts, inclusive_end_timestamp_label=end_ts, poll_parameters=tm11.PollRequest.poll_parameters()) poll_req_xml = poll_req.to_xml() client = tc.HttpClient() if args[0].taxii_ssl: client.set_use_https(True) if args[0].taxii_username: client.set_auth_type(1) if not args[0].taxii_password: args[0].taxii_password = getpass.getpass( 'Enter your taxii password: '******'username': args[0].taxii_username, 'password': args[0].taxii_password }) elif args[0].taxii_key and args[0].taxii_cert: client.set_auth_type(2) client.set_auth_credentials({ 'key': args[0].taxii_key, 'cert': args[0].taxii_cert }) if args[0].proxy: client.set_proxy(args[0].proxy) resp = client.call_taxii_service2( args[0].taxii, args[0].taxii_endpoint + '/poll/', t.VID_TAXII_XML_11, poll_req_xml, args[0].taxiiport) response_message = t.get_message_from_http_response(resp, '0') response_dict = response_message.to_dict() indicators = 0 if 'content_blocks' in response_dict: for content in response_dict['content_blocks']: bindingId = content['content_binding']['binding_id'] if bindingId and bindingId.startswith( 'urn:stix.mitre.org:xml'): #if args[0].verbose: # print >> sys.stderr, 'RECIEVED STIX DATA:\n' #print >> sys.stderr, content['content'] try: # Parse the information stix_pkg = STIXPackage.from_xml( io.BytesIO(content['content'])) stix_package = stix_pkg indicators += process_package_dict( args, stix_package.to_dict()) except ValueError: print( 'Could not parse STIX document: ', file=sys.stderr) print(content['content'], file=sys.stderr) raise print('Imported', indicators, 'indicators into set') else: print('Invalid reponse from TAXII server', file=sys.stderr) pprint.pprint(response_dict, sys.stderr) exit(255) # Import from a XML file on disk elif args[0].file: stix_package = STIXPackage.from_xml(args[0].file) indicators = process_package_dict(args, stix_package.to_dict()) print('Imported', indicators, 'indicators into set') else: print('Invalid arguments. Type --help for usage.\n', file=sys.stderr)
def _poll_taxii_11(self, args): '''Poll a TAXII 1.1 feed and return a generator over the content blocks found. Arguments: args - A dictionary of arguments. ''' app = args.get('app') cert_file = args.get('cert_file') key_file = args.get('key_file') username = args.get('username') password = args.get('password') if app and cert_file and key_file: cert_file_path = make_splunkhome_path( ["etc", "apps", app, "auth", cert_file]) key_file_path = make_splunkhome_path( ["etc", "apps", app, "auth", key_file]) if not (os.path.exists(cert_file_path) and os.path.exists(key_file_path)): logger.info( "Certificate not found - falling back to AUTH_BASIC.") cert_file_path = key_file_path = None else: logger.info( "Certificate information incomplete - falling back to AUTH_BASIC." ) cert_file_path = key_file_path = None client = tc.HttpClient() client.set_use_https(args.get('use_ssl')) # Add proxy parameters if present. if 'proxy' in args: client.set_proxy(args['proxy']) if cert_file_path and key_file_path and username: logger.info("Auth Type: AUTH_CERT_BASIC") client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC) client.set_auth_credentials({ 'username': username, 'password': password, 'cert_file': cert_file_path, 'key_file': key_file_path }) elif cert_file_path and key_file_path: logger.info("Auth Type: AUTH_CERT") client.set_auth_type(tc.HttpClient.AUTH_CERT) client.set_auth_credentials({ 'cert_file': cert_file_path, 'key_file': key_file_path }) else: logger.info("Auth Type: AUTH_BASIC") client.set_auth_type(tc.HttpClient.AUTH_BASIC) client.set_auth_credentials({ 'username': username, 'password': password }) poll_request = tm11.PollRequest( tm11.generate_message_id(), collection_name=args.get('collection'), exclusive_begin_timestamp_label=args.get('earliest'), #inclusive_end_timestamp_label=args.get('latest'), poll_parameters=tm11.PollParameters()) poll_xml = poll_request.to_xml() http_resp = client.call_taxii_service2(args.get('url'), args.get('service'), tm11.VID_TAXII_XML_11, poll_xml, port=args.get('port')) taxii_message = libtaxii.get_message_from_http_response( http_resp, poll_request.message_id) if http_resp.getcode() == 200: if hasattr(taxii_message, 'content_blocks'): return taxii_message.to_xml() else: raise TaxiiHandlerException( 'Exception when polling TAXII feed (no content returned): %s' % self._clean_taxii_message(taxii_message)) else: raise TaxiiHandlerException( 'Exception when polling TAXII feed: %s' % self._clean_taxii_message(taxii_message))
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, ssl_cert=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None): '''poll cti via taxii''' client = tc.HttpClient() client.set_use_https(use_ssl) if ssl_cert: client.setAuthType(client.AUTH_CERT_BASIC) client.setAuthCredentials({ 'username': user, 'password': passwd, 'key_file': ssl_cert, 'cert_file': ssl_cert }) else: client.setAuthType(client.AUTH_BASIC) client.setAuthCredentials({'username': user, 'password': passwd}) cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \ 'exploit_targets': set(), 'incidents': set(), \ 'indicators': set(), 'threat_actors': set(), \ 'ttps': set()} cooked_cybox_objs = { 'AccountObjectType': set(), 'AddressObjectType': set(), 'APIObjectType': set(), 'ArchiveFileObjectType': set(), 'ARPCacheObjectType': set(), 'ArtifactObjectType': set(), 'ASObjectType': set(), 'CodeObjectType': set(), 'CustomObjectType': set(), 'DeviceObjectType': set(), 'DiskObjectType': set(), 'DiskPartitionObjectType': set(), 'DNSCacheObjectType': set(), 'DNSQueryObjectType': set(), 'DNSRecordObjectType': set(), 'DomainNameObjectType': set(), 'EmailMessageObjectType': set(), 'FileObjectType': set(), 'GUIDialogboxObjectType': set(), 'GUIObjectType': set(), 'GUIWindowObjectType': set(), 'HostnameObjectType': set(), 'HTTPSessionObjectType': set(), 'ImageFileObjectType': set(), 'LibraryObjectType': set(), 'LinkObjectType': set(), 'LinuxPackageObjectType': set(), 'MemoryObjectType': set(), 'MutexObjectType': set(), 'NetworkConnectionObjectType': set(), 'NetworkFlowObjectType': set(), 'NetworkPacketObjectType': set(), 'NetworkRouteEntryObjectType': set(), 'NetRouteObjectType': set(), 'NetworkSocketObjectType': set(), 'NetworkSubnetObjectType': set(), 'PDFFileObjectType': set(), 'PipeObjectType': set(), 'PortObjectType': set(), 'ProcessObjectType': set(), 'ProductObjectType': set(), 'SemaphoreObjectType': set(), 'SMSMessageObjectType': set(), 'SocketAddressObjectType': set(), 'SystemObjectType': set(), 'UnixFileObjectType': set(), 'UnixNetworkRouteEntryObjectType': set(), 'UnixPipeObjectType': set(), 'UnixProcessObjectType': set(), 'UnixUserAccountObjectType': set(), 'UnixVolumeObjectType': set(), 'URIObjectType': set(), 'URLHistoryObjectType': set(), 'UserAccountObjectType': set(), 'UserSessionObjectType': set(), 'VolumeObjectType': set(), 'WhoisObjectType': set(), 'WindowsComputerAccountObjectType': set(), 'WindowsCriticalSectionObjectType': set(), 'WindowsDriverObjectType': set(), 'WindowsEventLogObjectType': set(), 'WindowsEventObjectType': set(), 'WindowsExecutableFileObjectType': set(), 'WindowsFilemappingObjectType': set(), 'WindowsFileObjectType': set(), 'WindowsHandleObjectType': set(), 'WindowsHookObjectType': set(), 'WindowsKernelHookObjectType': set(), 'WindowsKernelObjectType': set(), 'WindowsMailslotObjectType': set(), 'WindowsMemoryPageRegionObjectType': set(), 'WindowsMutexObjectType': set(), 'WindowsNetworkRouteEntryObjectType': set(), 'WindowsNetworkShareObjectType': set(), 'WindowsPipeObjectType': set(), 'WindowsPrefetchObjectType': set(), 'WindowsProcessObjectType': set(), 'WindowsRegistryKeyObjectType': set(), 'WindowsSemaphoreObjectType': set(), 'WindowsServiceObjectType': set(), 'WindowsSystemObjectType': set(), 'WindowsSystemRestoreObjectType': set(), 'WindowsTaskObjectType': set(), 'WindowsThreadObjectType': set(), 'WindowsUserAccountObjectType': set(), 'WindowsVolumeObjectType': set(), 'WindowsWaitableTimerObjectType': set(), 'X509CertificateObjectType': set(), } earliest = poll_start(time_range) latest = nowutc() poll_window = 43200 # 12 hour blocks seem reasonable total_windows = (latest - earliest) / poll_window if (latest - earliest) % poll_window: total_windows += 1 if not quiet: widgets = [ 'TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA() ] progress = ProgressBar(widgets=widgets, maxval=total_windows).start() window_latest = latest window_earliest = window_latest - poll_window for i in range(total_windows): window_latest -= poll_window if window_earliest - poll_window < earliest: window_earliest = earliest else: window_earliest -= poll_window if window_latest <= window_earliest: if not quiet: progress.update(i) break poll_params = tm11.PollParameters( allow_asynch=False, response_type=RT_FULL, content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)]) poll_request = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=collection, exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp( window_earliest).replace(tzinfo=pytz.utc), inclusive_end_timestamp_label=datetime.datetime.fromtimestamp( window_latest).replace(tzinfo=pytz.utc), poll_parameters=(poll_params)) try: http_response = client.callTaxiiService2(host, endpoint, t.VID_TAXII_XML_11, poll_request.to_xml(), port=port) taxii_message = t.get_message_from_http_response( http_response, poll_request.message_id) if isinstance(taxii_message, tm11.StatusMessage): print("TAXII connection error! %s" % (taxii_message.message)) elif isinstance(taxii_message, tm11.PollResponse): for content_block in taxii_message.content_blocks: try: stix_package = taxii_content_block_to_stix( content_block) (raw_stix_objs, raw_cybox_objs) = \ process_stix_pkg(stix_package) for k in raw_stix_objs.keys(): cooked_stix_objs[k].update(raw_stix_objs[k]) for k in raw_cybox_objs.keys(): if not k in cooked_cybox_objs.keys(): cooked_cybox_objs[k] = set() cooked_cybox_objs[k].update(raw_cybox_objs[k]) except: next except: next if not quiet: progress.update(i) if not quiet: progress.finish() return (cooked_stix_objs, cooked_cybox_objs)