def message_tests(taxii_message): if not isinstance(taxii_message, tm.TAXIIMessage): raise ValueError('taxii_message was not an instance of TAXIIMessage') print '***** Message type = %s; id = %s' % (taxii_message.message_type, taxii_message.message_id) xml_string = taxii_message.to_xml() valid = tm.validate_xml(xml_string) if not valid: raise Exception('\tFailure of test #1 - XML not schema valid') msg_from_xml = tm.get_message_from_xml(xml_string) dictionary = taxii_message.to_dict() msg_from_dict = tm.get_message_from_dict(dictionary) if taxii_message != msg_from_xml: print '\t Failure of test #2 - running equals w/ debug:' taxii_message.__eq__(msg_from_xml, True) raise Exception('Test #2 failed - taxii_message != msg_from_xml') if taxii_message != msg_from_dict: print '\t Failure of test #3 - running equals w/ debug:' taxii_message.__eq__(msg_from_dict, True) raise Exception('Test #3 failed - taxii_message != msg_from_dict') if msg_from_xml != msg_from_dict: print '\t Failure of test #4 - running equals w/ debug:' msg_from_xml.__eq__(msg_from_dict, True) raise Exception('Test #4 failed - msg_from_xml != msg_from_dict') print '***** All tests completed!'
def execute_taxii_agent(hostname=None, https=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None): ret = { 'Certificate': [], 'Domain': [], 'Email': [], 'Event': [], 'Indicator': [], 'IP': [], 'PCAP': [], 'RawData': [], 'Sample': [], 'successes': 0, 'failures': [], 'status': False, 'reason': '' } sc = get_config('taxii_service') # XXX: Validate these! if not hostname: hostname = str(sc['hostname']) if not keyfile: keyfile = str(sc['keyfile']) if not certfile: certfile = str(sc['certfile']) if not feed: feed = str(sc['data_feed']) if https == None: https = sc['https'] create_events = sc['create_events'] # Last document's end time is our start time. if not start: last = taxii.Taxii.get_last() if last: start = pytz.utc.localize(last.end) # If start is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(start, str): start = pytz.utc.localize(parse(start, fuzzy=True)) # store the current time as the time of this request runtime = datetime.now(tzutc()) # End time is always now, unless specified. if not end: end = runtime # If end is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(end, str): end = pytz.utc.localize(parse(end, fuzzy=True)) # compare start and end to make sure: # 1) start time is before end time # 2) end time is not in the future if (start != None and start >= end) and end > runtime: ret['reason'] = "Bad timestamp(s)" return ret client = tc.HttpClient() if https: client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile}) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy) crits_taxii = taxii.Taxii() crits_taxii.runtime = runtime crits_taxii.end = end # Poll using 1.1 then 1.0 if that fails. poll_msg = tm11.PollRequest( message_id=tm11.generate_message_id(), collection_name=feed, poll_parameters=tm11.PollRequest.PollParameters(), exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id) if response.getcode( ) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE: # Check if this is a TAXII 1.0 server and try again if response.info().getheader( 'X-TAXII-Content-Type') == t.VID_TAXII_XML_10: poll_msg = tm.PollRequest(message_id=tm.generate_message_id(), feed_name=feed, exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response( response, poll_msg.message_id) if response.getcode( ) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret else: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret valid = tm.validate_xml(taxii_msg.to_xml()) if valid != True: ret['reason'] = "Invalid XML: %s" % valid return ret if taxii_msg.message_type != tm.MSG_POLL_RESPONSE: ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type return ret ret['status'] = True if not taxii_msg.content_blocks: crits_taxii.save() return ret mid = taxii_msg.message_id for content_block in taxii_msg.content_blocks: data = parse_content_block(content_block, keyfile, certfile) if not data: ret['failures'].append(('No data found in content block', 'Data')) continue objs = import_standards_doc(data, analyst, method, ref=mid, make_event=create_events) for k in objs['imported']: ret['successes'] += 1 ret[k[0]].append(k[1]) for k in objs['failed']: ret['failures'].append(k) crits_taxii.save() return ret
def execute_taxii_agent(hostname=None, https=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None): ret = { 'Certificate': [], 'Domain': [], 'Email': [], 'Event': [], 'Indicator': [], 'IP': [], 'PCAP': [], 'RawData': [], 'Sample': [], 'successes': 0, 'failures': [], 'status': False, 'reason': '' } sc = manager.get_config('taxii_service') # XXX: Validate these! if not hostname: hostname = str(sc['hostname']) if not keyfile: keyfile = str(sc['keyfile']) if not certfile: certfile = str(sc['certfile']) if not feed: feed = str(sc['data_feed']) if https == None: https = sc['https'] create_events = sc['create_events'] # Last document's end time is our start time. if not start: last = taxii.Taxii.get_last() if last: start = pytz.utc.localize(last.end) # If start is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(start, str): start = pytz.utc.localize(parse(start, fuzzy=True)) # store the current time as the time of this request runtime = datetime.now(tzutc()) # End time is always now, unless specified. if not end: end = runtime # If end is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(end, str): end = pytz.utc.localize(parse(end, fuzzy=True)) # compare start and end to make sure: # 1) start time is before end time # 2) end time is not in the future if (start != None and start >= end) and end > runtime: ret['reason'] = "Bad timestamp(s)" return ret client = tc.HttpClient() if https: client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile}) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS) crits_taxii = taxii.Taxii() crits_taxii.runtime = runtime crits_taxii.end = end # Poll using 1.1 then 1.0 if that fails. poll_msg = tm11.PollRequest(message_id=tm11.generate_message_id(), collection_name=feed, poll_parameters=tm11.PollRequest.PollParameters(), exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id) if response.getcode() != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE: # Check if this is a TAXII 1.0 server and try again if response.info().getheader('X-TAXII-Content-Type') == t.VID_TAXII_XML_10: poll_msg = tm.PollRequest(message_id=tm.generate_message_id(), feed_name=feed, exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10, poll_msg.to_xml()) taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id) if response.getcode() != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret else: ret['reason'] = "%s: %s" % (taxii_msg.status_type, taxii_msg.message) return ret valid = tm.validate_xml(taxii_msg.to_xml()) if valid != True: ret['reason'] = "Invalid XML: %s" % valid return ret if taxii_msg.message_type != tm.MSG_POLL_RESPONSE: ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type return ret ret['status'] = True if not taxii_msg.content_blocks: crits_taxii.save() return ret mid = taxii_msg.message_id for content_block in taxii_msg.content_blocks: data = parse_content_block(content_block, keyfile, certfile) if not data: ret['failures'].append(('No data found in content block', 'Data')) continue objs = import_standards_doc(data, analyst, method, ref=mid, make_event=create_events) for k in objs['imported']: ret['successes'] += 1 ret[k[0]].append(k[1]) for k in objs['failed']: ret['failures'].append(k) crits_taxii.save() return ret
def execute_taxii_agent(hostname=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None): ret = { 'events': [], 'samples': [], 'emails': [], 'indicators': [], 'successes': 0, 'failures': 0, 'status': False, 'reason': '' } sc = manager.get_config('taxii_service') # XXX: Validate these! if not hostname: hostname = str(sc['hostname']) if not keyfile: keyfile = str(sc['keyfile']) if not certfile: certfile = str(sc['certfile']) if not feed: feed = str(sc['data_feed']) # Last document's end time is our start time. if not start: last = taxii.Taxii.get_last() if last: start = pytz.utc.localize(last.end) # If start is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(start, str): start = pytz.utc.localize(parse(start, fuzzy=True)) # store the current time as the time of this request runtime = datetime.now(tzutc()) # End time is always now, unless specified. if not end: end = runtime # If end is a string, convert it to a datetime # YYYY-MM-DD HH:MM:SS if isinstance(end, str): end = pytz.utc.localize(parse(end, fuzzy=True)) # compare start and end to make sure: # 1) start time is before end time # 2) end time is not in the future if (start != None and start >= end) and end > runtime: ret['reason'] = "Bad timestamp(s)" return ret client = tc.HttpClient() client.setUseHttps(True) client.setAuthType(tc.HttpClient.AUTH_CERT) client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile}) if settings.HTTP_PROXY: proxy = settings.HTTP_PROXY if not proxy.startswith('http://'): proxy = 'http://' + proxy client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS) crits_taxii = taxii.Taxii() crits_taxii.runtime = runtime crits_taxii.end = end poll_msg = tm.PollRequest(message_id=tm.generate_message_id(), feed_name=feed, exclusive_begin_timestamp_label=start, inclusive_end_timestamp_label=end) response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10, poll_msg.to_xml()) if response.getcode() != 200: ret['reason'] = "Response is not 200 OK" return ret taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id) valid = tm.validate_xml(taxii_msg.to_xml()) if valid != True: ret['reason'] = valid return ret if taxii_msg.message_type != tm.MSG_POLL_RESPONSE: ret['reason'] = "No poll response" return ret ret['status'] = True if not taxii_msg.content_blocks: crits_taxii.save() return ret mid = taxii_msg.message_id for content_block in taxii_msg.content_blocks: data = parse_content_block(content_block, keyfile, certfile) if not data: ret['failures'] += 1 continue objs = import_standards_doc(data, analyst, method, ref=mid, make_event=True) ret['successes'] += 1 for k in ["events", "samples", "emails", "indicators"]: for i in objs[k]: ret[k].append(i) crits_taxii.save() return ret