def __call__(self):
"""
Invoke a TAXII Service based on the arguments
"""
try:
parser = self.get_arg_parser(
parser_description=self.parser_description, path=self.path)
args = parser.parse_args()
TaxiiScript._parse_url_info(args, parser)
request_message = self.create_request_message(args)
url = urlparse(args.url)
client = self.create_client(url.scheme == 'https', args.proxy,
args.cert, args.key, args.username,
args.password, args.verify_server)
print("Request:\n")
if args.xml_output is False:
print(request_message.to_text())
else:
print(request_message.to_xml(pretty_print=True))
resp = client.call_taxii_service2(
url.hostname, url.path, self.taxii_version,
request_message.to_xml(pretty_print=True), url.port)
r = t.get_message_from_http_response(resp, '0')
self.handle_response(r, args)
except Exception as ex:
traceback.print_exc()
sys.exit(EXIT_FAILURE)
sys.exit(EXIT_SUCCESS)
def __call__(self):
"""
Invoke a TAXII Service based on the arguments
"""
try:
parser = self.get_arg_parser(parser_description=self.parser_description, path=self.path)
args = parser.parse_args()
request_message = self.create_request_message(args)
client = self.create_client(args)
print "Request:\n"
if args.xml_output is False:
print request_message.to_text()
else:
print request_message.to_xml(pretty_print=True)
resp = client.call_taxii_service2(args.host, args.path, self.taxii_version, request_message.to_xml(pretty_print=True), args.port)
r = t.get_message_from_http_response(resp, '0')
self.handle_response(r, args)
except Exception as ex:
traceback.print_exc()
sys.exit(EXIT_FAILURE)
sys.exit(EXIT_SUCCESS)
def send_poll_request(self, path, messages_vid, poll_request, status_type=None, sd_keys=None):
#status_type=None means expect a PollResponse
#status_type=<value> means expect a StatusMessage
if messages_vid not in (VID_TAXII_XML_11, VID_TAXII_XML_10):
raise ValueError("Wrong messages_vid!")
client = tc.HttpClient()
client.setProxy(tc.HttpClient.NO_PROXY)
resp = client.callTaxiiService2(HOST, path, messages_vid, poll_request.to_xml(), PORT)
msg = t.get_message_from_http_response(resp, poll_request.message_id)
if not status_type: # Expect a Poll Response
if msg.message_type != MSG_POLL_RESPONSE:
raise ValueError('Expected Poll Response. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
else: # Expect a Status Message
if msg.message_type != MSG_STATUS_MESSAGE:
raise ValueError('Expected Status Message. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
if sd_keys:
for key in sd_keys:
if key not in msg.status_detail:
raise ValueError('Expected status detail key was not present: %s\r\n%s' % \
(key, msg.to_xml(pretty_print=True)) )
return msg
def _discover_services(self, tc):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.DiscoveryRequest(msg_id)
request = request.to_xml()
up = urlparse.urlparse(self.discovery_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(
hostname,
path,
libtaxii.constants.VID_TAXII_XML_11,
request
)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('Discovery_Response {%s} %s',
type(tm), tm.to_xml(pretty_print=True))
self.collection_mgmt_service = None
for si in tm.service_instances:
if si.service_type != libtaxii.constants.SVC_COLLECTION_MANAGEMENT:
continue
self.collection_mgmt_service = si.service_address
break
if self.collection_mgmt_service is None:
raise RuntimeError('%s - collection management service not found' %
self.name)
def main():
parser = argparse.ArgumentParser(description="Inbox Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Inbox Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Inbox Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/inbox/default/", help="Path where the Inbox Service is hosted. Defaults to /services/inbox/default/.")
parser.add_argument("--content-binding", dest="content_binding", default=t.CB_STIX_XML_10, help="Content binding of the Content Block to send. Defaults to %s" % t.CB_STIX_XML_10 )
parser.add_argument("--content-file", dest="content_file", default=stix_watchlist, help="Content of the Content Block to send. Defaults to a STIX watchlist.")
args = parser.parse_args()
if args.content_file is stix_watchlist:
c = StringIO.StringIO(stix_watchlist)
else:
c = open(args.content_file, 'r')
cb = tm.ContentBlock(args.content_binding, c.read())
inbox_message = tm.InboxMessage(message_id = tm.generate_message_id(), content_blocks=[cb])
inbox_xml = inbox_message.to_xml()
print "Inbox Message: \r\n", inbox_xml
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, inbox_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
def log(self, event):
# converts from conpot log format to STIX compatible xml
stix_package = self.stix_transformer.transform(event)
# wrapping the stix message in a TAXII envelope
content_block = ContentBlock(libtaxii.CB_STIX_XML_11,
stix_package.encode('utf-8'))
inbox_message = InboxMessage(message_id=generate_message_id(),
content_blocks=[content_block])
inbox_xml = inbox_message.to_xml()
# the actual call to the TAXII web service
response = self.client.callTaxiiService2(self.host, self.inbox_path,
libtaxii.VID_TAXII_XML_11,
inbox_xml, self.port)
response_message = libtaxii.get_message_from_http_response(
response, '0')
if response_message.status_type != libtaxii.messages.ST_SUCCESS:
logger.error(
'Error while transmitting message to TAXII server: {0}'.format(
response_message.message))
return False
else:
return True
def gen_send(tm_, client, encrypted_block, hostname, t_xml, dcn=None, eh=None,
url="/inbox/"):
"""
Generate and send a TAXII message.
:param tm_: The TAXII version imported that we should use.
:type tm_: TAXII message class.
:param client: The TAXII client to use.
:type client: TAXII Client.
:param encrypted_block: The encrypted block to use.
:type encrypted_block: TAXII Encrypted Block
:param hostname: The TAXII server hostname to connect to.
:type hostname: str
:param t_xml: The TAXII XML Schema version we used.
:type t_xml: str
:param dcn: Destination Collection Names we are using.
:type dcn: list
:param eh: Extended Headers to use.
:type eh: dict
:param url: The URL suffix to locate the inbox service.
:type url: str
:returns: tuple (response, taxii_message) or (exception message)
"""
# Wrap encrypted block in content block
content_block = tm_.ContentBlock(
content_binding = "application/x-pks7-mime",
content = encrypted_block
)
# Create inbox message
if dcn:
inbox_message = tm_.InboxMessage(
message_id = tm_.generate_message_id(),
content_blocks = [content_block],
destination_collection_names = dcn
)
elif eh:
inbox_message = tm.InboxMessage(
message_id = tm.generate_message_id(),
content_blocks = [content_block],
extended_headers = eh
)
else:
#TODO: return better
return None
# send inbox message via TAXII service
try:
response = client.callTaxiiService2(
hostname,
url,
t_xml,
inbox_message.to_xml()
)
taxii_message = t.get_message_from_http_response(response,
inbox_message.message_id)
return (response, taxii_message)
# can happen if 'hostname' is reachable, but is not a TAXII server, etc
except Exception, e:
return (e)
def main():
parser = scripts.get_base_parser("Poll Fulfillment Client", path="/services/query_example/")
parser.add_argument("--collection", dest="collection", default="default_queryable", help="Data Collection that this Fulfillment request applies to. Defaults to 'default_queryable'.")
parser.add_argument("--result_id", dest="result_id", required=True, help="The result_id being requested.")
parser.add_argument("--result_part_number", dest="result_part_number", default=1, help="The part number being requested. Defaults to '1'.")
args = parser.parse_args()
poll_fulf_req = tm11.PollFulfillmentRequest(message_id=tm11.generate_message_id(),
collection_name=args.collection,
result_id=args.result_id,
result_part_number=args.result_part_number)
print "Request:\n"
if args.xml_output is False:
print poll_fulf_req.to_text()
else:
print poll_fulf_req.to_xml(pretty_print=True)
client = scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, poll_fulf_req.to_xml(pretty_print=True), args.port)
r = t.get_message_from_http_response(resp, '0')
print "Response:\n"
if args.xml_output is False:
print r.to_text()
else:
print r.to_xml(pretty_print=True)
def client_example():
# Create the TAXII HTTPS Client
client = tc.HttpClient()
# Uncomment to use HTTPS
client.set_use_https(True)
# Uncomment to use basic authentication
# client.set_auth_type(tc.HttpClient.AUTH_BASIC)
# client.set_auth_credentials({'username':'******', 'password':'******'})
# Uncomment to use certificate-based authentication
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({'key_file': 'keyfile',
'cert_file': 'certfile'})
# Uncomment to set a proxy
# client.set_proxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
# Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(), feed_name='TheFeedToPoll')
# Call without a proxy
http_response = client.call_taxii_service2('hostname', '/poll_service_path/', VID_TAXII_XML_10, poll_request1.to_xml())
print http_response.__class__.__name__
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print(taxii_message.to_xml())
def _discover_services(self, tc):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.DiscoveryRequest(msg_id)
request = request.to_xml()
up = urlparse.urlparse(self.discovery_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(hostname, path,
libtaxii.constants.VID_TAXII_XML_11,
request)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('Discovery_Response {%s} %s', type(tm),
tm.to_xml(pretty_print=True))
self.collection_mgmt_service = None
for si in tm.service_instances:
if si.service_type != libtaxii.constants.SVC_COLLECTION_MANAGEMENT:
continue
self.collection_mgmt_service = si.service_address
break
if self.collection_mgmt_service is None:
raise RuntimeError('%s - collection management service not found' %
self.name)
def main():
parser = scripts.get_base_parser("Inbox Client", path="/services/inbox/default/")
parser.add_argument("--content-binding", dest="content_binding", default=t.CB_STIX_XML_11, help="Content binding of the Content Block to send. Defaults to %s" % t.CB_STIX_XML_11 )
parser.add_argument("--subtype", dest="subtype", default=None, help="The subtype of the Content Binding. Defaults to None")
parser.add_argument("--content-file", dest="content_file", default=stix_watchlist, help="Content of the Content Block to send. Defaults to a STIX watchlist.")
args = parser.parse_args()
if args.content_file is stix_watchlist:
c = StringIO.StringIO(stix_watchlist)
else:
c = open(args.content_file, 'r')
cb = tm11.ContentBlock(tm11.ContentBinding(args.content_binding), c.read())
c.close()
if args.subtype is not None:
cb.subtypes.append(args.subtype)
inbox_message = tm11.InboxMessage(message_id = tm11.generate_message_id(), content_blocks=[cb])
inbox_xml = inbox_message.to_xml(pretty_print=True)
print "Inbox Message: \r\n", inbox_xml
client = scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, inbox_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml(pretty_print=True)
def gen_send(tm_, client, encrypted_block, hostname, t_xml, dcn=None, eh=None,
url="/inbox/"):
"""
Generate and send a TAXII message.
:param tm_: The TAXII version imported that we should use.
:type tm_: TAXII message class.
:param client: The TAXII client to use.
:type client: TAXII Client.
:param encrypted_block: The encrypted block to use.
:type encrypted_block: TAXII Encrypted Block
:param hostname: The TAXII server hostname to connect to.
:type hostname: str
:param t_xml: The TAXII XML Schema version we used.
:type t_xml: str
:param dcn: Destination Collection Names we are using.
:type dcn: list
:param eh: Extended Headers to use.
:type eh: dict
:param url: The URL suffix to locate the inbox service.
:type url: str
:returns: tuple (response, taxii_message) or (exception message)
"""
# Wrap encrypted block in content block
content_block = tm_.ContentBlock(
content_binding = "application/x-pks7-mime",
content = encrypted_block
)
# Create inbox message
if dcn:
inbox_message = tm_.InboxMessage(
message_id = tm_.generate_message_id(),
content_blocks = [content_block],
destination_collection_names = dcn
)
elif eh:
inbox_message = tm.InboxMessage(
message_id = tm.generate_message_id(),
content_blocks = [content_block],
extended_headers = eh
)
else:
#TODO: return better
return None
# send inbox message via TAXII service
try:
response = client.callTaxiiService2(
hostname,
url,
t_xml,
inbox_message.to_xml()
)
taxii_message = t.get_message_from_http_response(response,
inbox_message.message_id)
return (response, taxii_message)
# can happen if 'hostname' is reachable, but is not a TAXII server, etc
except Exception, e:
return (e)
def poll_feed(settings,subscription):
""" polls a TAXII feed"""
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(True)
client.set_auth_credentials({'username': settings['username'], 'password': settings['password']})
msg_id=uuid.uuid4().hex
poll_request1 = tm11.PollRequest(message_id=msg_id,collection_name=settings['subscriptions'][subscription]['collection_name'],subscription_id=settings['subscriptions'][subscription]['subscription_id'])
poll_xml=poll_request1.to_xml()
http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/', VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(http_resp, poll_request1.message_id)
observables={}
indicators = json.loads(taxii_message.to_json())
if 'content_blocks' in indicators.keys():
for indicator in indicators['content_blocks']:
open('/tmp/indicator.xml','w').write(indicator['content'])
indi=STIXPackage.from_xml('/tmp/indicator.xml').to_dict()
if 'observables' in indi.keys():
for obs in indi['observables']['observables']:
if 'object' in obs.keys():
ot=obs['object']['properties']['xsi:type']
if ot in settings['supported_objects'].keys() and not ot in observables.keys():
observables[ot]=[]
if ot in settings['supported_objects'].keys() and settings['supported_objects'][ot] in obs['object']['properties'].keys():
# note, you will only be able to process one property per object type, but you also know there's only one property you can process
try:
observables[ot].append(obs['object']['properties'][settings['supported_objects'][ot]])
except:
print "[-] you're dumb"
print supported_objects[ot], "not in:", obs['object']
return observables
def push_11(taxii_client, stix_file_doc):
if stix_file_doc.version.startswith('2'):
try:
content = stix_file_doc.get_slide_12()
except Exception as e:
traceback.print_exc()
raise e
else:
with open(stix_file_doc.origin_path, 'r', encoding='utf-8') as fp:
content = fp.read()
auth_credentials_dict, cert_fd, key_fd = _get_auth_credentials_dict(
taxii_client)
taxii_client._client.set_auth_credentials(auth_credentials_dict)
try:
subscription_information = tm11.SubscriptionInformation(
collection_name=taxii_client._collection_name,
subscription_id='subscripiton_id')
cb = tm11.ContentBlock(const.CB_STIX_XML_11, content)
im = tm11.InboxMessage(
message_id=tm11.generate_message_id(),
destination_collection_names=[taxii_client._collection_name],
subscription_information=subscription_information,
content_blocks=[cb])
http_resp = taxii_client._client.call_taxii_service2(
taxii_client._address,
taxii_client._path,
const.VID_TAXII_XML_11,
im.to_xml(),
port=taxii_client._port)
taxii_message = libtaxii.get_message_from_http_response(
http_resp, im.message_id)
if taxii_message.status_type != 'SUCCESS':
raise Exception('taxii_message.status_type is not SUCCESS')
return 'Success !!'
except Exception as e:
traceback.print_exc()
raise e
finally:
if cert_fd is not None:
try:
os.close(cert_fd)
except Exception:
pass
if key_fd is not None:
try:
os.close(key_fd)
except Exception:
pass
if 'cert_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['cert_file'])
except Exception:
pass
if 'key_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['key_file'])
except Exception:
pass
def envio_informacion(data_feed, host, path, port):
#Given the host, port and path of a TAXII client we sent the data_feed to that client.
logger = logging.getLogger('TAXIIApplication.rest.tasks.envio_informacion')
logger.debug('Get the subscription methods')
logger.debug('The parameters are: ')
logger.debug('Host: ' + host)
logger.debug('Path: ' + path)
logger.debug('Port: ' + str(port))
logger.debug('Data Feed name:' + data_feed.name)
content_blocks = data_feed.content_blocks
content = []
for content_block in content_blocks.all():
cb = tm11.ContentBlock(tm11.ContentBinding(content_block.content_binding.binding_id), content_block.content)
content.append(cb)
subInfo = tm11.InboxMessage.SubscriptionInformation(collection_name = data_feed.title + ' ' + ORGANIZATION_NAME, subscription_id = 1)
inbox_message = tm11.InboxMessage(message_id = tm11.generate_message_id(), content_blocks=content, subscription_information = subInfo)
inbox_xml = inbox_message.to_xml()
logger.debug('The message to be sent is: '+ inbox_xml)
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, inbox_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug('The response message was: ' + response_message.to_xml())
def push(s, collection, content, path="/"):
msg_id=tm11.generate_message_id()
content = tm11.ContentBlock(tm11.ContentBinding(t.CB_STIX_XML_11),
content)
# Create inbox request
req = tm11.InboxMessage(message_id=msg_id,
destination_collection_names=[collection],
content_blocks=[content])
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path, t.VID_TAXII_XML_11,
req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
if resp.status_type != tm11.ST_SUCCESS:
print("%s: %s" % (resp.status_type, resp.message))
def send_poll_request(self):
"""Send the poll request to the TAXII server."""
poll_request1 = self.create_poll_request()
self._logger.debug(
"Request generated: using collection name - %s",
self._collection)
http_response = self.call_taxii_service2(
self._hostname,
self._path,
VID_TAXII_XML_11,
poll_request1.to_xml(),
self._port,
)
self._logger.debug("TAXII response received")
self._logger.debug("HTTP response %s",
http_response.__class__.__name__)
self._poll_response = get_message_from_http_response(
http_response,
poll_request1.message_id,
)
if self._poll_response.message_type != MSG_POLL_RESPONSE:
raise Exception('TAXII response not a poll response as expected.')
def send_poll_request(self,
path,
messages_vid,
poll_request,
status_type=None,
sd_keys=None):
#status_type=None means expect a PollResponse
#status_type=<value> means expect a StatusMessage
if messages_vid not in (VID_TAXII_XML_11, VID_TAXII_XML_10):
raise ValueError("Wrong messages_vid!")
client = tc.HttpClient()
client.setProxy(tc.HttpClient.NO_PROXY)
resp = client.callTaxiiService2(HOST, path, messages_vid,
poll_request.to_xml(), PORT)
msg = t.get_message_from_http_response(resp, poll_request.message_id)
if not status_type: # Expect a Poll Response
if msg.message_type != MSG_POLL_RESPONSE:
raise ValueError('Expected Poll Response. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
else: # Expect a Status Message
if msg.message_type != MSG_STATUS_MESSAGE:
raise ValueError('Expected Status Message. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
if sd_keys:
for key in sd_keys:
if key not in msg.status_detail:
raise ValueError('Expected status detail key was not present: %s\r\n%s' % \
(key, msg.to_xml(pretty_print=True)) )
return msg
def poll_11(self):
auth_credentials_dict = self._get_auth_credentials_dict()
self._client.set_auth_credentials(auth_credentials_dict)
try:
poll_parameters = tm11.PollParameters()
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=self._collection_name,
exclusive_begin_timestamp_label=self._start,
inclusive_end_timestamp_label=self._end,
poll_parameters=poll_parameters,
)
last_requested = datetime.datetime.now(pytz.utc)
http_resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
poll_request.to_xml(),
port=self._port)
taxii_message = libtaxii.get_message_from_http_response(
http_resp, poll_request.message_id)
try:
if taxii_message.content_blocks is None:
return 0
except AttributeError:
return 0
count = 0
for cb in taxii_message.content_blocks:
_, stix_file_path = tempfile.mkstemp(suffix='.xml')
with open(stix_file_path, 'wb+') as fp:
fp.write(cb.content)
try:
from ctirs.core.stix.regist import regist
if self._community is not None:
regist(stix_file_path, self._community, self._via)
count += 1
except BaseException:
traceback.print_exc()
self._taxii.last_requested = last_requested
self._taxii.save()
return count
finally:
if 'cert_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['cert_file'])
except Exception:
pass
if 'key_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['key_file'])
except Exception:
pass
def _main(first=True):
if first:
begin = datetime.datetime.now(tzutc()) - timedelta(days=int(days))
else:
try:
begin = _readTimestamp()
except IOError:
print("[-] No timestamp file found have you 'first_run'?")
sys.exit(0)
end = datetime.datetime.now(tzutc())
poll_params1 = tm11.PollParameters(
allow_asynch=False,
response_type=RT_COUNT_ONLY,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)],
)
try:
poll_req3 = tm11.PollRequest(
message_id='PollReq03',
collection_name=collection,
poll_parameters=poll_params1,
exclusive_begin_timestamp_label=begin,
inclusive_end_timestamp_label=end,
)
except ValueError:
print("[-] Invalid timestamp file")
sys.exit(0)
except Exception:
print("[-] Error with PollRequest")
poll_xml = poll_req3.to_xml()
http_resp = client.call_taxii_service2(
server, path, VID_TAXII_XML_11,
poll_xml, port=port)
taxii_message = t.get_message_from_http_response(
http_resp, poll_req3.message_id)
if taxii_message.message_type == MSG_POLL_RESPONSE:
if taxii_message.content_blocks:
try:
for content in taxii_message.content_blocks:
package_io = StringIO(content.content)
pkg = STIXPackage.from_xml(package_io)
title = pkg.id_.split(':', 1)[-1]
with open(title + ".xml", "w") as text_file:
text_file.write(content.content)
print("[+] Successfully generated " + title)
except Exception:
print("[-] Error with TAXII response")
else:
print("[+] No content returned")
_saveTimestamp(str(end))
else:
print("[-] Error with TAXII response")
def obtener_remote_data_feeds(request):
#Given the id of a TAXII Service we make a FeedInformation request to that service address.
#The response is a list of the feed names of the TAXII client and a list of all protocol bindings, content binding and message binding.
feed_managment = TAXIIServices.objects.get(id = request.DATA.get('id'))
urlParsed = urlparse(feed_managment.feed_managment)
logger = logging.getLogger('TAXIIApplication.rest.tasks.obtener_remote_data_feeds')
logger.debug('We get the server data feeds')
logger.debug('Host: ' + urlParsed.hostname)
logger.debug('Path: ' + urlParsed.path)
logger.debug('Port: ' + str(urlParsed.port))
host = urlParsed.hostname
path = urlParsed.path
port = str(urlParsed.port)
feed_information = tm.FeedInformationRequest(message_id=tm.generate_message_id())
feed_info_xml = feed_information.to_xml()
logger.debug('The following message is sent: ' + feed_info_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_info_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug("The response was: " + response_message.to_xml())
try:
taxii_message = tm.get_message_from_xml(response_message.to_xml())
logger.debug("The JSON is: " + taxii_message.to_json())
feed_informations = taxii_message.feed_informations
feed_names = []
for feed in feed_informations:
feed_names.append({"name" : feed.feed_name})
protocolBindings = ProtocolBindingId.objects.all()
protocol_bindings = []
for proto in protocolBindings:
protocol_bindings.append({"binding_id" : proto.binding_id})
contentBindings = ContentBindingId.objects.all()
content_bindings = []
for content in contentBindings:
content_bindings.append({"binding_id" : content.binding_id})
messageBindings = MessageBindingId.objects.all()
message_bindings = []
for message in messageBindings:
message_bindings.append({"binding_id" : message.binding_id})
json_data = json.dumps({ "items" : feed_names, "protocol_bindings" : protocol_bindings, "content_bindings" : content_bindings, "message_bindings" : message_bindings })
logger.debug("The response is the following JSON: " + json_data)
return HttpResponse(json_data, content_type="application/json")
except Exception as ex:
logger.debug('The message could not be parsed:s', ex.message)
def poll_11(self):
#request xml作成
poll_parameters = tm11.PollParameters()
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=self._collection_name,
exclusive_begin_timestamp_label=self._start,
inclusive_end_timestamp_label=self._end,
poll_parameters=poll_parameters,
)
last_requested = datetime.datetime.now(pytz.utc)
#from xml.dom.minidom import parseString
#print( parseString(poll_request.to_xml()).toprettyxml(encoding='utf-8'))
#request
http_resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
poll_request.to_xml(),
port=self._port)
#taxii_message抽出
taxii_message = libtaxii.get_message_from_http_response(http_resp, poll_request.message_id)
#content_blocks が None or 存在しない場合は登録ししない
try:
if taxii_message.content_blocks is None:
return 0
except AttributeError:
return 0
#content_blocksごとに登録処理
count = 0
for cb in taxii_message.content_blocks:
#stixファイルを一時ファイルに出力
_,stix_file_path = tempfile.mkstemp(suffix='.xml')
with open(stix_file_path,'wb+') as fp:
#cb.contentがstixの中身
fp.write(cb.content)
#登録
try:
from ctirs.core.stix.regist import regist
if self._community is not None:
regist(stix_file_path,self._community,self._via)
count += 1
except:
#taxiiの場合は途中で失敗しても処理を継続する
traceback.print_exc()
#last_requested更新
self._taxii.last_requested = last_requested
self._taxii.save()
return count
def run(self):
request_message = self.create_request_message()
client = self.create_client()
if self.port:
resp = client.call_taxii_service2(self.host, self.url.path, self.taxii_version, request_message.to_xml(pretty_print=True), self.port)
else:
resp = client.call_taxii_service2(self.host, self.url.path, self.taxii_version, request_message.to_xml(pretty_print=True))
r = t.get_message_from_http_response(resp, '0')
self.handle_response(r)
def _poll_fulfillment_request(self, tc, result_id, result_part_number):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.PollFulfillmentRequest(
message_id=msg_id,
result_id=result_id,
result_part_number=result_part_number,
collection_name=self.collection)
request = request.to_xml()
resp = self._call_taxii_service(self.poll_service, tc, request)
return libtaxii.get_message_from_http_response(resp, msg_id)
def main():
parser = scripts.get_base_parser("Discovery Client", path="/services/discovery/")
args = parser.parse_args()
discovery_req = tm11.DiscoveryRequest(message_id=tm11.generate_message_id())
discovery_req_xml = discovery_req.to_xml(pretty_print=True)
print "Discovery Request: \r\n", discovery_req_xml
client = scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, discovery_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml(pretty_print=True)
def main():
parser = t.scripts.get_base_parser("TAXII 1.0 Poll Client (Deprecated)", path="/services/poll/")
parser.add_argument("--feed", dest="feed", default="default", help="Data Collection to poll. Defaults to 'default'.")
parser.add_argument("--begin_timestamp", dest="begin_ts", default=None, help="The begin timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
parser.add_argument("--end_timestamp", dest="end_ts", default=None, help="The end timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
parser.add_argument("--subscription-id", dest="subs_id", default=None, help="The subscription ID to use. Defaults to None")
args = parser.parse_args()
try:
if args.begin_ts:
begin_ts = dateutil.parser.parse(args.begin_ts)
if not begin_ts.tzinfo:
raise ValueError
else:
begin_ts = None
if args.end_ts:
end_ts = dateutil.parser.parse(args.end_ts)
if not end_ts.tzinfo:
raise ValueError
else:
end_ts = None
except ValueError:
print "Unable to parse timestamp value. Timestamp should include both date and time information along with a timezone or UTC offset (e.g., YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm). Aborting poll."
sys.exit()
poll_req = tm10.PollRequest(message_id=tm10.generate_message_id(),
feed_name=args.feed,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
subscription_id=args.subs_id)
poll_req_xml = poll_req.to_xml(pretty_print=True)
print "Poll Request: \r\n", poll_req_xml
client = tc.HttpClient()
client.setUseHttps(args.https)
client.setProxy(args.proxy)
tls = (args.cert is not None and args.key is not None)
basic = (args.username is not None and args.password is not None)
if tls and basic:
client.setAuthType(tc.HttpClient.AUTH_CERT_BASIC)
client.setAuthCredentials({'key_file': args.key, 'cert_file': args.cert, 'username': args.username, 'password': args.password})
elif tls:
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': args.key, 'cert_file': args.cert})
elif basic:
client.setAuthType(tc.HttpClient.AUTH_BASIC)
client.setAuthCredentials({'username': args.username, 'password': args.password})
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, poll_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml(pretty_print=True)
def send_taxii_message(self, request, host, path, port):
# Send the request message and return the response
http_response = self.call_taxii_service2(
host=host,
path=path,
message_binding=VID_TAXII_XML_11,
post_data=request.to_xml(),
port=port,
user_agent='{} (libtaxii)'.format(version_string))
response = get_message_from_http_response(
http_response=http_response,
in_response_to=request.message_id,
)
return response
def _check_collections(self, tc):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.CollectionInformationRequest(msg_id)
request = request.to_xml()
resp = self._call_taxii_service(self.collection_mgmt_service, tc,
request)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('Collection_Information_Response {%s} %s', type(tm),
tm.to_xml(pretty_print=True))
tci = None
for ci in tm.collection_informations:
if ci.collection_name != self.collection:
continue
tci = ci
break
if tci is None:
raise RuntimeError('%s - collection %s not found' %
(self.name, self.collection))
if tci.polling_service_instances is None or \
len(tci.polling_service_instances) == 0:
raise RuntimeError('%s - collection %s doesn\'t support polling' %
(self.name, self.collection))
if tci.collection_type != libtaxii.constants.CT_DATA_FEED:
raise RuntimeError(
'%s - collection %s is not a data feed (%s)' %
(self.name, self.collection, tci.collection_type))
for pi in tci.polling_service_instances:
LOG.info('{} - message binding: {}'.format(
self.name, pi.poll_message_bindings))
if pi.poll_message_bindings[
0] == libtaxii.constants.VID_TAXII_XML_11:
self.poll_service = pi.poll_address
LOG.info('{} - poll service found'.format(self.name))
break
else:
raise RuntimeError(
'%s - collection %s does not support TAXII 1.1 message binding (%s)'
% (self.name, self.collection, tci.collection_type))
LOG.debug('%s - poll service: %s', self.name, self.poll_service)
def push_11(self, stix_file_doc):
self.debug_print('>>> push_11: enter')
# stix_file_doc の versionが 2.0 ならスライド
if stix_file_doc.version == '2.0':
try:
content = stix_file_doc.get_slide_1_x()
except Exception as e:
traceback.print_exc()
raise e
else:
with open(stix_file_doc.origin_path, 'r', encoding='utf-8') as fp:
content = fp.read()
# Subscription Information xml作成
subscription_information = tm11.SubscriptionInformation(
collection_name=self._collection_name,
subscription_id='subscripiton_id')
cb = tm11.ContentBlock(const.CB_STIX_XML_11, content)
im = tm11.InboxMessage(
message_id=tm11.generate_message_id(),
destination_collection_names=[self._collection_name],
subscription_information=subscription_information,
content_blocks=[cb])
try:
# push
self._address = '10.0.3.100'
# self.debug_print('>>> push_11: self._address:%s' %(self._address))
# self.debug_print('>>> push_11: self._path:%s' %(self._path))
# self.debug_print('>>> push_11: self._port:%d' %(self._port))
# self.debug_print('>>> push_11: self._collection_name:%s' %(self._collection_name))
# self.debug_print('>>> push_11: verify_server:%s' %(str(self._client.verify_server)))
# self.debug_print('>>> push_11: use_https:%s' %(str(self._client.use_https)))
http_resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
im.to_xml(),
port=self._port)
taxii_message = libtaxii.get_message_from_http_response(
http_resp, im.message_id)
if taxii_message.status_type != 'SUCCESS':
self.debug_print('taxii_message.status_type is not SUCCESS')
self.debug_print(taxii_message.to_xml(pretty_print=True))
raise Exception('taxii_message.status_type is not SUCCESS')
except Exception as e:
traceback.print_exc()
self.debug_print(e)
raise e
def _poll_collection(self, tc, begin=None, end=None):
msg_id = libtaxii.messages_11.generate_message_id()
prargs = dict(
message_id=msg_id,
collection_name=self.collection,
exclusive_begin_timestamp_label=begin,
inclusive_end_timestamp_label=end,
)
if self.subscription_id_required:
prargs['subscription_id'] = self.subscription_id
else:
pps = libtaxii.messages_11.PollParameters(response_type='FULL',
allow_asynch=False)
prargs['poll_parameters'] = pps
request = libtaxii.messages_11.PollRequest(**prargs)
LOG.debug('%s - first poll request %s', self.name,
request.to_xml(pretty_print=True))
request = request.to_xml()
resp = self._call_taxii_service(self.poll_service, tc, request)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('%s - Poll_Response {%s} %s', self.name, type(tm),
tm.to_xml(pretty_print=True))
stix_objects = {'observables': {}, 'indicators': {}, 'ttps': {}}
self._handle_content_blocks(tm.content_blocks, stix_objects)
while tm.more:
tm = self._poll_fulfillment_request(
tc,
result_id=tm.result_id,
result_part_number=tm.result_part_number + 1)
self._handle_content_blocks(tm.content_blocks, stix_objects)
LOG.debug('%s - stix_objects: %s', self.name, stix_objects)
params = {
'ttps': stix_objects['ttps'],
'observables': stix_objects['observables']
}
return [[iid, iv, params]
for iid, iv in stix_objects['indicators'].iteritems()]
def main():
parser = argparse.ArgumentParser(description="Poll Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Poll Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Poll Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/poll/", help="Path where the Poll Service is hosted. Defaults to /services/poll/.")
parser.add_argument("--feed", dest="feed", default="default", help="Data Feed to poll. Defaults to 'default'.")
parser.add_argument("--begin_timestamp", dest="begin_ts", default=None, help="The begin timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
parser.add_argument("--end_timestamp", dest="end_ts", default=None, help="The end timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
parser.add_argument("--content-binding", dest="content_binding", default= t.CB_STIX_XML_10, help="Content binding of the Content Block to send. Defaults to %s" % t.CB_STIX_XML_10 )
args = parser.parse_args()
try:
if args.begin_ts:
begin_ts = dateutil.parser.parse(args.begin_ts)
if not begin_ts.tzinfo:
raise ValueError
else:
begin_ts = None
if args.end_ts:
end_ts = dateutil.parser.parse(args.end_ts)
if not end_ts.tzinfo:
raise ValueError
else:
end_ts = None
except ValueError:
print "Unable to parse timestamp value. Timestamp should include both date and time information along with a timezone or UTC offset (e.g., YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm). Aborting poll."
sys.exit()
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name = args.feed,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
subscription_id='1')
poll_req_xml = poll_req.to_xml()
print "Poll Request: \r\n", poll_req_xml
client = tc.HttpClient()
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, poll_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
message = tm11.get_message_from_xml(response_message.to_xml())
response_message.
def poll_feed(settings, subscription):
""" polls a TAXII feed"""
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(True)
client.set_auth_credentials({
'username': settings['username'],
'password': settings['password']
})
msg_id = uuid.uuid4().hex
poll_request1 = tm11.PollRequest(message_id=msg_id,
collection_name=settings['subscriptions']
[subscription]['collection_name'],
subscription_id=settings['subscriptions']
[subscription]['subscription_id'])
poll_xml = poll_request1.to_xml()
http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/',
VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(http_resp,
poll_request1.message_id)
observables = {}
indicators = json.loads(taxii_message.to_json())
if 'content_blocks' in indicators.keys():
for indicator in indicators['content_blocks']:
open('/tmp/indicator.xml', 'w').write(indicator['content'])
indi = STIXPackage.from_xml('/tmp/indicator.xml').to_dict()
if 'observables' in indi.keys():
for obs in indi['observables']['observables']:
if 'object' in obs.keys():
ot = obs['object']['properties']['xsi:type']
if ot in settings['supported_objects'].keys(
) and not ot in observables.keys():
observables[ot] = []
if ot in settings['supported_objects'].keys(
) and settings['supported_objects'][ot] in obs[
'object']['properties'].keys():
# note, you will only be able to process one property per object type, but you also know there's only one property you can process
try:
observables[ot].append(
obs['object']['properties'][
settings['supported_objects'][ot]])
except:
print "[-] you're dumb"
print supported_objects[ot], "not in:", obs[
'object']
return observables
def _poll_collection(self, tc, begin=None, end=None):
msg_id = libtaxii.messages_11.generate_message_id()
pps = libtaxii.messages_11.PollParameters(response_type='FULL',
allow_asynch=False)
request = libtaxii.messages_11.PollRequest(
message_id=msg_id,
collection_name=self.collection,
exclusive_begin_timestamp_label=begin,
inclusive_end_timestamp_label=end,
poll_parameters=pps)
LOG.debug('%s - first poll request %s', self.name,
request.to_xml(pretty_print=True))
request = request.to_xml()
up = urlparse.urlparse(self.poll_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(hostname, path,
libtaxii.constants.VID_TAXII_XML_11,
request)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('%s - Poll_Response {%s} %s', self.name, type(tm),
tm.to_xml(pretty_print=True))
stix_objects = {'observables': {}, 'indicators': {}, 'ttps': {}}
self._handle_content_blocks(tm.content_blocks, stix_objects)
while tm.more:
tm = self._poll_fulfillment_request(
tc,
result_id=tm.result_id,
result_part_number=tm.result_part_number + 1)
self._handle_content_blocks(tm.content_blocks, stix_objects)
LOG.debug('%s - stix_objects: %s', self.name, stix_objects)
params = {
'ttps': stix_objects['ttps'],
'observables': stix_objects['observables']
}
return [[iid, iv, params]
for iid, iv in stix_objects['indicators'].iteritems()]
def _check_collections(self, tc):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.CollectionInformationRequest(msg_id)
request = request.to_xml()
up = urlparse.urlparse(self.collection_mgmt_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(
hostname,
path,
libtaxii.constants.VID_TAXII_XML_11,
request
)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('Collection_Information_Response {%s} %s',
type(tm), tm.to_xml(pretty_print=True))
tci = None
for ci in tm.collection_informations:
if ci.collection_name != self.collection:
continue
tci = ci
break
if tci is None:
raise RuntimeError('%s - collection %s not found' %
(self.name, self.collection))
if tci.polling_service_instances is None or \
len(tci.polling_service_instances) == 0:
raise RuntimeError('%s - collection %s doesn\'t support polling' %
(self.name, self.collection))
if tci.collection_type != libtaxii.constants.CT_DATA_FEED:
raise RuntimeError(
'%s - collection %s is not a data feed (%s)' %
(self.name, self.collection, tci.collection_type)
)
self.poll_service = tci.polling_service_instances[0].poll_address
LOG.debug('%s - poll service: %s',
self.name, self.poll_service)
def obtener_data_feeds(host, port, path):
#Given the host, port and path of a TAXII client we get the data feeds of that client
logger = logging.getLogger('TAXIIApplication.rest.tasks.obtener_data_feeds')
logger.debug('We get the server data feeds')
logger.debug('Host: ' + host)
logger.debug('Path: ' + path)
logger.debug('Port: ' + str(port))
feed_information = tm.FeedInformationRequest(message_id=tm.generate_message_id())
feed_info_xml = feed_information.to_xml()
logger.debug('The following message is sent: ' + feed_info_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_info_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug('We get the following response: '+response_message.to_xml())
def main():
parser = argparse.ArgumentParser(description="Discovery Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Discovery Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Discovery Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/discovery/", help="Path where the Discovery Service is hosted. Defaults to /services/discovery/.")
args = parser.parse_args()
discovery_req = tm.DiscoveryRequest(message_id=tm.generate_message_id())
discovery_req_xml = discovery_req.to_xml()
print "Discovery Request: \r\n", discovery_req_xml
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, discovery_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
def push(self, content):
cb = self._get_content_block(content)
im = self._get_inbox_message(cb)
im_xml = im.to_xml()
resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
im_xml,
port=self._port
)
taxii_message = libtaxii.get_message_from_http_response(resp, im.message_id)
if taxii_message.status_type == 'SUCCESS':
# messageがない場合のみ成功
return '' if taxii_message.message is None else taxii_message.message
else:
return taxii_message.message
def indicators(self,
collection_name,
starttime=(datetime.now(tzutc()) - timedelta(hours=1)),
endtime=datetime.now(tzutc()),
subscription_id=None):
delivery_params = tm11.DeliveryParameters(VID_TAXII_HTTP_10,
self.remote,
VID_TAXII_XML_11)
"""
poll_params = tm11.PollParameters(response_type=RT_COUNT_ONLY,
#content_bindings=[tm11.ContentBinding(CB_STIX_XML_11)],
delivery_parameters=delivery_params
)
"""
poll_params = tm11.PollRequest.PollParameters()
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=collection_name,
exclusive_begin_timestamp_label=starttime,
inclusive_end_timestamp_label=endtime,
poll_parameters=poll_params,
subscription_id=subscription_id)
logger.debug('TAXII collection poll request: {}'.format(
poll_req.to_xml(pretty_print=True)))
poll_req_xml = poll_req.to_xml()
http_resp = self.client.call_taxii_service2(self.up.hostname,
self.up.path,
VID_TAXII_XML_11,
poll_req_xml, self.up.port)
taxii_message = t.get_message_from_http_response(
http_resp, poll_req.message_id)
logger.debug('TAXII collection poll response: {}'.format(
taxii_message.to_xml(pretty_print=True)))
if taxii_message.message_type == MSG_STATUS_MESSAGE:
if taxii_message.status_type == ST_SUCCESS:
logger.info(
'TAXII polled successfully but returned no results')
return []
raise RuntimeError('TAXII polling failed: {} - {}'.format(
taxii_message.status_type, taxii_message.message))
return self._parse_taxii_content(taxii_message.content_blocks)
def send_taxii_message(self, request, host, path, port):
# Send the request message and return the response
self._logger.debug('sending the taxii message now')
http_response = self.call_taxii_service2(
host=host,
path=path,
message_binding=VID_TAXII_XML_11,
post_data=request.to_xml(),
port=port,
user_agent='{} (libtaxii)'.format(version_string)
)
self._logger.debug('sent the taxii message')
response = get_message_from_http_response(
http_response=http_response,
in_response_to=request.message_id,
)
self._logger.debug('got a response to the taxii message')
return response
def log(self, event):
# converts from conpot log format to STIX compatible xml
stix_package = self.stix_transformer.transform(event)
# wrapping the stix message in a TAXII envelope
content_block = ContentBlock(libtaxii.CB_STIX_XML_10, stix_package)
inbox_message = InboxMessage(message_id=generate_message_id(), content_blocks=[content_block])
inbox_xml = inbox_message.to_xml()
# the actual call to the TAXII web service
response = self.client.callTaxiiService2(self.host, self.inbox_path, libtaxii.VID_TAXII_XML_10, inbox_xml, self.port)
response_message = libtaxii.get_message_from_http_response(response, '0')
if response_message.status_type != libtaxii.messages.ST_SUCCESS:
logger.error('Error while transmitting message to TAXII server: {0}'.format(response_message.status_detail))
return False
else:
return True
def _poll_fulfillment_request(self, tc, result_id, result_part_number):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.PollFulfillmentRequest(
message_id=msg_id,
result_id=result_id,
result_part_number=result_part_number,
collection_name=self.collection)
request = request.to_xml()
up = urlparse.urlparse(self.poll_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(hostname, path,
libtaxii.constants.VID_TAXII_XML_11,
request)
return libtaxii.get_message_from_http_response(resp, msg_id)
def poll(s, path="/", collection="default", query=None,
begin_ts=None, end_ts=None):
if query != None:
query=s.create_query(query)
poll_params=tm11.PollRequest.PollParameters(query=query)
else:
poll_params=tm11.PollRequest.PollParameters()
# Create poll request
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=poll_params)
# Convert to XML for request body
poll_req_xml = poll_req.to_xml(True)
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path,
t.VID_TAXII_XML_11,
poll_req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
pkgs = []
# Process each content block
for cb in resp.content_blocks:
content = cb.content
# Parse the payload, should be a STIX document.
package = STIXPackage.from_xml(StringIO.StringIO(content))
pkgs.append(package)
return resp.inclusive_end_timestamp_label, pkgs
def taxii_poll(src_feed, is_ssl_enabled, username, feed_path=common.TAXII_DATA, site_url="127.0.0.1", timestamp=None):
"""
pull stix from edge via taxii, currently requires trust_localhost enabled in taxii server
:param site_url:
:param src_feed:
:param is_ssl_enabled:
:param feed_path:
:param timestamp:
:return:
"""
client = tc.HttpClient()
client.setUseHttps(is_ssl_enabled)
# NOTE: IMPORTANT: ASSUMES TAXII CLIENT USES TRUST_LOCALHOST
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': username, 'password': ''})
if not timestamp:
earliest = epoch_start()
else:
earliest = timestamp
latest = nowutc()
poll_request = tm10.PollRequest(
message_id=tm10.generate_message_id(),
feed_name=src_feed,
inclusive_end_timestamp_label=latest,
content_bindings=[t.CB_STIX_XML_11])
http_response = client.callTaxiiService2(
site_url,
feed_path,
t.VID_TAXII_XML_10, poll_request.to_xml())
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm10.StatusMessage):
pass
elif isinstance(taxii_message, tm10.PollResponse):
indicators = dict()
observables = dict()
for content_block in taxii_message.content_blocks:
(indicators_, observables_) = \
process_taxii_content_blocks(content_block)
indicators.update(indicators_)
observables.update(observables_)
return latest, indicators, observables
def taxii_poll(config, src, dest, timestamp=None):
"""pull stix from edge via taxii"""
client = tc.HttpClient()
client.setUseHttps(config["edge"]["sites"][src]["taxii"]["ssl"])
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{
"username": config["edge"]["sites"][src]["taxii"]["user"],
"password": config["edge"]["sites"][src]["taxii"]["pass"],
}
)
if not timestamp:
earliest = util.epoch_start()
else:
earliest = timestamp
latest = util.nowutc()
poll_request = tm10.PollRequest(
message_id=tm10.generate_message_id(),
feed_name=config["edge"]["sites"][src]["taxii"]["collection"],
exclusive_begin_timestamp_label=earliest,
inclusive_end_timestamp_label=latest,
content_bindings=[t.CB_STIX_XML_11],
)
http_response = client.callTaxiiService2(
config["edge"]["sites"][src]["host"],
config["edge"]["sites"][src]["taxii"]["path"],
t.VID_TAXII_XML_10,
poll_request.to_xml(),
port=config["edge"]["sites"][src]["taxii"]["port"],
)
taxii_message = t.get_message_from_http_response(http_response, poll_request.message_id)
if isinstance(taxii_message, tm10.StatusMessage):
config["logger"].error(log.log_messages["polling_error"].format(type_="taxii", error=taxii_message.message))
elif isinstance(taxii_message, tm10.PollResponse):
incidents = dict()
indicators = dict()
observables = dict()
for content_block in taxii_message.content_blocks:
(incidents_, indicators_, observables_) = process_taxii_content_blocks(config, content_block)
incidents.update(incidents_)
indicators.update(indicators_)
observables.update(observables_)
return (latest, incidents, indicators, observables)
def _check_collections(self, tc):
msg_id = libtaxii.messages_11.generate_message_id()
request = libtaxii.messages_11.CollectionInformationRequest(msg_id)
request = request.to_xml()
up = urlparse.urlparse(self.collection_mgmt_service)
hostname = up.hostname
path = up.path
resp = tc.call_taxii_service2(hostname, path,
libtaxii.constants.VID_TAXII_XML_11,
request)
tm = libtaxii.get_message_from_http_response(resp, msg_id)
LOG.debug('Collection_Information_Response {%s} %s', type(tm),
tm.to_xml(pretty_print=True))
tci = None
for ci in tm.collection_informations:
if ci.collection_name != self.collection:
continue
tci = ci
break
if tci is None:
raise RuntimeError('%s - collection %s not found' %
(self.name, self.collection))
if tci.polling_service_instances is None or \
len(tci.polling_service_instances) == 0:
raise RuntimeError('%s - collection %s doesn\'t support polling' %
(self.name, self.collection))
if tci.collection_type != libtaxii.constants.CT_DATA_FEED:
raise RuntimeError(
'%s - collection %s is not a data feed (%s)' %
(self.name, self.collection, tci.collection_type))
self.poll_service = tci.polling_service_instances[0].poll_address
LOG.debug('%s - poll service: %s', self.name, self.poll_service)
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username': user,
'password': passwd})
earliest = epoch_start()
latest = nowutc()
poll_request = tm10.PollRequest(
message_id=tm10.generate_message_id(),
feed_name=collection,
exclusive_begin_timestamp_label=earliest,
inclusive_end_timestamp_label=latest,
content_bindings=[t.CB_STIX_XML_11])
http_response = client.callTaxiiService2(
host, endpoint,
t.VID_TAXII_XML_10, poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm10.StatusMessage):
print('''TAXII connection error! Exiting...
%s''' % (taxii_message.message))
elif isinstance(taxii_message, tm10.PollResponse):
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
for content_block in taxii_message.content_blocks:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
return(cooked_stix_objs, cooked_cybox_objs)
def perform_discovery(path="/"):
# Create discovery request
req = tm11.DiscoveryRequest(message_id=tm11.generate_message_id())
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_11,
req_xml, port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def enumerate_collections(self, _logger):
client = self.__instantiate_http_client()
collection_request = tm11.CollectionInformationRequest(tm11.generate_message_id())
collection_xml = collection_request.to_xml()
http_resp = client.callTaxiiService2(self.base_domain,
self.discovery_request_uri,
taxii.VID_TAXII_XML_11,
collection_xml)
message = taxii.get_message_from_http_response(http_resp, collection_request.message_id)
if type(message) == tm11.StatusMessage:
t = message.to_text()
x = getattr(message, 'status_type', None)
if x:
_logger.warn("Message response: %s" % x)
raise UnauthorizedException(t)
x = message.to_dict()
return x.get('collection_informations', [])
def subscripcion_data_feed(request):
#Given the id of a TAXII Service and the id of a Data Feed and the service id we make a Manage Feed Subscription request for that Data Feed.
logger = logging.getLogger('TAXIIApplication.taxii.views.subscripcion_data_feed')
logger.debug('The data feed subscription starts')
logger.debug(request.DATA)
data_feed = request.DATA.get('data_feed')
service = request.DATA.get('id')
inbox_protocol = request.DATA.get('protocol_binding')
message_binding = request.DATA.get('message_binding')
content_binding = request.DATA.get('content_binding')
feed_managment = TAXIIServices.objects.get(id = service)
urlParsed = urlparse(feed_managment.subscription)
logger.debug('Host: ' + urlParsed.hostname)
logger.debug('Path: ' + urlParsed.path)
logger.debug('Port: ' + str(urlParsed.port))
host = urlParsed.hostname
path = urlParsed.path
port = str(urlParsed.port)
delivery_parameters = tm.DeliveryParameters(inbox_protocol=inbox_protocol, inbox_address=INBOX_SERVICES_URL,
delivery_message_binding=message_binding, content_bindings=content_binding)
f = tm.ACT_TYPES
feed_subscription = tm.ManageFeedSubscriptionRequest(message_id=tm.generate_message_id(), feed_name=data_feed,
action= f[0], subscription_id='1', delivery_parameters=delivery_parameters)
feed_subscription_xml = feed_subscription.to_xml()
logger.debug('The message sent is: ' + feed_subscription_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_subscription_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug('The server respons: ' + response_message.to_xml())
return Response(status = status.HTTP_200_OK)
def main():
parser = t.scripts.get_base_parser("Poll Query Client", path="/services/query_example/")
parser.add_argument("--collection", dest="collection", default="default_queryable", help="Data Collection to poll. Defaults to 'default_queryable'.")
parser.add_argument("--allow_asynch", dest="allow_asynch", default=True, help="Indicate whether or not the client support Asynchronous Polling. Defaults to True")
parser.add_argument("--ip", dest="ip", default=None, help="The IP address to query")
parser.add_argument("--hash", dest="hash", default=None, help="The file hash to query")
args = parser.parse_args()
if args.ip is None and args.hash is None:
print "At least one of --ip or --hash must be specified!"
sys.exit()
criterion = []
if args.ip is not None:
tmp = tdq.DefaultQuery.Criterion.Test(capability_id=tdq.CM_CORE,
relationship='equals',
parameters={'value': args.ip, 'match_type': 'case_insensitive_string'})
criterion.append(tdq.DefaultQuery.Criterion(target='//Address_Value', test=tmp))
if args.hash is not None:
tmp = tdq.DefaultQuery.Criterion.Test(capability_id=tdq.CM_CORE,
relationship='equals',
parameters={'value': args.hash, 'match_type': 'case_insensitive_string'})
criterion.append(tdq.DefaultQuery.Criterion(target='//Hash/Simple_Hash_Value', test=tmp))
criteria = tdq.DefaultQuery.Criteria(operator=tdq.OP_AND, criterion=criterion)
q = tdq.DefaultQuery(t.CB_STIX_XML_11, criteria)
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=args.collection,
poll_parameters=tm11.PollRequest.PollParameters(allow_asynch=args.allow_asynch, query=q))
poll_req_xml = poll_req.to_xml(pretty_print=True)
print "Poll Request: \r\n", poll_req_xml
client = t.scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, poll_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml(pretty_print=True)
def run(self):
client = tc.HttpClient()
# TODO: handle authentication stuff
#client.set_auth_type(tc.AUTH_NONE)
client.set_use_https(False)
r = CollectionInformationRequest(generate_message_id())
col_xml = r.to_xml(pretty_print=True)
if ':' in self.url.netloc:
host = self.url.netloc.split(':')[0]
server_port = self.url.netloc.split(':')[1]
else:
host = self.url.netloc
server_port = None
if server_port:
http_resp = client.call_taxii_service2(host, self.url.path, VID_TAXII_XML_11, col_xml, port = server_port)
else:
http_resp = client.call_taxii_service2(host, self.url.path, VID_TAXII_XML_11, col_xml)
taxii_message = t.get_message_from_http_response(http_resp, r.message_id)
return taxii_message.to_dict()
def main():
parser = scripts.get_base_parser("Discovery Client", path="/services/discovery/")
args = parser.parse_args()
discovery_req = tm11.DiscoveryRequest(message_id=tm11.generate_message_id())
print "Request:\n"
if args.xml_output is False:
print discovery_req.to_text()
else:
print discovery_req.to_xml(pretty_print=True)
client = scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, discovery_req.to_xml(pretty_print=True), args.port)
r = t.get_message_from_http_response(resp, '0')
print "Response:\n"
if args.xml_output is False:
print r.to_text()
else:
print r.to_xml(pretty_print=True)
def main():
parser = scripts.get_base_parser("Collection Information Client", path="/services/collection-management/")
args = parser.parse_args()
collection_information_req = tm11.CollectionInformationRequest(message_id=tm11.generate_message_id())
print "Request:\n"
if args.xml_output is False:
print collection_information_req.to_text()
else:
print collection_information_req.to_xml(pretty_print=True)
client = scripts.create_client(args)
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11, collection_information_req.to_xml(pretty_print=True), args.port)
r = t.get_message_from_http_response(resp, '0')
print "Response:\n"
if args.xml_output is False:
print r.to_text()
else:
print r.to_xml(pretty_print=True)
def main(**args):
check_process(PID_FILE)
client = tc.HttpClient()
if PROXY_ENABLED:
client.proxy_type = PROXY_SCHEME
client.proxy_string = PROXY_STRING
msg_id, msg = '', ''
if args['data_type'] == 'string':
msg_id, msg = create_inbox_message(args['data'])
else:
print '[-] Please use a JSON string'
raise SystemExit
http_response = client.callTaxiiService2(
args['host'], args['path'],
t.VID_CERT_EU_JSON_10, msg, args['port'])
taxii_response = t.get_message_from_http_response(http_response, msg_id)
print(taxii_response.to_json())
def taxii_poll(config, src, dest, timestamp=None):
'''pull stix from edge via taxii'''
client = tc.HttpClient()
client.setUseHttps(config['edge']['sites'][src]['taxii']['ssl'])
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username': config['edge']['sites'][src]['taxii']['user'],
'password': config['edge']['sites'][src]['taxii']['pass']})
if not timestamp:
earliest = util.epoch_start()
else:
earliest = timestamp
latest = util.nowutc()
poll_request = tm10.PollRequest(
message_id=tm10.generate_message_id(),
feed_name=config['edge']['sites'][src]['taxii']['collection'],
exclusive_begin_timestamp_label=earliest,
inclusive_end_timestamp_label=latest,
content_bindings=[t.CB_STIX_XML_11])
http_response = client.callTaxiiService2(
config['edge']['sites'][src]['host'],
config['edge']['sites'][src]['taxii']['path'],
t.VID_TAXII_XML_10, poll_request.to_xml(),
port=config['edge']['sites'][src]['taxii']['port'])
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm10.StatusMessage):
config['logger'].error(log.log_messages['polling_error'].format(
type_='taxii', error=taxii_message.message))
elif isinstance(taxii_message, tm10.PollResponse):
incidents = dict()
indicators = dict()
observables = dict()
for content_block in taxii_message.content_blocks:
(incidents_, indicators_, observables_) = \
process_taxii_content_blocks(config, content_block)
incidents.update(incidents_)
indicators.update(indicators_)
observables.update(observables_)
return(latest, incidents, indicators, observables)
def subscribe(s, path="/", collection="default", query=None,
inbox="http://localhost:8888"):
if query != None:
query = s.create_query(query)
else:
query = None
params = tm11.SubscriptionParameters(query=query)
deliv = tm11.PushParameters(
inbox_protocol=t.VID_TAXII_HTTP_10,
inbox_address=inbox,
delivery_message_binding=t.VID_TAXII_XML_11)
# Create request
msg_id=tm11.generate_message_id()
req = tm11.ManageCollectionSubscriptionRequest(
message_id=msg_id,
collection_name=collection,
action=tm11.ACT_SUBSCRIBE,
subscription_parameters=params,
push_parameters=deliv
)
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path, t.VID_TAXII_XML_11,
req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def perform_manage_collection_subscription(path="/", act="status",
collection="default"):
if act == "subscribe":
action = tm11.ACT_SUBSCRIBE
elif act == "unsubscribe":
action = tm11.ACT_UNSUBSCRIBE
elif act == "pause":
action = tm11.ACT_PAUSE
elif act == "resume":
action = tm11.ACT_RESUME
elif act == "status":
action = tm11.ACT_STATUS
else:
print("Need a subscription action I recognise")
sys.exit(1)
# Create request
msg_id=tm11.generate_message_id()
req = tm11.ManageCollectionSubscriptionRequest(message_id=msg_id,
collection_name=collection,
action=action)
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_11,
req_xml, port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def client_example():
# Create the TAXII HTTPS Client
client = tc.HttpClient()
# Uncomment to use HTTPS
client.set_use_https(True)
# Uncomment to use basic authentication
# client.set_auth_type(tc.HttpClient.AUTH_BASIC)
# client.set_auth_credentials({'username':'******', 'password':'******'})
# Uncomment to use certificate-based authentication
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({
'key_file': 'keyfile',
'cert_file': 'certfile'
})
# Uncomment to set a proxy
# client.set_proxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
# Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name='TheFeedToPoll')
# Call without a proxy
http_response = client.call_taxii_service2('hostname',
'/poll_service_path/',
VID_TAXII_XML_10,
poll_request1.to_xml())
print(http_response.__class__.__name__)
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print((taxii_message.to_xml()))
def send_poll_request(self):
"""Send the poll request to the TAXII server."""
poll_request1 = self.create_poll_request()
self._logger.debug("Request generated: using collection name - %s",
self._collection)
http_response = self.call_taxii_service2(
self._hostname,
self._path,
VID_TAXII_XML_11,
poll_request1.to_xml(),
self._port,
)
self._logger.debug("TAXII response received")
self._logger.debug("HTTP response %s",
http_response.__class__.__name__)
self._poll_response = get_message_from_http_response(
http_response,
poll_request1.message_id,
)
if self._poll_response.message_type != MSG_POLL_RESPONSE:
raise Exception('TAXII response not a poll response as expected.')