def inbox_service(request, inbox_name):
"""Handles TAXII Inbox Service requests."""
logger = logging.getLogger('yeti.taxii_services.views.inbox_service')
logger.debug('Entering Inbox service')
resp = handlers.validate_taxii_request(request)
if resp: return resp # if validation failed, return the response
try:
taxii_message = tm.get_message_from_xml(request.body)
except Exception as ex:
logger.debug('Unable to parse inbound message: %s', ex.message)
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_BAD_MESSAGE, message='Message received could not be parsed')
return handlers.create_taxii_response(m, use_https=request.is_secure())
logger.debug('Inbox [%s] received TAXII message with id [%s] and type [%s]',
make_safe(inbox_name), make_safe(taxii_message.message_id), make_safe(taxii_message.message_type))
if taxii_message.message_type != tm.MSG_INBOX_MESSAGE:
logger.info('TAXII message with id [%s] was not Inbox type [%s]', make_safe(taxii_message.message_id), make_safe(taxii_message.message_type))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_FAILURE, message='Message sent to Inbox service did not have an inbox Message type')
return handlers.create_taxii_response(m, use_https=request.is_secure())
resp = handlers.inbox_add_content(request, inbox_name, taxii_message)
return resp
def poll_get_content(request, taxii_message):
"""Returns a Poll response for a given Poll Request Message"""
logger = logging.getLogger('taxii_services.utils.handlers.poll_get_content')
logger.debug('polling data from data feed [%s]' % taxii_message.feed_name)
logger.debug('begin_ts: %s, end_ts: %s' % (taxii_message.exclusive_begin_timestamp_label, taxii_message.inclusive_end_timestamp_label))
try:
data_feed = DataFeed.objects.get(name=taxii_message.feed_name)
except:
logger.debug('attempting to poll unknown data feed [%s]' % (taxii_message.feed_name))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_NOT_FOUND, message='Data feed does not exist [%s]' % (taxii_message.feed_name))
return create_taxii_response(m, use_https=request.is_secure())
# build query for poll results
query_params = {}
if taxii_message.exclusive_begin_timestamp_label:
query_params['timestamp_label__gt'] = taxii_message.exclusive_begin_timestamp_label
current_datetime = datetime.datetime.now(tzutc())
if taxii_message.inclusive_end_timestamp_label and (taxii_message.inclusive_end_timestamp_label < current_datetime):
query_params['timestamp_label__lte'] = taxii_message.inclusive_end_timestamp_label
else:
query_params['timestamp_label__lte'] = current_datetime
if taxii_message.content_bindings:
query_params['content_binding__in'] = taxii_message.content_bindings
content_blocks = data_feed.content_blocks.filter(**query_params).order_by('timestamp_label')
logger.debug('returned %d content blocks from data feed' % (len(content_blocks)))
# TAXII Poll Requests have exclusive begin timestamp label fields, while Poll Responses
# have *inclusive* begin timestamp label fields. To satisfy this, we add one millisecond
# to the Poll Request's begin timestamp label.
#
# This will be addressed in future versions of TAXII
inclusive_begin_ts = None
if taxii_message.exclusive_begin_timestamp_label:
inclusive_begin_ts = taxii_message.exclusive_begin_timestamp_label + datetime.timedelta(milliseconds=1)
# build poll response
poll_response_message = tm.PollResponse(tm.generate_message_id(),
taxii_message.message_id,
feed_name=data_feed.name,
inclusive_begin_timestamp_label=inclusive_begin_ts,
inclusive_end_timestamp_label=query_params['timestamp_label__lte'])
for content_block in content_blocks:
cb = tm.ContentBlock(content_block.content_binding.binding_id, content_block.content, content_block.timestamp_label)
if content_block.padding:
cb.padding = content_block.padding
poll_response_message.content_blocks.append(cb)
return create_taxii_response(poll_response_message, use_https=request.is_secure())
def inbox_add_content(request, inbox_name, taxii_message):
"""Adds content to inbox and associated data feeds"""
logger = logging.getLogger('taxii_services.utils.handlers.inbox_add_content')
logger.debug('Adding content to inbox [%s]', make_safe(inbox_name))
try:
inbox = Inbox.objects.get(name=inbox_name)
except:
logger.debug('Attempting to push content to unknown inbox [%s]', make_safe(inbox_name))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_NOT_FOUND, message='Inbox does not exist [%s]' % (make_safe(inbox_name)))
return create_taxii_response(m, use_https=request.is_secure())
logger.debug('TAXII message [%s] contains [%d] content blocks', make_safe(taxii_message.message_id), len(taxii_message.content_blocks))
for content_block in taxii_message.content_blocks:
try:
content_binding_id = ContentBindingId.objects.get(binding_id=content_block.content_binding)
except:
logger.debug('TAXII message [%s] contained unrecognized content binding [%s]', make_safe(taxii_message.message_id), make_safe(content_block.content_binding))
continue # cannot proceed - move on to the next content block
if content_binding_id not in inbox.supported_content_bindings.all():
logger.debug('Inbox [%s] does not accept content with binding id [%s]', make_safe(inbox_name), make_safe(content_block.content_binding))
else:
c = ContentBlock()
c.message_id = taxii_message.message_id
c.content_binding = content_binding_id
c.content = content_block.content
if content_block.padding:
c.padding = content_block.padding
if request.user.is_authenticated():
c.submitted_by = request.user
c.save()
inbox.content_blocks.add(c) # add content block to inbox
for data_feed in inbox.data_feeds.all():
if content_binding_id in data_feed.supported_content_bindings.all():
data_feed.content_blocks.add(c)
data_feed.save()
else:
logger.debug('Inbox [%s] received data using content binding [%s] - '
'associated data feed [%s] does not support this binding.',
make_safe(inbox_name), make_safe(content_block.content_binding), make_safe(data_feed.name))
inbox.save()
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type = tm.ST_SUCCESS)
return create_taxii_response(m, use_https=request.is_secure())
def validate_taxii_headers(request, request_message_id):
"""
Validates TAXII headers. It returns a response containing a TAXII
status message if the headers were not valid. It returns None if
the headers are valid.
"""
logger = logging.getLogger("taxii.utils.handlers.validate_taxii_headers")
# Make sure the required headers are present
logger.debug("TAXII headers are going to be validated")
missing_required_headers = set(DICT_REQUIRED_TAXII_HTTP_HEADERS).difference(set(request.META))
if missing_required_headers:
logger.debug("MIssing required headers")
required_headers = ', '.join(DICT_REVERSE_DJANGO_NORMALIZATION[x] for x in missing_required_headers)
msg = "Required headers not present: [%s]" % (required_headers)
m = tm.StatusMessage(tm.generate_message_id(), request_message_id, status_type=tm.ST_FAILURE, message=msg)
return create_taxii_response(m, use_https=request.is_secure())
#For headers that exist, make sure the values are supported
for header in DICT_TAXII_HTTP_HEADER_VALUES:
if header not in request.META:
continue
header_value = request.META[header]
supported_values = DICT_TAXII_HTTP_HEADER_VALUES[header]
if header_value not in supported_values:
logger.debug("A header value is not supported [%s]", header_value)
msg = 'The value of %s is not supported. Supported values are %s' % (DICT_REVERSE_DJANGO_NORMALIZATION[header], supported_values)
m = tm.StatusMessage(tm.generate_message_id(), request_message_id, status_type=tm.ST_FAILURE, message=msg)
return create_taxii_response(m, use_https=request.is_secure())
# Check to make sure the specified protocol matches the protocol used
if request.META[DJANGO_HTTP_HEADER_X_TAXII_PROTOCOL] == t.VID_TAXII_HTTPS_10:
header_proto = 'HTTPS'
elif request.META[DJANGO_HTTP_HEADER_X_TAXII_PROTOCOL] == t.VID_TAXII_HTTP_10:
header_proto = 'HTTP'
else:
header_proto = 'unknown'
request_proto = 'HTTPS' if request.is_secure() else 'HTTP' # Did the request come over HTTP or HTTPS?
if header_proto != request_proto:
msg = 'Protocol value incorrect. TAXII header specified %s but was sent over %s' % (header_proto, request_proto)
m = tm.StatusMessage(tm.generate_message_id(), request_message_id, status_type=tm.ST_FAILURE, message=msg)
return create_taxii_response(m, use_https=request.is_secure())
# At this point, the header values are known to be good.
return None
def gen_send(tm_, client, encrypted_block, hostname, t_xml, dcn=None, eh=None,
url="/inbox/"):
"""
Generate and send a TAXII message.
:param tm_: The TAXII version imported that we should use.
:type tm_: TAXII message class.
:param client: The TAXII client to use.
:type client: TAXII Client.
:param encrypted_block: The encrypted block to use.
:type encrypted_block: TAXII Encrypted Block
:param hostname: The TAXII server hostname to connect to.
:type hostname: str
:param t_xml: The TAXII XML Schema version we used.
:type t_xml: str
:param dcn: Destination Collection Names we are using.
:type dcn: list
:param eh: Extended Headers to use.
:type eh: dict
:param url: The URL suffix to locate the inbox service.
:type url: str
:returns: tuple (response, taxii_message) or (exception message)
"""
# Wrap encrypted block in content block
content_block = tm_.ContentBlock(
content_binding = "application/x-pks7-mime",
content = encrypted_block
)
# Create inbox message
if dcn:
inbox_message = tm_.InboxMessage(
message_id = tm_.generate_message_id(),
content_blocks = [content_block],
destination_collection_names = dcn
)
elif eh:
inbox_message = tm.InboxMessage(
message_id = tm.generate_message_id(),
content_blocks = [content_block],
extended_headers = eh
)
else:
#TODO: return better
return None
# send inbox message via TAXII service
try:
response = client.callTaxiiService2(
hostname,
url,
t_xml,
inbox_message.to_xml()
)
taxii_message = t.get_message_from_http_response(response,
inbox_message.message_id)
return (response, taxii_message)
# can happen if 'hostname' is reachable, but is not a TAXII server, etc
except Exception, e:
return (e)
def main():
parser = argparse.ArgumentParser(description="Inbox Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Inbox Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Inbox Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/inbox/default/", help="Path where the Inbox Service is hosted. Defaults to /services/inbox/default/.")
parser.add_argument("--content-binding", dest="content_binding", default=t.CB_STIX_XML_10, help="Content binding of the Content Block to send. Defaults to %s" % t.CB_STIX_XML_10 )
parser.add_argument("--content-file", dest="content_file", default=stix_watchlist, help="Content of the Content Block to send. Defaults to a STIX watchlist.")
args = parser.parse_args()
if args.content_file is stix_watchlist:
c = StringIO.StringIO(stix_watchlist)
else:
c = open(args.content_file, 'r')
cb = tm.ContentBlock(args.content_binding, c.read())
inbox_message = tm.InboxMessage(message_id = tm.generate_message_id(), content_blocks=[cb])
inbox_xml = inbox_message.to_xml()
print "Inbox Message: \r\n", inbox_xml
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, inbox_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
def client_example():
# Create the TAXII HTTPS Client
client = tc.HttpClient()
# Uncomment to use HTTPS
client.set_use_https(True)
# Uncomment to use basic authentication
# client.set_auth_type(tc.HttpClient.AUTH_BASIC)
# client.set_auth_credentials({'username':'******', 'password':'******'})
# Uncomment to use certificate-based authentication
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({'key_file': 'keyfile',
'cert_file': 'certfile'})
# Uncomment to set a proxy
# client.set_proxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
# Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(), feed_name='TheFeedToPoll')
# Call without a proxy
http_response = client.call_taxii_service2('hostname', '/poll_service_path/', VID_TAXII_XML_10, poll_request1.to_xml())
print http_response.__class__.__name__
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print(taxii_message.to_xml())
def discovery_get_services(request, taxii_message):
"""Returns a Discovery response for a given Discovery Request Message"""
logger = logging.getLogger('taxii_services.utils.handlers.discovery_get_services')
all_services = []
# Inbox Services
for inbox in Inbox.objects.all():
service_type = tm.SVC_INBOX
inbox_name = inbox.name
service_addr = request.build_absolute_uri(reverse('taxii_services.views.inbox_service', args=[inbox_name]))
proto_binding = TAXII_PROTO_HTTP_BINDING_ID
message_bindings = [x.binding_id for x in inbox.supported_message_bindings.all()]
content_bindings = [x.binding_id for x in inbox.supported_content_bindings.all()]
available = True # TODO: this should reflect whether or not the authenticated user has access to this inbox
service_instance = tm.DiscoveryResponse.ServiceInstance(service_type=service_type,
services_version=TAXII_SERVICES_VERSION_ID,
protocol_binding=proto_binding,
service_address=service_addr,
message_bindings=message_bindings,
inbox_service_accepted_content=content_bindings,
available=available)
all_services.append(service_instance)
# Poll Service
all_data_feeds = DataFeed.objects.all()
if all_data_feeds:
all_data_feed_msg_bindings = set()
for data_feed in all_data_feeds:
poll_svc_instances = data_feed.poll_service_instances.all()
for poll_svc_instance in poll_svc_instances:
message_bindings = [x.binding_id for x in poll_svc_instance.message_bindings.all()]
all_data_feed_msg_bindings.update(message_bindings)
service_instance = tm.DiscoveryResponse.ServiceInstance(service_type=tm.SVC_POLL,
services_version=TAXII_SERVICES_VERSION_ID,
protocol_binding=TAXII_PROTO_HTTP_BINDING_ID,
service_address=request.build_absolute_uri(reverse('taxii_services.views.poll_service')),
message_bindings=all_data_feed_msg_bindings,
available=True)
all_services.append(service_instance)
# Discovery Service
service_instance = tm.DiscoveryResponse.ServiceInstance(service_type=tm.SVC_DISCOVERY,
services_version=TAXII_SERVICES_VERSION_ID,
protocol_binding=TAXII_PROTO_HTTP_BINDING_ID,
service_address=request.build_absolute_uri(reverse('taxii_services.views.discovery_service')),
message_bindings=[TAXII_MESSAGE_XML_BINDING_ID],
available=True)
all_services.append(service_instance)
# Build response
discovery_response_message = tm.DiscoveryResponse(message_id=tm.generate_message_id(),
in_response_to=taxii_message.message_id,
service_instances=all_services)
return create_taxii_response(discovery_response_message, use_https=request.is_secure())
def gen_send(tm_, client, encrypted_block, hostname, t_xml, dcn=None, eh=None,
url="/inbox/"):
"""
Generate and send a TAXII message.
:param tm_: The TAXII version imported that we should use.
:type tm_: TAXII message class.
:param client: The TAXII client to use.
:type client: TAXII Client.
:param encrypted_block: The encrypted block to use.
:type encrypted_block: TAXII Encrypted Block
:param hostname: The TAXII server hostname to connect to.
:type hostname: str
:param t_xml: The TAXII XML Schema version we used.
:type t_xml: str
:param dcn: Destination Collection Names we are using.
:type dcn: list
:param eh: Extended Headers to use.
:type eh: dict
:param url: The URL suffix to locate the inbox service.
:type url: str
:returns: tuple (response, taxii_message) or (exception message)
"""
# Wrap encrypted block in content block
content_block = tm_.ContentBlock(
content_binding = "application/x-pks7-mime",
content = encrypted_block
)
# Create inbox message
if dcn:
inbox_message = tm_.InboxMessage(
message_id = tm_.generate_message_id(),
content_blocks = [content_block],
destination_collection_names = dcn
)
elif eh:
inbox_message = tm.InboxMessage(
message_id = tm.generate_message_id(),
content_blocks = [content_block],
extended_headers = eh
)
else:
#TODO: return better
return None
# send inbox message via TAXII service
try:
response = client.callTaxiiService2(
hostname,
url,
t_xml,
inbox_message.to_xml()
)
taxii_message = t.get_message_from_http_response(response,
inbox_message.message_id)
return (response, taxii_message)
# can happen if 'hostname' is reachable, but is not a TAXII server, etc
except Exception, e:
return (e)
def discovery_service():
"""The Discovery Service provides a requester with a list of TAXII Services
and how these Services can be invoked"""
# be conservative in what you send and liberal in what you accept
request_msg = tm.get_message_from_json(request.data)
inbox_instance = tm.DiscoveryResponse.ServiceInstance(
service_type=tm.SVC_INBOX,
services_version=t.VID_TAXII_SERVICES_10,
protocol_binding=t.VID_TAXII_HTTP_10,
service_address=TAXII_ROOT + '/inbox/',
message_bindings=[t.VID_CERT_EU_JSON_10],
inbox_service_accepted_content=[CB_EVENTS_JSON_10],
available=True,
message='Inbox service')
poll_instance = tm.DiscoveryResponse.ServiceInstance(
service_type=tm.SVC_POLL,
services_version=t.VID_TAXII_SERVICES_10,
protocol_binding=t.VID_TAXII_HTTP_10,
service_address=TAXII_ROOT + '/poll/',
message_bindings=[t.VID_TAXII_XML_10, t.VID_CERT_EU_JSON_10],
inbox_service_accepted_content=[t.CB_STIX_XML_10, CB_EVENTS_JSON_10],
available=True,
message='Poll service')
discovery_response = tm.DiscoveryResponse(
message_id=tm.generate_message_id(),
in_response_to=request_msg.message_id,
service_instances=[inbox_instance, poll_instance])
return discovery_response.to_json()
def inbox():
"""The Inbox Service is the mechanism by which a Consumer accepts messages from a Producer in
Producer-initiated exchanges (i.e., push messaging)"""
start = time.time()
msg = tm.get_message_from_json(request.data)
dict = json.loads(msg.to_json())
data = dict['content_blocks'][0]['content']
posted_data = json.loads(data)
api_key = posted_data['api_key']
server_url = posted_data['server_url']
if request.headers.get('X-TAXII-Content-Type') != t.VID_CERT_EU_JSON_10:
sm = tm.StatusMessage(
message_id=tm.generate_message_id(),
in_response_to=msg.message_id,
status_type=tm.ST_UNSUPPORTED_PROTOCOL,
message='This service only accepts %s ' % t.VID_CERT_EU_JSON_10)
return make_taxii_response(sm.to_json())
if request.headers.get('X-TAXII-Content-Type') == t.VID_CERT_EU_JSON_10:
headers = {'Authorization': api_key, 'Content-Type': 'application/json',
'Accept': 'application/json'}
req = urllib2.Request(server_url, data, headers)
try:
response = urllib2.urlopen(req)
except Exception as e:
print str(e)
status_message1 = tm.StatusMessage(
message_id=tm.generate_message_id(),
in_response_to=msg.message_id,
status_type=tm.ST_SUCCESS,
status_detail="Total Time: %s" % (time.time() - start),
message=str(response.read()))
return make_taxii_response(status_message1.to_json())
error_message = tm.StatusMessage(
message_id=tm.generate_message_id(),
in_response_to=msg.message_id,
message='This queue only accepts %s content type' % t.VID_CERT_EU_JSON_10,
status_type=tm.ST_FAILURE)
return make_taxii_response(error_message.to_json())
def inner(request, *args, **kwargs):
if not taxii.settings.AUTH_REQUIRED:
return func(request, *args, **kwargs)
elif request.user.is_authenticated():
return func(request, *args, **kwargs)
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_UNAUTHORIZED, message='You are not authorized to access this URL.')
return handlers.create_taxii_response(m)
def obtener_remote_data_feeds(request):
#Given the id of a TAXII Service we make a FeedInformation request to that service address.
#The response is a list of the feed names of the TAXII client and a list of all protocol bindings, content binding and message binding.
feed_managment = TAXIIServices.objects.get(id = request.DATA.get('id'))
urlParsed = urlparse(feed_managment.feed_managment)
logger = logging.getLogger('TAXIIApplication.rest.tasks.obtener_remote_data_feeds')
logger.debug('We get the server data feeds')
logger.debug('Host: ' + urlParsed.hostname)
logger.debug('Path: ' + urlParsed.path)
logger.debug('Port: ' + str(urlParsed.port))
host = urlParsed.hostname
path = urlParsed.path
port = str(urlParsed.port)
feed_information = tm.FeedInformationRequest(message_id=tm.generate_message_id())
feed_info_xml = feed_information.to_xml()
logger.debug('The following message is sent: ' + feed_info_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_info_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug("The response was: " + response_message.to_xml())
try:
taxii_message = tm.get_message_from_xml(response_message.to_xml())
logger.debug("The JSON is: " + taxii_message.to_json())
feed_informations = taxii_message.feed_informations
feed_names = []
for feed in feed_informations:
feed_names.append({"name" : feed.feed_name})
protocolBindings = ProtocolBindingId.objects.all()
protocol_bindings = []
for proto in protocolBindings:
protocol_bindings.append({"binding_id" : proto.binding_id})
contentBindings = ContentBindingId.objects.all()
content_bindings = []
for content in contentBindings:
content_bindings.append({"binding_id" : content.binding_id})
messageBindings = MessageBindingId.objects.all()
message_bindings = []
for message in messageBindings:
message_bindings.append({"binding_id" : message.binding_id})
json_data = json.dumps({ "items" : feed_names, "protocol_bindings" : protocol_bindings, "content_bindings" : content_bindings, "message_bindings" : message_bindings })
logger.debug("The response is the following JSON: " + json_data)
return HttpResponse(json_data, content_type="application/json")
except Exception as ex:
logger.debug('The message could not be parsed:s', ex.message)
def validate_taxii_request(request):
"""Validates the broader request parameters and request type for a TAXII exchange"""
logger = logging.getLogger("taxii_services.utils.handlers.validate_taxii_request")
logger.debug('attempting to validate request')
if request.method != 'POST':
logger.info('request was not POST - returning error')
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_FAILURE, message='Request must be POST')
return create_taxii_response(m, use_https=request.is_secure())
header_validation_resp = validate_taxii_headers(request, '0') # TODO: What to use for request message id?
if header_validation_resp: # If response is not None an validation of the TAXII headers failed
return header_validation_resp
if len(request.body) == 0:
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_FAILURE, message='No POST data')
logger.info('Request had a body length of 0. Returning error.')
return create_taxii_response(m, use_https=request.is_secure())
return None
def process_exception(self, request, exception):
logger = logging.getLogger('yeti.taxii_services.middleware.ProcessExceptionMiddleware.process_exception')
logger.exception('Server error occured')
if request.path.startswith('/services'):
logger.debug('Returning ST_FAILURE message')
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_FAILURE, message='An internal server error occurred')
return handlers.create_taxii_response(m, handlers.HTTP_STATUS_OK, use_https=request.is_secure())
resp = HttpResponseServerError()
resp.body = 'A server error occurred'
return resp
def create_inbox_message(data, content_binding=t.VID_CERT_EU_JSON_10):
"""Creates TAXII message from data"""
content_block = tm.ContentBlock(
content_binding=content_binding,
content=data,
timestamp_label=datetime.datetime.now(tzutc()))
msg_id = tm.generate_message_id()
inbox_message = tm.InboxMessage(
message_id=msg_id,
content_blocks=[content_block])
return msg_id, inbox_message.to_json()
def discovery_service(request):
"""Handles TAXII Discovery Service requests"""
logger = logging.getLogger('yeti.taxii_services.views.discovery_service')
logger.debug('entering discovery service')
resp = handlers.validate_taxii_request(request)
if resp: return resp # if validation fails, return the response
try:
taxii_message = tm.get_message_from_xml(request.body)
logger.debug('received taxii message [%s]' % (taxii_message.message_id))
except Exception as ex:
logger.debug('unable to parse inbound message: %s' % (ex.message))
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_BAD_MESSAGE, message='Message received could not be parsed')
return handlers.create_taxii_response(m, use_https=request.is_secure())
if taxii_message.message_type != tm.MSG_DISCOVERY_REQUEST:
logger.info('Message [%s] was not discovery request [%s]' % (taxii_message.message_id,taxii_message.message_type))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_FAILURE, message='Message sent to discovery service did not have a discovery request message type')
return handlers.create_taxii_response(m, use_https=request.is_secure())
resp = handlers.discovery_get_services(request, taxii_message)
return resp
def feed_managment_get_content(request, taxii_message):
"""Returns a feed managment response for a given Poll Request Message"""
logger = logging.getLogger('taxii.utils.handlers.feed_mangment_get_content')
logger.debug('Retriving data feed names')
data_feeds = DataFeed.objects.all()
feed_information_list = []
for data_feed in data_feeds:
push_methods = []
polling_service_instances = []
subscription_methods = []
supported_content_bindings = []
for subs_method in data_feed.subscription_methods.all():
subscription_method = tm.FeedInformationResponse.FeedInformation.SubscriptionMethod(subscription_protocol=subs_method.protocol_binding.title,
subscription_address=subs_method.title,
subscription_message_bindings=get_message_bindings(subs_method.message_bindings))
subscription_methods.append(subscription_method)
for poll_service_instance in data_feed.poll_service_instances.all():
poll_service = tm.FeedInformationResponse.FeedInformation.PollingServiceInstance(poll_protocol=poll_service_instance.address,
poll_address=poll_service_instance.address,
poll_message_bindings=get_message_bindings(poll_service_instance.message_bindings))
polling_service_instances.append(poll_service)
for push_method in data_feed.push_methods.all():
push_me = tm.FeedInformationResponse.FeedInformation.PushMethod(push_protocol=push_method.protocol_binding.title,
push_message_bindings=[push_method.message_binding.title])
push_methods.append(push_me)
for supported_content_binding in data_feed.supported_content_bindings.all():
sup_cont_bind = supported_content_binding.title
supported_content_bindings.append(sup_cont_bind)
feed = tm.FeedInformationResponse.FeedInformation(feed_name = data_feed.name,
feed_description = data_feed.description,
supported_contents=supported_content_bindings,
available=True,
push_methods=push_methods,
polling_service_instances=polling_service_instances,
subscription_methods=subscription_methods)
feed_information_list.append(feed)
feed_managment_response_message = tm.FeedInformationResponse(tm.generate_message_id(),
taxii_message.message_id,
feed_informations=feed_information_list)
return create_taxii_response(feed_managment_response_message, use_https=request.is_secure())
def poll_service(request):
"""Handles TAXII Poll Service requests."""
logger = logging.getLogger("yeti.taxii_services.views.poll_service")
logger.debug('Entering poll service')
resp = handlers.validate_taxii_request(request)
if resp: return resp # if validation failed, return the response
try:
taxii_message = tm.get_message_from_xml(request.body)
except Exception as ex:
logger.debug('Unable to parse inbound message: %s', ex.message)
m = tm.StatusMessage(tm.generate_message_id(), '0', status_type=tm.ST_BAD_MESSAGE, message='Message received could not be parsed')
return handlers.create_taxii_response(m, use_https=request.is_secure())
logger.debug('Poll service received TAXII message with id [%s] and type [%s]', make_safe(taxii_message.message_id), make_safe(taxii_message.message_type))
if taxii_message.message_type != tm.MSG_POLL_REQUEST:
logger.info('TAXII message with id [%s] was not Poll request [%s]', make_safe(taxii_message.message_id), make_safe(taxii_message.message_type))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_FAILURE, message='Message sent to Poll service did not have a poll request message type')
return handlers.create_taxii_response(m, use_https=request.is_secure())
resp = handlers.poll_get_content(request, taxii_message)
return resp
def obtener_data_feeds(host, port, path):
#Given the host, port and path of a TAXII client we get the data feeds of that client
logger = logging.getLogger('TAXIIApplication.rest.tasks.obtener_data_feeds')
logger.debug('We get the server data feeds')
logger.debug('Host: ' + host)
logger.debug('Path: ' + path)
logger.debug('Port: ' + str(port))
feed_information = tm.FeedInformationRequest(message_id=tm.generate_message_id())
feed_info_xml = feed_information.to_xml()
logger.debug('The following message is sent: ' + feed_info_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_info_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug('We get the following response: '+response_message.to_xml())
def main():
parser = argparse.ArgumentParser(description="Discovery Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Discovery Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Discovery Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/discovery/", help="Path where the Discovery Service is hosted. Defaults to /services/discovery/.")
args = parser.parse_args()
discovery_req = tm.DiscoveryRequest(message_id=tm.generate_message_id())
discovery_req_xml = discovery_req.to_xml()
print "Discovery Request: \r\n", discovery_req_xml
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, discovery_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
def log(self, event):
# converts from conpot log format to STIX compatible xml
stix_package = self.stix_transformer.transform(event)
# wrapping the stix message in a TAXII envelope
content_block = ContentBlock(libtaxii.CB_STIX_XML_10, stix_package)
inbox_message = InboxMessage(message_id=generate_message_id(), content_blocks=[content_block])
inbox_xml = inbox_message.to_xml()
# the actual call to the TAXII web service
response = self.client.callTaxiiService2(self.host, self.inbox_path, libtaxii.VID_TAXII_XML_10, inbox_xml, self.port)
response_message = libtaxii.get_message_from_http_response(response, '0')
if response_message.status_type != libtaxii.messages.ST_SUCCESS:
logger.error('Error while transmitting message to TAXII server: {0}'.format(response_message.status_detail))
return False
else:
return True
def subscripcion_data_feed(request):
#Given the id of a TAXII Service and the id of a Data Feed and the service id we make a Manage Feed Subscription request for that Data Feed.
logger = logging.getLogger('TAXIIApplication.taxii.views.subscripcion_data_feed')
logger.debug('The data feed subscription starts')
logger.debug(request.DATA)
data_feed = request.DATA.get('data_feed')
service = request.DATA.get('id')
inbox_protocol = request.DATA.get('protocol_binding')
message_binding = request.DATA.get('message_binding')
content_binding = request.DATA.get('content_binding')
feed_managment = TAXIIServices.objects.get(id = service)
urlParsed = urlparse(feed_managment.subscription)
logger.debug('Host: ' + urlParsed.hostname)
logger.debug('Path: ' + urlParsed.path)
logger.debug('Port: ' + str(urlParsed.port))
host = urlParsed.hostname
path = urlParsed.path
port = str(urlParsed.port)
delivery_parameters = tm.DeliveryParameters(inbox_protocol=inbox_protocol, inbox_address=INBOX_SERVICES_URL,
delivery_message_binding=message_binding, content_bindings=content_binding)
f = tm.ACT_TYPES
feed_subscription = tm.ManageFeedSubscriptionRequest(message_id=tm.generate_message_id(), feed_name=data_feed,
action= f[0], subscription_id='1', delivery_parameters=delivery_parameters)
feed_subscription_xml = feed_subscription.to_xml()
logger.debug('The message sent is: ' + feed_subscription_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_subscription_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug('The server respons: ' + response_message.to_xml())
return Response(status = status.HTTP_200_OK)
def main():
parser = argparse.ArgumentParser(description="Poll Client")
parser.add_argument("--host", dest="host", default="localhost", help="Host where the Poll Service is hosted. Defaults to localhost.")
parser.add_argument("--port", dest="port", default="8080", help="Port where the Poll Service is hosted. Defaults to 8080.")
parser.add_argument("--path", dest="path", default="/services/poll/", help="Path where the Poll Service is hosted. Defaults to /services/poll/.")
parser.add_argument("--feed", dest="feed", default="default", help="Data Feed to poll. Defaults to 'default'.")
parser.add_argument("--begin_timestamp", dest="begin_ts", default=None, help="The begin timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
parser.add_argument("--end_timestamp", dest="end_ts", default=None, help="The end timestamp (format: YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm) for the poll request. Defaults to None.")
args = parser.parse_args()
try:
if args.begin_ts:
begin_ts = dateutil.parser.parse(args.begin_ts)
if not begin_ts.tzinfo:
raise ValueError
else:
begin_ts = None
if args.end_ts:
end_ts = dateutil.parser.parse(args.end_ts)
if not end_ts.tzinfo:
raise ValueError
else:
end_ts = None
except ValueError:
print "Unable to parse timestamp value. Timestamp should include both date and time information along with a timezone or UTC offset (e.g., YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm). Aborting poll."
sys.exit()
poll_req = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=args.feed,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts)
poll_req_xml = poll_req.to_xml()
print "Poll Request: \r\n", poll_req_xml
client = tc.HttpClient()
client.setProxy('noproxy')
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_10, poll_req_xml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
print "Response Message: \r\n", response_message.to_xml()
def client_example():
# Create the TAXII HTTPS Client
client = tc.HttpClient()
# Uncomment to use HTTPS
client.set_use_https(True)
# Uncomment to use basic authentication
# client.set_auth_type(tc.HttpClient.AUTH_BASIC)
# client.set_auth_credentials({'username':'******', 'password':'******'})
# Uncomment to use certificate-based authentication
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({
'key_file': 'keyfile',
'cert_file': 'certfile'
})
# Uncomment to set a proxy
# client.set_proxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
# Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name='TheFeedToPoll')
# Call without a proxy
http_response = client.call_taxii_service2('hostname',
'/poll_service_path/',
VID_TAXII_XML_10,
poll_request1.to_xml())
print(http_response.__class__.__name__)
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print((taxii_message.to_xml()))
def log(self, event):
# converts from conpot log format to STIX compatible xml
stix_package = self.stix_transformer.transform(event)
# wrapping the stix message in a TAXII envelope
content_block = ContentBlock(libtaxii.CB_STIX_XML_10, stix_package)
inbox_message = InboxMessage(message_id=generate_message_id(),
content_blocks=[content_block])
inbox_xml = inbox_message.to_xml()
# the actual call to the TAXII web service
response = self.client.callTaxiiService2(self.host, self.inbox_path,
libtaxii.VID_TAXII_XML_10,
inbox_xml, self.port)
response_message = libtaxii.get_message_from_http_response(
response, '0')
if response_message.status_type != libtaxii.messages.ST_SUCCESS:
logger.error(
'Error while transmitting message to TAXII server: {0}'.format(
response_message.status_detail))
return False
else:
return True
def execute_taxii_agent(hostname=None,
https=None,
feed=None,
keyfile=None,
certfile=None,
start=None,
end=None,
analyst=None,
method=None):
ret = {
'Certificate': [],
'Domain': [],
'Email': [],
'Event': [],
'Indicator': [],
'IP': [],
'PCAP': [],
'RawData': [],
'Sample': [],
'successes': 0,
'failures': [],
'status': False,
'reason': ''
}
sc = get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
if https == None:
https = sc['https']
create_events = sc['create_events']
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
if https:
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
# Poll using 1.1 then 1.0 if that fails.
poll_msg = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=feed,
poll_parameters=tm11.PollRequest.PollParameters(),
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
# Check if this is a TAXII 1.0 server and try again
if response.info().getheader(
'X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/',
t.VID_TAXII_XML_10,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(
response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
else:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = "Invalid XML: %s" % valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'].append(('No data found in content block', 'Data'))
continue
objs = import_standards_doc(data,
analyst,
method,
ref=mid,
make_event=create_events)
for k in objs['imported']:
ret['successes'] += 1
ret[k[0]].append(k[1])
for k in objs['failed']:
ret['failures'].append(k)
crits_taxii.save()
return ret
def registrar_remote_data_feeds(request):
#Given the id of a TAXII service we get the data feeds of the TAXII Client and copy them to the current system.
feed_managment = TAXIIServices.objects.get(id = request.DATA.get('id'))
urlParsed = urlparse(feed_managment.feed_managment)
logger = logging.getLogger('TAXIIApplication.rest.tasks.obtener_remote_data_feeds')
logger.debug('We get the server data feeds')
logger.debug('Host: ' + urlParsed.hostname)
logger.debug('Path: ' + urlParsed.path)
logger.debug('Port: ' + str(urlParsed.port))
host = urlParsed.hostname
path = urlParsed.path
port = str(urlParsed.port)
feed_information = tm.FeedInformationRequest(message_id=tm.generate_message_id())
feed_info_xml = feed_information.to_xml()
logger.debug('The following message is sent: ' + feed_info_xml)
client = tc.HttpClient()
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_10, feed_info_xml, port)
response_message = t.get_message_from_http_response(resp, '0')
logger.debug("The response was: " + response_message.to_xml())
try:
taxii_message = tm.get_message_from_xml(response_message.to_xml())
logger.debug("Feed Information iteration")
feed_informations = taxii_message.feed_informations
for feed in feed_informations:
logger.debug("Create a new Remote Data Feed")
remote_df = RemoteDataFeed()
remote_df.name = feed.feed_name
logger.debug(feed.feed_name)
if feed.feed_description == None:
remote_df.description = "None"
else:
remote_df.description = feed.feed_description
remote_df.producer = host
remote_df.save()
i = 0
logger.debug('We get the subscription methods')
for sm in feed.subscription_methods:
protocol_binding = ProtocolBindingId(binding_id = sm.subscription_protocol)
protocol_binding.save()
dfsm = DataFeedSubscriptionMethod()
dfsm.title = feed.feed_name + "_"+str(i)
dfsm.address = sm.subscription_address
dfsm.protocol_binding=protocol_binding
dfsm.save()
for mb in sm.subscription_message_bindings:
msgb = MessageBindingId(binding_id = mb)
msgb.save()
dfsm.message_bindings.add(msgb)
dfsm.save()
remote_df.subscription_methods.add(dfsm)
logger.debug('We get the Content Bindings')
for sc in feed.supported_contents:
cb = ContentBindingId(binding_id = sc )
cb.save()
remote_df.supported_content_bindings.add(cb)
logger.debug('Obtengo los push methods')
for pm in feed.push_methods:
pb = ProtocolBindingId(binding_id = pm.push_protocol)
pb.save()
mb = MessageBindingId(binding_id = pm.push_message_bindings)
mb.save()
dpm = DataFeedPushMethod(protocol_binding = pb, message_binding = mb)
dpm.save()
remote_df.push_methods.add(dpm)
poll_service_instances = []
logger.debug('We get the Poll Service Instances')
for psi in feed.polling_service_instances:
rdfpi = RemoteDataFeedPollInformation()
rdfpi.address = psi.poll_address
pb = ProtocolBindingId(binding_id = psi.poll_protocol)
pb.save()
rdfpi.protocol_binding = pb
rdfpi.save()
logger.debug(psi.poll_message_bindings)
for msg in psi.poll_message_bindings:
msgb = MessageBindingId(binding_id = msg)
msgb.save()
rdfpi.message_bindings.add(msgb)
rdfpi.save()
remote_df.poll_service_instances.add(rdfpi)
logger.debug("Save the remote data feed")
remote_df.save()
return Response(status=status.HTTP_201_CREATED)
except Exception as ex:
logger.debug( ex)
return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def execute_taxii_agent(hostname=None, https=None, feed=None, keyfile=None,
certfile=None, start=None, end=None, analyst=None,
method=None):
ret = {
'Certificate': [],
'Domain': [],
'Email': [],
'Event': [],
'Indicator': [],
'IP': [],
'PCAP': [],
'RawData': [],
'Sample': [],
'successes': 0,
'failures': [],
'status': False,
'reason': ''
}
sc = manager.get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
if https == None:
https = sc['https']
create_events = sc['create_events']
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
if https:
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
# Poll using 1.1 then 1.0 if that fails.
poll_msg = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=feed,
poll_parameters=tm11.PollRequest.PollParameters(),
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
if response.getcode() != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
# Check if this is a TAXII 1.0 server and try again
if response.info().getheader('X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
if response.getcode() != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
else:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = "Invalid XML: %s" % valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'].append(('No data found in content block', 'Data'))
continue
objs = import_standards_doc(data, analyst, method, ref=mid,
make_event=create_events)
for k in objs['imported']:
ret['successes'] += 1
ret[k[0]].append(k[1])
for k in objs['failed']:
ret['failures'].append(k)
crits_taxii.save()
return ret
def _scan(self, context):
#TODO: not sure if this should come after we make the TAXII message
# so the check is closer to actual submission time?
if not resolve_taxii_server(self.hostname):
self._error("Cannot contact TAXII Server: %s" % self.hostname)
return
else:
self._info("TAXII Server Online: %s" % self.hostname)
self._notify()
client = tc.HttpClient()
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({
'key_file': self.keyfile,
'cert_file': self.certfile
})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
event_list = Event.objects(id=context._id)
if len(event_list) < 1:
self._info("Could not locate event in the database")
self._notify()
else:
event_data = event_list[0]
(stix_doc, final_sources,
final_objects) = event_data.to_stix(context.username)
if len(final_sources) < 1:
self._error(
"No sources to send to! Ensure all related content is marked as releasable!"
)
return
final_objects.append(event_data)
# collect the list of data feeds to send this message to
destination_feeds = []
for crtfile in self.certfiles:
(source, feed, filepath) = crtfile.split(',')
if source.strip() in final_sources:
destination_feeds.append(
(source.strip(), feed.strip(), filepath.strip()))
self._info("Generating STIX document(s).")
self._notify()
inbox_messages = []
# generate inbox messages
# for now we will send one message per feed to isolate failures to one
# feed submission and not prevent other messages from being sent.
for feed in destination_feeds:
# Create encrypted block
encrypted_block = encrypt_block(
tm.ContentBlock(content_binding=t.CB_STIX_XML_10,
content=stix_doc.to_xml()).to_xml(),
feed[2])
# Wrap encrypted block in content block
content_block = tm.ContentBlock(content_binding="SMIME",
content=encrypted_block)
# Create inbox message
inbox_message = tm.InboxMessage(
message_id=tm.generate_message_id(),
content_blocks=[content_block],
extended_headers={'TargetFeed': feed[1]})
inbox_messages.append((feed[0], inbox_message))
self._info("Sending TAXII message(s)")
self._notify()
# send messages
for (src, inbox_msg) in inbox_messages:
response = client.callTaxiiService2(
self.hostname, "/inbox/", t.VID_TAXII_XML_10,
inbox_message.to_xml())
taxii_message = t.get_message_from_http_response(
response, inbox_message.message_id)
if taxii_message.status_type == tm.ST_SUCCESS:
# update releasability for objects
date = datetime.datetime.now()
instance = Releasability.ReleaseInstance(
analyst=context.username, date=date)
for idx in enumerate(final_objects):
final_objects[idx[0]].add_releasability_instance(
name=src, instance=instance)
self._add_result(self.name, "Success",
{'recipient': src})
else:
self._add_result(self.name, "Failure",
{'recipient': src})
# save releasability to database
self._info(
"Updated releasability status for all related content.")
self._notify()
for obj in final_objects:
obj.save()
return
def execute_taxii_agent(hostname=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None):
ret = {
'events': [],
'samples': [],
'emails': [],
'indicators': [],
'successes': 0,
'failures': 0,
'status': False,
'reason': ''
}
sc = manager.get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
poll_msg.to_xml())
if response.getcode() != 200:
ret['reason'] = "Response is not 200 OK"
return ret
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response"
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'] += 1
continue
objs = import_standards_doc(data, analyst, method, ref=mid, make_event=True)
ret['successes'] += 1
for k in ["events", "samples", "emails", "indicators"]:
for i in objs[k]:
ret[k].append(i)
crits_taxii.save()
return ret
def inbox_add_content(request, inbox_name, taxii_message):
"""Adds content to inbox and associated data feeds"""
logger = logging.getLogger('taxii.utils.handlers.inbox_add_content')
logger.debug('Adding content to inbox [%s]', make_safe(inbox_name))
try:
inbox = Inbox.objects.get(name=inbox_name)
except:
logger.debug('Attempting to push content to unknown inbox [%s]', make_safe(inbox_name))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_NOT_FOUND, message='Inbox does not exist [%s]' % (make_safe(inbox_name)))
return create_taxii_response(m, use_https=request.is_secure())
logger.debug('TAXII message [%s] contains [%d] content blocks', make_safe(taxii_message.message_id), len(taxii_message.content_blocks))
datafeed = taxii_message.subscription_information.collection_name
for content_block in taxii_message.content_blocks:
try:
content_binding_id = ContentBindingId.objects.filter(binding_id=content_block.content_binding)
except:
logger.debug('TAXII message [%s] contained unrecognized content binding [%s]', make_safe(taxii_message.message_id), make_safe(content_block.content_binding))
continue # cannot proceed - move on to the next content block
if content_binding_id[0] not in inbox.supported_content_bindings.all():
logger.debug('Inbox [%s] does not accept content with binding id [%s]', make_safe(inbox_name), make_safe(content_block.content_binding))
else:
tree = etree.parse(c.content)
import stix.bindings.stix_core as stix_core_binding
stix_package_obj = stix_core_binding.STIXType().factory()
stix_package_obj.build(tree.getroot())
from stix.core import STIXPackage # resolve circular dependencies
stix_package = STIXPackage().from_obj(stix_package_obj)
info = ContentBlock.objects.filter(stix_id = stix_package._id)
if info.exists():
c = ContentBlock()
c.origen = datafeed
c.message_id = taxii_message.message_id
c.content_binding = content_binding_id[0]
c.content = content_block.content
if content_block.padding:
c.padding = content_block.padding
if request.user.is_authenticated():
c.submitted_by = request.user
c.save()
inbox.content_blocks.add(c) # add content block to inbox
for data_feed in inbox.data_feeds.all():
if content_binding_id[0] in data_feed.supported_content_bindings.all():
data_feed.content_blocks.add(c)
data_feed.save()
else:
logger.debug('Inbox [%s] received data using content binding [%s] - '
'associated data feed [%s] does not support this binding.',
make_safe(inbox_name), make_safe(content_block.content_binding), make_safe(data_feed.name))
inbox.save()
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type = tm.ST_SUCCESS)
return create_taxii_response(m, use_https=request.is_secure())
def _scan(self, context):
#TODO: not sure if this should come after we make the TAXII message
# so the check is closer to actual submission time?
if not resolve_taxii_server(self.hostname):
self._error("Cannot contact TAXII Server: %s" % self.hostname)
return
else:
self._info("TAXII Server Online: %s" % self.hostname)
self._notify()
client = tc.HttpClient()
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': self.keyfile,
'cert_file': self.certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
event_list = Event.objects(id=context._id)
if len(event_list) < 1:
self._info("Could not locate event in the database")
self._notify()
else:
event_data = event_list[0]
(stix_doc, final_sources, final_objects) = event_data.to_stix(context.username)
if len(final_sources) < 1:
self._error("No sources to send to! Ensure all related content is marked as releasable!")
return
final_objects.append(event_data)
# collect the list of data feeds to send this message to
destination_feeds = []
for crtfile in self.certfiles:
(source, feed, filepath) = crtfile.split(',')
if source.strip() in final_sources:
destination_feeds.append((source.strip(), feed.strip(), filepath.strip()))
self._info("Generating STIX document(s).")
self._notify()
inbox_messages = []
# generate inbox messages
# for now we will send one message per feed to isolate failures to one
# feed submission and not prevent other messages from being sent.
for feed in destination_feeds:
# Create encrypted block
encrypted_block = encrypt_block(
tm.ContentBlock(
content_binding = t.CB_STIX_XML_10,
content = stix_doc.to_xml()).to_xml(),
feed[2]
)
# Wrap encrypted block in content block
content_block = tm.ContentBlock(
content_binding = "SMIME",
content = encrypted_block
)
# Create inbox message
inbox_message = tm.InboxMessage(
message_id = tm.generate_message_id(),
content_blocks = [content_block],
extended_headers = {'TargetFeed': feed[1]}
)
inbox_messages.append((feed[0], inbox_message))
self._info("Sending TAXII message(s)")
self._notify()
# send messages
for (src, inbox_msg) in inbox_messages:
response = client.callTaxiiService2(self.hostname,
"/inbox/",
t.VID_TAXII_XML_10,
inbox_message.to_xml())
taxii_message = t.get_message_from_http_response(response, inbox_message.message_id)
if taxii_message.status_type == tm.ST_SUCCESS:
# update releasability for objects
date = datetime.datetime.now()
instance = Releasability.ReleaseInstance(analyst=context.username, date=date)
for idx in enumerate(final_objects):
final_objects[idx[0]].add_releasability_instance(name=src, instance=instance)
self._add_result(self.name, "Success", {'recipient': src})
else:
self._add_result(self.name, "Failure", {'recipient': src})
# save releasability to database
self._info("Updated releasability status for all related content.")
self._notify()
for obj in final_objects:
obj.save()
return
def execute_taxii_agent(hostname=None,
feed=None,
keyfile=None,
certfile=None,
start=None,
end=None,
analyst=None,
method=None):
ret = {
'events': [],
'samples': [],
'emails': [],
'indicators': [],
'successes': 0,
'failures': 0,
'status': False,
'reason': ''
}
sc = manager.get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
poll_msg.to_xml())
if response.getcode() != 200:
ret['reason'] = "Response is not 200 OK"
return ret
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response"
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'] += 1
continue
objs = import_standards_doc(data,
analyst,
method,
ref=mid,
make_event=True)
ret['successes'] += 1
for k in ["events", "samples", "emails", "indicators"]:
for i in objs[k]:
ret[k].append(i)
crits_taxii.save()
return ret
#Create the TAXII HTTPS Client
client = tc.HttpClient()
#Uncomment to use HTTPS
client.setUseHttps(True)
#Uncomment to use basic authentication
#client.setAuthType(tc.HttpClient.AUTH_BASIC)
#client.setAuthCredentials({'username':'******', 'password':'******'})
#Uncomment to use certificate-based authentication
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': 'keyfile',
'cert_file': 'certfile'})
#Uncomment to set a proxy
#client.setProxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
#Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(), feed_name='TheFeedToPoll')
#Call without a proxy
http_response = client.callTaxiiService2('hostname', '/poll_service_path/', t.VID_TAXII_XML_10, poll_request1.to_xml())
print http_response.__class__.__name__
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print(taxii_message.to_xml())
def feed_subscription_get_content(request, taxii_message):
"""Returns a feed subscription response for a given subscription feed Message"""
logger = logging.getLogger('taxii.utils.handlers.feed_subscription_get_content')
logger.debug('Retriving data feed names')
f = tm.ACT_TYPES
#For now we only accept subscriptions
if taxii_message.action == f[0]:
try:
logger.debug('Getting the data feed [%s]', make_safe(taxii_message.feed_name))
data_feed = DataFeed.objects.get(name=taxii_message.feed_name)
except:
logger.debug('Attempting to subscribe to unknown data feed [%s]', make_safe(taxii_message.feed_name))
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_NOT_FOUND, message='Data feed does not exist [%s]' % (make_safe(taxii_message.feed_name)))
return create_taxii_response(m, use_https=request.is_secure())
try:
binding_id = taxii_message.delivery_parameters.inbox_protocol
logger.debug('Getting the binding protocol [%s]', make_safe(binding_id))
protocol_binding = ProtocolBindingId.objects.get(title=binding_id)
message_binding = taxii_message.delivery_parameters.delivery_message_binding
logger.debug('Getting the binding message [%s]', make_safe(message_binding))
message_binding = MessageBindingId.objects.get(title=message_binding)
except:
logger.debug('Attempting to subscribe to use unknowon protocol or message bindings')
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_NOT_FOUND, message='Protocol or message bindings does not exist')
return create_taxii_response(m, use_https=request.is_secure())
logger.debug("Response message")
subscr_methods = DataFeedSubscriptionMethod()
subscr_methods.title = taxii_message.delivery_parameters.inbox_address
subscr_methods.description = taxii_message.delivery_parameters.inbox_address
subscr_methods.address = taxii_message.delivery_parameters.inbox_address
subscr_methods.protocol_binding = protocol_binding
subscr_methods.save()
subscr_methods.message_bindings.add(message_binding)
subscr_methods.save()
logger.debug("The information was correclty saved")
user = User.objects.get(id=1)
logger.debug("Get user")
data_feed_subscription = DataFeedSubscription()
data_feed_subscription.active = True
data_feed_subscription.expires = timezone.now() + datetime.timedelta(days=500)
data_feed_subscription.data_feed_method = subscr_methods
data_feed_subscription.data_feed = data_feed
data_feed_subscription.user = user
subscription_list = DataFeedSubscription.objects.all()
if len(subscription_list) == 0:
id_value = 1
else:
id_value = DataFeedSubscription.objects.latest('id').id + 1
data_feed_subscription.subscription_id = id_value
data_feed_subscription.save()
delivery_parameters = tm.DeliveryParameters(inbox_protocol=taxii_message.delivery_parameters.inbox_protocol,
inbox_address=taxii_message.delivery_parameters.inbox_address,
delivery_message_binding=taxii_message.delivery_parameters.delivery_message_binding,
content_bindings=taxii_message.delivery_parameters.content_bindings)
logger.debug("Return the Poll Service instances")
poll_instances = []
for poll_info in data_feed.poll_service_instances.all():
poll_inst = tm.ManageFeedSubscriptionResponse.PollInstance(poll_protocol = poll_info.protocol_binding.binding_id,
poll_address = poll_info.address)
mbindings = []
for mbinding_id in poll_info.message_bindings.all():
mbindings.append(mbinding_id.binding_id)
poll_inst.poll_message_bindings = mbindings
poll_instances.append(poll_inst)
subscription_instances = []
subscr_instance = tm.ManageFeedSubscriptionResponse.SubscriptionInstance(subscription_id = str(data_feed_subscription.id))
subscr_instance.delivery_parameters = [delivery_parameters]
subscr_instance.poll_instances = poll_instances
subscription_instances.append(subscr_instance)
logger.debug("Returns the response")
feed_subscription_response_message = tm.ManageFeedSubscriptionResponse(message_id = tm.generate_message_id(), in_response_to = taxii_message.message_id,
feed_name = taxii_message.feed_name, message='Subscription succeds', subscription_instances = subscription_instances)
return create_taxii_response(feed_subscription_response_message, use_https=request.is_secure())
else:
logger.debug('Modifing or deleteing subscription is not supported')
m = tm.StatusMessage(tm.generate_message_id(), taxii_message.message_id, status_type=tm.ST_FAILURE, message='Modifing subscriptions is not allowed')
return create_taxii_response(m, use_https=request.is_secure())
