def __init__(self, **kwargs):
self.username = kwargs.get('username')
self.password = kwargs.get('password')
self.remote = kwargs.get('remote')
self.key_file = kwargs.get('key_file')
self.cert_file = kwargs.get('cert_file')
self.client = tc.HttpClient()
self.up = urlparse(self.remote)
if self.up.scheme.startswith('https'):
self.client.set_use_https(True)
if self.username and self.password:
self.client.set_auth_type(tc.HttpClient.AUTH_BASIC)
self.client.set_auth_credentials({
'username': self.username,
'password': self.password
})
# support client cert, default disabled
if self.key_file and self.cert_file:
# auth_basic = 1; auth_cert = 2; auth_cert_basic = 3
if self.client.auth_type:
self.client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC)
else:
self.client.set_auth_type(tc.HttpClient.AUTH_CERT)
self.client.auth_credentials.update({
'key_file': self.key_file,
'cert_file': self.cert_file
})
def push(s, collection, content, path="/"):
msg_id=tm11.generate_message_id()
content = tm11.ContentBlock(tm11.ContentBinding(t.CB_STIX_XML_11),
content)
# Create inbox request
req = tm11.InboxMessage(message_id=msg_id,
destination_collection_names=[collection],
content_blocks=[content])
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path, t.VID_TAXII_XML_11,
req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
if resp.status_type != tm11.ST_SUCCESS:
print("%s: %s" % (resp.status_type, resp.message))
def create_client(self,
use_https,
proxy,
cert=None,
key=None,
username=None,
password=None,
verify_server=None):
client = tc.HttpClient()
client.set_use_https(use_https)
client.set_proxy(proxy)
tls = (cert is not None and key is not None)
basic = (username is not None and password is not None)
if tls and basic:
client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC)
client.set_auth_credentials({
'key_file': key,
'cert_file': cert,
'username': username,
'password': password
})
elif tls:
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({'key_file': key, 'cert_file': cert})
elif basic:
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_auth_credentials({
'username': username,
'password': password
})
if verify_server is not None:
client.set_verify_server(verify_server=True, ca_file=verify_server)
return client
def send_poll_request(self,
path,
messages_vid,
poll_request,
status_type=None,
sd_keys=None):
#status_type=None means expect a PollResponse
#status_type=<value> means expect a StatusMessage
if messages_vid not in (VID_TAXII_XML_11, VID_TAXII_XML_10):
raise ValueError("Wrong messages_vid!")
client = tc.HttpClient()
client.setProxy(tc.HttpClient.NO_PROXY)
resp = client.callTaxiiService2(HOST, path, messages_vid,
poll_request.to_xml(), PORT)
msg = t.get_message_from_http_response(resp, poll_request.message_id)
if not status_type: # Expect a Poll Response
if msg.message_type != MSG_POLL_RESPONSE:
raise ValueError('Expected Poll Response. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
else: # Expect a Status Message
if msg.message_type != MSG_STATUS_MESSAGE:
raise ValueError('Expected Status Message. Got: %s.\r\n%s' % \
(msg.message_type, msg.to_xml(pretty_print=True)) )
if sd_keys:
for key in sd_keys:
if key not in msg.status_detail:
raise ValueError('Expected status detail key was not present: %s\r\n%s' % \
(key, msg.to_xml(pretty_print=True)) )
return msg
def _gen_client(self):
if self.host:
tc.socket.setdefaulttimeout(60) # in seconds
client = tc.HttpClient()
if int(self.port) == 443 or int(self.port) == 8443:
client.setUseHttps(True)
if self.scheme:
if 'https' in self.scheme.lower():
client.setUseHttps(True)
if self.is_https:
client.setUseHttps(True)
if self.usr_pass:
client.setAuthType(client.AUTH_BASIC)
if self.usr_cert_prv:
client.setAuthType(client.AUTH_CERT_BASIC)
client.setAuthCredentials({
'username' : self.usr_name,
'password' : self.usr_pass,
'key_file' : self.usr_cert_prv,
'cert_file': self.usr_cert_pub
})
return client
def __init__(self,
taxii_name=None,
taxii_id=None,
community=None,
taxii_client=None):
taxii = None
if taxii_client is not None:
taxii = taxii_client
elif taxii_name is not None:
taxii = TaxiiClients.objects.get(name=taxii_name)
elif taxii_id is not None:
taxii = TaxiiClients.objects.get(id=taxii_id)
self._address = taxii.address
self._port = taxii.port
self._path = taxii.path
self._collection_name = taxii.collection
self._jobs = taxii.jobs
self._interval_job = taxii.interval_schedule_job
self._protocol_version = taxii.protocol_version
# taxii client設定
self._client = clients.HttpClient()
if taxii.is_use_cert:
self._auth_type = clients.HttpClient.AUTH_CERT_BASIC
self._key_file = taxii.key_file
self._cert_file = taxii.cert_file
else:
self._auth_type = clients.HttpClient.AUTH_BASIC
self._username = taxii.login_id
self._password = taxii.login_password
self._ssl = taxii.ssl
# proxy 設定があれば設定する
self._proxies = System.get_request_proxies()
if self._proxies is not None:
p = urlparse(self._address)
if p.scheme == 'https':
self._client.set_proxy(self._proxies['https'])
else:
self._client.set_proxy(self._proxies['http'])
try:
self._community = taxii.community
except BaseException:
self._community = None
self._via = Vias.get_via_taxii_poll(taxii, uploader=taxii.uploader)
self._taxii = taxii
self._start = None
self._end = None
self._schedule = CtirsScheduler()
# _scheduleのstartはschedule単位
# modify/resume/pauseはjob単位
# TAXII Protocol 1.1 なら Authentication 設定を行う
if self._protocol_version == '1.1':
self.set_taxii_11_authentication()
def __init__(self, community=None, taxii_client=None, taxii2_client=None):
if taxii2_client:
taxii = taxii2_client
elif taxii_client:
taxii = taxii_client
self._name = taxii.name
self._protocol_version = taxii.protocol_version
self._username = taxii.login_id
self._password = taxii.login_password
self._jobs = taxii.jobs
self._interval_job = taxii.interval_schedule_job
self._can_read = taxii.can_read
self._can_write = taxii.can_write
if taxii.is_use_cert:
self._auth_type = clients.HttpClient.AUTH_CERT_BASIC
self._key_file = taxii.key_file
self._cert_file = taxii.cert_file
else:
self._auth_type = clients.HttpClient.AUTH_BASIC
if taxii2_client:
self._api_root = taxii.api_root
self._collection = taxii.collection
self._via = Vias.get_via_taxii_poll(taxii2_client=taxii,
uploader=taxii.uploader)
else:
self._address = taxii.address
self._port = taxii.port
self._path = taxii.path
self._collection_name = taxii.collection
self._via = Vias.get_via_taxii_poll(taxii_client=taxii,
uploader=taxii.uploader)
self._client = clients.HttpClient()
self._ssl = taxii.ssl
self._client.set_use_https(self._ssl)
self._client.set_auth_type(self._auth_type)
self._proxies = System.get_request_proxies()
if self._proxies is not None:
p = urlparse(self._address)
if p.scheme == 'https':
self._client.set_proxy(self._proxies['https'])
else:
self._client.set_proxy(self._proxies['http'])
try:
self._community = taxii.community
except BaseException:
self._community = None
self._taxii = taxii
self._start = None
self._end = None
self._schedule = CtirsScheduler()
def main():
client = tc.HttpClient()
client.set_use_https(True)
client.set_auth_type(tc.HttpClient.AUTH_CERT)
response = client.call_taxii_service2(
host='HOSTNAME',
path='/path/to/service',
message_binding=VID_TAXII_XML_11,
post_data=request_message.to_xml(pretty_print=True),
port=args.port)
response_message = t.get_message_from_http_response(response, 0)
print response_message.to_xml(pretty_print=True)
def taxii_inbox(config, dest, stix_package=None, src=None, crits_id=None):
'''inbox a stix package via taxii'''
if src and crits_id:
# check whether this has already been ingested
sync_state = config['db'].get_object_id(src, dest, crits_id=crits_id)
if sync_state and sync_state.get('crits_id', None):
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['object_already_ingested'].format(
src_type='crits', src_id=crits_id, src=src,
dest_type='edge', dest=dest, dest_id=sync_state['edge_id']))
return(True)
if stix_package:
stixroot = lxml.etree.fromstring(stix_package.to_xml())
client = tc.HttpClient()
client.setUseHttps(config['edge']['sites'][dest]['taxii']['ssl'])
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username':
config['edge']['sites'][dest]['taxii']['user'],
'password':
config['edge']['sites'][dest]['taxii']['pass']})
message = tm11.InboxMessage(message_id=tm11.generate_message_id())
content_block = tm11.ContentBlock(content_binding=t.CB_STIX_XML_11,
content=stixroot)
if config['edge']['sites'][dest]['taxii']['collection'] != \
'system.Default':
message.destination_collection_names = \
[config['edge']['sites'][dest]['taxii']['collection']]
message.content_blocks.append(content_block)
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages[
'open_session'].format(type_='taxii', host=dest))
taxii_response = client.callTaxiiService2(
config['edge']['sites'][dest]['host'],
config['edge']['sites'][dest]['taxii']['path'],
t.VID_TAXII_XML_11, message.to_xml(),
port=config['edge']['sites'][dest]['taxii']['port'])
if taxii_response.code != 200 or taxii_response.msg != 'OK':
success = False
config['logger'].error(
log.log_messages[
'inbox_error'].format(type_='taxii', host=dest,
msg=taxii_response.msg))
else:
success = True
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['inbox_success'].format(type_='taxii',
host=dest))
return(success)
def poll_feed(settings, subscription):
""" polls a TAXII feed"""
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(True)
client.set_auth_credentials({
'username': settings['username'],
'password': settings['password']
})
msg_id = uuid.uuid4().hex
poll_request1 = tm11.PollRequest(message_id=msg_id,
collection_name=settings['subscriptions']
[subscription]['collection_name'],
subscription_id=settings['subscriptions']
[subscription]['subscription_id'])
poll_xml = poll_request1.to_xml()
http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/',
VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(http_resp,
poll_request1.message_id)
observables = {}
indicators = json.loads(taxii_message.to_json())
if 'content_blocks' in indicators.keys():
for indicator in indicators['content_blocks']:
open('/tmp/indicator.xml', 'w').write(indicator['content'])
indi = STIXPackage.from_xml('/tmp/indicator.xml').to_dict()
if 'observables' in indi.keys():
for obs in indi['observables']['observables']:
if 'object' in obs.keys():
ot = obs['object']['properties']['xsi:type']
if ot in settings['supported_objects'].keys(
) and not ot in observables.keys():
observables[ot] = []
if ot in settings['supported_objects'].keys(
) and settings['supported_objects'][ot] in obs[
'object']['properties'].keys():
# note, you will only be able to process one property per object type, but you also know there's only one property you can process
try:
observables[ot].append(
obs['object']['properties'][
settings['supported_objects'][ot]])
except:
print "[-] you're dumb"
print supported_objects[ot], "not in:", obs[
'object']
return observables
def __init__(self, taxii_name):
taxii = TaxiiModel.objects.get_from_taxii_name(taxii_name)
self._address = taxii.address
self._port = taxii.port
self._path = taxii.path
self._collection_name = taxii.collection
# taxii client設定
self._client = clients.HttpClient()
self._client.set_use_https(taxii.ssl)
self._client.set_auth_type(clients.HttpClient.AUTH_BASIC)
self._client.set_auth_credentials({'username': taxii.login_id, 'password': taxii.login_password})
# SubscriptionInformatin設定
self._subscription_information = tm11.SubscriptionInformation(
collection_name=self._collection_name,
subscription_id=self._SUBSCRIPTION_ID,
)
def poll(s, path="/", collection="default", query=None,
begin_ts=None, end_ts=None):
if query != None:
query=s.create_query(query)
poll_params=tm11.PollRequest.PollParameters(query=query)
else:
poll_params=tm11.PollRequest.PollParameters()
# Create poll request
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=poll_params)
# Convert to XML for request body
poll_req_xml = poll_req.to_xml(True)
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path,
t.VID_TAXII_XML_11,
poll_req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
pkgs = []
# Process each content block
for cb in resp.content_blocks:
content = cb.content
# Parse the payload, should be a STIX document.
package = STIXPackage.from_xml(StringIO.StringIO(content))
pkgs.append(package)
return resp.inclusive_end_timestamp_label, pkgs
def perform_discovery(path="/"):
# Create discovery request
req = tm11.DiscoveryRequest(message_id=tm11.generate_message_id())
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_11,
req_xml, port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def run(self):
client = tc.HttpClient()
# TODO: handle authentication stuff
#client.set_auth_type(tc.AUTH_NONE)
client.set_use_https(False)
r = CollectionInformationRequest(generate_message_id())
col_xml = r.to_xml(pretty_print=True)
if ':' in self.url.netloc:
host = self.url.netloc.split(':')[0]
server_port = self.url.netloc.split(':')[1]
else:
host = self.url.netloc
server_port = None
if server_port:
http_resp = client.call_taxii_service2(host, self.url.path, VID_TAXII_XML_11, col_xml, port = server_port)
else:
http_resp = client.call_taxii_service2(host, self.url.path, VID_TAXII_XML_11, col_xml)
taxii_message = t.get_message_from_http_response(http_resp, r.message_id)
return taxii_message.to_dict()
def subscribe(s, path="/", collection="default", query=None,
inbox="http://localhost:8888"):
if query != None:
query = s.create_query(query)
else:
query = None
params = tm11.SubscriptionParameters(query=query)
deliv = tm11.PushParameters(
inbox_protocol=t.VID_TAXII_HTTP_10,
inbox_address=inbox,
delivery_message_binding=t.VID_TAXII_XML_11)
# Create request
msg_id=tm11.generate_message_id()
req = tm11.ManageCollectionSubscriptionRequest(
message_id=msg_id,
collection_name=collection,
action=tm11.ACT_SUBSCRIBE,
subscription_parameters=params,
push_parameters=deliv
)
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path, t.VID_TAXII_XML_11,
req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def taxii_poll(config, src, dest, timestamp=None):
'''pull stix from edge via taxii'''
client = tc.HttpClient()
client.setUseHttps(config['edge']['sites'][src]['taxii']['ssl'])
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username': config['edge']['sites'][src]['taxii']['user'],
'password': config['edge']['sites'][src]['taxii']['pass']})
if not timestamp:
earliest = util.epoch_start()
else:
earliest = timestamp
latest = util.nowutc()
poll_request = tm10.PollRequest(
message_id=tm10.generate_message_id(),
feed_name=config['edge']['sites'][src]['taxii']['collection'],
exclusive_begin_timestamp_label=earliest,
inclusive_end_timestamp_label=latest,
content_bindings=[t.CB_STIX_XML_11])
http_response = client.callTaxiiService2(
config['edge']['sites'][src]['host'],
config['edge']['sites'][src]['taxii']['path'],
t.VID_TAXII_XML_10, poll_request.to_xml(),
port=config['edge']['sites'][src]['taxii']['port'])
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm10.StatusMessage):
config['logger'].error(log.log_messages['polling_error'].format(
type_='taxii', error=taxii_message.message))
elif isinstance(taxii_message, tm10.PollResponse):
incidents = dict()
indicators = dict()
observables = dict()
for content_block in taxii_message.content_blocks:
(incidents_, indicators_, observables_) = \
process_taxii_content_blocks(config, content_block)
incidents.update(incidents_)
indicators.update(indicators_)
observables.update(observables_)
return(latest, incidents, indicators, observables)
def perform_manage_collection_subscription(path="/", act="status",
collection="default"):
if act == "subscribe":
action = tm11.ACT_SUBSCRIBE
elif act == "unsubscribe":
action = tm11.ACT_UNSUBSCRIBE
elif act == "pause":
action = tm11.ACT_PAUSE
elif act == "resume":
action = tm11.ACT_RESUME
elif act == "status":
action = tm11.ACT_STATUS
else:
print("Need a subscription action I recognise")
sys.exit(1)
# Create request
msg_id=tm11.generate_message_id()
req = tm11.ManageCollectionSubscriptionRequest(message_id=msg_id,
collection_name=collection,
action=action)
# Convert to XML for request body
req_xml = req.to_xml()
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(host, path, t.VID_TAXII_XML_11,
req_xml, port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
print(resp.to_xml())
def client_example():
# Create the TAXII HTTPS Client
client = tc.HttpClient()
# Uncomment to use HTTPS
client.set_use_https(True)
# Uncomment to use basic authentication
# client.set_auth_type(tc.HttpClient.AUTH_BASIC)
# client.set_auth_credentials({'username':'******', 'password':'******'})
# Uncomment to use certificate-based authentication
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({
'key_file': 'keyfile',
'cert_file': 'certfile'
})
# Uncomment to set a proxy
# client.set_proxy(tc.HttpClient.PROXY_HTTP, 'http://proxy.company.com:80')
# Create the poll request
poll_request1 = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name='TheFeedToPoll')
# Call without a proxy
http_response = client.call_taxii_service2('hostname',
'/poll_service_path/',
VID_TAXII_XML_10,
poll_request1.to_xml())
print(http_response.__class__.__name__)
taxii_message = t.get_message_from_http_response(http_response,
poll_request1.message_id)
print((taxii_message.to_xml()))
def create_client(self):
""" create client for connection """
client = tc.HttpClient()
client.set_use_https(False)
# TODO: add authentication stuff
return client
def process_fsisac_stix_for_today(self):
today_str = datetime.datetime.today().strftime('%Y-%m-%d')
print "[*] Downloading stix for today (%s)..." % (today_str)
# Create a TAXII Client
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC
) # Username/password plus client cert auth
client.set_use_https(True) # Use HTTPS
# Update with your CIR credentials
client.auth_credentials['username'] = FSISAC_USERNAME
client.auth_credentials['password'] = FSISAC_PASSWORD
client.auth_credentials['key_file'] = FSISAC_KEY
client.auth_credentials['cert_file'] = FSISAC_CERT
taxii_server = 'analysis.fsisac.com'
taxii_service = '/taxii-discovery-service/'
feed = 'system.Default' # TAXII feed to be polled. Update to poll a custom TAXII feed.
# TAXII poll Exclusive Start Date and Inclusive End Date, as python datetime tuples.
toyear = datetime.datetime.today().year
tomonth = datetime.datetime.today().month
today = datetime.datetime.today().day
yesterday = datetime.datetime.today() + datetime.timedelta(days=-1)
yesterday = yesterday.day
# print "=" * 100
# print "DEBUGGING"
# print "=" * 100
# print toyear, tomonth, yesterday # debug
# print toyear, tomonth, today # debug
# print "=" * 100
if yesterday == 31:
start = datetime.datetime(toyear,
tomonth - 1,
yesterday,
tzinfo=pytz.UTC)
else:
start = datetime.datetime(toyear,
tomonth,
yesterday,
tzinfo=pytz.UTC)
end = datetime.datetime(toyear, tomonth, today, tzinfo=pytz.UTC)
# start = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC)
# end = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC)
# A TAXII poll can return a lot of data. For performance reasons, if the polling period spans multiple days,
# only poll for one day at a time within the polling period.
inc_start = start
inc_end = inc_start + datetime.timedelta(days=1)
while inc_start <= end:
params = tm11.PollParameters()
#Create the TAXII poll request
poll_request = tm11.PollRequest(
tm11.generate_message_id(),
collection_name=feed,
poll_parameters=params,
exclusive_begin_timestamp_label=inc_start,
inclusive_end_timestamp_label=inc_end)
poll_xml = poll_request.to_xml()
# Get the TAXII poll response
http_resp = client.call_taxii_service2(taxii_server, taxii_service,
VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(
http_resp, poll_request.message_id)
# Write each content block from the TAXII poll response to the "path" directory.
for cb in taxii_message.content_blocks:
#filename = gen_filename(taxii_message.collection_name, 'FSISAC_STIX111_', cb.timestamp_label.isoformat(), '.xml')
filename = gen_filename('FSISAC', '_STIX111_',
cb.timestamp_label.isoformat(), '.xml')
with open(STIX_DOWNLOADED_PATH + "/" + filename,
'w') as outfile:
outfile.write(cb.content)
print "Written to %s" % filename
# Increment to the next day in the specified date range.
inc_start = inc_start + datetime.timedelta(days=1)
inc_end = inc_end + datetime.timedelta(days=1)
def run_taxii_service(analyst,
obj,
rcpts,
preview,
relation_choices=[],
confirmed=False):
"""
:param analyst The analyst triggering this TAXII service call
:param obj The context object being shared
:param rcpts The list of sources to which the TAXII message is being sent
:param preview If true, generate and return the STIX doc, rather than sending via TAXII
:param relation_choices The list of items related to OBJ that have been chosen for sharing
:param confirmed True if user has accepted & approved releasability updates
"""
ret = {
'success':
False, # tells client whether any message was sent successfully
'rcpts': [], # list of sources the message was sent
'failed_rcpts':
[], # list of sources to which the message failed to be sent
}
if not obj: # no item (shouldn't occur unless someone is really trying to break things.)
ret['reason'] = "No object found."
return ret
if not rcpts: # no sources selected in TAXII form (validation prevents this, anyway)
ret['reason'] = "No recipients selected."
return ret
# If dealing with an event context, make sure at least one related item is
# selected. Events have no real sharing value without related information.
if obj._meta['crits_type'] == Event._meta['crits_type'] and len(
relation_choices) == 0:
ret['reason'] = "Need at least one related item to send."
return ret
# Get config and grab some stuff we need.
sc = get_config('taxii_service')
hostname = sc['hostname']
https = sc['https']
keyfile = sc['keyfile']
certfile = sc['certfile']
certfiles = sc['certfiles']
# collect the list of destination data feeds for the message
destination_feeds = []
for crtfile in certfiles:
(source, feed, filepath) = crtfile.split(',')
src = source.strip()
if src in rcpts:
destination_feeds.append((src, feed.strip(), filepath.strip()))
if not destination_feeds or len(destination_feeds) != len(rcpts):
# TAXII form ensures that this shouldn't happen, but just in case...
ret['reason'] = "Misconfigured TAXII service -- contact an administrator."
return ret
# The minimum required info has been provided by user via the TAXII form.
# Form configuration and validation ensures the form is valid.
# The TAXII service has also been confirmed to have config information
# for each selected recipient.
#
# NOTE: this does not guarantee that the message will send to
# each/any recipient feed successfully.
# Convert object and chosen related items to STIX/CybOX
stix_msg = to_stix(obj, relation_choices, bin_fmt="base64")
stix_doc = stix_msg['stix_obj']
# if doing a preview of content, return content now
if preview:
ret['preview'] = stix_doc.to_xml()
return ret
elif not confirmed: # if user has not accepted responsibility for releasability
release = verify_releasability(rcpts, stix_msg['final_objects'],
analyst, False)
if release: # if releasability needs to change
ret['release_changes'] = release
return ret # make user confirm changes, instead of sending messages
#TODO: this doesn't confirm that 'hostname' is a TAXII server...
if not resolve_taxii_server(hostname):
ret['reason'] = "Cannot contact TAXII Server at: %s" % hostname
return ret
client = tc.HttpClient()
if https:
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy)
# generate and send inbox messages
# one message per feed, with appropriate TargetFeed header specified
# Store each TAXII message in a list.
for feed in destination_feeds:
rcpt = feed[0]
# Create encrypted block
encrypted_block = encrypt_block(
tm.ContentBlock(content_binding=t.CB_STIX_XML_111,
content=stix_doc.to_xml()).to_xml(), feed[2])
# Try TAXII 1.1 first:
try_10 = False
current_status_type = None
failed = True
result = gen_send(tm11,
client,
encrypted_block,
hostname,
t.VID_TAXII_XML_11,
dcn=[feed[1]],
url="/services/inbox/")
if len(result) == 2:
res = result[1]
if res.status_type == tm11.ST_SUCCESS:
failed = False
ret['rcpts'].append(rcpt)
else:
try_10 = True
current_status_type = "<br>tm11: " + res.status_type
else:
try_10 = True
current_status_type = "<br>tm11: " + result[0]
# Try TAXII 1.0 since 1.1 seems to have failed.
if try_10:
result = gen_send(tm,
client,
encrypted_block,
hostname,
t.VID_TAXII_XML_10,
eh={'TargetFeed': feed[1]},
url="/inbox/")
if len(result) == 2:
res = result[1]
if res.status_type == tm11.ST_SUCCESS:
failed = False
ret['rcpts'].append(rcpt)
else:
err = "tm10: " + res.status_type
current_status_type += "<br>%s" % err
else:
err = "tm10: " + result[0]
current_status_type += "<br>%s" % err
if failed:
ret['failed_rcpts'].append((rcpt, current_status_type))
if ret['rcpts']: # update releasability for successful TAXII messages
verify_releasability(ret['rcpts'], stix_msg['final_objects'], analyst,
True)
ret['success'] = True
return ret
def execute_taxii_agent(hostname=None,
https=None,
feed=None,
keyfile=None,
certfile=None,
start=None,
end=None,
analyst=None,
method=None):
ret = {
'Certificate': [],
'Domain': [],
'Email': [],
'Event': [],
'Indicator': [],
'IP': [],
'PCAP': [],
'RawData': [],
'Sample': [],
'successes': 0,
'failures': [],
'status': False,
'reason': ''
}
sc = get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
if https == None:
https = sc['https']
create_events = sc['create_events']
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
if https:
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
# Poll using 1.1 then 1.0 if that fails.
poll_msg = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=feed,
poll_parameters=tm11.PollRequest.PollParameters(),
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
# Check if this is a TAXII 1.0 server and try again
if response.info().getheader(
'X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/',
t.VID_TAXII_XML_10,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(
response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
else:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = "Invalid XML: %s" % valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'].append(('No data found in content block', 'Data'))
continue
objs = import_standards_doc(data,
analyst,
method,
ref=mid,
make_event=create_events)
for k in objs['imported']:
ret['successes'] += 1
ret[k[0]].append(k[1])
for k in objs['failed']:
ret['failures'].append(k)
crits_taxii.save()
return ret
def main():
# This is a work-around for the fact that the 1.0 indicator type was removed from the STIX python
# library, even though it is the essentially the same as the 1.1 type. We want to still support 1.0
# indicators since they are out there, and there is no difference for the purposes of this script.
# vocabs._VOCAB_MAP["stixVocabs:IndicatorTypeVocab-1.0"] = IndicatorType
# Create XML parser that can strip namespaces
xmlParser = EntityParser()
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].referenceset and args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print >> sys.stderr, "Could not parse either start or end time"
raise
#Create Request Header
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.PollParameters())
poll_req_xml = poll_req.to_xml()
#Create request client object
client = tc.HttpClient()
#HTTPS Setting
if args[0].taxii_ssl:
client.set_use_https(True)
# Basic Auth
if args[0].taxii_username:
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
"Enter your taxii password: "******"content_blocks"]:
bindingId = content["content_binding"]["binding_id"]
if bindingId and bindingId.startswith(
"urn:stix.mitre.org:xml"):
if args[0].verbose:
print >> sys.stderr, "RECIEVED STIX DATA:\n"
print >> sys.stderr, content["content"]
try:
# This string replace is a workaround for some invalid documents in my test server, if you don't need it, remove it
xmlString = content["content"].replace(
'low',
'Low').replace('medium',
'Medium').replace('high', 'High')
stix_package = xmlParser.parse_xml(
io.BytesIO(xmlString), False)
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print >> sys.stderr, "Could not parse STIX document: "
print >> sys.stderr, content["content"]
raise
print "Imported", indicators, "indicators into reference set", args[
0].referenceset
else:
print >> sys.stderr, "Invalid reponse from TAXII server"
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].referenceset and args[0].file:
stix_package = xmlParser.parse_xml(args[0].file, False)
indicators = process_package_dict(args, stix_package.to_dict())
print "Imported", indicators, "indicators into reference set", args[
0].referenceset
else:
print >> sys.stderr, "Invalid arguments. Type 'python stix_import.py --help' for usage.\n"
def main():
config = ConfigParser.ConfigParser()
config.read('app.conf')
dstHost = config.get('taxii', 'dstHost')
dstPort = config.get('taxii', 'dstPort')
dstPath = config.get('taxii', 'dstPath')
username = config.get('taxii', 'username')
password = config.get('taxii', 'password')
parser = argparse.ArgumentParser(description="Inbox Client")
parser.add_argument(
"--host",
dest="host",
default=dstHost,
help="Host where the Inbox Service is hosted. Defaults to " + dstHost)
parser.add_argument(
"--port",
dest="port",
default=dstPort,
help="Port where the Inbox Service is hosted. Defaults to " + dstPort)
parser.add_argument(
"--path",
dest="path",
default=dstPath,
help="Path where the Inbox Service is hosted. Defaults to " + dstPath)
parser.add_argument(
"--content-binding",
dest="content_binding",
default=t.CB_STIX_XML_11,
help="Content binding of the Content Block to send. Defaults to %s" %
t.CB_STIX_XML_11)
parser.add_argument(
"--content-file",
dest="content_file",
default=stixContent,
help="File name of the Content Block to send. Required.")
args = parser.parse_args()
if args.content_file is '':
parser.print_help()
sys.exit(1)
_l.info('Starting...')
c = open(args.content_file, 'r')
cb = tmsg.ContentBlock(tmsg.ContentBinding(args.content_binding), c.read())
c.close()
taxiiMsg = tmsg.InboxMessage(message_id=tmsg.generate_message_id(),
content_blocks=[cb])
taxiiMsgXml = taxiiMsg.to_xml()
# send it
_l.debug('Uploading content.')
client = tc.HttpClient()
client.setProxy('noproxy')
client.setAuthType(tc.HttpClient.AUTH_BASIC)
client.setAuthCredentials({'username': username, 'password': password})
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11,
taxiiMsgXml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
_l.debug('Response was: ' + response_message.to_xml())
_l.info('Ended.')
def main():
# Create XML parser that can strip namespaces
xmlParser = EntityParser()
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print >> sys.stderr, "Could not parse either start or end time"
raise
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.PollParameters())
poll_req_xml = poll_req.to_xml()
client = tc.HttpClient()
if args[0].taxii_ssl:
client.setUseHttps(True)
if args[0].taxii_username:
client.setAuthType(1)
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
"Enter your taxii password: "******"/poll/",
t.VID_TAXII_XML_11, poll_req_xml,
args[0].taxiiport)
response_message = t.get_message_from_http_response(resp, '0')
response_dict = response_message.to_dict()
indicators = 0
if 'content_blocks' in response_dict:
for content in response_dict["content_blocks"]:
bindingId = content["content_binding"]["binding_id"]
if bindingId and bindingId.startswith(
"urn:stix.mitre.org:xml"):
#if args[0].verbose:
# print >> sys.stderr, "RECIEVED STIX DATA:\n"
#print >> sys.stderr, content["content"]
try:
# Parse the information
stix_pkg = STIXPackage.from_xml(
io.BytesIO(content["content"]))
stix_package = stix_pkg
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print >> sys.stderr, "Could not parse STIX document: "
print >> sys.stderr, content["content"]
raise
print "Imported", indicators, "indicators into set"
else:
print >> sys.stderr, "Invalid reponse from TAXII server"
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].file:
stix_package = STIXPackage.from_xml(args[0].file)
indicators = process_package_dict(args, stix_package.to_dict())
print "Imported", indicators, "indicators into set"
else:
print >> sys.stderr, "Invalid arguments. Type 'python stix_Arcsight.py --help' for usage.\n"
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
path = os.path.dirname(os.path.abspath(sys.argv[0]))
parser = SafeConfigParser()
parser.read(path + '/config.ini')
username = parser.get('TAXII', 'Username')
password = parser.get('TAXII', 'Password')
proxy = parser.getboolean('TAXII', 'Proxy')
proxyaddress = parser.get('TAXII', 'ProxyAddress')
ssl = parser.getboolean('TAXII', 'HTTPS')
collection = parser.get('TAXII', 'Collection')
server = parser.get('TAXII', 'Server')
port = parser.get('TAXII', 'Port')
path = parser.get('TAXII', 'Path')
days = parser.get('TAXII', 'Days')
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(ssl)
if proxy is True:
client.set_proxy(proxyaddress)
client.set_auth_credentials(
{'username': username, 'password': password})
def _saveTimestamp(timestamp=None):
mtimestamp = timestamp
if not timestamp:
mtimestamp = datetime.datetime.now(tzutc())
fname = "timestamp"
f = open(fname, "w")
f.write(mtimestamp)
def main():
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print('Could not parse either start or end time', file=sys.stderr)
raise
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.poll_parameters())
poll_req_xml = poll_req.to_xml()
client = tc.HttpClient()
if args[0].taxii_ssl:
client.set_use_https(True)
if args[0].taxii_username:
client.set_auth_type(1)
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
'Enter your taxii password: '******'username': args[0].taxii_username,
'password': args[0].taxii_password
})
elif args[0].taxii_key and args[0].taxii_cert:
client.set_auth_type(2)
client.set_auth_credentials({
'key': args[0].taxii_key,
'cert': args[0].taxii_cert
})
if args[0].proxy:
client.set_proxy(args[0].proxy)
resp = client.call_taxii_service2(
args[0].taxii, args[0].taxii_endpoint + '/poll/',
t.VID_TAXII_XML_11, poll_req_xml, args[0].taxiiport)
response_message = t.get_message_from_http_response(resp, '0')
response_dict = response_message.to_dict()
indicators = 0
if 'content_blocks' in response_dict:
for content in response_dict['content_blocks']:
bindingId = content['content_binding']['binding_id']
if bindingId and bindingId.startswith(
'urn:stix.mitre.org:xml'):
#if args[0].verbose:
# print >> sys.stderr, 'RECIEVED STIX DATA:\n'
#print >> sys.stderr, content['content']
try:
# Parse the information
stix_pkg = STIXPackage.from_xml(
io.BytesIO(content['content']))
stix_package = stix_pkg
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print(
'Could not parse STIX document: ', file=sys.stderr)
print(content['content'], file=sys.stderr)
raise
print('Imported', indicators, 'indicators into set')
else:
print('Invalid reponse from TAXII server', file=sys.stderr)
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].file:
stix_package = STIXPackage.from_xml(args[0].file)
indicators = process_package_dict(args, stix_package.to_dict())
print('Imported', indicators, 'indicators into set')
else:
print('Invalid arguments. Type --help for usage.\n', file=sys.stderr)
def _poll_taxii_11(self, args):
'''Poll a TAXII 1.1 feed and return a generator over the content blocks found.
Arguments:
args - A dictionary of arguments.
'''
app = args.get('app')
cert_file = args.get('cert_file')
key_file = args.get('key_file')
username = args.get('username')
password = args.get('password')
if app and cert_file and key_file:
cert_file_path = make_splunkhome_path(
["etc", "apps", app, "auth", cert_file])
key_file_path = make_splunkhome_path(
["etc", "apps", app, "auth", key_file])
if not (os.path.exists(cert_file_path)
and os.path.exists(key_file_path)):
logger.info(
"Certificate not found - falling back to AUTH_BASIC.")
cert_file_path = key_file_path = None
else:
logger.info(
"Certificate information incomplete - falling back to AUTH_BASIC."
)
cert_file_path = key_file_path = None
client = tc.HttpClient()
client.set_use_https(args.get('use_ssl'))
# Add proxy parameters if present.
if 'proxy' in args:
client.set_proxy(args['proxy'])
if cert_file_path and key_file_path and username:
logger.info("Auth Type: AUTH_CERT_BASIC")
client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC)
client.set_auth_credentials({
'username': username,
'password': password,
'cert_file': cert_file_path,
'key_file': key_file_path
})
elif cert_file_path and key_file_path:
logger.info("Auth Type: AUTH_CERT")
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({
'cert_file': cert_file_path,
'key_file': key_file_path
})
else:
logger.info("Auth Type: AUTH_BASIC")
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_auth_credentials({
'username': username,
'password': password
})
poll_request = tm11.PollRequest(
tm11.generate_message_id(),
collection_name=args.get('collection'),
exclusive_begin_timestamp_label=args.get('earliest'),
#inclusive_end_timestamp_label=args.get('latest'),
poll_parameters=tm11.PollParameters())
poll_xml = poll_request.to_xml()
http_resp = client.call_taxii_service2(args.get('url'),
args.get('service'),
tm11.VID_TAXII_XML_11,
poll_xml,
port=args.get('port'))
taxii_message = libtaxii.get_message_from_http_response(
http_resp, poll_request.message_id)
if http_resp.getcode() == 200:
if hasattr(taxii_message, 'content_blocks'):
return taxii_message.to_xml()
else:
raise TaxiiHandlerException(
'Exception when polling TAXII feed (no content returned): %s'
% self._clean_taxii_message(taxii_message))
else:
raise TaxiiHandlerException(
'Exception when polling TAXII feed: %s' %
self._clean_taxii_message(taxii_message))
def execute_taxii_agent(hostname=None,
feed=None,
keyfile=None,
certfile=None,
start=None,
end=None,
analyst=None,
method=None):
ret = {
'events': [],
'samples': [],
'emails': [],
'indicators': [],
'successes': 0,
'failures': 0,
'status': False,
'reason': ''
}
sc = manager.get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
poll_msg.to_xml())
if response.getcode() != 200:
ret['reason'] = "Response is not 200 OK"
return ret
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response"
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'] += 1
continue
objs = import_standards_doc(data,
analyst,
method,
ref=mid,
make_event=True)
ret['successes'] += 1
for k in ["events", "samples", "emails", "indicators"]:
for i in objs[k]:
ret[k].append(i)
crits_taxii.save()
return ret