def create_request_message(self, args):
try:
if args.begin_ts:
begin_ts = dateutil.parser.parse(args.begin_ts)
if not begin_ts.tzinfo:
raise ValueError
else:
begin_ts = None
if args.end_ts:
end_ts = dateutil.parser.parse(args.end_ts)
if not end_ts.tzinfo:
raise ValueError
else:
end_ts = None
except ValueError:
print("Unable to parse timestamp value. Timestamp should include both date and time " \
"information along with a timezone or UTC offset (e.g., YYYY-MM-DDTHH:MM:SS.ssssss+/-hh:mm). " \
"Aborting poll.")
sys.exit()
create_kwargs = {
'message_id': tm11.generate_message_id(),
'collection_name': args.collection,
'exclusive_begin_timestamp_label': begin_ts,
'inclusive_end_timestamp_label': end_ts
}
if args.subscription_id:
create_kwargs['subscription_id'] = args.subscription_id
else:
create_kwargs['poll_parameters'] = tm11.PollRequest.PollParameters(
)
poll_req = tm11.PollRequest(**create_kwargs)
return poll_req
def test_10(self):
"""
Test a query. Should match just the APT1 report.
"""
test = tdq.Test(capability_id=tdq.CM_CORE,
relationship=R_EQUALS,
parameters={
P_VALUE: 'Unit 61398',
P_MATCH_TYPE: 'case_sensitive_string'
})
criterion = tdq.Criterion(
target=
'STIX_Package/Threat_Actors/Threat_Actor/Identity/Specification/PartyName/OrganisationName/SubDivisionName',
test=test)
criteria = tdq.Criteria(OP_AND, criterion=[criterion])
q = tdq.DefaultQuery(CB_STIX_XML_111, criteria)
pp = tm11.PollParameters()
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
#msg = self.send_poll_request('/services/test_poll_1/', VID_TAXII_XML_11, pr)
msg = make_request('/services/test_poll_1/', pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_POLL_RESPONSE)
if len(msg.content_blocks) != 1:
raise ValueError('Got %s CBs' % len(msg.content_blocks))
def prepare_request(collection_name, version, count_only=False,
bindings=[], subscription_id=None):
if version == 11:
content_bindings = [tm11.ContentBinding(b) for b in bindings]
if subscription_id:
poll_parameters = None
else:
poll_parameters = tm11.PollParameters(
response_type=(
RT_FULL if not count_only else RT_COUNT_ONLY),
content_bindings=content_bindings,
)
return tm11.PollRequest(
message_id=MESSAGE_ID,
collection_name=collection_name,
subscription_id=subscription_id,
poll_parameters=poll_parameters
)
elif version == 10:
content_bindings = bindings
return tm10.PollRequest(
message_id=MESSAGE_ID,
feed_name=collection_name,
content_bindings=content_bindings,
subscription_id=subscription_id
)
def main():
# Create the test portion of the query
my_test = tdq.Test(capability_id=CM_CORE,
relationship=R_EQUALS,
parameters={P_VALUE: value,
P_MATCH_TYPE: 'case_insensitive_string'}
)
#Put the test into a Criterion
my_criterion = tdq.Criterion(target=target, test=my_test)
# Put the Criterion into a Criteria
my_criteria = tdq.Criteria(operator=OP_AND,
criterion=[my_criterion],
criteria=None)
# Create a query with the criteria
my_query = tdq.DefaultQuery(CB_STIX_XML_111, my_criteria)
# Create a Poll Parameters that indicates
# Only STIX 1.1.1 is accepted in response
# and with the query created previously
params = tm11.PollParameters(content_bindings=[tm11.ContentBinding(CB_STIX_XML_111)],
query=my_query)
poll_request = tm11.PollRequest(message_id=generate_message_id(),
collection_name='file_hash_reputation',
poll_parameters=params)
print poll_request.to_xml(pretty_print=True)
def poll_11(self):
auth_credentials_dict = self._get_auth_credentials_dict()
self._client.set_auth_credentials(auth_credentials_dict)
try:
poll_parameters = tm11.PollParameters()
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=self._collection_name,
exclusive_begin_timestamp_label=self._start,
inclusive_end_timestamp_label=self._end,
poll_parameters=poll_parameters,
)
last_requested = datetime.datetime.now(pytz.utc)
http_resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
poll_request.to_xml(),
port=self._port)
taxii_message = libtaxii.get_message_from_http_response(
http_resp, poll_request.message_id)
try:
if taxii_message.content_blocks is None:
return 0
except AttributeError:
return 0
count = 0
for cb in taxii_message.content_blocks:
_, stix_file_path = tempfile.mkstemp(suffix='.xml')
with open(stix_file_path, 'wb+') as fp:
fp.write(cb.content)
try:
from ctirs.core.stix.regist import regist
if self._community is not None:
regist(stix_file_path, self._community, self._via)
count += 1
except BaseException:
traceback.print_exc()
self._taxii.last_requested = last_requested
self._taxii.save()
return count
finally:
if 'cert_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['cert_file'])
except Exception:
pass
if 'key_file' in auth_credentials_dict:
try:
os.remove(auth_credentials_dict['key_file'])
except Exception:
pass
def _main(first=True):
if first:
begin = datetime.datetime.now(tzutc()) - timedelta(days=int(days))
else:
try:
begin = _readTimestamp()
except IOError:
print("[-] No timestamp file found have you 'first_run'?")
sys.exit(0)
end = datetime.datetime.now(tzutc())
poll_params1 = tm11.PollParameters(
allow_asynch=False,
response_type=RT_COUNT_ONLY,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)],
)
try:
poll_req3 = tm11.PollRequest(
message_id='PollReq03',
collection_name=collection,
poll_parameters=poll_params1,
exclusive_begin_timestamp_label=begin,
inclusive_end_timestamp_label=end,
)
except ValueError:
print("[-] Invalid timestamp file")
sys.exit(0)
except Exception:
print("[-] Error with PollRequest")
poll_xml = poll_req3.to_xml()
http_resp = client.call_taxii_service2(
server, path, VID_TAXII_XML_11,
poll_xml, port=port)
taxii_message = t.get_message_from_http_response(
http_resp, poll_req3.message_id)
if taxii_message.message_type == MSG_POLL_RESPONSE:
if taxii_message.content_blocks:
try:
for content in taxii_message.content_blocks:
package_io = StringIO(content.content)
pkg = STIXPackage.from_xml(package_io)
title = pkg.id_.split(':', 1)[-1]
with open(title + ".xml", "w") as text_file:
text_file.write(content.content)
print("[+] Successfully generated " + title)
except Exception:
print("[-] Error with TAXII response")
else:
print("[+] No content returned")
_saveTimestamp(str(end))
else:
print("[-] Error with TAXII response")
def poll_11(self):
#request xml作成
poll_parameters = tm11.PollParameters()
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=self._collection_name,
exclusive_begin_timestamp_label=self._start,
inclusive_end_timestamp_label=self._end,
poll_parameters=poll_parameters,
)
last_requested = datetime.datetime.now(pytz.utc)
#from xml.dom.minidom import parseString
#print( parseString(poll_request.to_xml()).toprettyxml(encoding='utf-8'))
#request
http_resp = self._client.call_taxii_service2(
self._address,
self._path,
const.VID_TAXII_XML_11,
poll_request.to_xml(),
port=self._port)
#taxii_message抽出
taxii_message = libtaxii.get_message_from_http_response(http_resp, poll_request.message_id)
#content_blocks が None or 存在しない場合は登録ししない
try:
if taxii_message.content_blocks is None:
return 0
except AttributeError:
return 0
#content_blocksごとに登録処理
count = 0
for cb in taxii_message.content_blocks:
#stixファイルを一時ファイルに出力
_,stix_file_path = tempfile.mkstemp(suffix='.xml')
with open(stix_file_path,'wb+') as fp:
#cb.contentがstixの中身
fp.write(cb.content)
#登録
try:
from ctirs.core.stix.regist import regist
if self._community is not None:
regist(stix_file_path,self._community,self._via)
count += 1
except:
#taxiiの場合は途中で失敗しても処理を継続する
traceback.print_exc()
#last_requested更新
self._taxii.last_requested = last_requested
self._taxii.save()
return count
def test_03(self):
"""
Test an invalid subscription ID
"""
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
subscription_id='jdslkajdlksajdlksajld')
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_NOT_FOUND,
sd_keys=[SD_ITEM])
def test_09(self):
"""
Tests that a single PollRequest succeeds.
"""
pp = tm11.PollParameters()
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/', pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_POLL_RESPONSE)
#msg = self.send_poll_request('/services/test_poll_1/', VID_TAXII_XML_11, pr)
if len(msg.content_blocks) != 5:
raise ValueError('Got %s CBs' % len(msg.content_blocks))
def test_04(self):
"""
Test a Content Binding ID not supported by the server
"""
pp = tm11.PollParameters(
content_bindings=[tm11.ContentBinding('some_random_binding')])
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_UNSUPPORTED_CONTENT_BINDING,
sd_keys=[SD_SUPPORTED_CONTENT])
def poll_feed(settings, subscription):
""" polls a TAXII feed"""
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_use_https(True)
client.set_auth_credentials({
'username': settings['username'],
'password': settings['password']
})
msg_id = uuid.uuid4().hex
poll_request1 = tm11.PollRequest(message_id=msg_id,
collection_name=settings['subscriptions']
[subscription]['collection_name'],
subscription_id=settings['subscriptions']
[subscription]['subscription_id'])
poll_xml = poll_request1.to_xml()
http_resp = client.call_taxii_service2(settings['server'], '/taxii-data/',
VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(http_resp,
poll_request1.message_id)
observables = {}
indicators = json.loads(taxii_message.to_json())
if 'content_blocks' in indicators.keys():
for indicator in indicators['content_blocks']:
open('/tmp/indicator.xml', 'w').write(indicator['content'])
indi = STIXPackage.from_xml('/tmp/indicator.xml').to_dict()
if 'observables' in indi.keys():
for obs in indi['observables']['observables']:
if 'object' in obs.keys():
ot = obs['object']['properties']['xsi:type']
if ot in settings['supported_objects'].keys(
) and not ot in observables.keys():
observables[ot] = []
if ot in settings['supported_objects'].keys(
) and settings['supported_objects'][ot] in obs[
'object']['properties'].keys():
# note, you will only be able to process one property per object type, but you also know there's only one property you can process
try:
observables[ot].append(
obs['object']['properties'][
settings['supported_objects'][ot]])
except:
print "[-] you're dumb"
print supported_objects[ot], "not in:", obs[
'object']
return observables
def test_02(self):
"""
Test a begin TS later than an end TS.
"""
begin_ts = datetime.now(tzutc())
end_ts = begin_ts - timedelta(days=-7)
pp = tm11.PollParameters()
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts)
msg = make_request('/services/test_poll_1/', pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
ST_FAILURE)
def test_05(self):
"""
Test a supported Content Binding ID with an unsupported subtype
"""
pp = tm11.PollParameters(content_bindings=[
tm11.ContentBinding(CB_STIX_XML_111, subtype_ids=['FakeSubtype'])
])
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_UNSUPPORTED_CONTENT_BINDING,
sd_keys=[SD_SUPPORTED_CONTENT])
def test_01(self):
"""
Test an invalid collection name
"""
pp = tm11.PollParameters()
pr = tm11.PollRequest(
message_id=generate_message_id(),
collection_name='INVALID_COLLECTION_NAME_13820198320',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_NOT_FOUND,
sd_keys=[SD_ITEM])
def indicators(self,
collection_name,
starttime=(datetime.now(tzutc()) - timedelta(hours=1)),
endtime=datetime.now(tzutc()),
subscription_id=None):
delivery_params = tm11.DeliveryParameters(VID_TAXII_HTTP_10,
self.remote,
VID_TAXII_XML_11)
"""
poll_params = tm11.PollParameters(response_type=RT_COUNT_ONLY,
#content_bindings=[tm11.ContentBinding(CB_STIX_XML_11)],
delivery_parameters=delivery_params
)
"""
poll_params = tm11.PollRequest.PollParameters()
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=collection_name,
exclusive_begin_timestamp_label=starttime,
inclusive_end_timestamp_label=endtime,
poll_parameters=poll_params,
subscription_id=subscription_id)
logger.debug('TAXII collection poll request: {}'.format(
poll_req.to_xml(pretty_print=True)))
poll_req_xml = poll_req.to_xml()
http_resp = self.client.call_taxii_service2(self.up.hostname,
self.up.path,
VID_TAXII_XML_11,
poll_req_xml, self.up.port)
taxii_message = t.get_message_from_http_response(
http_resp, poll_req.message_id)
logger.debug('TAXII collection poll response: {}'.format(
taxii_message.to_xml(pretty_print=True)))
if taxii_message.message_type == MSG_STATUS_MESSAGE:
if taxii_message.status_type == ST_SUCCESS:
logger.info(
'TAXII polled successfully but returned no results')
return []
raise RuntimeError('TAXII polling failed: {} - {}'.format(
taxii_message.status_type, taxii_message.message))
return self._parse_taxii_content(taxii_message.content_blocks)
def poll(s, path="/", collection="default", query=None,
begin_ts=None, end_ts=None):
if query != None:
query=s.create_query(query)
poll_params=tm11.PollRequest.PollParameters(query=query)
else:
poll_params=tm11.PollRequest.PollParameters()
# Create poll request
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=poll_params)
# Convert to XML for request body
poll_req_xml = poll_req.to_xml(True)
# Create HTTP client
client = tc.HttpClient()
client.setProxy('noproxy')
# Call TAXII service, using the body
resp = client.callTaxiiService2(s.host, path,
t.VID_TAXII_XML_11,
poll_req_xml, s.port)
# Get response
resp = t.get_message_from_http_response(resp, '0')
pkgs = []
# Process each content block
for cb in resp.content_blocks:
content = cb.content
# Parse the payload, should be a STIX document.
package = STIXPackage.from_xml(StringIO.StringIO(content))
pkgs.append(package)
return resp.inclusive_end_timestamp_label, pkgs
def _prepare_poll_request(
self,
collection_name,
begin_date=None,
end_date=None,
subscription_id=None,
inbox_service=None,
content_bindings=None,
count_only=False,
):
data = dict(
message_id=self._generate_id(),
collection_name=collection_name,
exclusive_begin_timestamp_label=begin_date,
inclusive_end_timestamp_label=end_date,
)
if subscription_id:
data["subscription_id"] = subscription_id
else:
_bindings = pack_content_bindings(content_bindings, version=11)
poll_params = {"content_bindings": _bindings}
if inbox_service:
message_bindings = (inbox_service.message_bindings[0]
if inbox_service.message_bindings else [])
poll_params["delivery_parameters"] = tm11.DeliveryParameters(
inbox_protocol=inbox_service.protocol,
inbox_address=inbox_service.address,
delivery_message_binding=message_bindings,
)
poll_params["allow_asynch"] = True
if count_only:
poll_params["response_type"] = const.RT_COUNT_ONLY
else:
poll_params["response_type"] = const.RT_FULL
data["poll_parameters"] = tm11.PollRequest.PollParameters(
**poll_params)
return tm11.PollRequest(**data)
def process_fsisac_stix_for_today(self):
today_str = datetime.datetime.today().strftime('%Y-%m-%d')
print "[*] Downloading stix for today (%s)..." % (today_str)
# Create a TAXII Client
client = tc.HttpClient()
client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC
) # Username/password plus client cert auth
client.set_use_https(True) # Use HTTPS
# Update with your CIR credentials
client.auth_credentials['username'] = FSISAC_USERNAME
client.auth_credentials['password'] = FSISAC_PASSWORD
client.auth_credentials['key_file'] = FSISAC_KEY
client.auth_credentials['cert_file'] = FSISAC_CERT
taxii_server = 'analysis.fsisac.com'
taxii_service = '/taxii-discovery-service/'
feed = 'system.Default' # TAXII feed to be polled. Update to poll a custom TAXII feed.
# TAXII poll Exclusive Start Date and Inclusive End Date, as python datetime tuples.
toyear = datetime.datetime.today().year
tomonth = datetime.datetime.today().month
today = datetime.datetime.today().day
yesterday = datetime.datetime.today() + datetime.timedelta(days=-1)
yesterday = yesterday.day
# print "=" * 100
# print "DEBUGGING"
# print "=" * 100
# print toyear, tomonth, yesterday # debug
# print toyear, tomonth, today # debug
# print "=" * 100
if yesterday == 31:
start = datetime.datetime(toyear,
tomonth - 1,
yesterday,
tzinfo=pytz.UTC)
else:
start = datetime.datetime(toyear,
tomonth,
yesterday,
tzinfo=pytz.UTC)
end = datetime.datetime(toyear, tomonth, today, tzinfo=pytz.UTC)
# start = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC)
# end = datetime.datetime(2019, 4, 5, tzinfo=pytz.UTC)
# A TAXII poll can return a lot of data. For performance reasons, if the polling period spans multiple days,
# only poll for one day at a time within the polling period.
inc_start = start
inc_end = inc_start + datetime.timedelta(days=1)
while inc_start <= end:
params = tm11.PollParameters()
#Create the TAXII poll request
poll_request = tm11.PollRequest(
tm11.generate_message_id(),
collection_name=feed,
poll_parameters=params,
exclusive_begin_timestamp_label=inc_start,
inclusive_end_timestamp_label=inc_end)
poll_xml = poll_request.to_xml()
# Get the TAXII poll response
http_resp = client.call_taxii_service2(taxii_server, taxii_service,
VID_TAXII_XML_11, poll_xml)
taxii_message = t.get_message_from_http_response(
http_resp, poll_request.message_id)
# Write each content block from the TAXII poll response to the "path" directory.
for cb in taxii_message.content_blocks:
#filename = gen_filename(taxii_message.collection_name, 'FSISAC_STIX111_', cb.timestamp_label.isoformat(), '.xml')
filename = gen_filename('FSISAC', '_STIX111_',
cb.timestamp_label.isoformat(), '.xml')
with open(STIX_DOWNLOADED_PATH + "/" + filename,
'w') as outfile:
outfile.write(cb.content)
print "Written to %s" % filename
# Increment to the next day in the specified date range.
inc_start = inc_start + datetime.timedelta(days=1)
inc_end = inc_end + datetime.timedelta(days=1)
def main():
# Create XML parser that can strip namespaces
xmlParser = EntityParser()
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print >> sys.stderr, "Could not parse either start or end time"
raise
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.PollParameters())
poll_req_xml = poll_req.to_xml()
client = tc.HttpClient()
if args[0].taxii_ssl:
client.setUseHttps(True)
if args[0].taxii_username:
client.setAuthType(1)
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
"Enter your taxii password: "******"/poll/",
t.VID_TAXII_XML_11, poll_req_xml,
args[0].taxiiport)
response_message = t.get_message_from_http_response(resp, '0')
response_dict = response_message.to_dict()
indicators = 0
if 'content_blocks' in response_dict:
for content in response_dict["content_blocks"]:
bindingId = content["content_binding"]["binding_id"]
if bindingId and bindingId.startswith(
"urn:stix.mitre.org:xml"):
#if args[0].verbose:
# print >> sys.stderr, "RECIEVED STIX DATA:\n"
#print >> sys.stderr, content["content"]
try:
# Parse the information
stix_pkg = STIXPackage.from_xml(
io.BytesIO(content["content"]))
stix_package = stix_pkg
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print >> sys.stderr, "Could not parse STIX document: "
print >> sys.stderr, content["content"]
raise
print "Imported", indicators, "indicators into set"
else:
print >> sys.stderr, "Invalid reponse from TAXII server"
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].file:
stix_package = STIXPackage.from_xml(args[0].file)
indicators = process_package_dict(args, stix_package.to_dict())
print "Imported", indicators, "indicators into set"
else:
print >> sys.stderr, "Invalid arguments. Type 'python stix_Arcsight.py --help' for usage.\n"
def execute_taxii_agent(hostname=None,
https=None,
feed=None,
keyfile=None,
certfile=None,
start=None,
end=None,
analyst=None,
method=None):
ret = {
'Certificate': [],
'Domain': [],
'Email': [],
'Event': [],
'Indicator': [],
'IP': [],
'PCAP': [],
'RawData': [],
'Sample': [],
'successes': 0,
'failures': [],
'status': False,
'reason': ''
}
sc = get_config('taxii_service')
# XXX: Validate these!
if not hostname:
hostname = str(sc['hostname'])
if not keyfile:
keyfile = str(sc['keyfile'])
if not certfile:
certfile = str(sc['certfile'])
if not feed:
feed = str(sc['data_feed'])
if https == None:
https = sc['https']
create_events = sc['create_events']
# Last document's end time is our start time.
if not start:
last = taxii.Taxii.get_last()
if last:
start = pytz.utc.localize(last.end)
# If start is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(start, str):
start = pytz.utc.localize(parse(start, fuzzy=True))
# store the current time as the time of this request
runtime = datetime.now(tzutc())
# End time is always now, unless specified.
if not end:
end = runtime
# If end is a string, convert it to a datetime
# YYYY-MM-DD HH:MM:SS
if isinstance(end, str):
end = pytz.utc.localize(parse(end, fuzzy=True))
# compare start and end to make sure:
# 1) start time is before end time
# 2) end time is not in the future
if (start != None and start >= end) and end > runtime:
ret['reason'] = "Bad timestamp(s)"
return ret
client = tc.HttpClient()
if https:
client.setUseHttps(True)
client.setAuthType(tc.HttpClient.AUTH_CERT)
client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})
if settings.HTTP_PROXY:
proxy = settings.HTTP_PROXY
if not proxy.startswith('http://'):
proxy = 'http://' + proxy
client.setProxy(proxy)
crits_taxii = taxii.Taxii()
crits_taxii.runtime = runtime
crits_taxii.end = end
# Poll using 1.1 then 1.0 if that fails.
poll_msg = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=feed,
poll_parameters=tm11.PollRequest.PollParameters(),
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
# Check if this is a TAXII 1.0 server and try again
if response.info().getheader(
'X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
feed_name=feed,
exclusive_begin_timestamp_label=start,
inclusive_end_timestamp_label=end)
response = client.callTaxiiService2(hostname, '/poll/',
t.VID_TAXII_XML_10,
poll_msg.to_xml())
taxii_msg = t.get_message_from_http_response(
response, poll_msg.message_id)
if response.getcode(
) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
else:
ret['reason'] = "%s: %s" % (taxii_msg.status_type,
taxii_msg.message)
return ret
valid = tm.validate_xml(taxii_msg.to_xml())
if valid != True:
ret['reason'] = "Invalid XML: %s" % valid
return ret
if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
return ret
ret['status'] = True
if not taxii_msg.content_blocks:
crits_taxii.save()
return ret
mid = taxii_msg.message_id
for content_block in taxii_msg.content_blocks:
data = parse_content_block(content_block, keyfile, certfile)
if not data:
ret['failures'].append(('No data found in content block', 'Data'))
continue
objs = import_standards_doc(data,
analyst,
method,
ref=mid,
make_event=create_events)
for k in objs['imported']:
ret['successes'] += 1
ret[k[0]].append(k[1])
for k in objs['failed']:
ret['failures'].append(k)
crits_taxii.save()
return ret
def main():
# This is a work-around for the fact that the 1.0 indicator type was removed from the STIX python
# library, even though it is the essentially the same as the 1.1 type. We want to still support 1.0
# indicators since they are out there, and there is no difference for the purposes of this script.
# vocabs._VOCAB_MAP["stixVocabs:IndicatorTypeVocab-1.0"] = IndicatorType
# Create XML parser that can strip namespaces
xmlParser = EntityParser()
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].referenceset and args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print >> sys.stderr, "Could not parse either start or end time"
raise
#Create Request Header
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.PollParameters())
poll_req_xml = poll_req.to_xml()
#Create request client object
client = tc.HttpClient()
#HTTPS Setting
if args[0].taxii_ssl:
client.set_use_https(True)
# Basic Auth
if args[0].taxii_username:
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
"Enter your taxii password: "******"content_blocks"]:
bindingId = content["content_binding"]["binding_id"]
if bindingId and bindingId.startswith(
"urn:stix.mitre.org:xml"):
if args[0].verbose:
print >> sys.stderr, "RECIEVED STIX DATA:\n"
print >> sys.stderr, content["content"]
try:
# This string replace is a workaround for some invalid documents in my test server, if you don't need it, remove it
xmlString = content["content"].replace(
'low',
'Low').replace('medium',
'Medium').replace('high', 'High')
stix_package = xmlParser.parse_xml(
io.BytesIO(xmlString), False)
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print >> sys.stderr, "Could not parse STIX document: "
print >> sys.stderr, content["content"]
raise
print "Imported", indicators, "indicators into reference set", args[
0].referenceset
else:
print >> sys.stderr, "Invalid reponse from TAXII server"
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].referenceset and args[0].file:
stix_package = xmlParser.parse_xml(args[0].file, False)
indicators = process_package_dict(args, stix_package.to_dict())
print "Imported", indicators, "indicators into reference set", args[
0].referenceset
else:
print >> sys.stderr, "Invalid arguments. Type 'python stix_import.py --help' for usage.\n"
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
args_sts = date_N_hours_ago.strftime("%Y-%m-%d %H:%M:%S")
args_ets = date_now.strftime("%Y-%m-%d %H:%M:%S")
structTime = time.strptime(args_sts,'%Y-%m-%d %H:%M:%S')
begin_ts = datetime(*structTime[:7])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
eTime = time.strptime(args_ets,'%Y-%m-%d %H:%M:%S')
end_ts = datetime(*eTime[:7])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
### CREATE THE POLL
## SET Collection Name
poll_req = tm11.PollRequest(message_id=tm11.generate_message_id(),
collection_name='system.Default',
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.PollParameters())
poll_req_xml = poll_req.to_xml()
### SET SOME HEADERS
headrs = [
"Content-Type: application/xml",
"Content-Length: " + str(len(poll_req_xml)),
"User-Agent: TAXII Client Application",
"Accept: application/xml",
"X-TAXII-Accept: urn:taxii.mitre.org:message:xml:1.1",
"X-TAXII-Content-Type: urn:taxii.mitre.org:message:xml:1.1",
"X-TAXII-Protocol: urn:taxii.mitre.org:protocol:https:1.0",
]
def main():
stix_package = None
argParser = get_parser()
args = argParser.parse_args()
if args[0].help:
print_help(argParser)
# Import from a TAXII server
elif args[0].taxii:
begin_ts = None
end_ts = None
try:
if args[0].begin_ts:
structTime = time.strptime(args[0].begin_ts,
'%Y-%m-%d %H:%M:%S')
begin_ts = datetime.datetime(*structTime[:6])
begin_ts = begin_ts.replace(tzinfo=pytz.UTC)
else:
begin_ts = None
if args[0].end_ts:
structTime = time.strptime(args[0].end_ts, '%Y-%m-%d %H:%M:%S')
end_ts = datetime.datetime(*structTime[:6])
end_ts = end_ts.replace(tzinfo=pytz.UTC)
else:
end_ts = None
except ValueError:
print('Could not parse either start or end time', file=sys.stderr)
raise
poll_req = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=args[0].collection,
exclusive_begin_timestamp_label=begin_ts,
inclusive_end_timestamp_label=end_ts,
poll_parameters=tm11.PollRequest.poll_parameters())
poll_req_xml = poll_req.to_xml()
client = tc.HttpClient()
if args[0].taxii_ssl:
client.set_use_https(True)
if args[0].taxii_username:
client.set_auth_type(1)
if not args[0].taxii_password:
args[0].taxii_password = getpass.getpass(
'Enter your taxii password: '******'username': args[0].taxii_username,
'password': args[0].taxii_password
})
elif args[0].taxii_key and args[0].taxii_cert:
client.set_auth_type(2)
client.set_auth_credentials({
'key': args[0].taxii_key,
'cert': args[0].taxii_cert
})
if args[0].proxy:
client.set_proxy(args[0].proxy)
resp = client.call_taxii_service2(
args[0].taxii, args[0].taxii_endpoint + '/poll/',
t.VID_TAXII_XML_11, poll_req_xml, args[0].taxiiport)
response_message = t.get_message_from_http_response(resp, '0')
response_dict = response_message.to_dict()
indicators = 0
if 'content_blocks' in response_dict:
for content in response_dict['content_blocks']:
bindingId = content['content_binding']['binding_id']
if bindingId and bindingId.startswith(
'urn:stix.mitre.org:xml'):
#if args[0].verbose:
# print >> sys.stderr, 'RECIEVED STIX DATA:\n'
#print >> sys.stderr, content['content']
try:
# Parse the information
stix_pkg = STIXPackage.from_xml(
io.BytesIO(content['content']))
stix_package = stix_pkg
indicators += process_package_dict(
args, stix_package.to_dict())
except ValueError:
print(
'Could not parse STIX document: ', file=sys.stderr)
print(content['content'], file=sys.stderr)
raise
print('Imported', indicators, 'indicators into set')
else:
print('Invalid reponse from TAXII server', file=sys.stderr)
pprint.pprint(response_dict, sys.stderr)
exit(255)
# Import from a XML file on disk
elif args[0].file:
stix_package = STIXPackage.from_xml(args[0].file)
indicators = process_package_dict(args, stix_package.to_dict())
print('Imported', indicators, 'indicators into set')
else:
print('Invalid arguments. Type --help for usage.\n', file=sys.stderr)
def _poll_taxii_11(self, args):
'''Poll a TAXII 1.1 feed and return a generator over the content blocks found.
Arguments:
args - A dictionary of arguments.
'''
app = args.get('app')
cert_file = args.get('cert_file')
key_file = args.get('key_file')
username = args.get('username')
password = args.get('password')
if app and cert_file and key_file:
cert_file_path = make_splunkhome_path(
["etc", "apps", app, "auth", cert_file])
key_file_path = make_splunkhome_path(
["etc", "apps", app, "auth", key_file])
if not (os.path.exists(cert_file_path)
and os.path.exists(key_file_path)):
logger.info(
"Certificate not found - falling back to AUTH_BASIC.")
cert_file_path = key_file_path = None
else:
logger.info(
"Certificate information incomplete - falling back to AUTH_BASIC."
)
cert_file_path = key_file_path = None
client = tc.HttpClient()
client.set_use_https(args.get('use_ssl'))
# Add proxy parameters if present.
if 'proxy' in args:
client.set_proxy(args['proxy'])
if cert_file_path and key_file_path and username:
logger.info("Auth Type: AUTH_CERT_BASIC")
client.set_auth_type(tc.HttpClient.AUTH_CERT_BASIC)
client.set_auth_credentials({
'username': username,
'password': password,
'cert_file': cert_file_path,
'key_file': key_file_path
})
elif cert_file_path and key_file_path:
logger.info("Auth Type: AUTH_CERT")
client.set_auth_type(tc.HttpClient.AUTH_CERT)
client.set_auth_credentials({
'cert_file': cert_file_path,
'key_file': key_file_path
})
else:
logger.info("Auth Type: AUTH_BASIC")
client.set_auth_type(tc.HttpClient.AUTH_BASIC)
client.set_auth_credentials({
'username': username,
'password': password
})
poll_request = tm11.PollRequest(
tm11.generate_message_id(),
collection_name=args.get('collection'),
exclusive_begin_timestamp_label=args.get('earliest'),
#inclusive_end_timestamp_label=args.get('latest'),
poll_parameters=tm11.PollParameters())
poll_xml = poll_request.to_xml()
http_resp = client.call_taxii_service2(args.get('url'),
args.get('service'),
tm11.VID_TAXII_XML_11,
poll_xml,
port=args.get('port'))
taxii_message = libtaxii.get_message_from_http_response(
http_resp, poll_request.message_id)
if http_resp.getcode() == 200:
if hasattr(taxii_message, 'content_blocks'):
return taxii_message.to_xml()
else:
raise TaxiiHandlerException(
'Exception when polling TAXII feed (no content returned): %s'
% self._clean_taxii_message(taxii_message))
else:
raise TaxiiHandlerException(
'Exception when polling TAXII feed: %s' %
self._clean_taxii_message(taxii_message))
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
ssl_cert=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.set_use_https(use_ssl)
if ssl_cert:
client.setAuthType(client.AUTH_CERT_BASIC)
client.setAuthCredentials({
'username': user,
'password': passwd,
'key_file': ssl_cert,
'cert_file': ssl_cert
})
else:
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {
'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
if window_latest <= window_earliest:
if not quiet:
progress.update(i)
break
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
try:
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(
content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
