def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username': user,
'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA()]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
http_response = client.callTaxiiService2(
host, endpoint,
t.VID_TAXII_XML_11, poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return(cooked_stix_objs, cooked_cybox_objs)
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, ssl_cert=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.set_use_https(use_ssl)
if ssl_cert:
client.setAuthType(client.AUTH_CERT_BASIC)
client.setAuthCredentials(
{'username': user,
'password': passwd,
'key_file': ssl_cert,
'cert_file': ssl_cert})
else:
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials(
{'username': user,
'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA()]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
if window_latest <= window_earliest:
if not quiet:
progress.update(i)
break
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
try:
http_response = client.callTaxiiService2(
host, endpoint,
t.VID_TAXII_XML_11, poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(http_response,
poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return(cooked_stix_objs, cooked_cybox_objs)
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
ssl_cert=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.set_use_https(use_ssl)
if ssl_cert:
client.setAuthType(client.AUTH_CERT_BASIC)
client.setAuthCredentials({
'username': user,
'password': passwd,
'key_file': ssl_cert,
'cert_file': ssl_cert
})
else:
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {
'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
if window_latest <= window_earliest:
if not quiet:
progress.update(i)
break
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
try:
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(
content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)