def handle(content, file):
print("Adding %s..." % file)
# Create content block.
cb = tm11.ContentBlock(tm11.ContentBinding(t.CB_STIX_XML_11),
content)
# Append content block to list.
cbs.append(cb)
def make_content(version, content_binding=CUSTOM_CONTENT_BINDING, content=CONTENT, subtype=None):
if version == 10:
return tm10.ContentBlock(content_binding, content)
elif version == 11:
content_block = tm11.ContentBlock(tm11.ContentBinding(content_binding), content)
if subtype:
content_block.content_binding.subtype_ids.append(subtype)
return content_block
else:
raise ValueError('Unknown TAXII message version: %s' % version)
def gen_post(self, msg_type, xml):
content_block = tm11.ContentBlock(tm11.ContentBinding(Client.BINDING_STIX), xml)
if msg_type.lower() == 'inbox':
post = tm11.InboxMessage(tm11.generate_message_id())
post.content_blocks.append(content_block)
post.message = Client.CLIENT_MSG
if self.subscription:
post.subscription_id = self.subscription
if self.collection:
post.destination_collection_names.append(self.collection)
return post
def test_04(self):
"""
Test a Content Binding ID not supported by the server
"""
pp = tm11.PollParameters(
content_bindings=[tm11.ContentBinding('some_random_binding')])
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_UNSUPPORTED_CONTENT_BINDING,
sd_keys=[SD_SUPPORTED_CONTENT])
def test_05(self):
"""
Test a supported Content Binding ID with an unsupported subtype
"""
pp = tm11.PollParameters(content_bindings=[
tm11.ContentBinding(CB_STIX_XML_111, subtype_ids=['FakeSubtype'])
])
pr = tm11.PollRequest(message_id=generate_message_id(),
collection_name='default',
poll_parameters=pp)
msg = make_request('/services/test_poll_1/',
pr.to_xml(),
get_headers(VID_TAXII_SERVICES_11, False),
MSG_STATUS_MESSAGE,
st=ST_UNSUPPORTED_CONTENT_BINDING,
sd_keys=[SD_SUPPORTED_CONTENT])
def create_request_message(self, args):
if args.content_file is self.stix_watchlist:
data = self.stix_watchlist
else:
with open(args.content_file, 'r') as f:
data = f.read()
cb = tm11.ContentBlock(tm11.ContentBinding(args.content_binding), data)
if args.subtype is not None:
cb.content_binding.subtype_ids.append(args.subtype)
inbox_message = tm11.InboxMessage(message_id=tm11.generate_message_id(), content_blocks=[cb])
if args.dcn:
inbox_message.destination_collection_names.append(args.dcn)
return inbox_message
def get_content_block(self, f):
binding_map = {
'.xlsx': ('application',
'vnd.openxmlformats-officedocument.spreadsheetml.sheet'),
'.xls': ('application', 'vnd.ms-excel'),
'.csv': ('text', 'csv'),
'.json': ('application', 'json'),
'.syslog': ('text', 'syslog')
}
# Check the file extension to see if it is a supported type
name_part, ext_part = os.path.splitext(f.name)
if ext_part.lower() == '.xml':
cb = tm11.ContentBlock(tm11.ContentBinding(t.CB_STIX_XML_101),
f.read())
else:
binding_tuple = binding_map.get(ext_part.lower(), None)
if not binding_tuple:
logger.error(
'File extension not supported: %s. Supported extensions: %s'
% (ext_part, binding_map.keys()))
return
# Read the file and create a MIME message for it
maintype = binding_tuple[0]
subtype = binding_tuple[
1] # Note: This is MIME subtype, not TAXII subtype
mime_msg = MIMENonMultipart(maintype, subtype)
mime_msg.add_header('Content-Disposition',
'attachment',
filename=f.name)
mime_msg.set_payload(f.read())
encode_base64(mime_msg)
cb = tm11.ContentBlock('%s/%s' % (maintype, subtype),
mime_msg.as_string())
return cb
def prepare_request(collection_name,
version,
count_only=False,
bindings=[],
subscription_id=None):
if version == 11:
content_bindings = [tm11.ContentBinding(b) for b in bindings]
if subscription_id:
poll_parameters = None
else:
poll_parameters = tm11.PollParameters(
response_type=(RT_FULL if not count_only else RT_COUNT_ONLY),
content_bindings=content_bindings)
return tm11.PollRequest(message_id=MESSAGE_ID,
collection_name=collection_name,
subscription_id=subscription_id,
poll_parameters=poll_parameters)
elif version == 10:
content_bindings = bindings
return tm10.PollRequest(message_id=MESSAGE_ID,
feed_name=collection_name,
content_bindings=content_bindings,
subscription_id=subscription_id)
def content_binding_entity_to_content_binding(content_binding, version):
if version == 10:
return content_binding.binding
elif version == 11:
return tm11.ContentBinding(binding_id=content_binding.binding,
subtype_ids=content_binding.subtypes)
def main():
config = ConfigParser.ConfigParser()
config.read('app.conf')
dstHost = config.get('taxii', 'dstHost')
dstPort = config.get('taxii', 'dstPort')
dstPath = config.get('taxii', 'dstPath')
username = config.get('taxii', 'username')
password = config.get('taxii', 'password')
parser = argparse.ArgumentParser(description="Inbox Client")
parser.add_argument(
"--host",
dest="host",
default=dstHost,
help="Host where the Inbox Service is hosted. Defaults to " + dstHost)
parser.add_argument(
"--port",
dest="port",
default=dstPort,
help="Port where the Inbox Service is hosted. Defaults to " + dstPort)
parser.add_argument(
"--path",
dest="path",
default=dstPath,
help="Path where the Inbox Service is hosted. Defaults to " + dstPath)
parser.add_argument(
"--content-binding",
dest="content_binding",
default=t.CB_STIX_XML_11,
help="Content binding of the Content Block to send. Defaults to %s" %
t.CB_STIX_XML_11)
parser.add_argument(
"--content-file",
dest="content_file",
default=stixContent,
help="File name of the Content Block to send. Required.")
args = parser.parse_args()
if args.content_file is '':
parser.print_help()
sys.exit(1)
_l.info('Starting...')
c = open(args.content_file, 'r')
cb = tmsg.ContentBlock(tmsg.ContentBinding(args.content_binding), c.read())
c.close()
taxiiMsg = tmsg.InboxMessage(message_id=tmsg.generate_message_id(),
content_blocks=[cb])
taxiiMsgXml = taxiiMsg.to_xml()
# send it
_l.debug('Uploading content.')
client = tc.HttpClient()
client.setProxy('noproxy')
client.setAuthType(tc.HttpClient.AUTH_BASIC)
client.setAuthCredentials({'username': username, 'password': password})
resp = client.callTaxiiService2(args.host, args.path, t.VID_TAXII_XML_11,
taxiiMsgXml, args.port)
response_message = t.get_message_from_http_response(resp, '0')
_l.debug('Response was: ' + response_message.to_xml())
_l.info('Ended.')
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.setUseHttps(use_ssl)
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
def taxii_poll(host=None,
port=None,
endpoint=None,
collection=None,
user=None,
passwd=None,
ssl_cert=None,
use_ssl=None,
attempt_validation=None,
time_range=None,
quiet=None):
'''poll cti via taxii'''
client = tc.HttpClient()
client.set_use_https(use_ssl)
if ssl_cert:
client.setAuthType(client.AUTH_CERT_BASIC)
client.setAuthCredentials({
'username': user,
'password': passwd,
'key_file': ssl_cert,
'cert_file': ssl_cert
})
else:
client.setAuthType(client.AUTH_BASIC)
client.setAuthCredentials({'username': user, 'password': passwd})
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = {
'AccountObjectType': set(),
'AddressObjectType': set(),
'APIObjectType': set(),
'ArchiveFileObjectType': set(),
'ARPCacheObjectType': set(),
'ArtifactObjectType': set(),
'ASObjectType': set(),
'CodeObjectType': set(),
'CustomObjectType': set(),
'DeviceObjectType': set(),
'DiskObjectType': set(),
'DiskPartitionObjectType': set(),
'DNSCacheObjectType': set(),
'DNSQueryObjectType': set(),
'DNSRecordObjectType': set(),
'DomainNameObjectType': set(),
'EmailMessageObjectType': set(),
'FileObjectType': set(),
'GUIDialogboxObjectType': set(),
'GUIObjectType': set(),
'GUIWindowObjectType': set(),
'HostnameObjectType': set(),
'HTTPSessionObjectType': set(),
'ImageFileObjectType': set(),
'LibraryObjectType': set(),
'LinkObjectType': set(),
'LinuxPackageObjectType': set(),
'MemoryObjectType': set(),
'MutexObjectType': set(),
'NetworkConnectionObjectType': set(),
'NetworkFlowObjectType': set(),
'NetworkPacketObjectType': set(),
'NetworkRouteEntryObjectType': set(),
'NetRouteObjectType': set(),
'NetworkSocketObjectType': set(),
'NetworkSubnetObjectType': set(),
'PDFFileObjectType': set(),
'PipeObjectType': set(),
'PortObjectType': set(),
'ProcessObjectType': set(),
'ProductObjectType': set(),
'SemaphoreObjectType': set(),
'SMSMessageObjectType': set(),
'SocketAddressObjectType': set(),
'SystemObjectType': set(),
'UnixFileObjectType': set(),
'UnixNetworkRouteEntryObjectType': set(),
'UnixPipeObjectType': set(),
'UnixProcessObjectType': set(),
'UnixUserAccountObjectType': set(),
'UnixVolumeObjectType': set(),
'URIObjectType': set(),
'URLHistoryObjectType': set(),
'UserAccountObjectType': set(),
'UserSessionObjectType': set(),
'VolumeObjectType': set(),
'WhoisObjectType': set(),
'WindowsComputerAccountObjectType': set(),
'WindowsCriticalSectionObjectType': set(),
'WindowsDriverObjectType': set(),
'WindowsEventLogObjectType': set(),
'WindowsEventObjectType': set(),
'WindowsExecutableFileObjectType': set(),
'WindowsFilemappingObjectType': set(),
'WindowsFileObjectType': set(),
'WindowsHandleObjectType': set(),
'WindowsHookObjectType': set(),
'WindowsKernelHookObjectType': set(),
'WindowsKernelObjectType': set(),
'WindowsMailslotObjectType': set(),
'WindowsMemoryPageRegionObjectType': set(),
'WindowsMutexObjectType': set(),
'WindowsNetworkRouteEntryObjectType': set(),
'WindowsNetworkShareObjectType': set(),
'WindowsPipeObjectType': set(),
'WindowsPrefetchObjectType': set(),
'WindowsProcessObjectType': set(),
'WindowsRegistryKeyObjectType': set(),
'WindowsSemaphoreObjectType': set(),
'WindowsServiceObjectType': set(),
'WindowsSystemObjectType': set(),
'WindowsSystemRestoreObjectType': set(),
'WindowsTaskObjectType': set(),
'WindowsThreadObjectType': set(),
'WindowsUserAccountObjectType': set(),
'WindowsVolumeObjectType': set(),
'WindowsWaitableTimerObjectType': set(),
'X509CertificateObjectType': set(),
}
earliest = poll_start(time_range)
latest = nowutc()
poll_window = 43200 # 12 hour blocks seem reasonable
total_windows = (latest - earliest) / poll_window
if (latest - earliest) % poll_window:
total_windows += 1
if not quiet:
widgets = [
'TAXII Poll: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
window_latest = latest
window_earliest = window_latest - poll_window
for i in range(total_windows):
window_latest -= poll_window
if window_earliest - poll_window < earliest:
window_earliest = earliest
else:
window_earliest -= poll_window
if window_latest <= window_earliest:
if not quiet:
progress.update(i)
break
poll_params = tm11.PollParameters(
allow_asynch=False,
response_type=RT_FULL,
content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
poll_request = tm11.PollRequest(
message_id=tm11.generate_message_id(),
collection_name=collection,
exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
window_earliest).replace(tzinfo=pytz.utc),
inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
window_latest).replace(tzinfo=pytz.utc),
poll_parameters=(poll_params))
try:
http_response = client.callTaxiiService2(host,
endpoint,
t.VID_TAXII_XML_11,
poll_request.to_xml(),
port=port)
taxii_message = t.get_message_from_http_response(
http_response, poll_request.message_id)
if isinstance(taxii_message, tm11.StatusMessage):
print("TAXII connection error! %s" % (taxii_message.message))
elif isinstance(taxii_message, tm11.PollResponse):
for content_block in taxii_message.content_blocks:
try:
stix_package = taxii_content_block_to_stix(
content_block)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
except:
next
except:
next
if not quiet:
progress.update(i)
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)