Exemple #1
0
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(use_ssl)
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials(
        {'username': user,
         'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = dict()
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200 # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
                   ' ', ETA()]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        http_response = client.callTaxiiService2(
            host, endpoint,
            t.VID_TAXII_XML_11, poll_request.to_xml(),
            port=port)
        taxii_message = t.get_message_from_http_response(http_response,
            poll_request.message_id)
        if isinstance(taxii_message, tm11.StatusMessage):
            print("TAXII connection error! %s" % (taxii_message.message))
        elif isinstance(taxii_message, tm11.PollResponse):
            for content_block in taxii_message.content_blocks:
                try:
                    stix_package = taxii_content_block_to_stix(content_block)
                    (raw_stix_objs, raw_cybox_objs) = \
                        process_stix_pkg(stix_package)
                    for k in raw_stix_objs.keys():
                        cooked_stix_objs[k].update(raw_stix_objs[k])
                    for k in raw_cybox_objs.keys():
                        if not k in cooked_cybox_objs.keys():
                            cooked_cybox_objs[k] = set()
                        cooked_cybox_objs[k].update(raw_cybox_objs[k])
                except:
                    next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return(cooked_stix_objs, cooked_cybox_objs)
Exemple #2
0
def taxii_poll(host=None,
               port=None,
               endpoint=None,
               collection=None,
               user=None,
               passwd=None,
               use_ssl=None,
               attempt_validation=None,
               time_range=None,
               quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(use_ssl)
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials({'username': user, 'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = dict()
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200  # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = [
            'TAXII Poll: ',
            Percentage(), ' ',
            Bar(marker=RotatingMarker()), ' ',
            ETA()
        ]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
                window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
                window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        http_response = client.callTaxiiService2(host,
                                                 endpoint,
                                                 t.VID_TAXII_XML_11,
                                                 poll_request.to_xml(),
                                                 port=port)
        taxii_message = t.get_message_from_http_response(
            http_response, poll_request.message_id)
        if isinstance(taxii_message, tm11.StatusMessage):
            print("TAXII connection error! %s" % (taxii_message.message))
        elif isinstance(taxii_message, tm11.PollResponse):
            for content_block in taxii_message.content_blocks:
                try:
                    stix_package = taxii_content_block_to_stix(content_block)
                    (raw_stix_objs, raw_cybox_objs) = \
                        process_stix_pkg(stix_package)
                    for k in raw_stix_objs.keys():
                        cooked_stix_objs[k].update(raw_stix_objs[k])
                    for k in raw_cybox_objs.keys():
                        if not k in cooked_cybox_objs.keys():
                            cooked_cybox_objs[k] = set()
                        cooked_cybox_objs[k].update(raw_cybox_objs[k])
                except:
                    next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return (cooked_stix_objs, cooked_cybox_objs)
Exemple #3
0
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, ssl_cert=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.set_use_https(use_ssl)
    if ssl_cert:
        client.setAuthType(client.AUTH_CERT_BASIC)
        client.setAuthCredentials(
            {'username': user,
             'password': passwd,
             'key_file': ssl_cert,
             'cert_file': ssl_cert})
    else:
        client.setAuthType(client.AUTH_BASIC)
        client.setAuthCredentials(
            {'username': user,
             'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = {'AccountObjectType': set(),
                      'AddressObjectType': set(),
                      'APIObjectType': set(),
                      'ArchiveFileObjectType': set(),
                      'ARPCacheObjectType': set(),
                      'ArtifactObjectType': set(),
                      'ASObjectType': set(),
                      'CodeObjectType': set(),
                      'CustomObjectType': set(),
                      'DeviceObjectType': set(),
                      'DiskObjectType': set(),
                      'DiskPartitionObjectType': set(),
                      'DNSCacheObjectType': set(),
                      'DNSQueryObjectType': set(),
                      'DNSRecordObjectType': set(),
                      'DomainNameObjectType': set(),
                      'EmailMessageObjectType': set(),
                      'FileObjectType': set(),
                      'GUIDialogboxObjectType': set(),
                      'GUIObjectType': set(),
                      'GUIWindowObjectType': set(),
                      'HostnameObjectType': set(),
                      'HTTPSessionObjectType': set(),
                      'ImageFileObjectType': set(),
                      'LibraryObjectType': set(),
                      'LinkObjectType': set(),
                      'LinuxPackageObjectType': set(),
                      'MemoryObjectType': set(),
                      'MutexObjectType': set(),
                      'NetworkConnectionObjectType': set(),
                      'NetworkFlowObjectType': set(),
                      'NetworkPacketObjectType': set(),
                      'NetworkRouteEntryObjectType': set(),
                      'NetRouteObjectType': set(),
                      'NetworkSocketObjectType': set(),
                      'NetworkSubnetObjectType': set(),
                      'PDFFileObjectType': set(),
                      'PipeObjectType': set(),
                      'PortObjectType': set(),
                      'ProcessObjectType': set(),
                      'ProductObjectType': set(),
                      'SemaphoreObjectType': set(),
                      'SMSMessageObjectType': set(),
                      'SocketAddressObjectType': set(),
                      'SystemObjectType': set(),
                      'UnixFileObjectType': set(),
                      'UnixNetworkRouteEntryObjectType': set(),
                      'UnixPipeObjectType': set(),
                      'UnixProcessObjectType': set(),
                      'UnixUserAccountObjectType': set(),
                      'UnixVolumeObjectType': set(),
                      'URIObjectType': set(),
                      'URLHistoryObjectType': set(),
                      'UserAccountObjectType': set(),
                      'UserSessionObjectType': set(),
                      'VolumeObjectType': set(),
                      'WhoisObjectType': set(),
                      'WindowsComputerAccountObjectType': set(),
                      'WindowsCriticalSectionObjectType': set(),
                      'WindowsDriverObjectType': set(),
                      'WindowsEventLogObjectType': set(),
                      'WindowsEventObjectType': set(),
                      'WindowsExecutableFileObjectType': set(),
                      'WindowsFilemappingObjectType': set(),
                      'WindowsFileObjectType': set(),
                      'WindowsHandleObjectType': set(),
                      'WindowsHookObjectType': set(),
                      'WindowsKernelHookObjectType': set(),
                      'WindowsKernelObjectType': set(),
                      'WindowsMailslotObjectType': set(),
                      'WindowsMemoryPageRegionObjectType': set(),
                      'WindowsMutexObjectType': set(),
                      'WindowsNetworkRouteEntryObjectType': set(),
                      'WindowsNetworkShareObjectType': set(),
                      'WindowsPipeObjectType': set(),
                      'WindowsPrefetchObjectType': set(),
                      'WindowsProcessObjectType': set(),
                      'WindowsRegistryKeyObjectType': set(),
                      'WindowsSemaphoreObjectType': set(),
                      'WindowsServiceObjectType': set(),
                      'WindowsSystemObjectType': set(),
                      'WindowsSystemRestoreObjectType': set(),
                      'WindowsTaskObjectType': set(),
                      'WindowsThreadObjectType': set(),
                      'WindowsUserAccountObjectType': set(),
                      'WindowsVolumeObjectType': set(),
                      'WindowsWaitableTimerObjectType': set(),
                      'X509CertificateObjectType': set(),
    }
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200 # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
                   ' ', ETA()]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        if window_latest <= window_earliest:
            if not quiet:
                progress.update(i)
            break
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        try:
            http_response = client.callTaxiiService2(
                host, endpoint,
                t.VID_TAXII_XML_11, poll_request.to_xml(),
                port=port)
            taxii_message = t.get_message_from_http_response(http_response,
                poll_request.message_id)
            if isinstance(taxii_message, tm11.StatusMessage):
                print("TAXII connection error! %s" % (taxii_message.message))
            elif isinstance(taxii_message, tm11.PollResponse):
                for content_block in taxii_message.content_blocks:
                    try:
                        stix_package = taxii_content_block_to_stix(content_block)
                        (raw_stix_objs, raw_cybox_objs) = \
                            process_stix_pkg(stix_package)
                        for k in raw_stix_objs.keys():
                            cooked_stix_objs[k].update(raw_stix_objs[k])
                        for k in raw_cybox_objs.keys():
                            if not k in cooked_cybox_objs.keys():
                                cooked_cybox_objs[k] = set()
                            cooked_cybox_objs[k].update(raw_cybox_objs[k])
                    except:
                        next
        except:
            next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return(cooked_stix_objs, cooked_cybox_objs)
Exemple #4
0
def taxii_poll(host=None,
               port=None,
               endpoint=None,
               collection=None,
               user=None,
               passwd=None,
               ssl_cert=None,
               use_ssl=None,
               attempt_validation=None,
               time_range=None,
               quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.set_use_https(use_ssl)
    if ssl_cert:
        client.setAuthType(client.AUTH_CERT_BASIC)
        client.setAuthCredentials({
            'username': user,
            'password': passwd,
            'key_file': ssl_cert,
            'cert_file': ssl_cert
        })
    else:
        client.setAuthType(client.AUTH_BASIC)
        client.setAuthCredentials({'username': user, 'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = {
        'AccountObjectType': set(),
        'AddressObjectType': set(),
        'APIObjectType': set(),
        'ArchiveFileObjectType': set(),
        'ARPCacheObjectType': set(),
        'ArtifactObjectType': set(),
        'ASObjectType': set(),
        'CodeObjectType': set(),
        'CustomObjectType': set(),
        'DeviceObjectType': set(),
        'DiskObjectType': set(),
        'DiskPartitionObjectType': set(),
        'DNSCacheObjectType': set(),
        'DNSQueryObjectType': set(),
        'DNSRecordObjectType': set(),
        'DomainNameObjectType': set(),
        'EmailMessageObjectType': set(),
        'FileObjectType': set(),
        'GUIDialogboxObjectType': set(),
        'GUIObjectType': set(),
        'GUIWindowObjectType': set(),
        'HostnameObjectType': set(),
        'HTTPSessionObjectType': set(),
        'ImageFileObjectType': set(),
        'LibraryObjectType': set(),
        'LinkObjectType': set(),
        'LinuxPackageObjectType': set(),
        'MemoryObjectType': set(),
        'MutexObjectType': set(),
        'NetworkConnectionObjectType': set(),
        'NetworkFlowObjectType': set(),
        'NetworkPacketObjectType': set(),
        'NetworkRouteEntryObjectType': set(),
        'NetRouteObjectType': set(),
        'NetworkSocketObjectType': set(),
        'NetworkSubnetObjectType': set(),
        'PDFFileObjectType': set(),
        'PipeObjectType': set(),
        'PortObjectType': set(),
        'ProcessObjectType': set(),
        'ProductObjectType': set(),
        'SemaphoreObjectType': set(),
        'SMSMessageObjectType': set(),
        'SocketAddressObjectType': set(),
        'SystemObjectType': set(),
        'UnixFileObjectType': set(),
        'UnixNetworkRouteEntryObjectType': set(),
        'UnixPipeObjectType': set(),
        'UnixProcessObjectType': set(),
        'UnixUserAccountObjectType': set(),
        'UnixVolumeObjectType': set(),
        'URIObjectType': set(),
        'URLHistoryObjectType': set(),
        'UserAccountObjectType': set(),
        'UserSessionObjectType': set(),
        'VolumeObjectType': set(),
        'WhoisObjectType': set(),
        'WindowsComputerAccountObjectType': set(),
        'WindowsCriticalSectionObjectType': set(),
        'WindowsDriverObjectType': set(),
        'WindowsEventLogObjectType': set(),
        'WindowsEventObjectType': set(),
        'WindowsExecutableFileObjectType': set(),
        'WindowsFilemappingObjectType': set(),
        'WindowsFileObjectType': set(),
        'WindowsHandleObjectType': set(),
        'WindowsHookObjectType': set(),
        'WindowsKernelHookObjectType': set(),
        'WindowsKernelObjectType': set(),
        'WindowsMailslotObjectType': set(),
        'WindowsMemoryPageRegionObjectType': set(),
        'WindowsMutexObjectType': set(),
        'WindowsNetworkRouteEntryObjectType': set(),
        'WindowsNetworkShareObjectType': set(),
        'WindowsPipeObjectType': set(),
        'WindowsPrefetchObjectType': set(),
        'WindowsProcessObjectType': set(),
        'WindowsRegistryKeyObjectType': set(),
        'WindowsSemaphoreObjectType': set(),
        'WindowsServiceObjectType': set(),
        'WindowsSystemObjectType': set(),
        'WindowsSystemRestoreObjectType': set(),
        'WindowsTaskObjectType': set(),
        'WindowsThreadObjectType': set(),
        'WindowsUserAccountObjectType': set(),
        'WindowsVolumeObjectType': set(),
        'WindowsWaitableTimerObjectType': set(),
        'X509CertificateObjectType': set(),
    }
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200  # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = [
            'TAXII Poll: ',
            Percentage(), ' ',
            Bar(marker=RotatingMarker()), ' ',
            ETA()
        ]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        if window_latest <= window_earliest:
            if not quiet:
                progress.update(i)
            break
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
                window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
                window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        try:
            http_response = client.callTaxiiService2(host,
                                                     endpoint,
                                                     t.VID_TAXII_XML_11,
                                                     poll_request.to_xml(),
                                                     port=port)
            taxii_message = t.get_message_from_http_response(
                http_response, poll_request.message_id)
            if isinstance(taxii_message, tm11.StatusMessage):
                print("TAXII connection error! %s" % (taxii_message.message))
            elif isinstance(taxii_message, tm11.PollResponse):
                for content_block in taxii_message.content_blocks:
                    try:
                        stix_package = taxii_content_block_to_stix(
                            content_block)
                        (raw_stix_objs, raw_cybox_objs) = \
                            process_stix_pkg(stix_package)
                        for k in raw_stix_objs.keys():
                            cooked_stix_objs[k].update(raw_stix_objs[k])
                        for k in raw_cybox_objs.keys():
                            if not k in cooked_cybox_objs.keys():
                                cooked_cybox_objs[k] = set()
                            cooked_cybox_objs[k].update(raw_cybox_objs[k])
                    except:
                        next
        except:
            next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return (cooked_stix_objs, cooked_cybox_objs)