Beispiel #1
0
def message_tests(taxii_message):
    if not isinstance(taxii_message, tm.TAXIIMessage):
        raise ValueError('taxii_message was not an instance of TAXIIMessage')

    print '***** Message type = %s; id = %s' % (taxii_message.message_type, taxii_message.message_id)

    xml_string = taxii_message.to_xml()
    valid = tm.validate_xml(xml_string)
    if not valid:
        raise Exception('\tFailure of test #1 - XML not schema valid')
    msg_from_xml = tm.get_message_from_xml(xml_string)
    dictionary = taxii_message.to_dict()
    msg_from_dict = tm.get_message_from_dict(dictionary)
    if taxii_message != msg_from_xml:
        print '\t Failure of test #2 - running equals w/ debug:'
        taxii_message.__eq__(msg_from_xml, True)
        raise Exception('Test #2 failed - taxii_message != msg_from_xml')

    if taxii_message != msg_from_dict:
        print '\t Failure of test #3 - running equals w/ debug:'
        taxii_message.__eq__(msg_from_dict, True)
        raise Exception('Test #3 failed - taxii_message != msg_from_dict')

    if msg_from_xml != msg_from_dict:
        print '\t Failure of test #4 - running equals w/ debug:'
        msg_from_xml.__eq__(msg_from_dict, True)
        raise Exception('Test #4 failed - msg_from_xml != msg_from_dict')

    print '***** All tests completed!'
Beispiel #2
0
def execute_taxii_agent(hostname=None,
                        https=None,
                        feed=None,
                        keyfile=None,
                        certfile=None,
                        start=None,
                        end=None,
                        analyst=None,
                        method=None):
    ret = {
        'Certificate': [],
        'Domain': [],
        'Email': [],
        'Event': [],
        'Indicator': [],
        'IP': [],
        'PCAP': [],
        'RawData': [],
        'Sample': [],
        'successes': 0,
        'failures': [],
        'status': False,
        'reason': ''
    }

    sc = get_config('taxii_service')
    # XXX: Validate these!
    if not hostname:
        hostname = str(sc['hostname'])
    if not keyfile:
        keyfile = str(sc['keyfile'])
    if not certfile:
        certfile = str(sc['certfile'])
    if not feed:
        feed = str(sc['data_feed'])
    if https == None:
        https = sc['https']
    create_events = sc['create_events']

    # Last document's end time is our start time.
    if not start:
        last = taxii.Taxii.get_last()
        if last:
            start = pytz.utc.localize(last.end)

    # If start is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(start, str):
        start = pytz.utc.localize(parse(start, fuzzy=True))

    # store the current time as the time of this request
    runtime = datetime.now(tzutc())

    # End time is always now, unless specified.
    if not end:
        end = runtime

    # If end is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(end, str):
        end = pytz.utc.localize(parse(end, fuzzy=True))

    # compare start and end to make sure:
    # 1) start time is before end time
    # 2) end time is not in the future
    if (start != None and start >= end) and end > runtime:
        ret['reason'] = "Bad timestamp(s)"
        return ret

    client = tc.HttpClient()
    if https:
        client.setUseHttps(True)
        client.setAuthType(tc.HttpClient.AUTH_CERT)
        client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})

    if settings.HTTP_PROXY:
        proxy = settings.HTTP_PROXY
        if not proxy.startswith('http://'):
            proxy = 'http://' + proxy
        client.setProxy(proxy)

    crits_taxii = taxii.Taxii()
    crits_taxii.runtime = runtime
    crits_taxii.end = end

    # Poll using 1.1 then 1.0 if that fails.
    poll_msg = tm11.PollRequest(
        message_id=tm11.generate_message_id(),
        collection_name=feed,
        poll_parameters=tm11.PollRequest.PollParameters(),
        exclusive_begin_timestamp_label=start,
        inclusive_end_timestamp_label=end)

    response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
                                        poll_msg.to_xml())
    taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)

    if response.getcode(
    ) != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
        # Check if this is a TAXII 1.0 server and try again
        if response.info().getheader(
                'X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
            poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
                                      feed_name=feed,
                                      exclusive_begin_timestamp_label=start,
                                      inclusive_end_timestamp_label=end)

            response = client.callTaxiiService2(hostname, '/poll/',
                                                t.VID_TAXII_XML_10,
                                                poll_msg.to_xml())
            taxii_msg = t.get_message_from_http_response(
                response, poll_msg.message_id)
            if response.getcode(
            ) != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
                ret['reason'] = "%s: %s" % (taxii_msg.status_type,
                                            taxii_msg.message)
                return ret
        else:
            ret['reason'] = "%s: %s" % (taxii_msg.status_type,
                                        taxii_msg.message)
            return ret

    valid = tm.validate_xml(taxii_msg.to_xml())
    if valid != True:
        ret['reason'] = "Invalid XML: %s" % valid
        return ret

    if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
        ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
        return ret

    ret['status'] = True

    if not taxii_msg.content_blocks:
        crits_taxii.save()
        return ret

    mid = taxii_msg.message_id
    for content_block in taxii_msg.content_blocks:
        data = parse_content_block(content_block, keyfile, certfile)
        if not data:
            ret['failures'].append(('No data found in content block', 'Data'))
            continue

        objs = import_standards_doc(data,
                                    analyst,
                                    method,
                                    ref=mid,
                                    make_event=create_events)

        for k in objs['imported']:
            ret['successes'] += 1
            ret[k[0]].append(k[1])
        for k in objs['failed']:
            ret['failures'].append(k)

    crits_taxii.save()
    return ret
Beispiel #3
0
def execute_taxii_agent(hostname=None, https=None, feed=None, keyfile=None,
                        certfile=None, start=None, end=None, analyst=None,
                        method=None):
    ret = {
            'Certificate': [],
            'Domain': [],
            'Email': [],
            'Event': [],
            'Indicator': [],
            'IP': [],
            'PCAP': [],
            'RawData': [],
            'Sample': [],
            'successes': 0,
            'failures': [],
            'status': False,
            'reason': ''
          }

    sc = manager.get_config('taxii_service')
    # XXX: Validate these!
    if not hostname:
        hostname = str(sc['hostname'])
    if not keyfile:
        keyfile = str(sc['keyfile'])
    if not certfile:
        certfile = str(sc['certfile'])
    if not feed:
        feed = str(sc['data_feed'])
    if https == None:
        https = sc['https']
    create_events = sc['create_events']


    # Last document's end time is our start time.
    if not start:
        last = taxii.Taxii.get_last()
        if last:
            start = pytz.utc.localize(last.end)

    # If start is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(start, str):
        start = pytz.utc.localize(parse(start, fuzzy=True))

    # store the current time as the time of this request
    runtime = datetime.now(tzutc())

    # End time is always now, unless specified.
    if not end:
        end = runtime

    # If end is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(end, str):
        end = pytz.utc.localize(parse(end, fuzzy=True))

    # compare start and end to make sure:
    # 1) start time is before end time
    # 2) end time is not in the future
    if (start != None and start >= end) and end > runtime:
        ret['reason'] = "Bad timestamp(s)"
        return ret

    client = tc.HttpClient()
    if https:
        client.setUseHttps(True)
        client.setAuthType(tc.HttpClient.AUTH_CERT)
        client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})

    if settings.HTTP_PROXY:
        proxy = settings.HTTP_PROXY
        if not proxy.startswith('http://'):
            proxy = 'http://' + proxy
        client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)

    crits_taxii = taxii.Taxii()
    crits_taxii.runtime = runtime
    crits_taxii.end = end

    # Poll using 1.1 then 1.0 if that fails.
    poll_msg = tm11.PollRequest(message_id=tm11.generate_message_id(),
                                collection_name=feed,
                                poll_parameters=tm11.PollRequest.PollParameters(),
                                exclusive_begin_timestamp_label=start,
                                inclusive_end_timestamp_label=end)

    response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_11,
                                        poll_msg.to_xml())
    taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)

    if response.getcode() != 200 or taxii_msg.message_type == tm11.MSG_STATUS_MESSAGE:
        # Check if this is a TAXII 1.0 server and try again
        if response.info().getheader('X-TAXII-Content-Type') == t.VID_TAXII_XML_10:
            poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
                                    feed_name=feed,
                                    exclusive_begin_timestamp_label=start,
                                    inclusive_end_timestamp_label=end)

            response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
                                            poll_msg.to_xml())
            taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)
            if response.getcode() != 200 or taxii_msg.message_type == tm.MSG_STATUS_MESSAGE:
                ret['reason'] = "%s: %s" % (taxii_msg.status_type,
                                            taxii_msg.message)
                return ret
        else:
            ret['reason'] = "%s: %s" % (taxii_msg.status_type,
                                        taxii_msg.message)
            return ret


    valid = tm.validate_xml(taxii_msg.to_xml())
    if valid != True:
        ret['reason'] = "Invalid XML: %s" % valid
        return ret

    if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
        ret['reason'] = "No poll response. Unexpected message type: %s" % taxii_msg.message_type
        return ret

    ret['status'] = True

    if not taxii_msg.content_blocks:
        crits_taxii.save()
        return ret

    mid = taxii_msg.message_id
    for content_block in taxii_msg.content_blocks:
        data = parse_content_block(content_block, keyfile, certfile)
        if not data:
            ret['failures'].append(('No data found in content block', 'Data'))
            continue

        objs = import_standards_doc(data, analyst, method, ref=mid,
                                    make_event=create_events)

        for k in objs['imported']:
            ret['successes'] += 1
            ret[k[0]].append(k[1])
        for k in objs['failed']:
            ret['failures'].append(k)


    crits_taxii.save()
    return ret
def execute_taxii_agent(hostname=None,
                        feed=None,
                        keyfile=None,
                        certfile=None,
                        start=None,
                        end=None,
                        analyst=None,
                        method=None):
    ret = {
        'events': [],
        'samples': [],
        'emails': [],
        'indicators': [],
        'successes': 0,
        'failures': 0,
        'status': False,
        'reason': ''
    }

    sc = manager.get_config('taxii_service')
    # XXX: Validate these!
    if not hostname:
        hostname = str(sc['hostname'])
    if not keyfile:
        keyfile = str(sc['keyfile'])
    if not certfile:
        certfile = str(sc['certfile'])
    if not feed:
        feed = str(sc['data_feed'])

    # Last document's end time is our start time.
    if not start:
        last = taxii.Taxii.get_last()
        if last:
            start = pytz.utc.localize(last.end)

    # If start is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(start, str):
        start = pytz.utc.localize(parse(start, fuzzy=True))

    # store the current time as the time of this request
    runtime = datetime.now(tzutc())

    # End time is always now, unless specified.
    if not end:
        end = runtime

    # If end is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(end, str):
        end = pytz.utc.localize(parse(end, fuzzy=True))

    # compare start and end to make sure:
    # 1) start time is before end time
    # 2) end time is not in the future
    if (start != None and start >= end) and end > runtime:
        ret['reason'] = "Bad timestamp(s)"
        return ret

    client = tc.HttpClient()
    client.setUseHttps(True)
    client.setAuthType(tc.HttpClient.AUTH_CERT)
    client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})

    if settings.HTTP_PROXY:
        proxy = settings.HTTP_PROXY
        if not proxy.startswith('http://'):
            proxy = 'http://' + proxy
        client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)

    crits_taxii = taxii.Taxii()
    crits_taxii.runtime = runtime
    crits_taxii.end = end

    poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
                              feed_name=feed,
                              exclusive_begin_timestamp_label=start,
                              inclusive_end_timestamp_label=end)
    response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
                                        poll_msg.to_xml())

    if response.getcode() != 200:
        ret['reason'] = "Response is not 200 OK"
        return ret

    taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)

    valid = tm.validate_xml(taxii_msg.to_xml())
    if valid != True:
        ret['reason'] = valid
        return ret

    if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
        ret['reason'] = "No poll response"
        return ret

    ret['status'] = True

    if not taxii_msg.content_blocks:
        crits_taxii.save()
        return ret

    mid = taxii_msg.message_id
    for content_block in taxii_msg.content_blocks:
        data = parse_content_block(content_block, keyfile, certfile)
        if not data:
            ret['failures'] += 1
            continue

        objs = import_standards_doc(data,
                                    analyst,
                                    method,
                                    ref=mid,
                                    make_event=True)

        ret['successes'] += 1

        for k in ["events", "samples", "emails", "indicators"]:
            for i in objs[k]:
                ret[k].append(i)

    crits_taxii.save()
    return ret
Beispiel #5
0
def execute_taxii_agent(hostname=None, feed=None, keyfile=None, certfile=None, start=None, end=None, analyst=None, method=None):
    ret = {
            'events': [],
            'samples': [],
            'emails': [],
            'indicators': [],
            'successes': 0,
            'failures': 0,
            'status': False,
            'reason': ''
          }

    sc = manager.get_config('taxii_service')
    # XXX: Validate these!
    if not hostname:
        hostname = str(sc['hostname'])
    if not keyfile:
        keyfile = str(sc['keyfile'])
    if not certfile:
        certfile = str(sc['certfile'])
    if not feed:
        feed = str(sc['data_feed'])

    # Last document's end time is our start time.
    if not start:
        last = taxii.Taxii.get_last()
        if last:
            start = pytz.utc.localize(last.end)

    # If start is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(start, str):
        start = pytz.utc.localize(parse(start, fuzzy=True))

    # store the current time as the time of this request
    runtime = datetime.now(tzutc())

    # End time is always now, unless specified.
    if not end:
        end = runtime

    # If end is a string, convert it to a datetime
    # YYYY-MM-DD HH:MM:SS
    if isinstance(end, str):
        end = pytz.utc.localize(parse(end, fuzzy=True))

    # compare start and end to make sure:
    # 1) start time is before end time
    # 2) end time is not in the future
    if (start != None and start >= end) and end > runtime:
        ret['reason'] = "Bad timestamp(s)"
        return ret 

    client = tc.HttpClient()
    client.setUseHttps(True)
    client.setAuthType(tc.HttpClient.AUTH_CERT)
    client.setAuthCredentials({'key_file': keyfile, 'cert_file': certfile})

    if settings.HTTP_PROXY:
        proxy = settings.HTTP_PROXY
        if not proxy.startswith('http://'):
            proxy = 'http://' + proxy
        client.setProxy(proxy, proxy_type=tc.HttpClient.PROXY_HTTPS)

    crits_taxii = taxii.Taxii()
    crits_taxii.runtime = runtime
    crits_taxii.end = end

    poll_msg = tm.PollRequest(message_id=tm.generate_message_id(),
                              feed_name=feed,
                              exclusive_begin_timestamp_label=start,
                              inclusive_end_timestamp_label=end)
    response = client.callTaxiiService2(hostname, '/poll/', t.VID_TAXII_XML_10,
                                        poll_msg.to_xml())

    if response.getcode() != 200:
        ret['reason'] = "Response is not 200 OK"
        return ret

    taxii_msg = t.get_message_from_http_response(response, poll_msg.message_id)

    valid = tm.validate_xml(taxii_msg.to_xml())
    if valid != True:
        ret['reason'] = valid
        return ret

    if taxii_msg.message_type != tm.MSG_POLL_RESPONSE:
        ret['reason'] = "No poll response"
        return ret

    ret['status'] = True

    if not taxii_msg.content_blocks:
        crits_taxii.save()
        return ret

    mid = taxii_msg.message_id
    for content_block in taxii_msg.content_blocks:
        data = parse_content_block(content_block, keyfile, certfile)
        if not data:
            ret['failures'] += 1
            continue

        objs = import_standards_doc(data, analyst, method, ref=mid, make_event=True)

        ret['successes'] += 1

        for k in ["events", "samples", "emails", "indicators"]:
            for i in objs[k]:
                ret[k].append(i)

    crits_taxii.save()
    return ret