def staged_parse_checkhostnetresult_geodns():
stage3 = geodns.GeoDNSStage(
kafka_consumer_topics=["dns-raw"],
kafka_producer_topics=["dns-parsed"],
kafka_consumer_group_id="parse_checkhostnetresult",
worker=geodns.GeoDNS(
compass_username="",
compass_password="",
).parse_checkhostnetresult,
max_read_count=10,
)
return stage3.run()
def test_geodns_parse_dns():
"""test we can parse and restructure a valid dns output"""
worker = geodns.GeoDNS(
compass_username="",
compass_password="",
dns_resolver=geodns.CheckHostNet(),
)
domain = "www.ip-echelon.com"
parsed_response = worker.parse_checkhostnetresult(_resolve_dns_res)
#TODO: not sure what to assert here, its a big dict?
assert parsed_response is not None, "can parse valid dns results"
def staged_resolve_domains_geodns(mock_resolve_dns):
# given a valid checkhostnet raw result
mock_resolve_dns.return_value = _resolve_dns_res
# first stage of pipeline (dns-domain to dns-raw) should run
stage2 = geodns.GeoDNSStage(
kafka_consumer_topics=["dns-domains"],
kafka_producer_topics=["dns-raw"],
kafka_consumer_group_id="resolve_domains_geodns",
worker=geodns.GeoDNS(
compass_username="",
compass_password="",
).resolve_dns,
max_read_count=10,
)
return stage2.run()
def test_resolved_geog_from_dns_geodns_empty(mock_compass_post):
"""test resolution from valid dns results to empty geog results"""
mock_compass_post.return_value.content = b'{"Error":"no routes"}\n'
mock_compass_post.return_value.status_code = 500
worker = geodns.GeoDNS(
compass_username="******",
compass_password="",
dns_resolver=geodns.CheckHostNet(),
)
# directly instantiate a parsed response
parsed_geog_results = worker.resolve_geog_from_dns(_parsed_dns_res)
assert parsed_geog_results is not None, "resolve_geog_from_dns(valid_dns_results) returns something"
assert len(parsed_geog_results["geog_results"].keys()) == 0, "missing compass data should give geog_results with 0 keys"
def staged_resolve_geog_from_dns_geodns(mock_compass_post):
# return the same compass result each time
mock_compass_post.return_value.content = _resolve_compass_res
mock_compass_post.return_value.status_coode = 200
stage4 = geodns.GeoDNSStage(
kafka_consumer_topics=["dns-parsed"],
kafka_producer_topics=["dns-geodns-parsed"],
kafka_consumer_group_id="resolve_geog_from_dns",
worker=geodns.GeoDNS(
compass_username="",
compass_password="",
).resolve_geog_from_dns,
max_read_count=10,
)
return stage4.run()
def test_geodns_geog_from_dns(mock_compass_post):
# return the same compass result each time
mock_compass_post.return_value.content = _resolve_compass_res
mock_compass_post.return_value.status_coode = 200
# TODO: fixtures for this object
worker = geodns.GeoDNS(
compass_username="",
compass_password="",
dns_resolver=geodns.CheckHostNet(),
)
# directly instantiate a parsed response
parsed_dns_results = worker.parse_checkhostnetresult(_resolve_dns_res)
parsed_geog_results = worker.resolve_geog_from_dns(parsed_dns_results)
# TODO: not sure what to assert for this
assert parsed_geog_results is not None, "resolve_geog_from_dns(valid_dns_results) returns something"
def test_staged_resolve_domains_geodns_with_exception(mock_resolve_dns):
# given a failing checkhostnet response
mock_resolve_dns.side_effect = CheckHostNetError("mocked exception")
# first stage of pipeline (dns-domain to dns-raw) should error out
stage2 = geodns.GeoDNSStage(
kafka_consumer_topics=["dns-domains"],
kafka_producer_topics=["dns-raw"],
kafka_consumer_group_id="resolve_domains_geodns_withexceptions",
worker=geodns.GeoDNS(
compass_username="",
compass_password="",
).resolve_dns,
retryable_exceptions=(CheckHostNetError,),
retryable_exceptions_count=2,
max_read_count=10,
)
try:
stage2.run()
except CheckHostNetError as exc:
# after having tried N times
assert stage2.metrics["retryable_exceptions"] == stage2.retryable_exceptions_count, "retried catchable exceptions"
def test_staged_resolve_domains_geodns_with_timeout(mock_resolve_dns):
# given a successful but slow checkhostnet response
mock_resolve_dns.side_effect = lambda *args, **kwargs: time.sleep(9999)
mock_resolve_dns.return_value = _resolve_dns_res
# first stage of pipeline (dns-domain to dns-raw) should error out
stage2 = geodns.GeoDNSStage(
kafka_consumer_topics=["dns-domains"],
kafka_producer_topics=["dns-raw"],
kafka_consumer_group_id="resolve_domains_geodns_withtimeout",
worker=geodns.GeoDNS(
compass_username="",
compass_password="",
).resolve_dns,
worker_timeout_seconds=5,
retryable_exceptions=(CheckHostNetError,),
retryable_exceptions_count=2,
max_read_count=10,
)
try:
stage2.run()
except WorkerTimedOut as exc:
# after having tried N times
assert stage2.metrics["retryable_exceptions"] == stage2.retryable_exceptions_count, "retried after timeout"