def test_enricher_load_from_db(mocker):
"""
Create a traceroute, then destroy the enricher and
create a new one. At this point, the IP Info entries
that were previously stored into the DB are loaded,
and when a new enrichment is performed, we shouldn't
see any queries performed towards the external
sources.
"""
raw = open("tests/data/traceroute/mtr_json_1.json").read()
create_traceroute(raw)
# After the first enrichment is performed, the function
# that pulls IP information from external sources is
# expected to be called 5 times.
assert len(get_ip_info_from_external_sources_mock.call_args_list) == 5
# Now destroy the enricher and create it from scratch.
# IP Info entries that were previously retrieved will
# be loaded from the DB.
_setup_enricher(mocker)
assert len(enricher.ip_info_db.prefixes()) == 5
# Run a second enrichment process for the same traceroute.
raw = open("tests/data/traceroute/mtr_json_1.json").read()
create_traceroute(raw)
# The number of calls to fetch info from external sources
# should now be 0. The enricher (and the MagicMock) were
# destroyed few steps above, thus the calls count reset
# to zero.
assert len(get_ip_info_from_external_sources_mock.call_args_list) == 0
def test_enricher_expired_cache():
"""
Create a traceroute, then manipulate the enrichers
ip_info_db PyRadix object to mark an entry as
expired, to verify that the code works properly and
the information for that prefix are retrieved again
from the external sources.
"""
raw = open("tests/data/traceroute/mtr_json_1.json").read()
create_traceroute(raw)
# After the first enrichment is performed, the function
# that pulls IP information from external sources is
# expected to be called 5 times.
assert len(get_ip_info_from_external_sources_mock.call_args_list) == 5
# Mark an enricher's cache entry as updated one year
# ago, so it will be discarded during the next
# enrichement process.
enricher.ip_info_db.search_exact("89.97.0.0/16").data[
"last_updated"] = datetime.datetime.utcnow() - datetime.timedelta(
days=365)
# Trigger a new enrichment for the same traceroute.
create_traceroute(raw)
# After marking one prefix as expired, the function
# that fetches IP info from external sources is
# expected to be called one more time, making the
# total count equal 6.
assert len(get_ip_info_from_external_sources_mock.call_args_list) == 6
def new():
raw = request.form["raw"]
if not raw.split():
return redirect(url_for("home.index", err_code=3))
if ReCaptcha.is_used():
user_recaptcha_data = request.form.get("g-recaptcha-response")
recaptcha_ver = int(request.form.get("recaptcha-ver", 3))
recaptcha = ReCaptcha(recaptcha_ver)
if not recaptcha.check_user_response(user_recaptcha_data):
if recaptcha_ver == 3:
return render_template("index.html",
recaptcha=ReCaptcha(2),
raw=raw)
else:
return render_template("index.html", err_code=1, raw=raw)
traceroute = create_traceroute(raw)
if not traceroute.parsed:
return render_template(
"index.html",
recaptcha=ReCaptcha(3) if ReCaptcha.is_used() else None,
err_code=2,
raw=raw)
return redirect(url_for("traceroute.t", traceroute_id=traceroute.id))
def test_enricher_basic():
raw = open("tests/data/traceroute/mtr_json_1.json").read()
create_traceroute(raw)
# Verify the calls to the function that's used to
# retrieve IP information from external sources.
# Please note: a call for hop 9 IP 216.239.50.241 should not
# be performed because an entry for 216.239.32.0/19 should
# be found while getting IPDBInfo for 216.239.51.9
assert get_ip_info_from_external_sources_mock.call_args_list == [
call(IPv4Address("89.97.200.190")),
call(IPv4Address("62.101.124.17")), # 62-101-124-17.fastres.net
call(IPv4Address("209.85.168.64")),
call(IPv4Address("216.239.51.9")),
# call(IPv4Address("216.239.50.241")), <<< expected to be missing
call(IPv4Address("8.8.8.8"))
]
# Verify that the traceroute is marked as parsed
# and enriched properly.
t = Traceroute.select()[0]
assert t.parsed is True
assert t.enriched is True
assert t.enrichment_started is not None
assert t.enrichment_completed is not None
assert t.enrichment_completed >= t.enrichment_started
assert len(t.hops) == 10
hop = t.get_hop_n(1)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "192.168.1.254"
assert str(host.ip) == "192.168.1.254"
assert host.name is None
assert host.enriched is True
assert len(host.origins) == 0
hop = t.get_hop_n(6)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "62-101-124-17.fastres.net"
assert str(host.ip) == "62.101.124.17"
assert host.name == "62-101-124-17.fastres.net"
assert host.enriched is True
assert len(host.origins) == 1
origin = host.origins[0]
assert origin.asn == 12874
assert origin.holder == "FASTWEB - Fastweb SpA"
hop = t.get_hop_n(10)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "dns.google"
assert str(host.ip) == "8.8.8.8"
assert host.name == "dns.google"
assert host.enriched is True
assert len(host.origins) == 1
origin = host.origins[0]
assert origin.asn == 15169
assert origin.holder == "GOOGLE"
# Verify the to_dict of Traceroute.
json_t = t.to_dict()
assert json_t["enriched"] is True
assert json_t["status"] == "enriched"
assert len(json_t["hops"]) == 10
json_host = json_t["hops"][1][0]
assert json_host["ip"] == "192.168.1.254"
assert json_host["ixp_network"] is None
assert json_host["origins"] is None
assert isinstance(json_host["avg_rtt"], float)
assert isinstance(json_host["loss"], float)
assert json_host["loss"] == 0
json_host = json_t["hops"][10][0]
assert json_host["ip"] == "8.8.8.8"
assert json_host["ixp_network"] is None
assert json_host["origins"] == [(15169, "GOOGLE")]
assert isinstance(json_host["avg_rtt"], float)
assert isinstance(json_host["loss"], float)
assert json_host["name"] == "dns.google"
# Just verify that the dict is serializable as JSON.
t.to_json()
def test_traceroute_to_text():
def _normalize_text(s):
# Normalizing the lines obtained from the Traceroute
# object and those expected by the test case.
# The expected text contains some blank lines, just
# added here to make the string looking a bit better
# inside this file. Also, since the editor removes
# any trailing blank space from that string, to make
# the expected text matching the real one I'm removing
# trailing spaces from everywhere.
return "\n".join(
[line.rstrip() for line in s.split("\n") if line.strip()])
raw = open("tests/data/traceroute/mtr_json_1.json").read()
t = create_traceroute(raw)
txt = t.to_text()
print(txt)
assert _normalize_text(txt) == _normalize_text("""
Hop IP Loss RTT Origin Reverse
1. 192.168.1.254 0% 5.48 ms
2. 10.1.131.181 0% 16.35 ms
3. 10.250.139.186 0% 11.60 ms
4. 10.254.0.217 0% 12.56 ms
5. 89.97.200.190 0% 11.43 ms AS12874 FASTWEB - Fastweb SpA
6. 62.101.124.17 0% 59.78 ms AS12874 FASTWEB - Fastweb SpA 62-101-124-17.fastres.net
7. 209.85.168.64 0% 19.72 ms AS15169 GOOGLE
8. 216.239.51.9 0% 21.97 ms AS15169 GOOGLE
9. 216.239.50.241 0% 19.91 ms AS15169 GOOGLE
10. 8.8.8.8 0% 22.86 ms AS15169 GOOGLE dns.google
""")
# A traceroute having some hops represented by multiple
# hosts.
raw = open("tests/data/traceroute/bsd_1.txt").read()
t = create_traceroute(raw)
txt = t.to_text()
print(txt)
assert _normalize_text(txt) == _normalize_text("""
Hop IP RTT Origin Reverse
1. 192.168.1.254 3.44 ms
2. 10.1.131.181 9.76 ms
3. 10.250.139.186 14.33 ms
4. 10.254.0.217 12.85 ms
10.254.0.221 13.18 ms
5. 89.97.200.186 11.10 ms AS12874 FASTWEB - Fastweb SpA
89.97.200.190 13.02 ms AS12874 FASTWEB - Fastweb SpA
89.97.200.201 12.93 ms AS12874 FASTWEB - Fastweb SpA
6. 93.57.68.145 12.91 ms AS12874 FASTWEB - Fastweb SpA
93.57.68.149 15.66 ms AS12874 FASTWEB - Fastweb SpA
7. 193.201.28.33 27.09 ms cloudflare-nap.namex.it
8. 172.68.197.130 32.96 ms AS13335 CLOUDFLARENET
172.68.197.8 33.98 ms AS13335 CLOUDFLARENET
9. *
10. *
""")
# Just adding an entry to the enricher's local cache
# so that we'll be able to see how a hop that traverses
# an IX LAN looks like.
enricher.add_ip_info_to_local_cache(IPDBInfo(
prefix=IPv4Network("217.29.66.0/23"),
origins=None,
ixp_network=IXPNetwork(lan_name="Test LAN",
ix_name="MIX-IT",
ix_description="Milan Internet Exchange")),
dispatch_to_others=False)
raw = open("tests/data/traceroute/mtr_json_2.json").read()
t = create_traceroute(raw)
txt = t.to_text()
print(txt)
assert _normalize_text(txt) == _normalize_text("""
Hop IP Loss RTT Origin Reverse
1. 192.168.1.254 0% 3.79 ms
2. 10.1.131.181 0% 14.78 ms
3. 10.250.139.190 0% 10.71 ms
4. 10.254.0.221 0% 10.69 ms
5. 89.97.200.201 0% 10.68 ms AS12874 FASTWEB - Fastweb SpA
6. 93.63.100.141 0% 19.02 ms AS12874 FASTWEB - Fastweb SpA 93-63-100-141.ip27.fastwebnet.it
7. 217.29.66.1 0% 22.22 ms IX: MIX-IT mix-1.mix-it.net
8. 217.29.76.16 0% 18.74 ms AS16004 MIXITA-AS - MIX S.r.L.... kroot-server1.mix-it.net
""")
# Changing the enricher's cache entry to fake a MOAS
# (Multiple-Origin AS) prefix for the 5th hop.
enricher.add_ip_info_to_local_cache(IPDBInfo(
prefix=IPv4Network("94.198.103.142"),
origins=[(65501, "Origin AS1 of a MOAS prefix"),
(65502, "Origin AS2 of a MOAS prefix")],
ixp_network=None),
dispatch_to_others=False)
raw = open("tests/data/traceroute/linux_2.txt").read()
t = create_traceroute(raw)
txt = t.to_text()
print(txt)
assert _normalize_text(txt) == _normalize_text("""
Hop IP RTT Origin Reverse
1. 72.14.232.198 19.60 ms AS15169 GOOGLE
2. 94.198.103.149 19.54 ms AS49367 ASSEFLOW - Seflow... google-demarc.seflow.it
3. *
4. *
5. 94.198.103.142 19.44 ms AS65501 Origin AS1 of a MOAS... mix.gw.mix-ddos.seflow.it
AS65502 Origin AS2 of a MOAS...
6. 217.29.72.146 13.63 ms AS16004 MIXITA-AS - MIX S.r.L.... fw.mix-it.net
7. *
8. *
""")
def test_consumers_mysql_goes_away():
"""
Create a traceroute and get it parsed and enriched
using consumers, then shut MySQL down and spin
it up again, and process another traceroute, to test
that consumers are actually able to reconnect to the
DB server properly if it goes down and comes back.
"""
assert len(CONSUMER_THREADS) == CONSUMER_THREADS_NUM
raw = open("tests/data/traceroute/mtr_json_1.json").read()
t1_id = create_traceroute(raw).id
_wait_for_completion()
# Compare the SocketIO records emitted by the enricher
# with those that we expect to see.
socketio_emitted_records = _get_socketio_emitted_records()
expected_socketio_emitted_records = _prefix_traceroute_id(
EXPECTED_SOCKETIO_EMIT_CALLS_TR1,
t1_id
)
assert socketio_emitted_records == expected_socketio_emitted_records
# Verify that the last call to SocketIO is the one
# that notifies about the completion of the
# enrichment process.
t = Traceroute.get(Traceroute.id == t1_id)
socketio_emit_mock.assert_called_with(
SOCKET_IO_ENRICHMENT_COMPLETED_EVENT,
{
"traceroute_id": t1_id,
"traceroute": t.to_dict(),
"text": t.to_text()
},
namespace=f"/t/{t1_id}"
)
t = Traceroute.get(Traceroute.id == t1_id)
assert t.parsed is True
assert t.enriched is True
MYSQL_CONTAINER.kill_existing_container()
MYSQL_CONTAINER.ensure_is_up()
MYSQL_CONTAINER.recreate_last_schema()
raw = open("tests/data/traceroute/mtr_json_1.json").read()
t2_id = create_traceroute(raw).id
_wait_for_completion()
# At this point, the records emitted via SocketIO
# should be those originated while parsing the first
# traceroute + those originated while parsing the
# second one.
socketio_emitted_records = _get_socketio_emitted_records()
expected_socketio_emitted_records = \
_prefix_traceroute_id(EXPECTED_SOCKETIO_EMIT_CALLS_TR1, t1_id) + \
_prefix_traceroute_id(EXPECTED_SOCKETIO_EMIT_CALLS_TR1, t2_id)
assert socketio_emitted_records == expected_socketio_emitted_records
t = Traceroute.get(Traceroute.id == t2_id)
assert t.parsed is True
assert t.enriched is True
assert len(CONSUMER_THREADS) == CONSUMER_THREADS_NUM
def test_ixp_networks_updater_integration(ixp_networks):
"""
Process a traceroute having some hops crossing an IXP network.
"""
raw = open("tests/data/traceroute/mtr_json_2.json").read()
t1_id = create_traceroute(raw).id
_wait_for_completion()
# Compare the SocketIO records emitted by the enricher
# with those that we expect to see.
socketio_emitted_records = _get_socketio_emitted_records()
expected_socketio_emitted_records = _prefix_traceroute_id(
EXPECTED_SOCKETIO_EMIT_CALLS_TR2,
t1_id
)
assert socketio_emitted_records == expected_socketio_emitted_records
# Verify that the last call to SocketIO is the one
# that notifies about the completion of the
# enrichment process.
t = Traceroute.get(Traceroute.id == t1_id)
socketio_emit_mock.assert_called_with(
SOCKET_IO_ENRICHMENT_COMPLETED_EVENT,
{
"traceroute_id": t1_id,
"traceroute": t.to_dict(),
"text": t.to_text()
},
namespace=f"/t/{t1_id}"
)
t = Traceroute.get(Traceroute.id == t1_id)
assert t.parsed is True
assert t.enriched is True
assert len(t.hops) == 8
# Check that the host inside the IXP network is correct.
hop = t.get_hop_n(7)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "217.29.66.1"
assert str(host.ip) == "217.29.66.1"
assert host.name == "mix-1.mix-it.net"
assert host.enriched is True
assert len(host.origins) == 0
assert host.ixp_network is not None
assert host.ixp_network.lan_name is None
assert host.ixp_network.ix_name == "MIX-IT"
assert host.ixp_network.ix_description == "Milan Internet eXchange"
# Now, let's verify that all the enrichers from
# the consumer threads got their IP info DB populated
# equally. This is to ensure that the IXPNetworksUpdater
# properly dispatch the IP info entries to all the
# consumers.
for thread in CONSUMER_THREADS:
for enricher in thread.enrichers:
ip_info_db = enricher.ip_info_db
assert len(ip_info_db.nodes()) == 4
assert sorted(ip_info_db.prefixes()) == sorted([
"89.97.0.0/16",
"93.62.0.0/15",
"217.29.66.0/23",
"217.29.72.0/21"
])
assert ip_info_db.search_exact(
"217.29.66.0/23"
).data["ip_db_info"] == IPDBInfo(
prefix=ipaddress.ip_network("217.29.66.0/23"),
origins=None,
ixp_network=IXPNetwork(
lan_name=None,
ix_name="MIX-IT",
ix_description="Milan Internet eXchange"
)
)
# Check now that the IP Info DB is populated properly.
db_prefixes = IPInfo_Prefix.select()
# Build a dict using DB records to make comparisons easier.
db_prefixes_dict = {
db_prefix.prefix: db_prefix.origins
for db_prefix in db_prefixes
}
assert len(db_prefixes_dict.keys()) == 4
assert sorted(db_prefixes_dict.keys()) == sorted([
ipaddress.IPv4Network("89.97.0.0/16"),
ipaddress.IPv4Network("93.62.0.0/15"),
ipaddress.IPv4Network("217.29.66.0/23"),
ipaddress.IPv4Network("217.29.72.0/21"),
])
db_prefix = IPInfo_Prefix.get(prefix="217.29.66.0/23")
assert db_prefix.to_ipdbinfo() == IPDBInfo(
prefix=ipaddress.ip_network("217.29.66.0/23"),
origins=None,
ixp_network=IXPNetwork(
lan_name=None,
ix_name="MIX-IT",
ix_description="Milan Internet eXchange"
)
)
def test_consumers_basic():
"""
Create a traceroute and get it parsed and enriched
using consumers.
"""
# Just to be sure that we're actually using the
# n. of thread we expect, just in case I'll change
# the way consumer threads are spun up while doing
# some debugging.
assert CONSUMER_THREADS_NUM > 1
assert len(CONSUMER_THREADS) == CONSUMER_THREADS_NUM
raw = open("tests/data/traceroute/mtr_json_1.json").read()
t_id = create_traceroute(raw).id
_wait_for_completion()
# Compare the SocketIO records emitted by the enricher
# with those that we expect to see.
socketio_emitted_records = _get_socketio_emitted_records()
expected_socketio_emitted_records = _prefix_traceroute_id(
EXPECTED_SOCKETIO_EMIT_CALLS_TR1,
t_id
)
assert socketio_emitted_records == expected_socketio_emitted_records
# Verify that the last call to SocketIO is the one
# that notifies about the completion of the
# enrichment process.
t = Traceroute.select()[0]
socketio_emit_mock.assert_called_with(
SOCKET_IO_ENRICHMENT_COMPLETED_EVENT,
{
"traceroute_id": t_id,
"traceroute": t.to_dict(),
"text": t.to_text()
},
namespace=f"/t/{t_id}"
)
# Let's check that the traceroute is in the expected
# state, and that hops and hosts were processed.
t = Traceroute.select()[0]
assert t.parsed is True
assert t.enriched is True
assert len(t.hops) == 10
hop = t.get_hop_n(1)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "192.168.1.254"
assert str(host.ip) == "192.168.1.254"
assert host.name is None
assert host.enriched is True
assert len(host.origins) == 0
assert host.ixp_network is None
hop = t.get_hop_n(6)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "62-101-124-17.fastres.net"
assert str(host.ip) == "62.101.124.17"
assert host.name == "62-101-124-17.fastres.net"
assert host.enriched is True
assert len(host.origins) == 1
origin = host.origins[0]
assert origin.asn == 12874
assert origin.holder == "FASTWEB - Fastweb SpA"
assert host.ixp_network is None
hop = t.get_hop_n(10)
assert len(hop.hosts) == 1
host = hop.hosts[0]
assert host.original_host == "dns.google"
assert str(host.ip) == "8.8.8.8"
assert host.name == "dns.google"
assert host.enriched is True
assert len(host.origins) == 1
origin = host.origins[0]
assert origin.asn == 15169
assert origin.holder == "GOOGLE"
assert host.ixp_network is None
# Now, let's verify that all the enrichers from
# the consumer threads got their IP info DB populated
# equally. This is to ensure that the IP info records
# are properly distributed across the consumers.
for thread in CONSUMER_THREADS:
for enricher in thread.enrichers:
ip_info_db = enricher.ip_info_db
assert len(ip_info_db.nodes()) == 5
assert sorted(ip_info_db.prefixes()) == sorted([
"89.97.0.0/16",
"62.101.124.0/22",
"209.85.128.0/17",
"216.239.32.0/19",
"8.8.8.0/24",
])
assert ip_info_db.search_exact(
"89.97.0.0/16"
).data["ip_db_info"] == IPDBInfo(
prefix=ipaddress.ip_network("89.97.0.0/16"),
origins=[
(12874, "FASTWEB - Fastweb SpA")
],
ixp_network=None
)
# Check now that the IP Info DB is populated properly.
db_prefixes = IPInfo_Prefix.select()
# Build a dict using DB records to make comparisons easier.
db_prefixes_dict = {
db_prefix.prefix: db_prefix.origins
for db_prefix in db_prefixes
}
assert len(db_prefixes_dict.keys()) == 5
assert sorted(db_prefixes_dict.keys()) == sorted([
ipaddress.IPv4Network("89.97.0.0/16"),
ipaddress.IPv4Network("62.101.124.0/22"),
ipaddress.IPv4Network("209.85.128.0/17"),
ipaddress.IPv4Network("216.239.32.0/19"),
ipaddress.IPv4Network("8.8.8.0/24")
])
db_prefix = IPInfo_Prefix.get(prefix="89.97.0.0/16")
assert db_prefix.to_ipdbinfo() == IPDBInfo(
prefix=ipaddress.ip_network("89.97.0.0/16"),
origins=[
(12874, "FASTWEB - Fastweb SpA")
],
ixp_network=None
)
# Verify that metrics logging is working properly.
# To see which metrics have been collected:
# metrics_mock_wrapper.mm.print_records()
mm_records = metrics_mock_wrapper.mm.get_records()
# Expecting 5 calls to the function that performs
# external queries to fetch IP info.
# Every time, we want the counter to be increased.
mm_ip_info_from_external_sources = filter(
lambda r: (
r[0] == "incr" and
r[1] == ("rich_traceroute.enrichers.enricher."
"ip_info_from_external_sources") and
r[2] == 1
),
mm_records
)
assert len(list(mm_ip_info_from_external_sources)) == 5
# Check that we're keeping track of how long those
# 5 upstream queries take to complete.
mm_ip_info_from_external_sources = filter(
lambda r: (
r[0] == "timing" and
r[1] == ("rich_traceroute.enrichers.enricher."
"ripestat.query_time")
),
mm_records
)
assert len(list(mm_ip_info_from_external_sources)) == 5