def run(self, serial_newest_seen: int, database_handler: DatabaseHandler):
serial_start = serial_newest_seen + 1
nrtm_host = get_setting(f'sources.{self.source}.nrtm_host')
nrtm_port = int(
get_setting(f'sources.{self.source}.nrtm_port',
DEFAULT_SOURCE_NRTM_PORT))
if not nrtm_host:
logger.debug(
f'Skipping NRTM updates for {self.source}, nrtm_host not set.')
return
end_markings = [
f'\n%END {self.source}\n',
f'\n% END {self.source}\n',
'\n%ERROR',
'\n% ERROR',
'\n% Warning: there are no newer updates available',
'\n% Warning (1): there are no newer updates available',
]
logger.info(
f'Retrieving NRTM updates for {self.source} from serial {serial_start} on {nrtm_host}:{nrtm_port}'
)
query = f'-g {self.source}:3:{serial_start}-LAST'
response = whois_query(nrtm_host, nrtm_port, query, end_markings)
logger.debug(
f'Received NRTM response for {self.source}: {response.strip()}')
stream_parser = NRTMStreamParser(self.source, response,
database_handler)
for operation in stream_parser.operations:
operation.save(database_handler)
def test_irrd_integration(self, tmpdir):
# IRRD_DATABASE_URL overrides the yaml config, so should be removed
if 'IRRD_DATABASE_URL' in os.environ:
del os.environ['IRRD_DATABASE_URL']
# PYTHONPATH needs to contain the twisted plugin path.
os.environ['PYTHONPATH'] = IRRD_ROOT_PATH
os.environ['IRRD_SCHEDULER_TIMER_OVERRIDE'] = '1'
self.tmpdir = tmpdir
self._start_mailserver()
self._start_irrds()
# Attempt to load a mntner with valid auth, but broken references.
self._submit_update(self.config_path1,
SAMPLE_MNTNER + '\n\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate FAILED: [mntner] TEST-MNT\n' in mail_text
assert '\nERROR: Object PERSON-TEST referenced in field admin-c not found in database TEST - must reference one of role, person.\n' in mail_text
assert '\nERROR: Object OTHER1-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text
assert '\nERROR: Object OTHER2-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text
assert 'email footer' in mail_text
assert 'Generated by IRRd version ' in mail_text
# Load a regular valid mntner and person into the DB, and verify
# the contents of the result.
self._submit_update(
self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON +
'\n\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text
assert 'email footer' in mail_text
assert 'Generated by IRRd version ' in mail_text
# Check whether the objects can be queried from irrd #1,
# whether the hash is masked, and whether encoding is correct.
mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# After three seconds, a new export should have been generated by irrd #1,
# loaded by irrd #2, and the objects should be available in irrd #2
time.sleep(3)
mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# Load a key-cert. This should cause notifications to mnt-nfy (2x).
# Change is authenticated by valid password.
self._submit_update(self.config_path1,
SAMPLE_KEY_CERT + '\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 3
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert 'Create succeeded: [key-cert] PGPKEY-80F238C6' in self._extract_message_body(
messages[0])
self._check_recipients_in_mails(
messages[1:], ['*****@*****.**', '*****@*****.**'])
self._check_text_in_mails(messages[1:], [
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nCreate succeeded for object below: [key-cert] PGPKEY-80F238C6:\n',
'email footer',
'Generated by IRRd version ',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Use the new PGP key to make an update to PERSON-TEST. Should
# again trigger mnt-nfy messages, and a mail to the notify address
# of PERSON-TEST.
self._submit_update(self.config_path1, SIGNED_PERSON_UPDATE_VALID)
messages = self._retrieve_mails()
assert len(messages) == 4
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nModify succeeded: [person] PERSON-TEST\n' in mail_text
self._check_recipients_in_mails(messages[1:], [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
])
self._check_text_in_mails(messages[1:], [
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nModify succeeded for object below: [person] PERSON-TEST:\n',
'\n@@ -1,4 +1,4 @@\n',
'\nNew version of this object:\n',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Check that the person is updated on irrd #1
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# After 2s, NRTM from irrd #2 should have picked up the change.
time.sleep(2)
person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit an update back to the original person object, with an invalid
# password and invalid override. Should trigger notification to upd-to.
self._submit_update(
self.config_path1,
SAMPLE_PERSON + '\npassword: invalid\noverride: invalid\n')
messages = self._retrieve_mails()
assert len(messages) == 2
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nModify FAILED: [person] PERSON-TEST\n' in mail_text
assert '\nERROR: Authorisation for person PERSON-TEST failed: must by authenticated by one of: TEST-MNT\n' in mail_text
mail_text = self._extract_message_body(messages[1])
assert messages[1][
'Subject'] == 'Notification of TEST database changes'
assert messages[1]['From'] == '*****@*****.**'
assert messages[1]['To'] == '*****@*****.**'
assert '\nModify FAILED AUTHORISATION for object below: [person] PERSON-TEST:\n' in mail_text
# Object should not have changed by latest update.
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit a delete with a valid password for PERSON-TEST.
# This should be rejected, because it creates a dangling reference.
# No mail should be sent to upd-to.
self._submit_update(
self.config_path1,
SAMPLE_PERSON + 'password: md5-password\ndelete: delete\n')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nDelete FAILED: [person] PERSON-TEST\n' in mail_text
assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by mntner TEST-MNT\n' in mail_text
assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by key-cert PGPKEY-80F238C6\n' in mail_text
# Object should not have changed by latest update.
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit a valid delete for all our new objects.
self._submit_update(
self.config_path1,
f'{SAMPLE_PERSON}delete: delete\n\n{SAMPLE_KEY_CERT}delete: delete\n\n'
+
f'{SAMPLE_MNTNER_CLEAN}delete: delete\npassword: crypt-password\n')
messages = self._retrieve_mails()
# Expected mails are status, mnt-nfy on mntner (2x), and notify on mntner
# (notify on PERSON-TEST was removed in the PGP signed update)
assert len(messages) == 4
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nDelete succeeded: [person] PERSON-TEST\n' in mail_text
assert '\nDelete succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nDelete succeeded: [key-cert] PGPKEY-80F238C6\n' in mail_text
self._check_recipients_in_mails(messages[1:], [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
])
mnt_nfy_msgs = [
msg for msg in messages
if msg['To'] in ['*****@*****.**', '*****@*****.**']
]
self._check_text_in_mails(
mnt_nfy_msgs,
[
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nDelete succeeded for object below: [person] PERSON-TEST:\n',
'\nDelete succeeded for object below: [mntner] TEST-MNT:\n',
'\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n',
'unįcöde tæst 🌈🦄\n',
# The object submitted to be deleted has the original name,
# but when sending delete notifications, they should include the
# object as currently in the DB, not as submitted in the email.
'Test person changed by PGP signed update\n',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Notify attribute mails are only about the objects concerned.
notify_msg = [
msg for msg in messages if msg['To'] == '*****@*****.**'
][0]
mail_text = self._extract_message_body(notify_msg)
assert notify_msg['Subject'] == 'Notification of TEST database changes'
assert notify_msg['From'] == '*****@*****.**'
assert '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n' in mail_text
assert '\nDelete succeeded for object below: [person] PERSON-TEST:\n' not in mail_text
assert '\nDelete succeeded for object below: [mntner] TEST-MNT:\n' in mail_text
assert '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n' not in mail_text
# Object should be deleted
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'No entries found for the selected source(s)' in person_text
assert 'PERSON-TEST' not in person_text
# Object should be deleted from irrd #2 as well through NRTM.
time.sleep(2)
person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST')
assert 'No entries found for the selected source(s)' in person_text
assert 'PERSON-TEST' not in person_text
# Load samples of all known objects.
self._submit_update(self.config_path1,
LARGE_UPDATE + '\n\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text
assert '\nINFO: AS number as065537 was reformatted as AS65537\n' in mail_text
assert '\nCreate succeeded: [filter-set] FLTR-SETTEST\n' in mail_text
assert '\nINFO: Address range 192.0.2.0 - 192.0.02.255 was reformatted as 192.0.2.0 - 192.0.2.255\n' in mail_text
assert '\nINFO: Address prefix 192.0.02.0/24 was reformatted as 192.0.2.0/24\n' in mail_text
assert '\nINFO: Route set member 2001:0dB8::/48 was reformatted as 2001:db8::/48\n' in mail_text
# Check whether the objects can be queried from irrd #1,
# and whether the hash is masked.
mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# (This is the first instance of an object with unicode chars
# appearing on the NRTM stream.)
time.sleep(3)
mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# Test every major query type on both instances.
for port in self.port_whois1, self.port_whois2:
query_result = whois_query_irrd('127.0.0.1', port, '!gAS65537')
assert query_result == '192.0.2.0/24'
query_result = whois_query_irrd('127.0.0.1', port, '!6AS65537')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', port, '!iRS-TEST')
assert query_result == '192.0.2.0/24 2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', port, '!iAS-SETTEST')
assert query_result == 'AS65537 AS65538 AS65539'
query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF')
assert query_result == 'AS-SETTEST AS65540'
query_result = whois_query_irrd('127.0.0.1', port,
'!iAS-TESTREF,1')
assert query_result == 'AS65537 AS65538 AS65539 AS65540'
query_result = whois_query_irrd('127.0.0.1', port, '!aAS-SETTEST')
assert query_result == '192.0.2.0/24 2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', port, '!aAS-TESTREF')
assert query_result == '192.0.2.0/24 2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', port, '!a4AS-TESTREF')
assert query_result == '192.0.2.0/24'
query_result = whois_query_irrd('127.0.0.1', port, '!a6AS-TESTREF')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', port,
'!maut-num,as65537')
assert 'AS65537' in query_result
assert 'TEST-AS' in query_result
query_result = whois_query_irrd('127.0.0.1', port, '!oTEST-MNT')
assert 'AS65537' in query_result
assert 'TEST-AS' in query_result
assert 'AS65536 - AS65538' in query_result
assert 'rtrs-settest' in query_result
query_result = whois_query_irrd('127.0.0.1', port,
'!r192.0.2.0/24')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', port,
'!r192.0.2.0/25,l')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', port,
'!r192.0.2.0/24,L')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', port,
'!r192.0.2.0/23,M')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', port,
'!r192.0.2.0/24,o')
assert query_result == 'AS65537'
query_result = whois_query('127.0.0.1', port, '-x 192.0.02.0/24')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', port, '-l 192.0.02.0/25')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', port, '-L 192.0.02.0/24')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', port, '-M 192.0.02.0/23')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', port,
'-i member-of RS-test')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', port,
'-T route6 -i member-of RS-TEST')
assert 'No entries found for the selected source(s)' in query_result
query_result = whois_query('127.0.0.1', port, 'dashcare')
assert 'ROLE-TEST' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!j-*')
assert query_result == 'TEST:Y:1-29:29'
# irrd #2 missed the first update from NRTM, as they were done at
# the same time and loaded from the full export, so its serial should
# start at 2 rather than 1.
query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*')
assert query_result == 'TEST:Y:2-29:29'
def test_irrd_integration(self, tmpdir):
# IRRD_DATABASE_URL and IRRD_REDIS_URL override the yaml config, so should be removed
if 'IRRD_DATABASE_URL' in os.environ:
del os.environ['IRRD_DATABASE_URL']
if 'IRRD_REDIS_URL' in os.environ:
del os.environ['IRRD_REDIS_URL']
# PYTHONPATH needs to contain the twisted plugin path to support the mailserver.
os.environ['PYTHONPATH'] = IRRD_ROOT_PATH
os.environ['IRRD_SCHEDULER_TIMER_OVERRIDE'] = '1'
self.tmpdir = tmpdir
self._start_mailserver()
self._start_irrds()
# Attempt to load a mntner with valid auth, but broken references.
self._submit_update(self.config_path1,
SAMPLE_MNTNER + '\n\noverride: override-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate FAILED: [mntner] TEST-MNT\n' in mail_text
assert '\nERROR: Object PERSON-TEST referenced in field admin-c not found in database TEST - must reference one of role, person.\n' in mail_text
assert '\nERROR: Object OTHER1-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text
assert '\nERROR: Object OTHER2-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text
assert 'email footer' in mail_text
assert 'Generated by IRRd version ' in mail_text
# Load a regular valid mntner and person into the DB, and verify
# the contents of the result.
self._submit_update(
self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON +
'\n\noverride: override-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text
assert 'email footer' in mail_text
assert 'Generated by IRRd version ' in mail_text
# Check whether the objects can be queried from irrd #1,
# whether the hash is masked, and whether encoding is correct.
mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# After three seconds, a new export should have been generated by irrd #1,
# loaded by irrd #2, and the objects should be available in irrd #2
time.sleep(3)
mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# Load a key-cert. This should cause notifications to mnt-nfy (2x).
# Change is authenticated by valid password.
self._submit_update(self.config_path1,
SAMPLE_KEY_CERT + '\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 3
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert 'Create succeeded: [key-cert] PGPKEY-80F238C6' in self._extract_message_body(
messages[0])
self._check_recipients_in_mails(
messages[1:], ['*****@*****.**', '*****@*****.**'])
self._check_text_in_mails(messages[1:], [
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nCreate succeeded for object below: [key-cert] PGPKEY-80F238C6:\n',
'email footer',
'Generated by IRRd version ',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Use the new PGP key to make an update to PERSON-TEST. Should
# again trigger mnt-nfy messages, and a mail to the notify address
# of PERSON-TEST.
self._submit_update(self.config_path1, SIGNED_PERSON_UPDATE_VALID)
messages = self._retrieve_mails()
assert len(messages) == 4
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nModify succeeded: [person] PERSON-TEST\n' in mail_text
self._check_recipients_in_mails(messages[1:], [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
])
self._check_text_in_mails(messages[1:], [
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nModify succeeded for object below: [person] PERSON-TEST:\n',
'\n@@ -1,4 +1,4 @@\n',
'\nNew version of this object:\n',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Check that the person is updated on irrd #1
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# After 2s, NRTM from irrd #2 should have picked up the change.
time.sleep(2)
person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit an update back to the original person object, with an invalid
# password and invalid override. Should trigger notification to upd-to.
self._submit_update(
self.config_path1,
SAMPLE_PERSON + '\npassword: invalid\noverride: invalid\n')
messages = self._retrieve_mails()
assert len(messages) == 2
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nModify FAILED: [person] PERSON-TEST\n' in mail_text
assert '\nERROR: Authorisation for person PERSON-TEST failed: must by authenticated by one of: TEST-MNT\n' in mail_text
mail_text = self._extract_message_body(messages[1])
assert messages[1][
'Subject'] == 'Notification of TEST database changes'
assert messages[1]['From'] == '*****@*****.**'
assert messages[1]['To'] == '*****@*****.**'
assert '\nModify FAILED AUTHORISATION for object below: [person] PERSON-TEST:\n' in mail_text
# Object should not have changed by latest update.
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit a delete with a valid password for PERSON-TEST.
# This should be rejected, because it creates a dangling reference.
# No mail should be sent to upd-to.
self._submit_update(
self.config_path1,
SAMPLE_PERSON + 'password: md5-password\ndelete: delete\n')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'FAILED: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nDelete FAILED: [person] PERSON-TEST\n' in mail_text
assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by mntner TEST-MNT\n' in mail_text
assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by key-cert PGPKEY-80F238C6\n' in mail_text
# Object should not have changed by latest update.
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'PERSON-TEST' in person_text
assert 'Test person changed by PGP signed update' in person_text
# Submit a valid delete for all our new objects.
self._submit_update(
self.config_path1,
f'{SAMPLE_PERSON}delete: delete\n\n{SAMPLE_KEY_CERT}delete: delete\n\n'
+
f'{SAMPLE_MNTNER_CLEAN}delete: delete\npassword: crypt-password\n')
messages = self._retrieve_mails()
# Expected mails are status, mnt-nfy on mntner (2x), and notify on mntner
# (notify on PERSON-TEST was removed in the PGP signed update)
assert len(messages) == 4
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nDelete succeeded: [person] PERSON-TEST\n' in mail_text
assert '\nDelete succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nDelete succeeded: [key-cert] PGPKEY-80F238C6\n' in mail_text
self._check_recipients_in_mails(messages[1:], [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
])
mnt_nfy_msgs = [
msg for msg in messages
if msg['To'] in ['*****@*****.**', '*****@*****.**']
]
self._check_text_in_mails(
mnt_nfy_msgs,
[
'\n> Message-ID: <1325754288.4989.6.camel@hostname>\n',
'\nDelete succeeded for object below: [person] PERSON-TEST:\n',
'\nDelete succeeded for object below: [mntner] TEST-MNT:\n',
'\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n',
'unįcöde tæst 🌈🦄\n',
# The object submitted to be deleted has the original name,
# but when sending delete notifications, they should include the
# object as currently in the DB, not as submitted in the email.
'Test person changed by PGP signed update\n',
])
for message in messages[1:]:
assert message[
'Subject'] == 'Notification of TEST database changes'
assert message['From'] == '*****@*****.**'
# Notify attribute mails are only about the objects concerned.
notify_msg = [
msg for msg in messages if msg['To'] == '*****@*****.**'
][0]
mail_text = self._extract_message_body(notify_msg)
assert notify_msg['Subject'] == 'Notification of TEST database changes'
assert notify_msg['From'] == '*****@*****.**'
assert '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n' in mail_text
assert '\nDelete succeeded for object below: [person] PERSON-TEST:\n' not in mail_text
assert '\nDelete succeeded for object below: [mntner] TEST-MNT:\n' in mail_text
assert '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n' not in mail_text
# Object should be deleted
person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST')
assert 'No entries found for the selected source(s)' in person_text
assert 'PERSON-TEST' not in person_text
# Object should be deleted from irrd #2 as well through NRTM.
time.sleep(2)
person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST')
assert 'No entries found for the selected source(s)' in person_text
assert 'PERSON-TEST' not in person_text
# Load the mntner and person again, using the override password
# Note that the route/route6 objects are RPKI valid on IRRd #1,
# and RPKI-invalid on IRRd #2
self._submit_update(
self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON +
'\n\noverride: override-password')
messages = self._retrieve_mails()
assert len(messages) == 1
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text
assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text
assert 'email footer' in mail_text
assert 'Generated by IRRd version ' in mail_text
# Load samples of all known objects, using the mntner password
self._submit_update(self.config_path1,
LARGE_UPDATE + '\n\npassword: md5-password')
messages = self._retrieve_mails()
assert len(messages) == 3
mail_text = self._extract_message_body(messages[0])
assert messages[0]['Subject'] == 'SUCCESS: my subject'
assert messages[0]['From'] == '*****@*****.**'
assert messages[0]['To'] == 'Sasha <*****@*****.**>'
assert '\nINFO: AS number as065537 was reformatted as AS65537\n' in mail_text
assert '\nCreate succeeded: [filter-set] FLTR-SETTEST\n' in mail_text
assert '\nINFO: Address range 192.0.2.0 - 192.0.02.255 was reformatted as 192.0.2.0 - 192.0.2.255\n' in mail_text
assert '\nINFO: Address prefix 192.0.02.0/24 was reformatted as 192.0.2.0/24\n' in mail_text
assert '\nINFO: Route set member 2001:0dB8::/48 was reformatted as 2001:db8::/48\n' in mail_text
# Check whether the objects can be queried from irrd #1,
# and whether the hash is masked.
mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# (This is the first instance of an object with unicode chars
# appearing on the NRTM stream.)
time.sleep(3)
mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT')
assert 'TEST-MNT' in mntner_text
assert PASSWORD_HASH_DUMMY_VALUE in mntner_text
assert 'unįcöde tæst 🌈🦄' in mntner_text
assert 'PERSON-TEST' in mntner_text
# These queries have different responses on #1 than #2,
# as all IPv4 routes are RPKI invalid on #2.
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!gAS65537')
assert query_result == '192.0.2.0/24'
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!gAS65547')
assert query_result == '192.0.2.0/32' # Pseudo-IRR object from RPKI
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!6AS65537')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!iRS-TEST')
assert set(
query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'}
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!aAS-SETTEST')
assert set(
query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'}
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!aAS-TESTREF')
assert set(
query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'}
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!a4AS-TESTREF')
assert query_result == '192.0.2.0/24'
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!a6AS-TESTREF')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/24')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/25,l')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/24,L')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/23,M')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/24,M')
assert 'RPKI' in query_result # Does not match the /24, does match the RPKI pseudo-IRR /32
query_result = whois_query_irrd('127.0.0.1', self.port_whois1,
'!r192.0.2.0/24,o')
assert query_result == 'AS65537'
query_result = whois_query('127.0.0.1', self.port_whois1,
'-x 192.0.02.0/24')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', self.port_whois1,
'-l 192.0.02.0/25')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', self.port_whois1,
'-L 192.0.02.0/24')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', self.port_whois1,
'-M 192.0.02.0/23')
assert 'example route' in query_result
query_result = whois_query('127.0.0.1', self.port_whois1,
'-i member-of RS-test')
assert 'example route' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!gAS65537')
assert not query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!6AS65537')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!iRS-TEST')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!aAS-SETTEST')
assert query_result == '2001:db8::/48'
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!aAS-TESTREF')
assert query_result == '2001:db8::/48'
query_result = whois_query('127.0.0.1', self.port_whois2,
'-x 192.0.02.0/24')
assert 'example route' not in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!r192.0.2.0/24,L')
assert 'RPKI' in query_result # Pseudo-IRR object 0/0 from RPKI
# RPKI invalid object should not be in journal
query_result = whois_query('127.0.0.1', self.port_whois2,
'-g TEST:3:1-LAST')
assert 'route:192.0.2.0/24' not in query_result.replace(' ', '')
# These queries should produce identical answers on both instances.
for port in self.port_whois1, self.port_whois2:
query_result = whois_query_irrd('127.0.0.1', port, '!iAS-SETTEST')
assert set(
query_result.split(' ')) == {'AS65537', 'AS65538', 'AS65539'}
query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF')
assert set(query_result.split(' ')) == {'AS-SETTEST', 'AS65540'}
query_result = whois_query_irrd('127.0.0.1', port,
'!iAS-TESTREF,1')
assert set(query_result.split(' ')) == {
'AS65537', 'AS65538', 'AS65539', 'AS65540'
}
query_result = whois_query_irrd('127.0.0.1', port,
'!maut-num,as65537')
assert 'AS65537' in query_result
assert 'TEST-AS' in query_result
query_result = whois_query_irrd('127.0.0.1', port, '!oTEST-MNT')
assert 'AS65537' in query_result
assert 'TEST-AS' in query_result
assert 'AS65536 - AS65538' in query_result
assert 'rtrs-settest' in query_result
query_result = whois_query('127.0.0.1', port,
'-T route6 -i member-of RS-TEST')
assert 'No entries found for the selected source(s)' in query_result
query_result = whois_query('127.0.0.1', port, 'dashcare')
assert 'ROLE-TEST' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!j-*')
assert query_result == 'TEST:Y:1-29:29\nRPKI:N:-'
# irrd #2 missed the first update from NRTM, as they were done at
# the same time and loaded from the full export, and one RPKI-invalid object
# was not recorded in the journal, so its serial should
# is lower by three
query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*')
assert query_result == 'TEST:Y:1-26:26\nRPKI:N:-'
# Make the v4 route in irrd2 valid
with open(self.roa_source2, 'w') as roa_file:
ujson.dump(
{
'roas': [{
'prefix': '198.51.100.0/24',
'asn': 'AS0',
'maxLength': '32',
'ta': 'TA'
}]
}, roa_file)
time.sleep(2)
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!gAS65537')
assert query_result == '192.0.2.0/24'
# RPKI invalid object should now be added in the journal
query_result = whois_query('127.0.0.1', self.port_whois2,
'-g TEST:3:27-27')
assert 'ADD 27' in query_result
assert '192.0.2.0/24' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*')
assert query_result == 'TEST:Y:1-27:27\nRPKI:N:-'
# Make the v4 route in irrd2 invalid again
with open(self.roa_source2, 'w') as roa_file:
ujson.dump(
{
'roas': [{
'prefix': '128/1',
'asn': 'AS0',
'maxLength': '32',
'ta': 'TA'
}]
}, roa_file)
time.sleep(2)
query_result = whois_query_irrd('127.0.0.1', self.port_whois2,
'!gAS65537')
assert not query_result
# RPKI invalid object should now be deleted in the journal
query_result = whois_query('127.0.0.1', self.port_whois2,
'-g TEST:3:28-28')
assert 'DEL 28' in query_result
assert '192.0.2.0/24' in query_result
query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*')
assert query_result == 'TEST:Y:1-28:28\nRPKI:N:-'
# Make the v4 route in irrd1 invalid, triggering a mail
with open(self.roa_source1, 'w') as roa_file:
ujson.dump(
{
'roas': [{
'prefix': '128/1',
'asn': 'AS0',
'maxLength': '32',
'ta': 'TA'
}]
}, roa_file)
# irrd1 is authoritative for the now invalid v4 route, should have sent mail
time.sleep(2)
messages = self._retrieve_mails()
assert len(messages) == 3
mail_text = self._extract_message_body(messages[0])
assert messages[0][
'Subject'] == 'route(6) objects in TEST marked RPKI invalid'
expected_recipients = {
'*****@*****.**', '*****@*****.**', '*****@*****.**'
}
assert {m['To'] for m in messages} == expected_recipients
assert '192.0.2.0/24' in mail_text
status1 = requests.get(
f'http://127.0.0.1:{self.port_http1}/v1/status/')
status2 = requests.get(
f'http://127.0.0.1:{self.port_http2}/v1/status/')
assert status1.status_code == 200
assert status2.status_code == 200
assert 'IRRD version' in status1.text
assert 'IRRD version' in status2.text
assert 'TEST' in status1.text
assert 'TEST' in status2.text
assert 'RPKI' in status1.text
assert 'RPKI' in status2.text
assert 'Authoritative: Yes' in status1.text
assert 'Authoritative: Yes' not in status2.text
def __init__(self, input_file, host_reference, port_reference, host_tested,
port_tested):
self.host_reference = host_reference
self.port_reference = port_reference
self.host_tested = host_tested
self.port_tested = port_tested
if input_file == '-':
f = sys.stdin
else:
f = open(input_file, encoding='utf-8', errors='backslashreplace')
for query in f.readlines():
query = query.strip() + '\n'
if query == '!!\n':
continue
self.queries_run += 1
error_reference = None
error_tested = None
response_reference = None
response_tested = None
# ignore version or singular source queries
if query.lower().startswith('!v') or query.lower().startswith(
'!s'):
continue
if (query.startswith('-x')
and not query.startswith('-x ')) or re.search(
ASDOT_RE, query):
self.queries_invalid += 1
continue
# ignore queries asking for NRTM data or mirror serial status
if query.lower().startswith('-g ') or query.lower().startswith(
'!j'):
self.queries_mirror += 1
continue
if query.startswith('!'): # IRRD style query
try:
response_reference = whois_query_irrd(
self.host_reference, self.port_reference, query)
except ConnectionError as ce:
error_reference = str(ce)
except WhoisQueryError as wqe:
error_reference = str(wqe)
except ValueError:
print(f'Query response to {query} invalid')
continue
try:
response_tested = whois_query_irrd(self.host_tested,
self.port_tested, query)
except WhoisQueryError as wqe:
error_tested = str(wqe)
except ValueError:
print(f'Query response to {query} invalid')
continue
else: # RIPE style query
try:
response_reference = whois_query(self.host_reference,
self.port_reference,
query)
except ConnectionError as ce:
error_reference = str(ce)
response_tested = whois_query(self.host_tested,
self.port_tested, query)
# If both produce error messages, don't compare them
both_error = error_reference and error_tested
both_comment = (response_reference and response_tested
and response_reference.strip()
and response_tested.strip()
and response_reference.strip()[0] == '%'
and response_tested.strip()[0] == '%')
if both_error or both_comment:
self.queries_both_error += 1
continue
try:
cleaned_reference = self.clean(query, response_reference)
except ValueError as ve:
print(
f'Invalid reference response to query {query.strip()}: {response_reference}: {ve}'
)
continue
try:
cleaned_tested = self.clean(query, response_tested)
except ValueError as ve:
print(
f'Invalid tested response to query {query.strip()}: {response_tested}: {ve}'
)
continue
if cleaned_reference != cleaned_tested:
self.queries_different += 1
self.write_inconsistency_report(query, cleaned_reference,
cleaned_tested)
print(
f'Ran {self.queries_run} objects, {self.queries_different} had different results, '
f'{self.queries_both_error} produced errors on both instances, '
f'{self.queries_invalid} invalid queries were skipped, '
f'{self.queries_mirror} NRTM queries were skipped')
