def run(self, serial_newest_seen: int, database_handler: DatabaseHandler): serial_start = serial_newest_seen + 1 nrtm_host = get_setting(f'sources.{self.source}.nrtm_host') nrtm_port = int( get_setting(f'sources.{self.source}.nrtm_port', DEFAULT_SOURCE_NRTM_PORT)) if not nrtm_host: logger.debug( f'Skipping NRTM updates for {self.source}, nrtm_host not set.') return end_markings = [ f'\n%END {self.source}\n', f'\n% END {self.source}\n', '\n%ERROR', '\n% ERROR', '\n% Warning: there are no newer updates available', '\n% Warning (1): there are no newer updates available', ] logger.info( f'Retrieving NRTM updates for {self.source} from serial {serial_start} on {nrtm_host}:{nrtm_port}' ) query = f'-g {self.source}:3:{serial_start}-LAST' response = whois_query(nrtm_host, nrtm_port, query, end_markings) logger.debug( f'Received NRTM response for {self.source}: {response.strip()}') stream_parser = NRTMStreamParser(self.source, response, database_handler) for operation in stream_parser.operations: operation.save(database_handler)
def test_irrd_integration(self, tmpdir): # IRRD_DATABASE_URL overrides the yaml config, so should be removed if 'IRRD_DATABASE_URL' in os.environ: del os.environ['IRRD_DATABASE_URL'] # PYTHONPATH needs to contain the twisted plugin path. os.environ['PYTHONPATH'] = IRRD_ROOT_PATH os.environ['IRRD_SCHEDULER_TIMER_OVERRIDE'] = '1' self.tmpdir = tmpdir self._start_mailserver() self._start_irrds() # Attempt to load a mntner with valid auth, but broken references. self._submit_update(self.config_path1, SAMPLE_MNTNER + '\n\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate FAILED: [mntner] TEST-MNT\n' in mail_text assert '\nERROR: Object PERSON-TEST referenced in field admin-c not found in database TEST - must reference one of role, person.\n' in mail_text assert '\nERROR: Object OTHER1-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text assert '\nERROR: Object OTHER2-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text assert 'email footer' in mail_text assert 'Generated by IRRd version ' in mail_text # Load a regular valid mntner and person into the DB, and verify # the contents of the result. self._submit_update( self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON + '\n\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text assert 'email footer' in mail_text assert 'Generated by IRRd version ' in mail_text # Check whether the objects can be queried from irrd #1, # whether the hash is masked, and whether encoding is correct. mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # After three seconds, a new export should have been generated by irrd #1, # loaded by irrd #2, and the objects should be available in irrd #2 time.sleep(3) mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # Load a key-cert. This should cause notifications to mnt-nfy (2x). # Change is authenticated by valid password. self._submit_update(self.config_path1, SAMPLE_KEY_CERT + '\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 3 assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert 'Create succeeded: [key-cert] PGPKEY-80F238C6' in self._extract_message_body( messages[0]) self._check_recipients_in_mails( messages[1:], ['*****@*****.**', '*****@*****.**']) self._check_text_in_mails(messages[1:], [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nCreate succeeded for object below: [key-cert] PGPKEY-80F238C6:\n', 'email footer', 'Generated by IRRd version ', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Use the new PGP key to make an update to PERSON-TEST. Should # again trigger mnt-nfy messages, and a mail to the notify address # of PERSON-TEST. self._submit_update(self.config_path1, SIGNED_PERSON_UPDATE_VALID) messages = self._retrieve_mails() assert len(messages) == 4 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nModify succeeded: [person] PERSON-TEST\n' in mail_text self._check_recipients_in_mails(messages[1:], [ '*****@*****.**', '*****@*****.**', '*****@*****.**', ]) self._check_text_in_mails(messages[1:], [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nModify succeeded for object below: [person] PERSON-TEST:\n', '\n@@ -1,4 +1,4 @@\n', '\nNew version of this object:\n', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Check that the person is updated on irrd #1 person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # After 2s, NRTM from irrd #2 should have picked up the change. time.sleep(2) person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit an update back to the original person object, with an invalid # password and invalid override. Should trigger notification to upd-to. self._submit_update( self.config_path1, SAMPLE_PERSON + '\npassword: invalid\noverride: invalid\n') messages = self._retrieve_mails() assert len(messages) == 2 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nModify FAILED: [person] PERSON-TEST\n' in mail_text assert '\nERROR: Authorisation for person PERSON-TEST failed: must by authenticated by one of: TEST-MNT\n' in mail_text mail_text = self._extract_message_body(messages[1]) assert messages[1][ 'Subject'] == 'Notification of TEST database changes' assert messages[1]['From'] == '*****@*****.**' assert messages[1]['To'] == '*****@*****.**' assert '\nModify FAILED AUTHORISATION for object below: [person] PERSON-TEST:\n' in mail_text # Object should not have changed by latest update. person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit a delete with a valid password for PERSON-TEST. # This should be rejected, because it creates a dangling reference. # No mail should be sent to upd-to. self._submit_update( self.config_path1, SAMPLE_PERSON + 'password: md5-password\ndelete: delete\n') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nDelete FAILED: [person] PERSON-TEST\n' in mail_text assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by mntner TEST-MNT\n' in mail_text assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by key-cert PGPKEY-80F238C6\n' in mail_text # Object should not have changed by latest update. person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit a valid delete for all our new objects. self._submit_update( self.config_path1, f'{SAMPLE_PERSON}delete: delete\n\n{SAMPLE_KEY_CERT}delete: delete\n\n' + f'{SAMPLE_MNTNER_CLEAN}delete: delete\npassword: crypt-password\n') messages = self._retrieve_mails() # Expected mails are status, mnt-nfy on mntner (2x), and notify on mntner # (notify on PERSON-TEST was removed in the PGP signed update) assert len(messages) == 4 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nDelete succeeded: [person] PERSON-TEST\n' in mail_text assert '\nDelete succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nDelete succeeded: [key-cert] PGPKEY-80F238C6\n' in mail_text self._check_recipients_in_mails(messages[1:], [ '*****@*****.**', '*****@*****.**', '*****@*****.**', ]) mnt_nfy_msgs = [ msg for msg in messages if msg['To'] in ['*****@*****.**', '*****@*****.**'] ] self._check_text_in_mails( mnt_nfy_msgs, [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nDelete succeeded for object below: [person] PERSON-TEST:\n', '\nDelete succeeded for object below: [mntner] TEST-MNT:\n', '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n', 'unįcöde tæst 🌈🦄\n', # The object submitted to be deleted has the original name, # but when sending delete notifications, they should include the # object as currently in the DB, not as submitted in the email. 'Test person changed by PGP signed update\n', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Notify attribute mails are only about the objects concerned. notify_msg = [ msg for msg in messages if msg['To'] == '*****@*****.**' ][0] mail_text = self._extract_message_body(notify_msg) assert notify_msg['Subject'] == 'Notification of TEST database changes' assert notify_msg['From'] == '*****@*****.**' assert '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n' in mail_text assert '\nDelete succeeded for object below: [person] PERSON-TEST:\n' not in mail_text assert '\nDelete succeeded for object below: [mntner] TEST-MNT:\n' in mail_text assert '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n' not in mail_text # Object should be deleted person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'No entries found for the selected source(s)' in person_text assert 'PERSON-TEST' not in person_text # Object should be deleted from irrd #2 as well through NRTM. time.sleep(2) person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST') assert 'No entries found for the selected source(s)' in person_text assert 'PERSON-TEST' not in person_text # Load samples of all known objects. self._submit_update(self.config_path1, LARGE_UPDATE + '\n\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text assert '\nINFO: AS number as065537 was reformatted as AS65537\n' in mail_text assert '\nCreate succeeded: [filter-set] FLTR-SETTEST\n' in mail_text assert '\nINFO: Address range 192.0.2.0 - 192.0.02.255 was reformatted as 192.0.2.0 - 192.0.2.255\n' in mail_text assert '\nINFO: Address prefix 192.0.02.0/24 was reformatted as 192.0.2.0/24\n' in mail_text assert '\nINFO: Route set member 2001:0dB8::/48 was reformatted as 2001:db8::/48\n' in mail_text # Check whether the objects can be queried from irrd #1, # and whether the hash is masked. mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # (This is the first instance of an object with unicode chars # appearing on the NRTM stream.) time.sleep(3) mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # Test every major query type on both instances. for port in self.port_whois1, self.port_whois2: query_result = whois_query_irrd('127.0.0.1', port, '!gAS65537') assert query_result == '192.0.2.0/24' query_result = whois_query_irrd('127.0.0.1', port, '!6AS65537') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', port, '!iRS-TEST') assert query_result == '192.0.2.0/24 2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', port, '!iAS-SETTEST') assert query_result == 'AS65537 AS65538 AS65539' query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF') assert query_result == 'AS-SETTEST AS65540' query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF,1') assert query_result == 'AS65537 AS65538 AS65539 AS65540' query_result = whois_query_irrd('127.0.0.1', port, '!aAS-SETTEST') assert query_result == '192.0.2.0/24 2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', port, '!aAS-TESTREF') assert query_result == '192.0.2.0/24 2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', port, '!a4AS-TESTREF') assert query_result == '192.0.2.0/24' query_result = whois_query_irrd('127.0.0.1', port, '!a6AS-TESTREF') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', port, '!maut-num,as65537') assert 'AS65537' in query_result assert 'TEST-AS' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!oTEST-MNT') assert 'AS65537' in query_result assert 'TEST-AS' in query_result assert 'AS65536 - AS65538' in query_result assert 'rtrs-settest' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/25,l') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24,L') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/23,M') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24,o') assert query_result == 'AS65537' query_result = whois_query('127.0.0.1', port, '-x 192.0.02.0/24') assert 'example route' in query_result query_result = whois_query('127.0.0.1', port, '-l 192.0.02.0/25') assert 'example route' in query_result query_result = whois_query('127.0.0.1', port, '-L 192.0.02.0/24') assert 'example route' in query_result query_result = whois_query('127.0.0.1', port, '-M 192.0.02.0/23') assert 'example route' in query_result query_result = whois_query('127.0.0.1', port, '-i member-of RS-test') assert 'example route' in query_result query_result = whois_query('127.0.0.1', port, '-T route6 -i member-of RS-TEST') assert 'No entries found for the selected source(s)' in query_result query_result = whois_query('127.0.0.1', port, 'dashcare') assert 'ROLE-TEST' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!j-*') assert query_result == 'TEST:Y:1-29:29' # irrd #2 missed the first update from NRTM, as they were done at # the same time and loaded from the full export, so its serial should # start at 2 rather than 1. query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*') assert query_result == 'TEST:Y:2-29:29'
def test_irrd_integration(self, tmpdir): # IRRD_DATABASE_URL and IRRD_REDIS_URL override the yaml config, so should be removed if 'IRRD_DATABASE_URL' in os.environ: del os.environ['IRRD_DATABASE_URL'] if 'IRRD_REDIS_URL' in os.environ: del os.environ['IRRD_REDIS_URL'] # PYTHONPATH needs to contain the twisted plugin path to support the mailserver. os.environ['PYTHONPATH'] = IRRD_ROOT_PATH os.environ['IRRD_SCHEDULER_TIMER_OVERRIDE'] = '1' self.tmpdir = tmpdir self._start_mailserver() self._start_irrds() # Attempt to load a mntner with valid auth, but broken references. self._submit_update(self.config_path1, SAMPLE_MNTNER + '\n\noverride: override-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate FAILED: [mntner] TEST-MNT\n' in mail_text assert '\nERROR: Object PERSON-TEST referenced in field admin-c not found in database TEST - must reference one of role, person.\n' in mail_text assert '\nERROR: Object OTHER1-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text assert '\nERROR: Object OTHER2-MNT referenced in field mnt-by not found in database TEST - must reference mntner.\n' in mail_text assert 'email footer' in mail_text assert 'Generated by IRRd version ' in mail_text # Load a regular valid mntner and person into the DB, and verify # the contents of the result. self._submit_update( self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON + '\n\noverride: override-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text assert 'email footer' in mail_text assert 'Generated by IRRd version ' in mail_text # Check whether the objects can be queried from irrd #1, # whether the hash is masked, and whether encoding is correct. mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # After three seconds, a new export should have been generated by irrd #1, # loaded by irrd #2, and the objects should be available in irrd #2 time.sleep(3) mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # Load a key-cert. This should cause notifications to mnt-nfy (2x). # Change is authenticated by valid password. self._submit_update(self.config_path1, SAMPLE_KEY_CERT + '\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 3 assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert 'Create succeeded: [key-cert] PGPKEY-80F238C6' in self._extract_message_body( messages[0]) self._check_recipients_in_mails( messages[1:], ['*****@*****.**', '*****@*****.**']) self._check_text_in_mails(messages[1:], [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nCreate succeeded for object below: [key-cert] PGPKEY-80F238C6:\n', 'email footer', 'Generated by IRRd version ', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Use the new PGP key to make an update to PERSON-TEST. Should # again trigger mnt-nfy messages, and a mail to the notify address # of PERSON-TEST. self._submit_update(self.config_path1, SIGNED_PERSON_UPDATE_VALID) messages = self._retrieve_mails() assert len(messages) == 4 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nModify succeeded: [person] PERSON-TEST\n' in mail_text self._check_recipients_in_mails(messages[1:], [ '*****@*****.**', '*****@*****.**', '*****@*****.**', ]) self._check_text_in_mails(messages[1:], [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nModify succeeded for object below: [person] PERSON-TEST:\n', '\n@@ -1,4 +1,4 @@\n', '\nNew version of this object:\n', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Check that the person is updated on irrd #1 person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # After 2s, NRTM from irrd #2 should have picked up the change. time.sleep(2) person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit an update back to the original person object, with an invalid # password and invalid override. Should trigger notification to upd-to. self._submit_update( self.config_path1, SAMPLE_PERSON + '\npassword: invalid\noverride: invalid\n') messages = self._retrieve_mails() assert len(messages) == 2 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nModify FAILED: [person] PERSON-TEST\n' in mail_text assert '\nERROR: Authorisation for person PERSON-TEST failed: must by authenticated by one of: TEST-MNT\n' in mail_text mail_text = self._extract_message_body(messages[1]) assert messages[1][ 'Subject'] == 'Notification of TEST database changes' assert messages[1]['From'] == '*****@*****.**' assert messages[1]['To'] == '*****@*****.**' assert '\nModify FAILED AUTHORISATION for object below: [person] PERSON-TEST:\n' in mail_text # Object should not have changed by latest update. person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit a delete with a valid password for PERSON-TEST. # This should be rejected, because it creates a dangling reference. # No mail should be sent to upd-to. self._submit_update( self.config_path1, SAMPLE_PERSON + 'password: md5-password\ndelete: delete\n') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'FAILED: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nDelete FAILED: [person] PERSON-TEST\n' in mail_text assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by mntner TEST-MNT\n' in mail_text assert '\nERROR: Object PERSON-TEST to be deleted, but still referenced by key-cert PGPKEY-80F238C6\n' in mail_text # Object should not have changed by latest update. person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'PERSON-TEST' in person_text assert 'Test person changed by PGP signed update' in person_text # Submit a valid delete for all our new objects. self._submit_update( self.config_path1, f'{SAMPLE_PERSON}delete: delete\n\n{SAMPLE_KEY_CERT}delete: delete\n\n' + f'{SAMPLE_MNTNER_CLEAN}delete: delete\npassword: crypt-password\n') messages = self._retrieve_mails() # Expected mails are status, mnt-nfy on mntner (2x), and notify on mntner # (notify on PERSON-TEST was removed in the PGP signed update) assert len(messages) == 4 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nDelete succeeded: [person] PERSON-TEST\n' in mail_text assert '\nDelete succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nDelete succeeded: [key-cert] PGPKEY-80F238C6\n' in mail_text self._check_recipients_in_mails(messages[1:], [ '*****@*****.**', '*****@*****.**', '*****@*****.**', ]) mnt_nfy_msgs = [ msg for msg in messages if msg['To'] in ['*****@*****.**', '*****@*****.**'] ] self._check_text_in_mails( mnt_nfy_msgs, [ '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n', '\nDelete succeeded for object below: [person] PERSON-TEST:\n', '\nDelete succeeded for object below: [mntner] TEST-MNT:\n', '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n', 'unįcöde tæst 🌈🦄\n', # The object submitted to be deleted has the original name, # but when sending delete notifications, they should include the # object as currently in the DB, not as submitted in the email. 'Test person changed by PGP signed update\n', ]) for message in messages[1:]: assert message[ 'Subject'] == 'Notification of TEST database changes' assert message['From'] == '*****@*****.**' # Notify attribute mails are only about the objects concerned. notify_msg = [ msg for msg in messages if msg['To'] == '*****@*****.**' ][0] mail_text = self._extract_message_body(notify_msg) assert notify_msg['Subject'] == 'Notification of TEST database changes' assert notify_msg['From'] == '*****@*****.**' assert '\n> Message-ID: <1325754288.4989.6.camel@hostname>\n' in mail_text assert '\nDelete succeeded for object below: [person] PERSON-TEST:\n' not in mail_text assert '\nDelete succeeded for object below: [mntner] TEST-MNT:\n' in mail_text assert '\nDelete succeeded for object below: [key-cert] PGPKEY-80F238C6:\n' not in mail_text # Object should be deleted person_text = whois_query('127.0.0.1', self.port_whois1, 'PERSON-TEST') assert 'No entries found for the selected source(s)' in person_text assert 'PERSON-TEST' not in person_text # Object should be deleted from irrd #2 as well through NRTM. time.sleep(2) person_text = whois_query('127.0.0.1', self.port_whois2, 'PERSON-TEST') assert 'No entries found for the selected source(s)' in person_text assert 'PERSON-TEST' not in person_text # Load the mntner and person again, using the override password # Note that the route/route6 objects are RPKI valid on IRRd #1, # and RPKI-invalid on IRRd #2 self._submit_update( self.config_path1, SAMPLE_MNTNER_CLEAN + '\n\n' + SAMPLE_PERSON + '\n\noverride: override-password') messages = self._retrieve_mails() assert len(messages) == 1 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nCreate succeeded: [mntner] TEST-MNT\n' in mail_text assert '\nCreate succeeded: [person] PERSON-TEST\n' in mail_text assert 'email footer' in mail_text assert 'Generated by IRRd version ' in mail_text # Load samples of all known objects, using the mntner password self._submit_update(self.config_path1, LARGE_UPDATE + '\n\npassword: md5-password') messages = self._retrieve_mails() assert len(messages) == 3 mail_text = self._extract_message_body(messages[0]) assert messages[0]['Subject'] == 'SUCCESS: my subject' assert messages[0]['From'] == '*****@*****.**' assert messages[0]['To'] == 'Sasha <*****@*****.**>' assert '\nINFO: AS number as065537 was reformatted as AS65537\n' in mail_text assert '\nCreate succeeded: [filter-set] FLTR-SETTEST\n' in mail_text assert '\nINFO: Address range 192.0.2.0 - 192.0.02.255 was reformatted as 192.0.2.0 - 192.0.2.255\n' in mail_text assert '\nINFO: Address prefix 192.0.02.0/24 was reformatted as 192.0.2.0/24\n' in mail_text assert '\nINFO: Route set member 2001:0dB8::/48 was reformatted as 2001:db8::/48\n' in mail_text # Check whether the objects can be queried from irrd #1, # and whether the hash is masked. mntner_text = whois_query('127.0.0.1', self.port_whois1, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # (This is the first instance of an object with unicode chars # appearing on the NRTM stream.) time.sleep(3) mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT') assert 'TEST-MNT' in mntner_text assert PASSWORD_HASH_DUMMY_VALUE in mntner_text assert 'unįcöde tæst 🌈🦄' in mntner_text assert 'PERSON-TEST' in mntner_text # These queries have different responses on #1 than #2, # as all IPv4 routes are RPKI invalid on #2. query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!gAS65537') assert query_result == '192.0.2.0/24' query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!gAS65547') assert query_result == '192.0.2.0/32' # Pseudo-IRR object from RPKI query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!6AS65537') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!iRS-TEST') assert set( query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'} query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!aAS-SETTEST') assert set( query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'} query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!aAS-TESTREF') assert set( query_result.split(' ')) == {'192.0.2.0/24', '2001:db8::/48'} query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!a4AS-TESTREF') assert query_result == '192.0.2.0/24' query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!a6AS-TESTREF') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/24') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/25,l') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/24,L') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/23,M') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/24,M') assert 'RPKI' in query_result # Does not match the /24, does match the RPKI pseudo-IRR /32 query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!r192.0.2.0/24,o') assert query_result == 'AS65537' query_result = whois_query('127.0.0.1', self.port_whois1, '-x 192.0.02.0/24') assert 'example route' in query_result query_result = whois_query('127.0.0.1', self.port_whois1, '-l 192.0.02.0/25') assert 'example route' in query_result query_result = whois_query('127.0.0.1', self.port_whois1, '-L 192.0.02.0/24') assert 'example route' in query_result query_result = whois_query('127.0.0.1', self.port_whois1, '-M 192.0.02.0/23') assert 'example route' in query_result query_result = whois_query('127.0.0.1', self.port_whois1, '-i member-of RS-test') assert 'example route' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!gAS65537') assert not query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!6AS65537') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!iRS-TEST') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!aAS-SETTEST') assert query_result == '2001:db8::/48' query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!aAS-TESTREF') assert query_result == '2001:db8::/48' query_result = whois_query('127.0.0.1', self.port_whois2, '-x 192.0.02.0/24') assert 'example route' not in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!r192.0.2.0/24,L') assert 'RPKI' in query_result # Pseudo-IRR object 0/0 from RPKI # RPKI invalid object should not be in journal query_result = whois_query('127.0.0.1', self.port_whois2, '-g TEST:3:1-LAST') assert 'route:192.0.2.0/24' not in query_result.replace(' ', '') # These queries should produce identical answers on both instances. for port in self.port_whois1, self.port_whois2: query_result = whois_query_irrd('127.0.0.1', port, '!iAS-SETTEST') assert set( query_result.split(' ')) == {'AS65537', 'AS65538', 'AS65539'} query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF') assert set(query_result.split(' ')) == {'AS-SETTEST', 'AS65540'} query_result = whois_query_irrd('127.0.0.1', port, '!iAS-TESTREF,1') assert set(query_result.split(' ')) == { 'AS65537', 'AS65538', 'AS65539', 'AS65540' } query_result = whois_query_irrd('127.0.0.1', port, '!maut-num,as65537') assert 'AS65537' in query_result assert 'TEST-AS' in query_result query_result = whois_query_irrd('127.0.0.1', port, '!oTEST-MNT') assert 'AS65537' in query_result assert 'TEST-AS' in query_result assert 'AS65536 - AS65538' in query_result assert 'rtrs-settest' in query_result query_result = whois_query('127.0.0.1', port, '-T route6 -i member-of RS-TEST') assert 'No entries found for the selected source(s)' in query_result query_result = whois_query('127.0.0.1', port, 'dashcare') assert 'ROLE-TEST' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!j-*') assert query_result == 'TEST:Y:1-29:29\nRPKI:N:-' # irrd #2 missed the first update from NRTM, as they were done at # the same time and loaded from the full export, and one RPKI-invalid object # was not recorded in the journal, so its serial should # is lower by three query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*') assert query_result == 'TEST:Y:1-26:26\nRPKI:N:-' # Make the v4 route in irrd2 valid with open(self.roa_source2, 'w') as roa_file: ujson.dump( { 'roas': [{ 'prefix': '198.51.100.0/24', 'asn': 'AS0', 'maxLength': '32', 'ta': 'TA' }] }, roa_file) time.sleep(2) query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!gAS65537') assert query_result == '192.0.2.0/24' # RPKI invalid object should now be added in the journal query_result = whois_query('127.0.0.1', self.port_whois2, '-g TEST:3:27-27') assert 'ADD 27' in query_result assert '192.0.2.0/24' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*') assert query_result == 'TEST:Y:1-27:27\nRPKI:N:-' # Make the v4 route in irrd2 invalid again with open(self.roa_source2, 'w') as roa_file: ujson.dump( { 'roas': [{ 'prefix': '128/1', 'asn': 'AS0', 'maxLength': '32', 'ta': 'TA' }] }, roa_file) time.sleep(2) query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!gAS65537') assert not query_result # RPKI invalid object should now be deleted in the journal query_result = whois_query('127.0.0.1', self.port_whois2, '-g TEST:3:28-28') assert 'DEL 28' in query_result assert '192.0.2.0/24' in query_result query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*') assert query_result == 'TEST:Y:1-28:28\nRPKI:N:-' # Make the v4 route in irrd1 invalid, triggering a mail with open(self.roa_source1, 'w') as roa_file: ujson.dump( { 'roas': [{ 'prefix': '128/1', 'asn': 'AS0', 'maxLength': '32', 'ta': 'TA' }] }, roa_file) # irrd1 is authoritative for the now invalid v4 route, should have sent mail time.sleep(2) messages = self._retrieve_mails() assert len(messages) == 3 mail_text = self._extract_message_body(messages[0]) assert messages[0][ 'Subject'] == 'route(6) objects in TEST marked RPKI invalid' expected_recipients = { '*****@*****.**', '*****@*****.**', '*****@*****.**' } assert {m['To'] for m in messages} == expected_recipients assert '192.0.2.0/24' in mail_text status1 = requests.get( f'http://127.0.0.1:{self.port_http1}/v1/status/') status2 = requests.get( f'http://127.0.0.1:{self.port_http2}/v1/status/') assert status1.status_code == 200 assert status2.status_code == 200 assert 'IRRD version' in status1.text assert 'IRRD version' in status2.text assert 'TEST' in status1.text assert 'TEST' in status2.text assert 'RPKI' in status1.text assert 'RPKI' in status2.text assert 'Authoritative: Yes' in status1.text assert 'Authoritative: Yes' not in status2.text
def __init__(self, input_file, host_reference, port_reference, host_tested, port_tested): self.host_reference = host_reference self.port_reference = port_reference self.host_tested = host_tested self.port_tested = port_tested if input_file == '-': f = sys.stdin else: f = open(input_file, encoding='utf-8', errors='backslashreplace') for query in f.readlines(): query = query.strip() + '\n' if query == '!!\n': continue self.queries_run += 1 error_reference = None error_tested = None response_reference = None response_tested = None # ignore version or singular source queries if query.lower().startswith('!v') or query.lower().startswith( '!s'): continue if (query.startswith('-x') and not query.startswith('-x ')) or re.search( ASDOT_RE, query): self.queries_invalid += 1 continue # ignore queries asking for NRTM data or mirror serial status if query.lower().startswith('-g ') or query.lower().startswith( '!j'): self.queries_mirror += 1 continue if query.startswith('!'): # IRRD style query try: response_reference = whois_query_irrd( self.host_reference, self.port_reference, query) except ConnectionError as ce: error_reference = str(ce) except WhoisQueryError as wqe: error_reference = str(wqe) except ValueError: print(f'Query response to {query} invalid') continue try: response_tested = whois_query_irrd(self.host_tested, self.port_tested, query) except WhoisQueryError as wqe: error_tested = str(wqe) except ValueError: print(f'Query response to {query} invalid') continue else: # RIPE style query try: response_reference = whois_query(self.host_reference, self.port_reference, query) except ConnectionError as ce: error_reference = str(ce) response_tested = whois_query(self.host_tested, self.port_tested, query) # If both produce error messages, don't compare them both_error = error_reference and error_tested both_comment = (response_reference and response_tested and response_reference.strip() and response_tested.strip() and response_reference.strip()[0] == '%' and response_tested.strip()[0] == '%') if both_error or both_comment: self.queries_both_error += 1 continue try: cleaned_reference = self.clean(query, response_reference) except ValueError as ve: print( f'Invalid reference response to query {query.strip()}: {response_reference}: {ve}' ) continue try: cleaned_tested = self.clean(query, response_tested) except ValueError as ve: print( f'Invalid tested response to query {query.strip()}: {response_tested}: {ve}' ) continue if cleaned_reference != cleaned_tested: self.queries_different += 1 self.write_inconsistency_report(query, cleaned_reference, cleaned_tested) print( f'Ran {self.queries_run} objects, {self.queries_different} had different results, ' f'{self.queries_both_error} produced errors on both instances, ' f'{self.queries_invalid} invalid queries were skipped, ' f'{self.queries_mirror} NRTM queries were skipped')