def update(source, filename) -> int: if any([ get_setting(f'sources.{source}.import_source'), get_setting(f'sources.{source}.import_serial_source') ]): print( f'Error: to use this command, import_source and import_serial_source ' f'for source {source} must not be set.') return 2 dh = DatabaseHandler() roa_validator = BulkRouteROAValidator(dh) parser = MirrorUpdateFileImportParser(source, filename, database_handler=dh, direct_error_return=True, roa_validator=roa_validator) error = parser.run_import() if error: dh.rollback() else: dh.commit() dh.close() if error: print(f'Error occurred while processing object:\n{error}') return 1 return 0
def load(source, filename, serial) -> int: if any([ get_setting(f'sources.{source}.import_source'), get_setting(f'sources.{source}.import_serial_source') ]): print(f'Error: to use this command, import_source and import_serial_source ' f'for source {source} must not be set.') return 2 dh = DatabaseHandler() roa_validator = BulkRouteROAValidator(dh) dh.delete_all_rpsl_objects_with_journal(source) dh.disable_journaling() parser = MirrorFileImportParser( source=source, filename=filename, serial=serial, database_handler=dh, direct_error_return=True, roa_validator=roa_validator) error = parser.run_import() if error: dh.rollback() else: dh.commit() dh.close() if error: print(f'Error occurred while processing object:\n{error}') return 1 return 0
def set_last_modified(): dh = DatabaseHandler() auth_sources = [ k for k, v in get_setting('sources').items() if v.get('authoritative') ] q = RPSLDatabaseQuery(column_names=['pk', 'object_text', 'updated'], enable_ordering=False) q = q.sources(auth_sources) results = list(dh.execute_query(q)) print(f'Updating {len(results)} objects in sources {auth_sources}') for result in results: rpsl_obj = rpsl_object_from_text(result['object_text'], strict_validation=False) if rpsl_obj.messages.errors(): # pragma: no cover print( f'Failed to process {rpsl_obj}: {rpsl_obj.messages.errors()}') continue new_text = rpsl_obj.render_rpsl_text(result['updated']) stmt = RPSLDatabaseObject.__table__.update().where( RPSLDatabaseObject.__table__.c.pk == result['pk']).values( object_text=new_text, ) dh.execute_statement(stmt) dh.commit() dh.close()
class SourceExportRunner: """ This SourceExportRunner is the entry point for the expect process for a single source. A gzipped file will be created in the export_destination directory with the contents of the source, along with a CURRENTSERIAL file. The contents of the source are first written to a temporary file, and then moved in place. """ def __init__(self, source: str) -> None: self.source = source def run(self) -> None: self.database_handler = DatabaseHandler() try: export_destination = get_setting(f'sources.{self.source}.export_destination') logger.info(f'Starting a source export for {self.source} to {export_destination}') self._export(export_destination) self.database_handler.commit() except Exception as exc: logger.error(f'An exception occurred while attempting to run an export ' f'for {self.source}: {exc}', exc_info=exc) finally: self.database_handler.close() def _export(self, export_destination): filename_export = Path(export_destination) / f'{self.source.lower()}.db.gz' export_tmpfile = NamedTemporaryFile(delete=False) filename_serial = Path(export_destination) / f'{self.source.upper()}.CURRENTSERIAL' query = DatabaseStatusQuery().source(self.source) try: serial = next(self.database_handler.execute_query(query))['serial_newest_seen'] except StopIteration: logger.error(f'Unable to run export for {self.source}, internal database status is empty.') return with gzip.open(export_tmpfile, 'wb') as fh: query = RPSLDatabaseQuery().sources([self.source]) for obj in self.database_handler.execute_query(query): object_bytes = remove_auth_hashes(obj['object_text']).encode('utf-8') fh.write(object_bytes + b'\n') if filename_export.exists(): os.unlink(filename_export) if filename_serial.exists(): os.unlink(filename_serial) shutil.move(export_tmpfile.name, filename_export) if serial is not None: with open(filename_serial, 'w') as fh: fh.write(str(serial)) self.database_handler.record_serial_exported(self.source, serial) logger.info(f'Export for {self.source} complete, stored in {filename_export} / {filename_serial}')
class RPSLParse: obj_parsed = 0 obj_errors = 0 obj_unknown = 0 unknown_object_classes: Set[str] = set() database_handler = None def main(self, filename, strict_validation, database, show_info=True): self.show_info = show_info if database: self.database_handler = DatabaseHandler(journaling_enabled=False) if filename == '-': # pragma: no cover f = sys.stdin else: f = open(filename, encoding='utf-8', errors='backslashreplace') for paragraph in split_paragraphs_rpsl(f): self.parse_object(paragraph, strict_validation) print( f'Processed {self.obj_parsed} objects, {self.obj_errors} with errors' ) if self.obj_unknown: unknown_formatted = ', '.join(self.unknown_object_classes) print( f'Ignored {self.obj_unknown} objects due to unknown object classes: {unknown_formatted}' ) if self.database_handler: self.database_handler.commit() self.database_handler.close() def parse_object(self, rpsl_text, strict_validation): try: self.obj_parsed += 1 obj = rpsl_object_from_text(rpsl_text.strip(), strict_validation=strict_validation) if (obj.messages.messages() and self.show_info) or obj.messages.errors(): if obj.messages.errors(): self.obj_errors += 1 print(rpsl_text.strip()) print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') print(obj.messages) print('\n=======================================\n') if self.database_handler and obj and not obj.messages.errors(): self.database_handler.upsert_rpsl_object(obj) except UnknownRPSLObjectClassException as e: self.obj_unknown += 1 self.unknown_object_classes.add(str(e).split(':')[1].strip()) except Exception as e: # pragma: no cover print('=======================================') print(rpsl_text) print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') raise e
class RPSLMirrorImportUpdateRunner: """ This RPSLMirrorImportUpdateRunner is the entry point for updating a single database mirror, depending on current state. If there is no current mirrored data, will call RPSLMirrorFullImportRunner to run a new import from full export files. Otherwise, will call NRTMImportUpdateStreamRunner to retrieve new updates from NRTM. """ def __init__(self, source: str) -> None: self.source = source self.full_import_runner = RPSLMirrorFullImportRunner(source) self.update_stream_runner = NRTMImportUpdateStreamRunner(source) def run(self) -> None: self.database_handler = DatabaseHandler() try: serial_newest_mirror, force_reload = self._status() nrtm_enabled = bool( get_setting(f'sources.{self.source}.nrtm_host')) logger.debug( f'Most recent mirrored serial for {self.source}: {serial_newest_mirror}, ' f'force_reload: {force_reload}, nrtm enabled: {nrtm_enabled}') if force_reload or not serial_newest_mirror or not nrtm_enabled: self.full_import_runner.run( database_handler=self.database_handler, serial_newest_mirror=serial_newest_mirror, force_reload=force_reload) else: self.update_stream_runner.run( serial_newest_mirror, database_handler=self.database_handler) self.database_handler.commit() except OSError as ose: # I/O errors can occur and should not log a full traceback (#177) logger.error( f'An error occurred while attempting a mirror update or initial import ' f'for {self.source}: {ose}') except Exception as exc: logger.error( f'An exception occurred while attempting a mirror update or initial import ' f'for {self.source}: {exc}', exc_info=exc) finally: self.database_handler.close() def _status(self) -> Tuple[Optional[int], Optional[bool]]: query = DatabaseStatusQuery().source(self.source) result = self.database_handler.execute_query(query) try: status = next(result) return status['serial_newest_mirror'], status['force_reload'] except StopIteration: return None, None
def generate_status(self) -> str: """ Generate a human-readable overview of database status. """ database_handler = DatabaseHandler() statistics_query = RPSLDatabaseObjectStatisticsQuery() self.statistics_results = list(database_handler.execute_query(statistics_query)) status_query = DatabaseStatusQuery() self.status_results = list(database_handler.execute_query(status_query)) results = [self._generate_header(), self._generate_statistics_table(), self._generate_source_detail()] database_handler.close() return '\n\n'.join(results)
def load(source, filename, serial) -> int: dh = DatabaseHandler() dh.delete_all_rpsl_objects_with_journal(source) dh.disable_journaling() parser = MirrorFileImportParser(source, filename, serial=serial, database_handler=dh, direct_error_return=True) error = parser.run_import() if error: dh.rollback() else: dh.commit() dh.close() if error: print(f'Error occurred while processing object:\n{error}') return 1 return 0
def load_pgp_keys(source: str) -> None: dh = DatabaseHandler() query = RPSLDatabaseQuery(column_names=['rpsl_pk', 'object_text']) query = query.sources([source]).object_classes(['key-cert']) keycerts = dh.execute_query(query) for keycert in keycerts: rpsl_pk = keycert["rpsl_pk"] print(f'Loading key-cert {rpsl_pk}') # Parsing the keycert in strict mode will load it into the GPG keychain result = rpsl_object_from_text(keycert['object_text'], strict_validation=True) if result.messages.errors(): print(f'Errors in PGP key {rpsl_pk}: {result.messages.errors()}') print('All valid key-certs loaded into the GnuPG keychain.') dh.close()
class MirrorUpdateRunner: """ This MirrorUpdateRunner is the entry point for updating a single database mirror, depending on current state. If there is no current mirrored data, will call MirrorFullImportRunner to run a new import from full export files. Otherwise, will call NRTMUpdateStreamRunner to retrieve new updates from NRTM. """ def __init__(self, source: str) -> None: self.source = source self.full_import_runner = MirrorFullImportRunner(source) self.update_stream_runner = NRTMUpdateStreamRunner(source) def run(self) -> None: self.database_handler = DatabaseHandler() try: serial_newest_seen, force_reload = self._status() logger.debug( f'Most recent serial seen for {self.source}: {serial_newest_seen}, force_reload: {force_reload}' ) if not serial_newest_seen or force_reload: self.full_import_runner.run( database_handler=self.database_handler) else: self.update_stream_runner.run( serial_newest_seen, database_handler=self.database_handler) self.database_handler.commit() except Exception as exc: logger.critical( f'An exception occurred while attempting a mirror update or initial import ' f'for {self.source}: {exc}', exc_info=exc) finally: self.database_handler.close() def _status(self) -> Tuple[Optional[int], Optional[bool]]: query = DatabaseStatusQuery().source(self.source) result = self.database_handler.execute_query(query) try: status = next(result) return status['serial_newest_seen'], status['force_reload'] except StopIteration: return None, None
class ScopeFilterUpdateRunner: """ Update the scope filter status for all objects. This runner does not actually import anything, the scope filter is in the configuration. """ # API consistency with other importers, source is actually ignored def __init__(self, source=None): pass def run(self): self.database_handler = DatabaseHandler() try: validator = ScopeFilterValidator() status = validator.validate_all_rpsl_objects(self.database_handler) rpsl_objs_now_in_scope, rpsl_objs_now_out_scope_as, rpsl_objs_now_out_scope_prefix = status self.database_handler.update_scopefilter_status( rpsl_objs_now_in_scope=rpsl_objs_now_in_scope, rpsl_objs_now_out_scope_as=rpsl_objs_now_out_scope_as, rpsl_objs_now_out_scope_prefix=rpsl_objs_now_out_scope_prefix, ) self.database_handler.commit() logger.info( f'Scopefilter status updated for all routes, ' f'{len(rpsl_objs_now_in_scope)} newly in scope, ' f'{len(rpsl_objs_now_out_scope_as)} newly out of scope AS, ' f'{len(rpsl_objs_now_out_scope_prefix)} newly out of scope prefix' ) except Exception as exc: logger.error( f'An exception occurred while attempting a scopefilter status update: {exc}', exc_info=exc) finally: self.database_handler.close()
class ROAImportRunner(FileImportRunnerBase): """ This runner performs a full import of ROA objects. The URL file for the ROA export in JSON format is provided in the configuration. """ # API consistency with other importers, source is actually ignored def __init__(self, source=None): pass def run(self): self.database_handler = DatabaseHandler() try: self.database_handler.disable_journaling() roa_objs = self._import_roas() # Do an early commit to make the new ROAs available to other processes. self.database_handler.commit() # The ROA import does not use journaling, but updating the RPKI # status may create journal entries. self.database_handler.enable_journaling() validator = BulkRouteROAValidator(self.database_handler, roa_objs) objs_now_valid, objs_now_invalid, objs_now_not_found = validator.validate_all_routes() self.database_handler.update_rpki_status( rpsl_objs_now_valid=objs_now_valid, rpsl_objs_now_invalid=objs_now_invalid, rpsl_objs_now_not_found=objs_now_not_found, ) self.database_handler.commit() notified = notify_rpki_invalid_owners(self.database_handler, objs_now_invalid) logger.info(f'RPKI status updated for all routes, {len(objs_now_valid)} newly valid, ' f'{len(objs_now_invalid)} newly invalid, ' f'{len(objs_now_not_found)} newly not_found routes, ' f'{notified} emails sent to contacts of newly invalid authoritative objects') except OSError as ose: # I/O errors can occur and should not log a full traceback (#177) logger.error(f'An error occurred while attempting a ROA import: {ose}') except ROAParserException as rpe: logger.error(f'An exception occurred while attempting a ROA import: {rpe}') except Exception as exc: logger.error(f'An exception occurred while attempting a ROA import: {exc}', exc_info=exc) finally: self.database_handler.close() def _import_roas(self): roa_source = get_setting('rpki.roa_source') slurm_source = get_setting('rpki.slurm_source') logger.info(f'Running full ROA import from: {roa_source}, SLURM {slurm_source}') self.database_handler.delete_all_roa_objects() self.database_handler.delete_all_rpsl_objects_with_journal(RPKI_IRR_PSEUDO_SOURCE) slurm_data = None if slurm_source: slurm_data, _ = self._retrieve_file(slurm_source, return_contents=True) roa_filename, roa_to_delete = self._retrieve_file(roa_source, return_contents=False) with open(roa_filename) as fh: roa_importer = ROADataImporter(fh.read(), slurm_data, self.database_handler) if roa_to_delete: os.unlink(roa_filename) logger.info(f'ROA import from {roa_source}, SLURM {slurm_source}, imported {len(roa_importer.roa_objs)} ROAs, running validator') return roa_importer.roa_objs
class ChangeSubmissionHandler: """ The ChangeSubmissionHandler handles the text of one or more requested RPSL changes (create, modify or delete), parses, validates and eventually saves them. This includes validating references between objects, including those part of the same message, and checking authentication. """ def load_text_blob(self, object_texts_blob: str, pgp_fingerprint: str = None, request_meta: Dict[str, Optional[str]] = None): self.database_handler = DatabaseHandler() self.request_meta = request_meta if request_meta else {} self._pgp_key_id = self._resolve_pgp_key_id( pgp_fingerprint) if pgp_fingerprint else None reference_validator = ReferenceValidator(self.database_handler) auth_validator = AuthValidator(self.database_handler, self._pgp_key_id) change_requests = parse_change_requests(object_texts_blob, self.database_handler, auth_validator, reference_validator) self._handle_change_requests(change_requests, reference_validator, auth_validator) self.database_handler.commit() self.database_handler.close() return self def load_change_submission(self, data: RPSLChangeSubmission, delete=False, request_meta: Dict[str, Optional[str]] = None): self.database_handler = DatabaseHandler() self.request_meta = request_meta if request_meta else {} reference_validator = ReferenceValidator(self.database_handler) auth_validator = AuthValidator(self.database_handler) change_requests: List[Union[ChangeRequest, SuspensionRequest]] = [] delete_reason = None if delete: delete_reason = data.delete_reason auth_validator.passwords = data.passwords auth_validator.overrides = [data.override] if data.override else [] for rpsl_obj in data.objects: object_text = rpsl_obj.object_text if rpsl_obj.attributes: # We don't have a neat way to process individual attribute pairs, # so construct a pseudo-object by appending the text. composite_object = [] for attribute in rpsl_obj.attributes: composite_object.append(attribute.name + ': ' + attribute.value) # type: ignore object_text = '\n'.join(composite_object) + '\n' assert object_text # enforced by pydantic change_requests.append( ChangeRequest(object_text, self.database_handler, auth_validator, reference_validator, delete_reason)) self._handle_change_requests(change_requests, reference_validator, auth_validator) self.database_handler.commit() self.database_handler.close() return self def load_suspension_submission(self, data: RPSLSuspensionSubmission, request_meta: Dict[str, Optional[str]] = None): self.database_handler = DatabaseHandler() self.request_meta = request_meta if request_meta else {} reference_validator = ReferenceValidator(self.database_handler) auth_validator = AuthValidator(self.database_handler) change_requests: List[Union[ChangeRequest, SuspensionRequest]] = [] auth_validator.overrides = [data.override] if data.override else [] for rpsl_obj in data.objects: # We don't have a neat way to process individual attribute pairs, # so construct a pseudo-object by appending the text. object_text = f"mntner: {rpsl_obj.mntner}\nsource: {rpsl_obj.source}\n" change_requests.append( SuspensionRequest( object_text, self.database_handler, auth_validator, rpsl_obj.request_type.value, )) self._handle_change_requests(change_requests, reference_validator, auth_validator) self.database_handler.commit() self.database_handler.close() return self def _handle_change_requests(self, change_requests: List[ Union[ChangeRequest, SuspensionRequest]], reference_validator: ReferenceValidator, auth_validator: AuthValidator) -> None: objects = ', '.join([ f'{request.rpsl_obj_new} (request {id(request)})' for request in change_requests ]) logger.info( f'Processing change requests for {objects}, metadata is {self.request_meta}' ) # When an object references another object, e.g. tech-c referring a person or mntner, # an add/update is only valid if those referred objects exist. To complicate matters, # the object referred to may be part of this very same submission. For this reason, the # reference validator can be provided with all new objects to be added in this submission. # However, a possible scenario is that A, B and C are submitted. Object A refers to B, # B refers to C, C refers to D and D does not exist - or C fails authentication. # At a first scan, A is valid because B exists, B is valid because C exists. C # becomes invalid on the first scan, which is why another scan is performed, which # will mark B invalid due to the reference to an invalid C, etc. This continues until # all references are resolved and repeated scans lead to the same conclusions. valid_changes = [r for r in change_requests if r.is_valid()] previous_valid_changes: List[Union[ChangeRequest, SuspensionRequest]] = [] loop_count = 0 loop_max = len(change_requests) + 10 while valid_changes != previous_valid_changes: previous_valid_changes = valid_changes reference_validator.preload(valid_changes) valid_potential_new_mntners = [ r.rpsl_obj_new for r in valid_changes if r.request_type == UpdateRequestType.CREATE and isinstance(r.rpsl_obj_new, RPSLMntner) ] auth_validator.pre_approve(valid_potential_new_mntners) for result in valid_changes: result.validate() valid_changes = [r for r in change_requests if r.is_valid()] loop_count += 1 if loop_count > loop_max: # pragma: no cover msg = f'Update validity resolver ran an excessive amount of loops, may be stuck, aborting ' \ f'processing. Message metadata: {self.request_meta}' logger.error(msg) raise ValueError(msg) for result in change_requests: if result.is_valid(): result.save() self.results = change_requests def _resolve_pgp_key_id(self, pgp_fingerprint: str) -> Optional[str]: """ Find a PGP key ID for a given fingerprint. This method looks for an actual matching object in the database, and then returns the object's PK. """ clean_fingerprint = pgp_fingerprint.replace(' ', '') key_id = 'PGPKEY-' + clean_fingerprint[-8:] query = RPSLDatabaseQuery().object_classes(['key-cert' ]).rpsl_pk(key_id) results = list(self.database_handler.execute_query(query)) for result in results: if result['parsed_data'].get('fingerpr', '').replace(' ', '') == clean_fingerprint: return key_id logger.info( f'Message was signed with key {key_id}, but key was not found in the database. Treating message ' f'as unsigned. Message metadata: {self.request_meta}') return None def status(self) -> str: """Provide a simple SUCCESS/FAILED string based - former used if all objects were saved.""" if all([ result.status == UpdateRequestStatus.SAVED for result in self.results ]): return 'SUCCESS' return 'FAILED' def submitter_report_human(self) -> str: """Produce a human-readable report for the submitter.""" # flake8: noqa: W293 successful = [ r for r in self.results if r.status == UpdateRequestStatus.SAVED ] failed = [ r for r in self.results if r.status != UpdateRequestStatus.SAVED ] number_successful_create = len([ r for r in successful if r.request_type == UpdateRequestType.CREATE ]) number_successful_modify = len([ r for r in successful if r.request_type == UpdateRequestType.MODIFY ]) number_successful_delete = len([ r for r in successful if r.request_type == UpdateRequestType.DELETE ]) number_failed_create = len( [r for r in failed if r.request_type == UpdateRequestType.CREATE]) number_failed_modify = len( [r for r in failed if r.request_type == UpdateRequestType.MODIFY]) number_failed_delete = len( [r for r in failed if r.request_type == UpdateRequestType.DELETE]) user_report = self._request_meta_str() + textwrap.dedent(f""" SUMMARY OF UPDATE: Number of objects found: {len(self.results):3} Number of objects processed successfully: {len(successful):3} Create: {number_successful_create:3} Modify: {number_successful_modify:3} Delete: {number_successful_delete:3} Number of objects processed with errors: {len(failed):3} Create: {number_failed_create:3} Modify: {number_failed_modify:3} Delete: {number_failed_delete:3} DETAILED EXPLANATION: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """) for result in self.results: user_report += '---\n' user_report += result.submitter_report_human() user_report += '\n' user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' return user_report def submitter_report_json(self): """Produce a JSON-ready report for the submitter.""" successful = [ r for r in self.results if r.status == UpdateRequestStatus.SAVED ] failed = [ r for r in self.results if r.status != UpdateRequestStatus.SAVED ] number_successful_create = len([ r for r in successful if r.request_type == UpdateRequestType.CREATE ]) number_successful_modify = len([ r for r in successful if r.request_type == UpdateRequestType.MODIFY ]) number_successful_delete = len([ r for r in successful if r.request_type == UpdateRequestType.DELETE ]) number_failed_create = len( [r for r in failed if r.request_type == UpdateRequestType.CREATE]) number_failed_modify = len( [r for r in failed if r.request_type == UpdateRequestType.MODIFY]) number_failed_delete = len( [r for r in failed if r.request_type == UpdateRequestType.DELETE]) return { 'request_meta': self.request_meta, 'summary': { 'objects_found': len(self.results), 'successful': len(successful), 'successful_create': number_successful_create, 'successful_modify': number_successful_modify, 'successful_delete': number_successful_delete, 'failed': len(failed), 'failed_create': number_failed_create, 'failed_modify': number_failed_modify, 'failed_delete': number_failed_delete, }, 'objects': [result.submitter_report_json() for result in self.results], } def send_notification_target_reports(self): # First key is e-mail address of recipient, second is UpdateRequestStatus.SAVED # or UpdateRequestStatus.ERROR_AUTH reports_per_recipient: Dict[str, Dict[UpdateRequestStatus, OrderedSet]] = defaultdict(dict) sources: OrderedSet[str] = OrderedSet() for result in self.results: for target in result.notification_targets(): if result.status in [ UpdateRequestStatus.SAVED, UpdateRequestStatus.ERROR_AUTH ]: if result.status not in reports_per_recipient[target]: reports_per_recipient[target][ result.status] = OrderedSet() reports_per_recipient[target][result.status].add( result.notification_target_report()) sources.add(result.rpsl_obj_new.source()) sources_str = '/'.join(sources) subject = f'Notification of {sources_str} database changes' header = get_setting('email.notification_header', '').format(sources_str=sources_str) header += '\nThis message is auto-generated.\n' header += 'The request was made with the following details:\n' header_saved = textwrap.dedent(""" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some objects in which you are referenced have been created, deleted or changed. """) header_failed = textwrap.dedent(""" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some objects in which you are referenced were requested to be created, deleted or changed, but *failed* the proper authorisation for any of the referenced maintainers. """) for recipient, reports_per_status in reports_per_recipient.items(): user_report = header + self._request_meta_str() if UpdateRequestStatus.ERROR_AUTH in reports_per_status: user_report += header_failed for report in reports_per_status[ UpdateRequestStatus.ERROR_AUTH]: user_report += f'---\n{report}\n' user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n' if UpdateRequestStatus.SAVED in reports_per_status: user_report += header_saved for report in reports_per_status[UpdateRequestStatus.SAVED]: user_report += f'---\n{report}\n' user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n' email.send_email(recipient, subject, user_report) def _request_meta_str(self): request_meta_str = '\n'.join( [f'> {k}: {v}' for k, v in self.request_meta.items() if v]) if request_meta_str: request_meta_str = '\n' + request_meta_str + '\n\n' return request_meta_str
class WhoisQueryParser: """ Parser for all whois-style queries. This parser distinguishes RIPE-style, e.g. '-K 192.0.2.1' or '-i mnt-by FOO' from IRRD-style, e.g. '!oFOO'. Some query flags, particularly -k/!! and -s/!s retain state across queries, so a single instance of this object should be created per session, with handle_query() being called for each individual query. """ lookup_field_names = lookup_field_names() database_handler: DatabaseHandler _current_set_root_object_class: Optional[str] def __init__(self, client_ip: str, client_str: str) -> None: self.all_valid_sources = list(get_setting('sources', {}).keys()) self.sources_default = get_setting('sources_default') self.sources: List[ str] = self.sources_default if self.sources_default else self.all_valid_sources if get_setting('rpki.roa_source'): self.all_valid_sources.append(RPKI_IRR_PSEUDO_SOURCE) self.object_classes: List[str] = [] self.user_agent: Optional[str] = None self.multiple_command_mode = False self.rpki_aware = bool(get_setting('rpki.roa_source')) self.rpki_invalid_filter_enabled = self.rpki_aware self.timeout = 30 self.key_fields_only = False self.client_ip = client_ip self.client_str = client_str self.preloader = Preloader() self._preloaded_query_count = 0 def handle_query(self, query: str) -> WhoisQueryResponse: """ Process a single query. Always returns a WhoisQueryResponse object. Not thread safe - only one call must be made to this method at the same time. """ # These flags are reset with every query. self.database_handler = DatabaseHandler() self.key_fields_only = False self.object_classes = [] if query.startswith('!'): try: return self.handle_irrd_command(query[1:]) except WhoisQueryParserException as exc: logger.info( f'{self.client_str}: encountered parsing error while parsing query "{query}": {exc}' ) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.IRRD, result=str(exc)) except Exception as exc: logger.error( f'An exception occurred while processing whois query "{query}": {exc}', exc_info=exc) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.IRRD, result= 'An internal error occurred while processing this query.') finally: self.database_handler.close() try: return self.handle_ripe_command(query) except WhoisQueryParserException as exc: logger.info( f'{self.client_str}: encountered parsing error while parsing query "{query}": {exc}' ) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.RIPE, result=str(exc)) except Exception as exc: logger.error( f'An exception occurred while processing whois query "{query}": {exc}', exc_info=exc) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.RIPE, result='An internal error occurred while processing this query.' ) finally: self.database_handler.close() def handle_irrd_command(self, full_command: str) -> WhoisQueryResponse: """Handle an IRRD-style query. full_command should not include the first exclamation mark. """ if not full_command: raise WhoisQueryParserException(f'Missing IRRD command') command = full_command[0].upper() parameter = full_command[1:] response_type = WhoisQueryResponseType.SUCCESS result = None # A is not tested here because it is already handled in handle_irrd_routes_for_as_set queries_with_parameter = list('TG6IJMNORS') if command in queries_with_parameter and not parameter: raise WhoisQueryParserException( f'Missing parameter for {command} query') if command == '!': self.multiple_command_mode = True result = None response_type = WhoisQueryResponseType.NO_RESPONSE elif full_command.upper() == 'FNO-RPKI-FILTER': self.rpki_invalid_filter_enabled = False result = 'Filtering out RPKI invalids is disabled for !r and RIPE style ' \ 'queries for the rest of this connection.' elif command == 'V': result = self.handle_irrd_version() elif command == 'T': self.handle_irrd_timeout_update(parameter) elif command == 'G': result = self.handle_irrd_routes_for_origin_v4(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == '6': result = self.handle_irrd_routes_for_origin_v6(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'A': result = self.handle_irrd_routes_for_as_set(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'I': result = self.handle_irrd_set_members(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'J': result = self.handle_irrd_database_serial_range(parameter) elif command == 'M': result = self.handle_irrd_exact_key(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'N': self.handle_user_agent(parameter) elif command == 'O': result = self.handle_inverse_attr_search('mnt-by', parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'R': result = self.handle_irrd_route_search(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'S': result = self.handle_irrd_sources_list(parameter) else: raise WhoisQueryParserException(f'Unrecognised command: {command}') return WhoisQueryResponse( response_type=response_type, mode=WhoisQueryResponseMode.IRRD, result=result, ) def handle_irrd_timeout_update(self, timeout: str) -> None: """!timeout query - update timeout in connection""" try: timeout_value = int(timeout) except ValueError: raise WhoisQueryParserException( f'Invalid value for timeout: {timeout}') if timeout_value > 0 and timeout_value <= 1000: self.timeout = timeout_value else: raise WhoisQueryParserException( f'Invalid value for timeout: {timeout}') def handle_irrd_routes_for_origin_v4(self, origin: str) -> str: """!g query - find all originating IPv4 prefixes from an origin, e.g. !gAS65537""" return self._routes_for_origin(origin, 4) def handle_irrd_routes_for_origin_v6(self, origin: str) -> str: """!6 query - find all originating IPv6 prefixes from an origin, e.g. !6as65537""" return self._routes_for_origin(origin, 6) def _routes_for_origin(self, origin: str, ip_version: Optional[int] = None) -> str: """ Resolve all route(6)s prefixes for an origin, returning a space-separated list of all originating prefixes, not including duplicates. """ try: origin_formatted, _ = parse_as_number(origin) except ValidationError as ve: raise WhoisQueryParserException(str(ve)) self._preloaded_query_called() prefixes = self.preloader.routes_for_origins([origin_formatted], self.sources, ip_version=ip_version) return ' '.join(prefixes) def handle_irrd_routes_for_as_set(self, set_name: str) -> str: """ !a query - find all originating prefixes for all members of an AS-set, e.g. !a4AS-FOO or !a6AS-FOO """ ip_version: Optional[int] = None if set_name.startswith('4'): set_name = set_name[1:] ip_version = 4 elif set_name.startswith('6'): set_name = set_name[1:] ip_version = 6 if not set_name: raise WhoisQueryParserException( f'Missing required set name for A query') self._preloaded_query_called() self._current_set_root_object_class = 'as-set' members = self._recursive_set_resolve({set_name}) prefixes = self.preloader.routes_for_origins(members, self.sources, ip_version=ip_version) return ' '.join(prefixes) def handle_irrd_set_members(self, parameter: str) -> str: """ !i query - find all members of an as-set or route-set, possibly recursively. e.g. !iAS-FOO for non-recursive, !iAS-FOO,1 for recursive """ self._preloaded_query_called() recursive = False if parameter.endswith(',1'): recursive = True parameter = parameter[:-2] self._current_set_root_object_class = None if not recursive: members, leaf_members = self._find_set_members({parameter}) members.update(leaf_members) else: members = self._recursive_set_resolve({parameter}) if parameter in members: members.remove(parameter) if get_setting('compatibility.ipv4_only_route_set_members'): original_members = set(members) for member in original_members: try: IP(member) except ValueError: continue # This is not a prefix, ignore. try: IP(member, ipversion=4) except ValueError: # This was a valid prefix, but not a valid IPv4 prefix, # and should be removed. members.remove(member) return ' '.join(sorted(members)) def _recursive_set_resolve(self, members: Set[str], sets_seen=None) -> Set[str]: """ Resolve all members of a number of sets, recursively. For each set in members, determines whether it has been seen already (to prevent infinite recursion), ignores it if already seen, and then either adds it directly or adds it to a set that requires further resolving. """ if not sets_seen: sets_seen = set() if all([member in sets_seen for member in members]): return set() sets_seen.update(members) set_members = set() resolved_as_members = set() sub_members, leaf_members = self._find_set_members(members) for sub_member in sub_members: if self._current_set_root_object_class is None or self._current_set_root_object_class == 'route-set': try: IP(sub_member) set_members.add(sub_member) continue except ValueError: pass # AS numbers are permitted in route-sets and as-sets, per RFC 2622 5.3. # When an AS number is encountered as part of route-set resolving, # the prefixes originating from that AS should be added to the response. try: as_number_formatted, _ = parse_as_number(sub_member) if self._current_set_root_object_class == 'route-set': set_members.update( self.preloader.routes_for_origins( [as_number_formatted], self.sources)) resolved_as_members.add(sub_member) else: set_members.add(sub_member) continue except ValueError: pass further_resolving_required = sub_members - set_members - sets_seen - resolved_as_members new_members = self._recursive_set_resolve(further_resolving_required, sets_seen) set_members.update(new_members) return set_members def _find_set_members(self, set_names: Set[str]) -> Tuple[Set[str], Set[str]]: """ Find all members of a number of route-sets or as-sets. Includes both direct members listed in members attribute, but also members included by mbrs-by-ref/member-of. Returns a tuple of two sets: - members found of the sets included in set_names, both references to other sets and direct AS numbers, etc. - leaf members that were included in set_names, i.e. names for which no further data could be found - for example references to non-existent other sets """ members: Set[str] = set() sets_already_resolved: Set[str] = set() columns = ['parsed_data', 'rpsl_pk', 'source', 'object_class'] query = self._prepare_query(column_names=columns) object_classes = ['as-set', 'route-set'] # Per RFC 2622 5.3, route-sets can refer to as-sets, # but as-sets can only refer to other as-sets. if self._current_set_root_object_class == 'as-set': object_classes = [self._current_set_root_object_class] query = query.object_classes(object_classes).rpsl_pks(set_names) query_result = list(self.database_handler.execute_query(query)) if not query_result: # No sub-members are found, and apparantly all inputs were leaf members. return set(), set_names # Track the object class of the root object set. # In one case self._current_set_root_object_class may already be set # on the first run: when the set resolving should be fixed to one # type of set object. if not self._current_set_root_object_class: self._current_set_root_object_class = query_result[0][ 'object_class'] for result in query_result: rpsl_pk = result['rpsl_pk'] # The same PK may occur in multiple sources, but we are # only interested in the first matching object, prioritised # according to the source order. This priority is part of the # query ORDER BY, so basically we only process an RPSL pk once. if rpsl_pk in sets_already_resolved: continue sets_already_resolved.add(rpsl_pk) object_class = result['object_class'] object_data = result['parsed_data'] mbrs_by_ref = object_data.get('mbrs-by-ref', None) for members_attr in ['members', 'mp-members']: if members_attr in object_data: members.update(set(object_data[members_attr])) if not rpsl_pk or not object_class or not mbrs_by_ref: continue # If mbrs-by-ref is set, find any objects with member-of pointing to the route/as-set # under query, and include a maintainer listed in mbrs-by-ref, unless mbrs-by-ref # is set to ANY. query_object_class = [ 'route', 'route6' ] if object_class == 'route-set' else ['aut-num'] query = self._prepare_query( column_names=columns).object_classes(query_object_class) query = query.lookup_attrs_in(['member-of'], [rpsl_pk]) if 'ANY' not in [m.strip().upper() for m in mbrs_by_ref]: query = query.lookup_attrs_in(['mnt-by'], mbrs_by_ref) referring_objects = self.database_handler.execute_query(query) for result in referring_objects: member_object_class = result['object_class'] members.add(result['parsed_data'][member_object_class]) leaf_members = set_names - sets_already_resolved return members, leaf_members def handle_irrd_database_serial_range(self, parameter: str) -> str: """!j query - database serial range""" if parameter == '-*': sources = self.sources_default if self.sources_default else self.all_valid_sources else: sources = [s.upper() for s in parameter.split(',')] invalid_sources = [ s for s in sources if s not in self.all_valid_sources ] query = DatabaseStatusQuery().sources(sources) query_results = self.database_handler.execute_query(query) result_txt = '' for query_result in query_results: source = query_result['source'].upper() keep_journal = 'Y' if get_setting( f'sources.{source}.keep_journal') else 'N' serial_oldest = query_result['serial_oldest_seen'] serial_newest = query_result['serial_newest_seen'] fields = [ source, keep_journal, f'{serial_oldest}-{serial_newest}' if serial_oldest and serial_newest else '-', ] if query_result['serial_last_export']: fields.append(str(query_result['serial_last_export'])) result_txt += ':'.join(fields) + '\n' for invalid_source in invalid_sources: result_txt += f'{invalid_source.upper()}:X:Database unknown\n' return result_txt.strip() def handle_irrd_exact_key(self, parameter: str): """!m query - exact object key lookup, e.g. !maut-num,AS65537""" try: object_class, rpsl_pk = parameter.split(',', maxsplit=1) except ValueError: raise WhoisQueryParserException( f'Invalid argument for object lookup: {parameter}') query = self._prepare_query().object_classes( [object_class]).rpsl_pk(rpsl_pk).first_only() return self._execute_query_flatten_output(query) def handle_irrd_route_search(self, parameter: str): """ !r query - route search with various options: !r192.0.2.0/24 returns all exact matching objects !r192.0.2.0/24,o returns space-separated origins of all exact matching objects !r192.0.2.0/24,l returns all one-level less specific objects, not including exact !r192.0.2.0/24,L returns all less specific objects, including exact !r192.0.2.0/24,M returns all more specific objects, not including exact """ option: Optional[str] = None if ',' in parameter: address, option = parameter.split(',') else: address = parameter try: address = IP(address) except ValueError: raise WhoisQueryParserException( f'Invalid input for route search: {parameter}') query = self._prepare_query(ordered_by_sources=False).object_classes( ['route', 'route6']) if option is None or option == 'o': query = query.ip_exact(address) elif option == 'l': query = query.ip_less_specific_one_level(address) elif option == 'L': query = query.ip_less_specific(address) elif option == 'M': query = query.ip_more_specific(address) else: raise WhoisQueryParserException( f'Invalid route search option: {option}') if option == 'o': query_result = self.database_handler.execute_query(query) prefixes = [r['parsed_data']['origin'] for r in query_result] return ' '.join(prefixes) return self._execute_query_flatten_output(query) def handle_irrd_sources_list(self, parameter: str) -> Optional[str]: """ !s query - set used sources !s-lc returns all enabled sources, space separated !sripe,nttcom limits sources to ripe and nttcom """ if parameter == '-lc': return ','.join(self.sources) sources = parameter.upper().split(',') if not all([source in self.all_valid_sources for source in sources]): raise WhoisQueryParserException( 'One or more selected sources are unavailable.') self.sources = sources return None def handle_irrd_version(self): """!v query - return version""" return f'IRRd -- version {__version__}' def handle_ripe_command(self, full_query: str) -> WhoisQueryResponse: """ Process RIPE-style queries. Any query that is not explicitly an IRRD-style query (i.e. starts with exclamation mark) is presumed to be a RIPE query. """ full_query = re.sub(' +', ' ', full_query) components = full_query.strip().split(' ') result = None response_type = WhoisQueryResponseType.SUCCESS while len(components): component = components.pop(0) if component.startswith('-'): command = component[1:] try: if command == 'k': self.multiple_command_mode = True elif command in ['l', 'L', 'M', 'x']: result = self.handle_ripe_route_search( command, components.pop(0)) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND break elif command == 'i': result = self.handle_inverse_attr_search( components.pop(0), components.pop(0)) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND break elif command == 's': self.handle_ripe_sources_list(components.pop(0)) elif command == 'a': self.handle_ripe_sources_list(None) elif command == 'T': self.handle_ripe_restrict_object_class( components.pop(0)) elif command == 't': result = self.handle_ripe_request_object_template( components.pop(0)) break elif command == 'K': self.handle_ripe_key_fields_only() elif command == 'V': self.handle_user_agent(components.pop(0)) elif command == 'g': result = self.handle_nrtm_request(components.pop(0)) elif command in ['F', 'r']: continue # These flags disable recursion, but IRRd never performs recursion anyways else: raise WhoisQueryParserException( f'Unrecognised flag/search: {command}') except IndexError: raise WhoisQueryParserException( f'Missing argument for flag/search: {command}') else: # assume query to be a free text search result = self.handle_ripe_text_search(component) return WhoisQueryResponse( response_type=response_type, mode=WhoisQueryResponseMode.RIPE, result=result, ) def handle_ripe_route_search(self, command: str, parameter: str) -> str: """ -l/L/M/x query - route search for: -x 192.0.2.0/2 returns all exact matching objects -l 192.0.2.0/2 returns all one-level less specific objects, not including exact -L 192.0.2.0/2 returns all less specific objects, including exact -M 192.0.2.0/2 returns all more specific objects, not including exact """ try: address = IP(parameter) except ValueError: raise WhoisQueryParserException( f'Invalid input for route search: {parameter}') query = self._prepare_query(ordered_by_sources=False).object_classes( ['route', 'route6']) if command == 'x': query = query.ip_exact(address) elif command == 'l': query = query.ip_less_specific_one_level(address) elif command == 'L': query = query.ip_less_specific(address) elif command == 'M': query = query.ip_more_specific(address) return self._execute_query_flatten_output(query) def handle_ripe_sources_list(self, sources_list: Optional[str]) -> None: """-s/-a parameter - set sources list. Empty list enables all sources. """ if sources_list: sources = sources_list.upper().split(',') if not all( [source in self.all_valid_sources for source in sources]): raise WhoisQueryParserException( 'One or more selected sources are unavailable.') self.sources = sources else: self.sources = self.sources_default if self.sources_default else self.all_valid_sources def handle_ripe_restrict_object_class(self, object_classes) -> None: """-T parameter - restrict object classes for this query, comma-seperated""" self.object_classes = object_classes.split(',') def handle_ripe_request_object_template(self, object_class) -> str: """-t query - return the RPSL template for an object class""" try: return OBJECT_CLASS_MAPPING[object_class]().generate_template() except KeyError: raise WhoisQueryParserException( f'Unknown object class: {object_class}') def handle_ripe_key_fields_only(self) -> None: """-K paramater - only return primary key and members fields""" self.key_fields_only = True def handle_ripe_text_search(self, value: str) -> str: query = self._prepare_query( ordered_by_sources=False).text_search(value) return self._execute_query_flatten_output(query) def handle_user_agent(self, user_agent: str): """-V/!n parameter/query - set a user agent for the client""" self.user_agent = user_agent logger.info(f'{self.client_str}: user agent set to: {user_agent}') def handle_nrtm_request(self, param): try: source, version, serial_range = param.split(':') except ValueError: raise WhoisQueryParserException( f'Invalid parameter: must contain three elements') try: serial_start, serial_end = serial_range.split('-') serial_start = int(serial_start) if serial_end == 'LAST': serial_end = None else: serial_end = int(serial_end) except ValueError: raise WhoisQueryParserException( f'Invalid serial range: {serial_range}') if version not in ['1', '3']: raise WhoisQueryParserException(f'Invalid NRTM version: {version}') source = source.upper() if source not in self.all_valid_sources: raise WhoisQueryParserException(f'Unknown source: {source}') if not is_client_permitted(self.client_ip, f'sources.{source}.nrtm_access_list'): raise WhoisQueryParserException(f'Access denied') try: return NRTMGenerator().generate(source, version, serial_start, serial_end, self.database_handler) except NRTMGeneratorException as nge: raise WhoisQueryParserException(str(nge)) def handle_inverse_attr_search(self, attribute: str, value: str) -> str: """ -i/!o query - inverse search for attribute values e.g. `-i mnt-by FOO` finds all objects where (one of the) maintainer(s) is FOO, as does `!oFOO`. Restricted to designated lookup fields. """ if attribute not in self.lookup_field_names: readable_lookup_field_names = ', '.join(self.lookup_field_names) msg = ( f'Inverse attribute search not supported for {attribute},' + f'only supported for attributes: {readable_lookup_field_names}' ) raise WhoisQueryParserException(msg) query = self._prepare_query(ordered_by_sources=False).lookup_attr( attribute, value) return self._execute_query_flatten_output(query) def _prepare_query(self, column_names=None, ordered_by_sources=True) -> RPSLDatabaseQuery: """Prepare an RPSLDatabaseQuery by applying relevant sources/class filters.""" query = RPSLDatabaseQuery(column_names, ordered_by_sources) if self.sources and self.sources != self.all_valid_sources: query.sources(self.sources) else: default = list(get_setting('sources_default', [])) if default: query.sources(list(default)) if self.object_classes: query.object_classes(self.object_classes) if self.rpki_invalid_filter_enabled: query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid]) return query def _execute_query_flatten_output(self, query: RPSLDatabaseQuery) -> str: """ Execute an RPSLDatabaseQuery, and flatten the output into a string with object text for easy passing to a WhoisQueryResponse. """ query_response = self.database_handler.execute_query(query) if self.key_fields_only: result = self._filter_key_fields(query_response) else: result = '' for obj in query_response: result += obj['object_text'] if (self.rpki_aware and obj['source'] != RPKI_IRR_PSEUDO_SOURCE and obj['object_class'] in RPKI_RELEVANT_OBJECT_CLASSES): comment = '' if obj['rpki_status'] == RPKIStatus.not_found: comment = ' # No ROAs found, or RPKI validation not enabled for source' result += f'rpki-ov-state: {obj["rpki_status"].name}{comment}\n' result += '\n' return result.strip('\n\r') def _filter_key_fields(self, query_response) -> str: results: OrderedSet[str] = OrderedSet() for obj in query_response: result = '' rpsl_object_class = OBJECT_CLASS_MAPPING[obj['object_class']] fields_included = rpsl_object_class.pk_fields + [ 'members', 'mp-members' ] for field_name in fields_included: field_data = obj['parsed_data'].get(field_name) if field_data: if isinstance(field_data, list): for item in field_data: result += f'{field_name}: {item}\n' else: result += f'{field_name}: {field_data}\n' results.add(result) return '\n'.join(results) def _preloaded_query_called(self): """ Called each time the user runs a query that can be preloaded. After the 5th, load the preload store into memory to speed up expected further queries. """ self._preloaded_query_count += 1 if self._preloaded_query_count > 5: self.preloader.load_routes_into_memory()
def set_force_reload(source) -> None: dh = DatabaseHandler(enable_preload_update=False) dh.set_force_reload(source) dh.commit() dh.close()
class ChangeSubmissionHandler: """ The ChangeSubmissionHandler handles the text of one or more requested RPSL changes (create, modify or delete), parses, validates and eventually saves them. This includes validating references between objects, including those part of the same message, and checking authentication. """ def __init__(self, object_texts: str, pgp_fingerprint: str=None, request_meta: Dict[str, Optional[str]]=None) -> None: self.database_handler = DatabaseHandler() self.request_meta = request_meta if request_meta else {} self._pgp_key_id = self._resolve_pgp_key_id(pgp_fingerprint) if pgp_fingerprint else None self._handle_object_texts(object_texts) self.database_handler.commit() self.database_handler.close() def _handle_object_texts(self, object_texts: str) -> None: reference_validator = ReferenceValidator(self.database_handler) auth_validator = AuthValidator(self.database_handler, self._pgp_key_id) results = parse_change_requests(object_texts, self.database_handler, auth_validator, reference_validator) # When an object references another object, e.g. tech-c referring a person or mntner, # an add/update is only valid if those referred objects exist. To complicate matters, # the object referred to may be part of this very same submission. For this reason, the # reference validator can be provided with all new objects to be added in this submission. # However, a possible scenario is that A, B and C are submitted. Object A refers to B, # B refers to C, C refers to D and D does not exist - or C fails authentication. # At a first scan, A is valid because B exists, B is valid because C exists. C # becomes invalid on the first scan, which is why another scan is performed, which # will mark B invalid due to the reference to an invalid C, etc. This continues until # all references are resolved and repeated scans lead to the same conclusions. valid_changes = [r for r in results if r.is_valid()] previous_valid_changes: List[ChangeRequest] = [] loop_count = 0 loop_max = len(results) + 10 while valid_changes != previous_valid_changes: previous_valid_changes = valid_changes reference_validator.preload(valid_changes) auth_validator.pre_approve(valid_changes) for result in valid_changes: result.validate() valid_changes = [r for r in results if r.is_valid()] loop_count += 1 if loop_count > loop_max: # pragma: no cover msg = f'Update validity resolver ran an excessive amount of loops, may be stuck, aborting ' \ f'processing. Message metadata: {self.request_meta}' logger.error(msg) raise ValueError(msg) for result in results: if result.is_valid(): result.save(self.database_handler) self.results = results def _resolve_pgp_key_id(self, pgp_fingerprint: str) -> Optional[str]: """ Find a PGP key ID for a given fingerprint. This method looks for an actual matching object in the database, and then returns the object's PK. """ clean_fingerprint = pgp_fingerprint.replace(' ', '') key_id = "PGPKEY-" + clean_fingerprint[-8:] query = RPSLDatabaseQuery().object_classes(['key-cert']).rpsl_pk(key_id) results = list(self.database_handler.execute_query(query)) for result in results: if result['parsed_data'].get('fingerpr', '').replace(' ', '') == clean_fingerprint: return key_id logger.info(f'Message was signed with key {key_id}, but key was not found in the database. Treating message ' f'as unsigned. Message metadata: {self.request_meta}') return None def status(self) -> str: """Provide a simple SUCCESS/FAILED string based - former used if all objects were saved.""" if all([result.status == UpdateRequestStatus.SAVED for result in self.results]): return "SUCCESS" return "FAILED" def submitter_report(self) -> str: """Produce a human-readable report for the submitter.""" # flake8: noqa: W293 successful = [r for r in self.results if r.status == UpdateRequestStatus.SAVED] failed = [r for r in self.results if r.status != UpdateRequestStatus.SAVED] number_successful_create = len([r for r in successful if r.request_type == UpdateRequestType.CREATE]) number_successful_modify = len([r for r in successful if r.request_type == UpdateRequestType.MODIFY]) number_successful_delete = len([r for r in successful if r.request_type == UpdateRequestType.DELETE]) number_failed_create = len([r for r in failed if r.request_type == UpdateRequestType.CREATE]) number_failed_modify = len([r for r in failed if r.request_type == UpdateRequestType.MODIFY]) number_failed_delete = len([r for r in failed if r.request_type == UpdateRequestType.DELETE]) user_report = self._request_meta_str() + textwrap.dedent(f""" SUMMARY OF UPDATE: Number of objects found: {len(self.results):3} Number of objects processed successfully: {len(successful):3} Create: {number_successful_create:3} Modify: {number_successful_modify:3} Delete: {number_successful_delete:3} Number of objects processed with errors: {len(failed):3} Create: {number_failed_create:3} Modify: {number_failed_modify:3} Delete: {number_failed_delete:3} DETAILED EXPLANATION: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """) for result in self.results: user_report += "---\n" user_report += result.submitter_report() user_report += "\n" user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' return user_report def send_notification_target_reports(self): # First key is e-mail address of recipient, second is UpdateRequestStatus.SAVED # or UpdateRequestStatus.ERROR_AUTH reports_per_recipient: Dict[str, Dict[UpdateRequestStatus, OrderedSet]] = defaultdict(dict) sources: OrderedSet[str] = OrderedSet() for result in self.results: for target in result.notification_targets(): if result.status in [UpdateRequestStatus.SAVED, UpdateRequestStatus.ERROR_AUTH]: if result.status not in reports_per_recipient[target]: reports_per_recipient[target][result.status] = OrderedSet() reports_per_recipient[target][result.status].add(result.notification_target_report()) sources.add(result.rpsl_obj_new.source()) sources_str = '/'.join(sources) subject = f'Notification of {sources_str} database changes' header = textwrap.dedent(f""" This is to notify you of changes in the {sources_str} database or object authorisation failures. You may receive this message because you are listed in the notify attribute on the changed object(s), or because you are listed in the mnt-nfy or upd-to attribute on a maintainer of the object(s). This message is auto-generated. The request was made by email, with the following details: """) header_saved = textwrap.dedent(""" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some objects in which you are referenced have been created, deleted or changed. """) header_failed = textwrap.dedent(""" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some objects in which you are referenced were requested to be created, deleted or changed, but *failed* the proper authorisation for any of the referenced maintainers. """) for recipient, reports_per_status in reports_per_recipient.items(): user_report = header + self._request_meta_str() if UpdateRequestStatus.ERROR_AUTH in reports_per_status: user_report += header_failed for report in reports_per_status[UpdateRequestStatus.ERROR_AUTH]: user_report += f"---\n{report}\n" user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n' if UpdateRequestStatus.SAVED in reports_per_status: user_report += header_saved for report in reports_per_status[UpdateRequestStatus.SAVED]: user_report += f"---\n{report}\n" user_report += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n' email.send_email(recipient, subject, user_report) def _request_meta_str(self): request_meta_str = '\n'.join([f"> {k}: {v}" for k, v in self.request_meta.items() if v]) if request_meta_str: request_meta_str = "\n" + request_meta_str + "\n\n" return request_meta_str
class WhoisQueryParser: """ Parser for all whois-style queries. This parser distinguishes RIPE-style, e.g. "-K 192.0.2.1" or "-i mnt-by FOO" from IRRD-style, e.g. "!oFOO". Some query flags, particularly -k/!! and -s/!s retain state across queries, so a single instance of this object should be created per session, with handle_query() being called for each individual query. """ lookup_field_names = lookup_field_names() database_handler: DatabaseHandler def __init__(self, peer: str) -> None: self.all_valid_sources = list(get_setting('sources').keys()) self.sources: List[str] = [] self.object_classes: List[str] = [] self.user_agent: Optional[str] = None self.multiple_command_mode = False self.key_fields_only = False self.peer = peer # The WhoisQueryParser itself should not run concurrently, # as answers could arrive out of order. self.safe_to_run_query = threading.Event() self.safe_to_run_query.set() def handle_query(self, query: str) -> WhoisQueryResponse: """Process a single query. Always returns a WhoisQueryResponse object.""" # These flags are reset with every query. self.safe_to_run_query.wait() self.safe_to_run_query.clear() self.database_handler = DatabaseHandler() self.key_fields_only = False self.object_classes = [] if query.startswith('!'): try: return self.handle_irrd_command(query[1:]) except WhoisQueryParserException as exc: logger.info(f'{self.peer}: encountered parsing error while parsing query "{query}": {exc}') return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.IRRD, result=str(exc) ) except Exception as exc: logger.critical(f'An exception occurred while processing whois query "{query}": {exc}', exc_info=exc) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.IRRD, result='An internal error occurred while processing this query.' ) finally: self.database_handler.close() self.safe_to_run_query.set() try: return self.handle_ripe_command(query) except WhoisQueryParserException as exc: logger.info(f'{self.peer}: encountered parsing error while parsing query "{query}": {exc}') return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.RIPE, result=str(exc) ) except Exception as exc: logger.critical(f'An exception occurred while processing whois query "{query}": {exc}', exc_info=exc) return WhoisQueryResponse( response_type=WhoisQueryResponseType.ERROR, mode=WhoisQueryResponseMode.RIPE, result='An internal error occurred while processing this query.' ) finally: self.database_handler.close() self.safe_to_run_query.set() def handle_irrd_command(self, full_command: str) -> WhoisQueryResponse: """Handle an IRRD-style query. full_command should not include the first exclamation mark. """ if not full_command: raise WhoisQueryParserException(f'Missing IRRD command') command = full_command[0].upper() parameter = full_command[1:] response_type = WhoisQueryResponseType.SUCCESS result = None if command == '!': self.multiple_command_mode = True result = None elif command == 'G': result = self.handle_irrd_routes_for_origin_v4(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == '6': result = self.handle_irrd_routes_for_origin_v6(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'I': result = self.handle_irrd_set_members(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'J': result = self.handle_irrd_database_serial_range(parameter) elif command == 'M': result = self.handle_irrd_exact_key(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'N': self.handle_user_agent(parameter) elif command == 'O': result = self.handle_inverse_attr_search('mnt-by', parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'R': result = self.handle_irrd_route_search(parameter) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND elif command == 'S': result = self.handle_irrd_sources_list(parameter) elif command == 'V': result = self.handle_irrd_version() else: raise WhoisQueryParserException(f'Unrecognised command: {command}') return WhoisQueryResponse( response_type=response_type, mode=WhoisQueryResponseMode.IRRD, result=result, ) def handle_irrd_routes_for_origin_v4(self, origin: str) -> str: """!g query - find all originating IPv4 prefixes from an origin, e.g. !gAS65537""" return self._routes_for_origin('route', origin) def handle_irrd_routes_for_origin_v6(self, origin: str) -> str: """!6 query - find all originating IPv6 prefixes from an origin, e.g. !6as65537""" return self._routes_for_origin('route6', origin) def _routes_for_origin(self, object_class: str, origin: str) -> str: """ Resolve all route(6)s for an origin, returning a space-separated list of all originating prefixes, not including duplicates. """ try: _, asn = parse_as_number(origin) except ValidationError as ve: raise WhoisQueryParserException(str(ve)) query = self._prepare_query().object_classes([object_class]).asn(asn) query_result = self.database_handler.execute_query(query) prefixes = [r['parsed_data'][object_class] for r in query_result] return ' '.join(OrderedSet(prefixes)) def handle_irrd_set_members(self, parameter: str) -> str: """ !i query - find all members of an as-set or route-set, possibly recursively. e.g. !iAS-FOO for non-recursive, !iAS-FOO,1 for recursive """ recursive = False if parameter.endswith(',1'): recursive = True parameter = parameter[:-2] self._current_set_priority_source = None if not recursive: members, leaf_members = self._find_set_members({parameter}) members.update(leaf_members) else: members = self._recursive_set_resolve({parameter}) if parameter in members: members.remove(parameter) return ' '.join(sorted(members)) def _recursive_set_resolve(self, members: Set[str], sets_seen=None) -> Set[str]: """ Resolve all members of a number of sets, recursively. For each set in members, determines whether it has been seen already (to prevent infinite recursion), ignores it if already seen, and then either adds it directly or adds it to a set that requires further resolving. """ if not sets_seen: sets_seen = set() if all([member in sets_seen for member in members]): return set() sets_seen.update(members) set_members = set() sub_members, leaf_members = self._find_set_members(members) for sub_member in sub_members: try: IP(sub_member) set_members.add(sub_member) continue except ValueError: pass try: parse_as_number(sub_member) set_members.add(sub_member) continue except ValueError: pass further_resolving_required = sub_members - set_members - sets_seen new_members = self._recursive_set_resolve(further_resolving_required, sets_seen) set_members.update(new_members) return set_members def _find_set_members(self, set_names: Set[str]) -> Tuple[Set[str], Set[str]]: """ Find all members of a number of route-sets or as-sets. Includes both direct members listed in members attribute, but also members included by mbrs-by-ref/member-of. Returns a tuple of two sets: - members found of the sets included in set_names, both references to other sets and direct AS numbers, etc. - leaf members that were included in set_names, i.e. names for which no further data could be found - for example references to non-existent other sets """ members: Set[str] = set() sets_already_resolved: Set[str] = set() query = self._prepare_query().object_classes(['as-set', 'route-set']).rpsl_pks(set_names) if self._current_set_priority_source: query.prioritise_source(self._current_set_priority_source) query_result = list(self.database_handler.execute_query(query)) if not query_result: # No sub-members are found, and apparantly all inputs were leaf members. return set(), set_names # Track the source of the root object set if not self._current_set_priority_source: self._current_set_priority_source = query_result[0]['source'] for result in query_result: rpsl_pk = result['rpsl_pk'] # The same PK may occur in multiple sources, but we are # only interested in the first matching object, prioritised # to look for the same source as the root object. This priority # is part of the query ORDER BY, so basically we only process # an RPSL pk once. if rpsl_pk in sets_already_resolved: continue sets_already_resolved.add(rpsl_pk) object_class = result['object_class'] object_data = result['parsed_data'] mbrs_by_ref = object_data.get('mbrs-by-ref', None) for members_attr in ['members', 'mp-members']: if members_attr in object_data: members.update(set(object_data[members_attr])) if not rpsl_pk or not object_class or not mbrs_by_ref: continue # If mbrs-by-ref is set, find any objects with member-of pointing to the route/as-set # under query, and include a maintainer listed in mbrs-by-ref, unless mbrs-by-ref # is set to ANY. query_object_class = ['route', 'route6'] if object_class == 'route-set' else ['aut-num'] query = self._prepare_query().object_classes(query_object_class) query = query.lookup_attrs_in(['member-of'], [rpsl_pk]) if 'ANY' not in [m.strip().upper() for m in mbrs_by_ref]: query = query.lookup_attrs_in(['mnt-by'], mbrs_by_ref) referring_objects = self.database_handler.execute_query(query) for result in referring_objects: member_object_class = result['object_class'] members.add(result['parsed_data'][member_object_class]) leaf_members = set_names - sets_already_resolved return members, leaf_members def handle_irrd_database_serial_range(self, parameter: str) -> str: """!j query - database serial range""" if parameter == '-*': default = get_setting('sources_default') sources = default if default else self.all_valid_sources else: sources = [s.upper() for s in parameter.split(',')] invalid_sources = [s for s in sources if s not in self.all_valid_sources] query = DatabaseStatusQuery().sources(sources) query_results = self.database_handler.execute_query(query) result_txt = '' for query_result in query_results: source = query_result['source'].upper() keep_journal = 'Y' if get_setting(f'sources.{source}.keep_journal') else 'N' serial_oldest = query_result['serial_oldest_journal'] serial_newest = query_result['serial_newest_journal'] fields = [ source, keep_journal, f'{serial_oldest}-{serial_newest}' if serial_oldest and serial_newest else '-', ] if query_result['serial_last_export']: fields.append(str(query_result['serial_last_export'])) result_txt += ':'.join(fields) + '\n' for invalid_source in invalid_sources: result_txt += f'{invalid_source.upper()}:X:Database unknown\n' return result_txt.strip() def handle_irrd_exact_key(self, parameter: str): """!m query - exact object key lookup, e.g. !maut-num,AS65537""" try: object_class, rpsl_pk = parameter.split(',', maxsplit=1) except ValueError: raise WhoisQueryParserException(f'Invalid argument for object lookup: {parameter}') query = self._prepare_query().object_classes([object_class]).rpsl_pk(rpsl_pk).first_only() return self._execute_query_flatten_output(query) def handle_irrd_route_search(self, parameter: str): """ !r query - route search with various options: !r192.0.2.0/24 returns all exact matching objects !r192.0.2.0/24,o returns space-separated origins of all exact matching objects !r192.0.2.0/24,l returns all one-level less specific objects, not including exact !r192.0.2.0/24,L returns all less specific objects, including exact !r192.0.2.0/24,M returns all more specific objects, not including exact """ option: Optional[str] = None if ',' in parameter: address, option = parameter.split(',') else: address = parameter try: address = IP(address) except ValueError: raise WhoisQueryParserException(f'Invalid input for route search: {parameter}') query = self._prepare_query().object_classes(['route', 'route6']) if option is None or option == 'o': query = query.ip_exact(address) elif option == 'l': query = query.ip_less_specific_one_level(address) elif option == 'L': query = query.ip_less_specific(address) elif option == 'M': query = query.ip_more_specific(address) else: raise WhoisQueryParserException(f'Invalid route search option: {option}') if option == 'o': query_result = self.database_handler.execute_query(query) prefixes = [r['parsed_data']['origin'] for r in query_result] return ' '.join(prefixes) return self._execute_query_flatten_output(query) def handle_irrd_sources_list(self, parameter: str) -> Optional[str]: """ !s query - set used sources !s-lc returns all enabled sources, space separated !sripe,nttcom limits sources to ripe and nttcom """ if parameter == '-lc': default = get_setting('sources_default') sources_selected = default if default else self.all_valid_sources return ','.join(sources_selected) if parameter: sources = parameter.upper().split(',') if not all([source in self.all_valid_sources for source in sources]): raise WhoisQueryParserException("One or more selected sources are unavailable.") self.sources = sources else: raise WhoisQueryParserException("One or more selected sources are unavailable.") return None def handle_irrd_version(self): """!v query - return version""" return f'IRRD -- version {__version__}' def handle_ripe_command(self, full_query: str) -> WhoisQueryResponse: """ Process RIPE-style queries. Any query that is not explicitly an IRRD-style query (i.e. starts with exclamation mark) is presumed to be a RIPE query. """ full_query = re.sub(' +', ' ', full_query) components = full_query.strip().split(' ') result = None response_type = WhoisQueryResponseType.SUCCESS while len(components): component = components.pop(0) if component.startswith('-'): command = component[1:] try: if command == 'k': self.multiple_command_mode = True elif command in ['l', 'L', 'M', 'x']: result = self.handle_ripe_route_search(command, components.pop(0)) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND break elif command == 'i': result = self.handle_inverse_attr_search(components.pop(0), components.pop(0)) if not result: response_type = WhoisQueryResponseType.KEY_NOT_FOUND break elif command == 's': self.handle_ripe_sources_list(components.pop(0)) elif command == 'a': self.handle_ripe_sources_list(None) elif command == 'T': self.handle_ripe_restrict_object_class(components.pop(0)) elif command == 't': result = self.handle_ripe_request_object_template(components.pop(0)) break elif command == 'K': self.handle_ripe_key_fields_only() elif command == 'V': self.handle_user_agent(components.pop(0)) elif command in ['F', 'r']: continue # These flags disable recursion, but IRRd never performs recursion anyways else: raise WhoisQueryParserException(f'Unrecognised flag/search: {command}') except IndexError: raise WhoisQueryParserException(f'Missing argument for flag/search: {command}') else: # assume query to be a free text search result = self.handle_ripe_text_search(component) return WhoisQueryResponse( response_type=response_type, mode=WhoisQueryResponseMode.RIPE, result=result, ) def handle_ripe_route_search(self, command: str, parameter: str) -> str: """ -l/L/M/x query - route search for: -x 192.0.2.0/2 returns all exact matching objects -l 192.0.2.0/2 returns all one-level less specific objects, not including exact -L 192.0.2.0/2 returns all less specific objects, including exact -M 192.0.2.0/2 returns all more specific objects, not including exact """ try: address = IP(parameter) except ValueError: raise WhoisQueryParserException(f'Invalid input for route search: {parameter}') query = self._prepare_query().object_classes(['route', 'route6']) if command == 'x': query = query.ip_exact(address) elif command == 'l': query = query.ip_less_specific_one_level(address) elif command == 'L': query = query.ip_less_specific(address) elif command == 'M': query = query.ip_more_specific(address) return self._execute_query_flatten_output(query) def handle_ripe_sources_list(self, sources_list: Optional[str]) -> None: """-s/-a parameter - set sources list. Empty list enables all sources. """ if sources_list: sources = sources_list.upper().split(',') if not all([source in self.all_valid_sources for source in sources]): raise WhoisQueryParserException("One or more selected sources are unavailable.") self.sources = sources else: self.sources = [] def handle_ripe_restrict_object_class(self, object_classes) -> None: """-T parameter - restrict object classes for this query, comma-seperated""" self.object_classes = object_classes.split(',') def handle_ripe_request_object_template(self, object_class) -> str: """-t query - return the RPSL template for an object class""" try: return OBJECT_CLASS_MAPPING[object_class]().generate_template() except KeyError: raise WhoisQueryParserException(f'Unknown object class: {object_class}') def handle_ripe_key_fields_only(self) -> None: """-K paramater - only return primary key and members fields""" self.key_fields_only = True def handle_ripe_text_search(self, value: str) -> str: query = self._prepare_query().text_search(value) return self._execute_query_flatten_output(query) def handle_user_agent(self, user_agent: str): """-V/!n parameter/query - set a user agent for the client""" self.user_agent = user_agent logger.info(f'{self.peer}: user agent set to: {user_agent}') def handle_inverse_attr_search(self, attribute: str, value: str) -> str: """ -i/!o query - inverse search for attribute values e.g. `-i mnt-by FOO` finds all objects where (one of the) maintainer(s) is FOO, as does `!oFOO`. Restricted to designated lookup fields. """ if attribute not in self.lookup_field_names: readable_lookup_field_names = ", ".join(self.lookup_field_names) msg = (f'Inverse attribute search not supported for {attribute},' + f'only supported for attributes: {readable_lookup_field_names}') raise WhoisQueryParserException(msg) query = self._prepare_query().lookup_attr(attribute, value) return self._execute_query_flatten_output(query) def _prepare_query(self) -> RPSLDatabaseQuery: """Prepare an RPSLDatabaseQuery by applying relevant sources/class filters.""" query = RPSLDatabaseQuery() if self.sources: query.sources(self.sources) else: default = get_setting('sources_default') if default: query.sources(list(default)) if self.object_classes: query.object_classes(self.object_classes) return query def _execute_query_flatten_output(self, query: RPSLDatabaseQuery) -> str: """ Execute an RPSLDatabaseQuery, and flatten the output into a string with object text for easy passing to a WhoisQueryResponse. """ query_response = self.database_handler.execute_query(query) if self.key_fields_only: result = self._filter_key_fields(query_response) else: result = '' for obj in query_response: result += obj['object_text'] + '\n' return result.strip('\n\r') def _filter_key_fields(self, query_response) -> str: results = OrderedSet() for obj in query_response: result = '' rpsl_object_class = OBJECT_CLASS_MAPPING[obj['object_class']] fields_included = rpsl_object_class.pk_fields + ['members', 'mp-members'] for field_name in fields_included: field_data = obj['parsed_data'].get(field_name) if field_data: if isinstance(field_data, list): for item in field_data: result += f'{field_name}: {item}\n' else: result += f'{field_name}: {field_data}\n' results.add(result) return '\n'.join(results)
def set_force_reload(source) -> None: dh = DatabaseHandler() dh.set_force_reload(source) dh.commit() dh.close()
class SourceExportRunner: """ This SourceExportRunner is the entry point for the export process for a single source. A gzipped file will be created in the export_destination directory with the contents of the source, along with a CURRENTSERIAL file. The contents of the source are first written to a temporary file, and then moved in place. """ def __init__(self, source: str) -> None: self.source = source def run(self) -> None: self.database_handler = DatabaseHandler() try: export_destination = get_setting( f'sources.{self.source}.export_destination') if export_destination: logger.info( f'Starting a source export for {self.source} to {export_destination}' ) self._export(export_destination) export_destination_unfiltered = get_setting( f'sources.{self.source}.export_destination_unfiltered') if export_destination_unfiltered: logger.info( f'Starting an unfiltered source export for {self.source} ' f'to {export_destination_unfiltered}') self._export(export_destination_unfiltered, remove_auth_hashes=False) self.database_handler.commit() except Exception as exc: logger.error( f'An exception occurred while attempting to run an export ' f'for {self.source}: {exc}', exc_info=exc) finally: self.database_handler.close() def _export(self, export_destination, remove_auth_hashes=True): filename_export = Path( export_destination) / f'{self.source.lower()}.db.gz' export_tmpfile = NamedTemporaryFile(delete=False) filename_serial = Path( export_destination) / f'{self.source.upper()}.CURRENTSERIAL' query = DatabaseStatusQuery().source(self.source) try: serial = next(self.database_handler.execute_query( query))['serial_newest_seen'] except StopIteration: serial = None with gzip.open(export_tmpfile.name, 'wb') as fh: query = RPSLDatabaseQuery().sources([self.source]) query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid]) query = query.scopefilter_status([ScopeFilterStatus.in_scope]) for obj in self.database_handler.execute_query(query): object_text = obj['object_text'] if remove_auth_hashes: object_text = remove_auth_hashes_func(object_text) object_bytes = object_text.encode('utf-8') fh.write(object_bytes + b'\n') fh.write(b'# EOF\n') os.chmod(export_tmpfile.name, EXPORT_PERMISSIONS) if filename_export.exists(): os.unlink(filename_export) if filename_serial.exists(): os.unlink(filename_serial) shutil.move(export_tmpfile.name, filename_export) if serial is not None: with open(filename_serial, 'w') as fh: fh.write(str(serial)) os.chmod(filename_serial, EXPORT_PERMISSIONS) self.database_handler.record_serial_exported(self.source, serial) logger.info( f'Export for {self.source} complete at serial {serial}, stored in {filename_export} / {filename_serial}' )