def __init__(self,
rpsl_text_submitted: str,
database_handler: DatabaseHandler,
auth_validator: AuthValidator,
reference_validator: ReferenceValidator,
delete_reason=Optional[str]) -> None:
"""
Initialise a new change request for a single RPSL object.
:param rpsl_text_submitted: the object text
:param database_handler: a DatabaseHandler instance
:param auth_validator: a AuthValidator instance, to resolve authentication requirements
:param reference_validator: a ReferenceValidator instance, to resolve references between objects
:param delete_reason: a string with the deletion reason, if this was a deletion request
The rpsl_text passed into this function should be cleaned from any
meta attributes like delete/override/password. Those should be passed
into this method as delete_reason, or provided to the AuthValidator.
The auth_validator and reference_validator must be shared between
different instances, to benefit from caching, and to resolve references
between different objects that are part of the same submission with
possibly multiple changes.
"""
self.database_handler = database_handler
self.auth_validator = auth_validator
self.reference_validator = reference_validator
self.rpsl_text_submitted = rpsl_text_submitted
self.mntners_notify = []
self.used_override = False
self._cached_roa_validity: Optional[bool] = None
self.roa_validator = SingleRouteROAValidator(database_handler)
try:
self.rpsl_obj_new = rpsl_object_from_text(rpsl_text_submitted,
strict_validation=True)
if self.rpsl_obj_new.messages.errors():
self.status = UpdateRequestStatus.ERROR_PARSING
self.error_messages = self.rpsl_obj_new.messages.errors()
self.info_messages = self.rpsl_obj_new.messages.infos()
logger.debug(
f'{id(self)}: Processing new ChangeRequest for object {self.rpsl_obj_new}: request {id(self)}'
)
except UnknownRPSLObjectClassException as exc:
self.rpsl_obj_new = None
self.request_type = None
self.status = UpdateRequestStatus.ERROR_UNKNOWN_CLASS
self.info_messages = []
self.error_messages = [str(exc)]
if self.is_valid() and self.rpsl_obj_new:
source = self.rpsl_obj_new.source()
if not get_setting(f'sources.{source}.authoritative'):
logger.debug(
f'{id(self)}: change is for non-authoritative source {source}, rejected'
)
self.error_messages.append(
f'This instance is not authoritative for source {source}')
self.status = UpdateRequestStatus.ERROR_NON_AUTHORITIVE
return
self._retrieve_existing_version()
if delete_reason:
self.request_type = UpdateRequestType.DELETE
if not self.rpsl_obj_current:
self.status = UpdateRequestStatus.ERROR_PARSING
self.error_messages.append(
'Can not delete object: no object found for this key in this database.'
)
logger.debug(
f'{id(self)}: Request attempts to delete object {self.rpsl_obj_new}, '
f'but no existing object found.')
def save(self, database_handler: DatabaseHandler) -> bool:
default_source = self.source if self.operation == DatabaseOperation.delete else None
try:
object_text = self.object_text.strip()
# If an object turns out to be a key-cert, and strict_import_keycert_objects
# is set, parse it again with strict validation to load it in the GPG keychain.
obj = rpsl_object_from_text(object_text,
strict_validation=False,
default_source=default_source)
if self.strict_validation_key_cert and obj.__class__ == RPSLKeyCert:
obj = rpsl_object_from_text(object_text,
strict_validation=True,
default_source=default_source)
except UnknownRPSLObjectClassException as exc:
# Unknown object classes are only logged if they have not been filtered out.
if not self.object_class_filter or exc.rpsl_object_class.lower(
) in self.object_class_filter:
logger.info(f'Ignoring NRTM operation {str(self)}: {exc}')
return False
if self.object_class_filter and obj.rpsl_object_class.lower(
) not in self.object_class_filter:
return False
if obj.messages.errors():
errors = '; '.join(obj.messages.errors())
logger.critical(
f'Parsing errors occurred while processing NRTM operation {str(self)}. '
f'This operation is ignored, causing potential data inconsistencies. '
f'A new operation for this update, without errors, '
f'will still be processed and cause the inconsistency to be resolved. '
f'Parser error messages: {errors}; original object text follows:\n{self.object_text}'
)
database_handler.record_mirror_error(
self.source, f'Parsing errors: {obj.messages.errors()}, '
f'original object text follows:\n{self.object_text}')
return False
if 'source' in obj.parsed_data and obj.parsed_data['source'].upper(
) != self.source:
msg = (
f'Incorrect source in NRTM object: stream has source {self.source}, found object with '
f'source {obj.source()} in operation {self.serial}/{self.operation.value}/{obj.pk()}. '
f'This operation is ignored, causing potential data inconsistencies.'
)
database_handler.record_mirror_error(self.source, msg)
logger.critical(msg)
return False
if self.operation == DatabaseOperation.add_or_update:
if self.rpki_aware and obj.rpki_relevant and obj.prefix and obj.asn_first:
roa_validator = SingleRouteROAValidator(database_handler)
obj.rpki_status = roa_validator.validate_route(
obj.prefix, obj.asn_first, obj.source())
scope_validator = ScopeFilterValidator()
obj.scopefilter_status, _ = scope_validator.validate_rpsl_object(
obj)
database_handler.upsert_rpsl_object(obj,
JournalEntryOrigin.mirror,
source_serial=self.serial)
elif self.operation == DatabaseOperation.delete:
database_handler.delete_rpsl_object(
rpsl_object=obj,
origin=JournalEntryOrigin.mirror,
source_serial=self.serial)
log = f'Completed NRTM operation {str(self)}/{obj.rpsl_object_class}/{obj.pk()}'
if self.rpki_aware and obj.rpki_relevant:
log += f', RPKI status {obj.rpki_status.value}'
logger.info(log)
return True
class ChangeRequest:
"""
A ChangeRequest tracks and processes a request for a single change.
In this context, a change can be creating, modifying or deleting an
RPSL object.
"""
rpsl_text_submitted: str
rpsl_obj_new: Optional[RPSLObject]
rpsl_obj_current: Optional[RPSLObject] = None
status = UpdateRequestStatus.PROCESSING
request_type: Optional[UpdateRequestType] = None
mntners_notify: List[RPSLMntner]
error_messages: List[str]
info_messages: List[str]
def __init__(self,
rpsl_text_submitted: str,
database_handler: DatabaseHandler,
auth_validator: AuthValidator,
reference_validator: ReferenceValidator,
delete_reason=Optional[str]) -> None:
"""
Initialise a new change request for a single RPSL object.
:param rpsl_text_submitted: the object text
:param database_handler: a DatabaseHandler instance
:param auth_validator: a AuthValidator instance, to resolve authentication requirements
:param reference_validator: a ReferenceValidator instance, to resolve references between objects
:param delete_reason: a string with the deletion reason, if this was a deletion request
The rpsl_text passed into this function should be cleaned from any
meta attributes like delete/override/password. Those should be passed
into this method as delete_reason, or provided to the AuthValidator.
The auth_validator and reference_validator must be shared between
different instances, to benefit from caching, and to resolve references
between different objects that are part of the same submission with
possibly multiple changes.
"""
self.database_handler = database_handler
self.auth_validator = auth_validator
self.reference_validator = reference_validator
self.rpsl_text_submitted = rpsl_text_submitted
self.mntners_notify = []
self.used_override = False
self._cached_roa_validity: Optional[bool] = None
self.roa_validator = SingleRouteROAValidator(database_handler)
try:
self.rpsl_obj_new = rpsl_object_from_text(rpsl_text_submitted,
strict_validation=True)
if self.rpsl_obj_new.messages.errors():
self.status = UpdateRequestStatus.ERROR_PARSING
self.error_messages = self.rpsl_obj_new.messages.errors()
self.info_messages = self.rpsl_obj_new.messages.infos()
logger.debug(
f'{id(self)}: Processing new ChangeRequest for object {self.rpsl_obj_new}: request {id(self)}'
)
except UnknownRPSLObjectClassException as exc:
self.rpsl_obj_new = None
self.request_type = None
self.status = UpdateRequestStatus.ERROR_UNKNOWN_CLASS
self.info_messages = []
self.error_messages = [str(exc)]
if self.is_valid() and self.rpsl_obj_new:
source = self.rpsl_obj_new.source()
if not get_setting(f'sources.{source}.authoritative'):
logger.debug(
f'{id(self)}: change is for non-authoritative source {source}, rejected'
)
self.error_messages.append(
f'This instance is not authoritative for source {source}')
self.status = UpdateRequestStatus.ERROR_NON_AUTHORITIVE
return
self._retrieve_existing_version()
if delete_reason:
self.request_type = UpdateRequestType.DELETE
if not self.rpsl_obj_current:
self.status = UpdateRequestStatus.ERROR_PARSING
self.error_messages.append(
'Can not delete object: no object found for this key in this database.'
)
logger.debug(
f'{id(self)}: Request attempts to delete object {self.rpsl_obj_new}, '
f'but no existing object found.')
def _retrieve_existing_version(self):
"""
Retrieve the current version of this object, if any, and store it in rpsl_obj_current.
Update self.status appropriately.
"""
query = RPSLDatabaseQuery().sources([self.rpsl_obj_new.source()])
query = query.object_classes([self.rpsl_obj_new.rpsl_object_class
]).rpsl_pk(self.rpsl_obj_new.pk())
results = list(self.database_handler.execute_query(query))
if not results:
self.request_type = UpdateRequestType.CREATE
logger.debug(
f'{id(self)}: Did not find existing version for object {self.rpsl_obj_new}, request is CREATE'
)
elif len(results) == 1:
self.request_type = UpdateRequestType.MODIFY
self.rpsl_obj_current = rpsl_object_from_text(
results[0]['object_text'], strict_validation=False)
logger.debug(f'{id(self)}: Retrieved existing version for object '
f'{self.rpsl_obj_current}, request is MODIFY/DELETE')
else: # pragma: no cover
# This should not be possible, as rpsl_pk/source are a composite unique value in the database scheme.
# Therefore, a query should not be able to affect more than one row.
affected_pks = ', '.join([r['pk'] for r in results])
msg = f'{id(self)}: Attempted to retrieve current version of object {self.rpsl_obj_new.pk()}/'
msg += f'{self.rpsl_obj_new.source()}, but multiple '
msg += f'objects were found, internal pks found: {affected_pks}'
logger.error(msg)
raise ValueError(msg)
def save(self, database_handler: DatabaseHandler) -> None:
"""Save the change to the database."""
if self.status != UpdateRequestStatus.PROCESSING or not self.rpsl_obj_new:
raise ValueError(
'ChangeRequest can only be saved in status PROCESSING')
if self.request_type == UpdateRequestType.DELETE and self.rpsl_obj_current is not None:
logger.info(
f'{id(self)}: Saving change for {self.rpsl_obj_new}: deleting current object'
)
database_handler.delete_rpsl_object(
rpsl_object=self.rpsl_obj_current,
origin=JournalEntryOrigin.auth_change)
else:
if not self.used_override:
self.rpsl_obj_new.overwrite_date_new_changed_attributes(
self.rpsl_obj_current)
# This call may have emitted a new info message.
self._import_new_rpsl_obj_info_messages()
logger.info(
f'{id(self)}: Saving change for {self.rpsl_obj_new}: inserting/updating current object'
)
database_handler.upsert_rpsl_object(self.rpsl_obj_new,
JournalEntryOrigin.auth_change)
self.status = UpdateRequestStatus.SAVED
def is_valid(self):
return self.status in [
UpdateRequestStatus.SAVED, UpdateRequestStatus.PROCESSING
]
def submitter_report(self) -> str:
"""Produce a string suitable for reporting back status and messages to the human submitter."""
status = 'succeeded' if self.is_valid() else 'FAILED'
report = f'{self.request_type_str().title()} {status}: [{self.object_class_str()}] {self.object_pk_str()}\n'
if self.info_messages or self.error_messages:
if not self.rpsl_obj_new or self.error_messages:
report += '\n' + self.rpsl_text_submitted + '\n'
else:
report += '\n' + self.rpsl_obj_new.render_rpsl_text() + '\n'
report += ''.join([f'ERROR: {e}\n' for e in self.error_messages])
report += ''.join([f'INFO: {e}\n' for e in self.info_messages])
return report
def notification_target_report(self):
"""
Produce a string suitable for reporting back status and messages
to a human notification target, i.e. someone listed
in notify/upd-to/mnt-nfy.
"""
if not self.is_valid(
) and self.status != UpdateRequestStatus.ERROR_AUTH:
raise ValueError(
'Notification reports can only be made for changes that are valid '
'or have failed authorisation.')
status = 'succeeded' if self.is_valid() else 'FAILED AUTHORISATION'
report = f'{self.request_type_str().title()} {status} for object below: '
report += f'[{self.object_class_str()}] {self.object_pk_str()}:\n\n'
if self.request_type == UpdateRequestType.MODIFY:
current_text = list(
splitline_unicodesafe(
self.rpsl_obj_current.render_rpsl_text()))
new_text = list(
splitline_unicodesafe(self.rpsl_obj_new.render_rpsl_text()))
diff = list(
difflib.unified_diff(current_text, new_text, lineterm=''))
report += '\n'.join(
diff[2:]
) # skip the lines from the diff which would have filenames
if self.status == UpdateRequestStatus.ERROR_AUTH:
report += '\n\n*Rejected* new version of this object:\n\n'
else:
report += '\n\nNew version of this object:\n\n'
if self.request_type == UpdateRequestType.DELETE:
report += self.rpsl_obj_current.render_rpsl_text()
else:
report += self.rpsl_obj_new.render_rpsl_text()
return report
def request_type_str(self) -> str:
return self.request_type.value if self.request_type else 'request'
def object_pk_str(self) -> str:
return self.rpsl_obj_new.pk(
) if self.rpsl_obj_new else '(unreadable object key)'
def object_class_str(self) -> str:
return self.rpsl_obj_new.rpsl_object_class if self.rpsl_obj_new else '(unreadable object class)'
def notification_targets(self) -> Set[str]:
"""
Produce a set of e-mail addresses that should be notified
about the change to this object.
May include mntner upd-to or mnt-nfy, and notify of existing object.
"""
targets: Set[str] = set()
status_qualifies_notification = self.is_valid(
) or self.status == UpdateRequestStatus.ERROR_AUTH
if self.used_override or not status_qualifies_notification:
return targets
mntner_attr = 'upd-to' if self.status == UpdateRequestStatus.ERROR_AUTH else 'mnt-nfy'
for mntner in self.mntners_notify:
for email in mntner.parsed_data.get(mntner_attr, []):
targets.add(email)
if self.rpsl_obj_current:
for email in self.rpsl_obj_current.parsed_data.get('notify', []):
targets.add(email)
return targets
def validate(self) -> bool:
auth_valid = self._check_auth()
if not auth_valid:
return False
references_valid = self._check_references()
self.notification_targets()
rpki_valid = self._check_conflicting_roa()
return references_valid and rpki_valid
def _check_auth(self) -> bool:
assert self.rpsl_obj_new
auth_result = self.auth_validator.process_auth(self.rpsl_obj_new,
self.rpsl_obj_current)
self.info_messages += auth_result.info_messages
self.mntners_notify = auth_result.mntners_notify
if not auth_result.is_valid():
self.status = UpdateRequestStatus.ERROR_AUTH
self.error_messages += auth_result.error_messages
logger.debug(
f'{id(self)}: Authentication check failed: {auth_result.error_messages}'
)
return False
self.used_override = auth_result.used_override
logger.debug(f'{id(self)}: Authentication check succeeded')
return True
def _check_references(self) -> bool:
"""
Check all references from this object to or from other objects.
For deletions, only references to the deleted object matter, as
they now become invalid. For other operations, only the validity
of references from the new object to others matter.
"""
if self.request_type == UpdateRequestType.DELETE and self.rpsl_obj_current is not None:
assert self.rpsl_obj_new
references_result = self.reference_validator.check_references_from_others(
self.rpsl_obj_current)
else:
assert self.rpsl_obj_new
references_result = self.reference_validator.check_references_to_others(
self.rpsl_obj_new)
self.info_messages += references_result.info_messages
if not references_result.is_valid():
self.error_messages += references_result.error_messages
logger.debug(
f'{id(self)}: Reference check failed: {references_result.error_messages}'
)
if self.is_valid(
): # Only change the status if this object was valid prior, so this is the first failure
self.status = UpdateRequestStatus.ERROR_REFERENCE
return False
logger.debug(f'{id(self)}: Reference check succeeded')
return True
def _check_conflicting_roa(self) -> bool:
"""
Check whether there are any conflicting ROAs with the new object.
Result is cached, as validate() may be called multiple times,
but the result of this check will not change. Always returns
True when not in RPKI-aware mode.
"""
assert self.rpsl_obj_new
if self._cached_roa_validity is not None:
return self._cached_roa_validity
if not get_setting(
'rpki.roa_source') or not self.rpsl_obj_new.rpki_relevant:
return True
# Deletes are permitted for RPKI-invalids, other operations are not
if self.request_type == UpdateRequestType.DELETE:
return True
assert self.rpsl_obj_new.asn_first
validation_result = self.roa_validator.validate_route(
self.rpsl_obj_new.prefix, self.rpsl_obj_new.asn_first,
self.rpsl_obj_new.source())
if validation_result == RPKIStatus.invalid:
import_timer = get_setting('rpki.roa_import_timer')
user_message = 'RPKI ROAs were found that conflict with this object. '
user_message += f'(This IRRd refreshes ROAs every {import_timer} seconds.)'
logger.debug(f'{id(self)}: Conflicting ROAs found')
if self.is_valid(
): # Only change the status if this object was valid prior, so this is first failure
self.status = UpdateRequestStatus.ERROR_ROA
self.error_messages.append(user_message)
self._cached_roa_validity = False
return False
else:
logger.debug(f'{id(self)}: No conflicting ROAs found')
self._cached_roa_validity = True
return True
def _import_new_rpsl_obj_info_messages(self):
"""
Import new info messages from self.rpsl_obj_new.
This is used after overwrite_date_new_changed_attributes()
is called, as it's called just before saving, but may
emit a new info message.
"""
for info_message in self.rpsl_obj_new.messages.infos():
if info_message not in self.info_messages:
self.info_messages.append(info_message)
def reactivate_for_mntner(
database_handler: DatabaseHandler,
reactivated_mntner: RPSLMntner) -> Tuple[List[RPSLObject], List[str]]:
"""
Reactivate previously suspended mntners and return the restored objects.
Revives objects that were previously suspended with suspend_for_mntner.
All RPSL objects that had `reactivated_mntner` as one of their mnt-by's at
the time of suspension are restored. Note that this is potentially different
from "all objects that were suspended at the time `reactivated_mntner` was
suspended". Reactivated objects are removed from the suspended store.
If an object is to be reactivated, but there is already another RPSL object
with the same class and primary key in the same source, the reactivation
is skipped. The object remains in the suspended store.
Throws a ValueError if not authoritative for this source or reactivated_mntner
does not exist in the suspended store.
Returns a tuple of all reactivated RPSL objects and a list
of info messages about reactivated and skipped objects.
"""
log_prelude = f'reactivation {reactivated_mntner.pk()}'
source = reactivated_mntner.source()
scopefilter_validator = ScopeFilterValidator()
roa_validator = SingleRouteROAValidator(database_handler)
if not get_setting(f'sources.{source}.authoritative'):
raise ValueError(f'Not authoritative for source {source}')
logger.info(
f"{log_prelude}: Starting reactivation for for {reactivated_mntner}")
def pk_exists(pk: str, rpsl_object_class: str) -> bool:
existing_object_query = RPSLDatabaseQuery(column_names=['pk']).sources(
[source])
existing_object_query = existing_object_query.rpsl_pk(
pk).object_classes([rpsl_object_class])
return bool(list(
database_handler.execute_query(existing_object_query)))
if pk_exists(reactivated_mntner.pk(), 'mntner'):
msg = f"source {source} has a currently active mntner {reactivated_mntner.pk()} - can not restore the suspended one"
logger.info(f"{log_prelude}: error: {msg}")
raise ValueError(msg)
# This is both a check to make sure the mntner is actually suspended,
# but also to catch cases where a suspended mntner did not have itself as mnt-by.
query = RPSLDatabaseSuspendedQuery().sources([source]).rpsl_pk(
reactivated_mntner.pk()).object_classes(['mntner'])
results = list(database_handler.execute_query(query))
if not results:
msg = f"mntner {reactivated_mntner.pk()} not found in suspended store for source {source}"
logger.info(f"{log_prelude}: error: {msg}")
raise ValueError(msg)
query = RPSLDatabaseSuspendedQuery().sources([source]).mntner(
reactivated_mntner.pk())
results += list(database_handler.execute_query(query))
restored_row_pk_uuids = set()
restored_objects = []
info_messages: List[str] = []
for result in results:
if result['pk'] in restored_row_pk_uuids:
continue
reactivating_obj = rpsl_object_from_text(result['object_text'],
strict_validation=False)
if pk_exists(reactivating_obj.pk(),
reactivating_obj.rpsl_object_class):
msg = f"Skipping restore of object {reactivating_obj} - an object already exists with the same key"
logger.info(f"{log_prelude}: {msg}")
info_messages.append(msg)
continue
reactivating_obj.scopefilter_status, _ = scopefilter_validator.validate_rpsl_object(
reactivating_obj)
if get_setting(
'rpki.roa_source'
) and reactivating_obj.rpki_relevant and reactivating_obj.asn_first:
reactivating_obj.rpki_status = roa_validator.validate_route(
reactivating_obj.prefix, reactivating_obj.asn_first, source)
database_handler.upsert_rpsl_object(
reactivating_obj,
JournalEntryOrigin.suspension,
forced_created_value=result['original_created'])
restored_row_pk_uuids.add(result['pk'])
restored_objects.append(reactivating_obj)
logger.info(f"{log_prelude}: Restoring object {reactivating_obj}")
database_handler.delete_suspended_rpsl_objects(restored_row_pk_uuids)
return restored_objects, info_messages