def check_references_from_others(self,
rpsl_obj: RPSLObject) -> ValidatorResult:
"""
Check for any references to this object in the DB.
Used for validating deletions.
Checks self._preload_deleted, because a reference from an object
that is also about to be deleted, is acceptable.
"""
result = ValidatorResult()
if not rpsl_obj.references_strong_inbound():
return result
query = RPSLDatabaseQuery().sources([rpsl_obj.source()])
query = query.lookup_attrs_in(rpsl_obj.references_strong_inbound(),
[rpsl_obj.pk()])
query_results = self.database_handler.execute_query(query)
for query_result in query_results:
reference_to_be_deleted = (
query_result['object_class'], query_result['rpsl_pk'],
query_result['source']) in self._preloaded_deleted
if not reference_to_be_deleted:
result.error_messages.add(
f'Object {rpsl_obj.pk()} to be deleted, but still referenced '
f'by {query_result["object_class"]} {query_result["rpsl_pk"]}'
)
return result
def pk_exists(pk: str, rpsl_object_class: str) -> bool:
existing_object_query = RPSLDatabaseQuery(column_names=['pk']).sources(
[source])
existing_object_query = existing_object_query.rpsl_pk(
pk).object_classes([rpsl_object_class])
return bool(list(
database_handler.execute_query(existing_object_query)))
def set_last_modified():
dh = DatabaseHandler()
auth_sources = [
k for k, v in get_setting('sources').items() if v.get('authoritative')
]
q = RPSLDatabaseQuery(column_names=['pk', 'object_text', 'updated'],
enable_ordering=False)
q = q.sources(auth_sources)
results = list(dh.execute_query(q))
print(f'Updating {len(results)} objects in sources {auth_sources}')
for result in results:
rpsl_obj = rpsl_object_from_text(result['object_text'],
strict_validation=False)
if rpsl_obj.messages.errors(): # pragma: no cover
print(
f'Failed to process {rpsl_obj}: {rpsl_obj.messages.errors()}')
continue
new_text = rpsl_obj.render_rpsl_text(result['updated'])
stmt = RPSLDatabaseObject.__table__.update().where(
RPSLDatabaseObject.__table__.c.pk == result['pk']).values(
object_text=new_text, )
dh.execute_statement(stmt)
dh.commit()
dh.close()
def _retrieve_existing_version(self):
"""
Retrieve the current version of this object, if any, and store it in rpsl_obj_current.
Update self.status appropriately.
"""
query = RPSLDatabaseQuery().sources([self.rpsl_obj_new.source()])
query = query.object_classes([self.rpsl_obj_new.rpsl_object_class
]).rpsl_pk(self.rpsl_obj_new.pk())
results = list(self.database_handler.execute_query(query))
if not results:
self.request_type = UpdateRequestType.CREATE
logger.debug(
f'{id(self)}: Did not find existing version for object {self.rpsl_obj_new}, request is CREATE'
)
elif len(results) == 1:
self.request_type = UpdateRequestType.MODIFY
self.rpsl_obj_current = rpsl_object_from_text(
results[0]['object_text'], strict_validation=False)
logger.debug(f'{id(self)}: Retrieved existing version for object '
f'{self.rpsl_obj_current}, request is MODIFY/DELETE')
else: # pragma: no cover
# This should not be possible, as rpsl_pk/source are a composite unique value in the database scheme.
# Therefore, a query should not be able to affect more than one row.
affected_pks = ', '.join([r['pk'] for r in results])
msg = f'{id(self)}: Attempted to retrieve current version of object {self.rpsl_obj_new.pk()}/'
msg += f'{self.rpsl_obj_new.source()}, but multiple '
msg += f'objects were found, internal pks found: {affected_pks}'
logger.error(msg)
raise ValueError(msg)
def _check_mntners(self, mntner_pk_list: List[str], source: str) -> Tuple[bool, List[RPSLMntner]]:
"""
Check whether authentication passes for a list of maintainers.
Returns True if at least one of the mntners in mntner_list
passes authentication, given self.passwords and
self.keycert_obj_pk. Updates and checks self.passed_mntner_cache
to prevent double checking of maintainers.
"""
mntner_pk_set = set(mntner_pk_list)
mntner_objs: List[RPSLMntner] = [m for m in self._mntner_db_cache if m.pk() in mntner_pk_set and m.source() == source]
mntner_pks_to_resolve: Set[str] = mntner_pk_set - {m.pk() for m in mntner_objs}
if mntner_pks_to_resolve:
query = RPSLDatabaseQuery().sources([source])
query = query.object_classes(['mntner']).rpsl_pks(mntner_pks_to_resolve)
results = list(self.database_handler.execute_query(query))
retrieved_mntner_objs: List[RPSLMntner] = [rpsl_object_from_text(r['object_text']) for r in results] # type: ignore
self._mntner_db_cache.update(retrieved_mntner_objs)
mntner_objs += retrieved_mntner_objs
for mntner_name in mntner_pk_list:
if mntner_name in self._pre_approved:
return True, mntner_objs
for mntner_obj in mntner_objs:
if mntner_obj.verify_auth(self.passwords, self.keycert_obj_pk):
return True, mntner_objs
return False, mntner_objs
def _prepare_query(self,
column_names=None,
ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources:
query.sources(self.sources)
if self.object_class_filter:
query.object_classes(self.object_class_filter)
if self.rpki_invalid_filter_enabled:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
if self.out_scope_filter_enabled:
query.scopefilter_status([ScopeFilterStatus.in_scope])
self.object_class_filter = []
return query
def load_pgp_keys(source: str) -> None:
dh = DatabaseHandler()
query = RPSLDatabaseQuery(column_names=['rpsl_pk', 'object_text'])
query = query.sources([source]).object_classes(['key-cert'])
keycerts = dh.execute_query(query)
for keycert in keycerts:
rpsl_pk = keycert["rpsl_pk"]
print(f'Loading key-cert {rpsl_pk}')
# Parsing the keycert in strict mode will load it into the GPG keychain
result = rpsl_object_from_text(keycert['object_text'], strict_validation=True)
if result.messages.errors():
print(f'Errors in PGP key {rpsl_pk}: {result.messages.errors()}')
print('All valid key-certs loaded into the GnuPG keychain.')
dh.close()
def _prepare_query(self,
column_names=None,
ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources and self.sources != self.all_valid_sources:
query.sources(self.sources)
else:
default = list(get_setting('sources_default', []))
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
if self.rpki_invalid_filter_enabled:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
return query
def _export(self, export_destination):
filename_export = Path(export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(query))['serial_newest_seen']
except StopIteration:
logger.error(f'Unable to run export for {self.source}, internal database status is empty.')
return
with gzip.open(export_tmpfile, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
for obj in self.database_handler.execute_query(query):
object_bytes = remove_auth_hashes(obj['object_text']).encode('utf-8')
fh.write(object_bytes + b'\n')
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
self.database_handler.record_serial_exported(self.source, serial)
logger.info(f'Export for {self.source} complete, stored in {filename_export} / {filename_serial}')
def _check_reference_to_others(self, object_classes: List[str],
object_pk: str, source: str) -> bool:
"""
Check whether one reference to a particular object class/source/PK is valid,
i.e. such an object exists in the database.
Object classes is a list of classes to which the reference may point, e.g.
person/role for admin-c, route for route-set members, or route/route6 for mp-members.
"""
for object_class in object_classes:
if (object_class, object_pk, source) in self._cache:
return True
if (object_class, object_pk, source) in self._preloaded_new:
return True
if (object_class, object_pk, source) in self._preloaded_deleted:
return False
query = RPSLDatabaseQuery().sources(
[source]).object_classes(object_classes).rpsl_pk(object_pk)
results = list(self.database_handler.execute_query(query))
for result in results:
self._cache.add((result['object_class'], object_pk, source))
if len(results):
return True
return False
def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \
Tuple[List[Dict[str, str]], List[Dict[str, str]], List[Dict[str, str]]]:
"""
Apply the scope filter to all relevant objects.
Retrieves all routes from the DB, and aggregates the validation results.
Returns a tuple of three sets:
- one with routes that should be set to status in_scope, but are not now
- one with routes that should be set to status out_scope_as, but are not now
- one with routes that should be set to status out_scope_prefix, but are not now
Each object is recorded as a dict, which has the fields shown
in "columns" below.
Objects where their current status in the DB matches the new
validation result, are not included in the return value.
"""
columns = [
'rpsl_pk', 'ip_first', 'prefix_length', 'asn_first', 'source',
'object_class', 'object_text', 'scopefilter_status'
]
objs_changed: Dict[ScopeFilterStatus,
List[Dict[str, str]]] = defaultdict(list)
q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False)
q = q.object_classes(['route', 'route6'])
results = database_handler.execute_query(q)
for result in results:
current_status = result['scopefilter_status']
result['old_status'] = current_status
prefix = None
if result['ip_first']:
prefix = IP(result['ip_first'] + '/' +
str(result['prefix_length']))
new_status, _ = self._validate_rpsl_data(
result['source'],
result['object_class'],
prefix,
result['asn_first'],
)
if new_status != current_status:
result['scopefilter_status'] = new_status
objs_changed[new_status].append(result)
return (objs_changed[ScopeFilterStatus.in_scope],
objs_changed[ScopeFilterStatus.out_scope_as],
objs_changed[ScopeFilterStatus.out_scope_prefix])
def validate_all_routes(self, sources: List[str]=None) -> \
Tuple[List[Dict[str, str]], List[Dict[str, str]], List[Dict[str, str]]]:
"""
Validate all RPSL route/route6 objects.
Retrieves all routes from the DB, and aggregates the validation results.
Returns a tuple of three sets of RPSL route(6)'s:
- one with routes that should be set to status VALID, but are not now
- one with routes that should be set to status INVALID, but are not now
- one with routes that should be set to status UNKNOWN, but are not now
Each route is recorded as a dict, which has the fields shown
in "columns" below.
Routes where their current validation status in the DB matches the new
validation result, are not included in the return value.
"""
columns = [
'rpsl_pk', 'ip_first', 'prefix_length', 'asn_first', 'source',
'object_class', 'object_text', 'rpki_status'
]
q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False)
q = q.object_classes(['route', 'route6'])
if sources:
q = q.sources(sources)
routes = self.database_handler.execute_query(q)
objs_changed: Dict[RPKIStatus, List[Dict[str,
str]]] = defaultdict(list)
for result in routes:
# RPKI_IRR_PSEUDO_SOURCE objects are ROAs, and don't need validation.
if result['source'] == RPKI_IRR_PSEUDO_SOURCE:
continue
current_status = result['rpki_status']
result['old_status'] = current_status
new_status = self.validate_route(result['ip_first'],
result['prefix_length'],
result['asn_first'],
result['source'])
if new_status != current_status:
result['rpki_status'] = new_status
objs_changed[new_status].append(result)
return objs_changed[RPKIStatus.valid], objs_changed[
RPKIStatus.invalid], objs_changed[RPKIStatus.not_found]
def _prepare_query(self, column_names=None, ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources:
query.sources(self.sources)
else:
default = get_setting('sources_default')
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
return query
def _prepare_query(self) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery()
if self.sources:
query.sources(self.sources)
else:
default = get_setting('sources_default')
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
return query
def _resolve_pgp_key_id(self, pgp_fingerprint: str) -> Optional[str]:
"""
Find a PGP key ID for a given fingerprint.
This method looks for an actual matching object in the database,
and then returns the object's PK.
"""
clean_fingerprint = pgp_fingerprint.replace(' ', '')
key_id = "PGPKEY-" + clean_fingerprint[-8:]
query = RPSLDatabaseQuery().object_classes(['key-cert']).rpsl_pk(key_id)
results = list(self.database_handler.execute_query(query))
for result in results:
if result['parsed_data'].get('fingerpr', '').replace(' ', '') == clean_fingerprint:
return key_id
logger.info(f'Message was signed with key {key_id}, but key was not found in the database. Treating message '
f'as unsigned. Message metadata: {self.request_meta}')
return None
def _export(self, export_destination, remove_auth_hashes=True):
filename_export = Path(
export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(
export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(
query))['serial_newest_seen']
except StopIteration:
serial = None
with gzip.open(export_tmpfile.name, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
query = query.scopefilter_status([ScopeFilterStatus.in_scope])
for obj in self.database_handler.execute_query(query):
object_text = obj['object_text']
if remove_auth_hashes:
object_text = remove_auth_hashes_func(object_text)
object_bytes = object_text.encode('utf-8')
fh.write(object_bytes + b'\n')
fh.write(b'# EOF\n')
os.chmod(export_tmpfile.name, EXPORT_PERMISSIONS)
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
os.chmod(filename_serial, EXPORT_PERMISSIONS)
self.database_handler.record_serial_exported(self.source, serial)
logger.info(
f'Export for {self.source} complete at serial {serial}, stored in {filename_export} / {filename_serial}'
)
def _columns_for_graphql_selection(info: GraphQLResolveInfo) -> Set[str]:
"""
Based on the selected GraphQL fields, determine which database
columns should be retrieved.
"""
# Some columns are always retrieved
columns = {'object_class', 'source', 'parsed_data', 'rpsl_pk'}
fields = _collect_predicate_names(
info.field_nodes[0].selection_set.selections) # type: ignore
requested_fields = {ariadne.convert_camel_case_to_snake(f) for f in fields}
for field in requested_fields:
if field in RPSLDatabaseQuery().columns:
columns.add(field)
if field == 'asn':
columns.add('asn_first')
columns.add('asn_last')
if field == 'prefix':
columns.add('ip_first')
columns.add('prefix_length')
return columns
def _resolve_subquery(rpsl_object,
info: GraphQLResolveInfo,
object_classes: List[str],
pk_field: str,
sticky_source=True):
"""
Resolve a subquery, like techCobjs, on an RPSL object, considering
a number of object classes, extracting the PK from pk_field.
If sticky_source is set, the referred object must be from the same source.
"""
pks = rpsl_object.get(pk_field)
if not pks:
return []
if not isinstance(pks, list):
pks = [pks]
query = RPSLDatabaseQuery(
column_names=_columns_for_graphql_selection(info),
ordered_by_sources=False,
enable_ordering=False)
query.object_classes(object_classes).rpsl_pks(pks)
if sticky_source:
query.sources([rpsl_object['source']])
return _rpsl_db_query_to_graphql_out(query, info)
def notify_rpki_invalid_owners(
database_handler: DatabaseHandler,
rpsl_dicts_now_invalid: List[Dict[str, str]]) -> int:
"""
Notify the owners/contacts of newly RPKI invalid objects.
Expects a list of objects, each a dict with their properties.
Contacts are resolved as any mnt-nfy, or any email address on any
tech-c or admin-c, of any maintainer of the object.
One email is sent per email address.
"""
if not get_setting('rpki.notify_invalid_enabled'):
return 0
rpsl_objs = []
for obj in rpsl_dicts_now_invalid:
source = obj['source']
authoritative = get_setting(f'sources.{source}.authoritative')
if authoritative and obj['rpki_status'] == RPKIStatus.invalid:
rpsl_objs.append(rpsl_object_from_text(obj['object_text']))
if not rpsl_objs:
return 0
sources = set([obj.parsed_data['source'] for obj in rpsl_objs])
mntner_emails_by_source = {}
for source in sources:
# For each source, a multi-step process is run to fill this
# dict with the contact emails for each mntner.
mntner_emails = defaultdict(set)
# Step 1: retrieve all relevant maintainers from the DB
mntner_pks = set(
itertools.chain(*[
obj.parsed_data.get('mnt-by', []) for obj in rpsl_objs
if obj.parsed_data['source'] == source
]))
query = RPSLDatabaseQuery(['rpsl_pk', 'parsed_data']).sources(
[source]).rpsl_pks(mntner_pks).object_classes(['mntner'])
mntners = list(database_handler.execute_query(query))
# Step 2: any mnt-nfy on these maintainers is a contact address
for mntner in mntners:
mntner_emails[mntner['rpsl_pk']].update(mntner['parsed_data'].get(
'mnt-nfy', []))
# Step 3: extract the contact handles for each maintainer
mntner_contacts = {
m['rpsl_pk']: m['parsed_data'].get('tech-c', []) +
m['parsed_data'].get('admin-c', [])
for m in mntners
}
# Step 4: retrieve all these contacts from the DB in bulk,
# and extract their e-mail addresses
contact_pks = set(itertools.chain(*mntner_contacts.values()))
query = RPSLDatabaseQuery(['rpsl_pk', 'parsed_data']).sources(
[source]).rpsl_pks(contact_pks).object_classes(['role', 'person'])
contacts = {
r['rpsl_pk']: r['parsed_data'].get('e-mail', [])
for r in database_handler.execute_query(query)
}
# Step 5: use the contacts per maintainer, and emails per contact
# to create a flattened list of emails per maintainer
for mntner_pk, mntner_contacts in mntner_contacts.items():
for contact_pk in mntner_contacts:
try:
mntner_emails[mntner_pk].update(contacts[contact_pk])
except KeyError:
pass
mntner_emails_by_source[source] = mntner_emails
# With mntners_emails_by_source filled with per source, per maintainer,
# all relevant emails, categorise the RPSL objects on which email
# addresses they need to be sent to.
objs_per_email: Dict[str, Set[RPSLObject]] = defaultdict(set)
for rpsl_obj in rpsl_objs:
mntners = rpsl_obj.parsed_data.get('mnt-by', [])
source = rpsl_obj.parsed_data['source']
for mntner_pk in mntners:
try:
for email in mntner_emails_by_source[source][mntner_pk]:
objs_per_email[email].add(rpsl_obj)
except KeyError: # pragma: no cover
pass
header_template = get_setting('rpki.notify_invalid_header', '')
subject_template = get_setting('rpki.notify_invalid_subject',
'').replace('\n', ' ')
for email, objs in objs_per_email.items():
sources_str = ', '.join(
set([obj.parsed_data['source'] for obj in objs]))
subject = subject_template.format(sources_str=sources_str,
object_count=len(objs))
body = header_template.format(sources_str=sources_str,
object_count=len(objs))
body += '\nThe following objects are affected:\n'
body += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n'
for rpsl_obj in objs:
body += rpsl_obj.render_rpsl_text() + '\n'
body += '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
try:
send_email(email, subject, body)
except Exception as e: # pragma: no cover
logger.warning(
f'Unable to send RPKI invalid notification to {email}: {e}')
return len(objs_per_email.keys())
def suspend_for_mntner(database_handler: DatabaseHandler,
suspended_mntner: RPSLMntner) -> List[Dict[str, str]]:
"""
Suspend all RPSL objects for a mntner and return details of suspended objects.
This will move RPSL objects to the suspended table, where they are never
included in query responses. RPSL objects suspended are:
- The mntner with primary key and source of `suspended_mntner`
- Any object in the same source, that has `suspended_mntner` as its only
active mnt-by, i.e. there are no other entries in mnt-by or those mntners
do not currently exist.
Throws a ValueError if not authoritative for this source or suspended_mntner
does not exist. Returns database rows for all suspended objects.
"""
log_prelude = f'suspension {suspended_mntner.pk()}'
source = suspended_mntner.source()
if not get_setting(f'sources.{source}.authoritative'):
raise ValueError(f'Not authoritative for source {source}')
logger.info(f"{log_prelude}: Starting suspension for {suspended_mntner}")
@functools.lru_cache(maxsize=50)
def mntner_active(rpsl_pk: str):
q = RPSLDatabaseQuery(column_names=['pk']).sources(
[source]).rpsl_pk(rpsl_pk).object_classes(['mntner'])
return bool(list(database_handler.execute_query(q.first_only())))
# This runs two queries, to account for the suspension of a mntner
# who is not an mnt-by for itself. In that case, query1 will not retrieve it,
# but query2 will.
query1 = RPSLDatabaseQuery(column_names=[
'pk', 'rpsl_pk', 'object_class', 'source', 'parsed_data'
])
query1 = query1.sources([source]).lookup_attr('mnt-by',
suspended_mntner.pk())
query1_result = list(database_handler.execute_query(query1))
query2 = RPSLDatabaseQuery(column_names=[
'pk', 'rpsl_pk', 'object_class', 'source', 'parsed_data'
])
query2 = query2.sources([source]).rpsl_pk(
suspended_mntner.pk()).object_classes(['mntner'])
query2_result = list(database_handler.execute_query(query2))
if not query2_result:
msg = f"mntner {suspended_mntner.pk()} does not exist in {source} (or may already be suspended)"
logger.info(f"{log_prelude}: error: {msg}")
raise ValueError(msg)
suspended_objects = []
for row in query1_result + query2_result:
if row in suspended_objects:
continue
mntners_active = [
m for m in set(row['parsed_data']['mnt-by'])
if m != suspended_mntner.pk() and mntner_active(m)
]
if row['rpsl_pk'] != suspended_mntner.pk() and mntners_active:
logger.info(
f"{log_prelude}: Skipping suspension of {row['object_class']}/{row['rpsl_pk']} because of remaining active mntners {mntners_active}"
)
continue
logger.info(
f"{log_prelude}: Suspending {row['object_class']}/{row['rpsl_pk']}"
)
database_handler.suspend_rpsl_object(row['pk'])
suspended_objects.append(row)
return suspended_objects
def mntner_active(rpsl_pk: str):
q = RPSLDatabaseQuery(column_names=['pk']).sources(
[source]).rpsl_pk(rpsl_pk).object_classes(['mntner'])
return bool(list(database_handler.execute_query(q.first_only())))
def resolve_rpsl_objects(_, info: GraphQLResolveInfo, **kwargs):
"""
Resolve a `rpslObjects` query. This query has a considerable
number of parameters, each of which is applied to an RPSL
database query.
"""
low_specificity_kwargs = {
'object_class', 'rpki_status', 'scope_filter_status', 'sources',
'sql_trace'
}
# A query is sufficiently specific if it has other fields than listed above,
# except that rpki_status is sufficient if it is exclusively selecting on
# valid or invalid.
low_specificity = all([
not (set(kwargs.keys()) - low_specificity_kwargs),
kwargs.get('rpki_status', []) not in [[RPKIStatus.valid],
[RPKIStatus.invalid]],
])
if low_specificity:
raise ValueError('Your query must be more specific.')
if kwargs.get('sql_trace'):
info.context['sql_trace'] = True
query = RPSLDatabaseQuery(
column_names=_columns_for_graphql_selection(info),
ordered_by_sources=False,
enable_ordering=False)
if 'record_limit' in kwargs:
query.limit(kwargs['record_limit'])
if 'rpsl_pk' in kwargs:
query.rpsl_pks(kwargs['rpsl_pk'])
if 'object_class' in kwargs:
query.object_classes(kwargs['object_class'])
if 'asn' in kwargs:
query.asns_first(kwargs['asn'])
if 'text_search' in kwargs:
query.text_search(kwargs['text_search'])
if 'rpki_status' in kwargs:
query.rpki_status(kwargs['rpki_status'])
else:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
if 'scope_filter_status' in kwargs:
query.scopefilter_status(kwargs['scope_filter_status'])
else:
query.scopefilter_status([ScopeFilterStatus.in_scope])
all_valid_sources = set(get_setting('sources', {}).keys())
if get_setting('rpki.roa_source'):
all_valid_sources.add(RPKI_IRR_PSEUDO_SOURCE)
sources_default = set(get_setting('sources_default', []))
if 'sources' in kwargs:
query.sources(kwargs['sources'])
elif sources_default and sources_default != all_valid_sources:
query.sources(list(sources_default))
# All other parameters are generic lookup fields, like `members`
for attr, value in kwargs.items():
attr = attr.replace('_', '-')
if attr in lookup_fields:
query.lookup_attrs_in([attr], value)
ip_filters = [
'ip_exact', 'ip_less_specific', 'ip_more_specific',
'ip_less_specific_one_level', 'ip_any'
]
for ip_filter in ip_filters:
if ip_filter in kwargs:
getattr(query, ip_filter)(IP(kwargs[ip_filter]))
return _rpsl_db_query_to_graphql_out(query, info)
def init_query(rpsl_object_class: str) -> RPSLDatabaseQuery:
query = RPSLDatabaseQuery().sources([rpsl_obj_new.source()])
query = query.object_classes([rpsl_object_class])
return query.first_only()
def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \
Tuple[List[Dict[str, str]], List[Dict[str, str]], List[Dict[str, str]]]:
"""
Apply the scope filter to all relevant objects.
Retrieves all routes from the DB, and aggregates the validation results.
Returns a tuple of three sets:
- one with routes that should be set to status in_scope, but are not now
- one with routes that should be set to status out_scope_as, but are not now
- one with routes that should be set to status out_scope_prefix, but are not now
Each object is recorded as a dict, which has the fields shown
in "columns" below.
Objects where their current status in the DB matches the new
validation result, are not included in the return value.
"""
columns = [
'pk', 'rpsl_pk', 'prefix', 'asn_first', 'source', 'object_class',
'scopefilter_status', 'rpki_status'
]
objs_changed: Dict[ScopeFilterStatus,
List[Dict[str, str]]] = defaultdict(list)
q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False)
q = q.object_classes(['route', 'route6', 'aut-num'])
results = database_handler.execute_query(q)
for result in results:
current_status = result['scopefilter_status']
result['old_status'] = current_status
prefix = None
if result.get('prefix'):
prefix = IP(result['prefix'])
new_status, _ = self._validate_rpsl_data(
result['source'],
result['object_class'],
prefix,
result['asn_first'],
)
if new_status != current_status:
result['scopefilter_status'] = new_status
objs_changed[new_status].append(result)
# Object text is only retrieved for objects with state changes
pks_to_enrich = [
obj['pk'] for objs in objs_changed.values() for obj in objs
]
query = RPSLDatabaseQuery(['pk', 'object_text'],
enable_ordering=False).pks(pks_to_enrich)
rows_per_pk = {
row['pk']: row
for row in database_handler.execute_query(query)
}
for rpsl_objs in objs_changed.values():
for rpsl_obj in rpsl_objs:
rpsl_obj.update(rows_per_pk[rpsl_obj['pk']])
return (objs_changed[ScopeFilterStatus.in_scope],
objs_changed[ScopeFilterStatus.out_scope_as],
objs_changed[ScopeFilterStatus.out_scope_prefix])
def _init_related_object_query(rpsl_object_class: str,
rpsl_obj_new: RPSLObject) -> RPSLDatabaseQuery:
query = RPSLDatabaseQuery().sources([rpsl_obj_new.source()])
query = query.object_classes([rpsl_object_class])
return query.first_only()
