def _prepare_query(self) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery()
if self.sources:
query.sources(self.sources)
else:
default = get_setting('sources_default')
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
return query
def _prepare_query(self, column_names=None, ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources:
query.sources(self.sources)
else:
default = get_setting('sources_default')
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
return query
def _prepare_query(self,
column_names=None,
ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources:
query.sources(self.sources)
if self.object_class_filter:
query.object_classes(self.object_class_filter)
if self.rpki_invalid_filter_enabled:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
if self.out_scope_filter_enabled:
query.scopefilter_status([ScopeFilterStatus.in_scope])
self.object_class_filter = []
return query
def set_last_modified():
dh = DatabaseHandler()
auth_sources = [
k for k, v in get_setting('sources').items() if v.get('authoritative')
]
q = RPSLDatabaseQuery(column_names=['pk', 'object_text', 'updated'],
enable_ordering=False)
q = q.sources(auth_sources)
results = list(dh.execute_query(q))
print(f'Updating {len(results)} objects in sources {auth_sources}')
for result in results:
rpsl_obj = rpsl_object_from_text(result['object_text'],
strict_validation=False)
if rpsl_obj.messages.errors(): # pragma: no cover
print(
f'Failed to process {rpsl_obj}: {rpsl_obj.messages.errors()}')
continue
new_text = rpsl_obj.render_rpsl_text(result['updated'])
stmt = RPSLDatabaseObject.__table__.update().where(
RPSLDatabaseObject.__table__.c.pk == result['pk']).values(
object_text=new_text, )
dh.execute_statement(stmt)
dh.commit()
dh.close()
def _prepare_query(self,
column_names=None,
ordered_by_sources=True) -> RPSLDatabaseQuery:
"""Prepare an RPSLDatabaseQuery by applying relevant sources/class filters."""
query = RPSLDatabaseQuery(column_names, ordered_by_sources)
if self.sources and self.sources != self.all_valid_sources:
query.sources(self.sources)
else:
default = list(get_setting('sources_default', []))
if default:
query.sources(list(default))
if self.object_classes:
query.object_classes(self.object_classes)
if self.rpki_invalid_filter_enabled:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
return query
def _resolve_subquery(rpsl_object,
info: GraphQLResolveInfo,
object_classes: List[str],
pk_field: str,
sticky_source=True):
"""
Resolve a subquery, like techCobjs, on an RPSL object, considering
a number of object classes, extracting the PK from pk_field.
If sticky_source is set, the referred object must be from the same source.
"""
pks = rpsl_object.get(pk_field)
if not pks:
return []
if not isinstance(pks, list):
pks = [pks]
query = RPSLDatabaseQuery(
column_names=_columns_for_graphql_selection(info),
ordered_by_sources=False,
enable_ordering=False)
query.object_classes(object_classes).rpsl_pks(pks)
if sticky_source:
query.sources([rpsl_object['source']])
return _rpsl_db_query_to_graphql_out(query, info)
def load_pgp_keys(source: str) -> None:
dh = DatabaseHandler()
query = RPSLDatabaseQuery(column_names=['rpsl_pk', 'object_text'])
query = query.sources([source]).object_classes(['key-cert'])
keycerts = dh.execute_query(query)
for keycert in keycerts:
rpsl_pk = keycert["rpsl_pk"]
print(f'Loading key-cert {rpsl_pk}')
# Parsing the keycert in strict mode will load it into the GPG keychain
result = rpsl_object_from_text(keycert['object_text'], strict_validation=True)
if result.messages.errors():
print(f'Errors in PGP key {rpsl_pk}: {result.messages.errors()}')
print('All valid key-certs loaded into the GnuPG keychain.')
dh.close()
def validate_all_routes(self, sources: List[str]=None) -> \
Tuple[List[Dict[str, str]], List[Dict[str, str]], List[Dict[str, str]]]:
"""
Validate all RPSL route/route6 objects.
Retrieves all routes from the DB, and aggregates the validation results.
Returns a tuple of three sets of RPSL route(6)'s:
- one with routes that should be set to status VALID, but are not now
- one with routes that should be set to status INVALID, but are not now
- one with routes that should be set to status UNKNOWN, but are not now
Each route is recorded as a dict, which has the fields shown
in "columns" below.
Routes where their current validation status in the DB matches the new
validation result, are not included in the return value.
"""
columns = [
'rpsl_pk', 'ip_first', 'prefix_length', 'asn_first', 'source',
'object_class', 'object_text', 'rpki_status'
]
q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False)
q = q.object_classes(['route', 'route6'])
if sources:
q = q.sources(sources)
routes = self.database_handler.execute_query(q)
objs_changed: Dict[RPKIStatus, List[Dict[str,
str]]] = defaultdict(list)
for result in routes:
# RPKI_IRR_PSEUDO_SOURCE objects are ROAs, and don't need validation.
if result['source'] == RPKI_IRR_PSEUDO_SOURCE:
continue
current_status = result['rpki_status']
result['old_status'] = current_status
new_status = self.validate_route(result['ip_first'],
result['prefix_length'],
result['asn_first'],
result['source'])
if new_status != current_status:
result['rpki_status'] = new_status
objs_changed[new_status].append(result)
return objs_changed[RPKIStatus.valid], objs_changed[
RPKIStatus.invalid], objs_changed[RPKIStatus.not_found]
def resolve_rpsl_objects(_, info: GraphQLResolveInfo, **kwargs):
"""
Resolve a `rpslObjects` query. This query has a considerable
number of parameters, each of which is applied to an RPSL
database query.
"""
low_specificity_kwargs = {
'object_class', 'rpki_status', 'scope_filter_status', 'sources',
'sql_trace'
}
# A query is sufficiently specific if it has other fields than listed above,
# except that rpki_status is sufficient if it is exclusively selecting on
# valid or invalid.
low_specificity = all([
not (set(kwargs.keys()) - low_specificity_kwargs),
kwargs.get('rpki_status', []) not in [[RPKIStatus.valid],
[RPKIStatus.invalid]],
])
if low_specificity:
raise ValueError('Your query must be more specific.')
if kwargs.get('sql_trace'):
info.context['sql_trace'] = True
query = RPSLDatabaseQuery(
column_names=_columns_for_graphql_selection(info),
ordered_by_sources=False,
enable_ordering=False)
if 'record_limit' in kwargs:
query.limit(kwargs['record_limit'])
if 'rpsl_pk' in kwargs:
query.rpsl_pks(kwargs['rpsl_pk'])
if 'object_class' in kwargs:
query.object_classes(kwargs['object_class'])
if 'asn' in kwargs:
query.asns_first(kwargs['asn'])
if 'text_search' in kwargs:
query.text_search(kwargs['text_search'])
if 'rpki_status' in kwargs:
query.rpki_status(kwargs['rpki_status'])
else:
query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
if 'scope_filter_status' in kwargs:
query.scopefilter_status(kwargs['scope_filter_status'])
else:
query.scopefilter_status([ScopeFilterStatus.in_scope])
all_valid_sources = set(get_setting('sources', {}).keys())
if get_setting('rpki.roa_source'):
all_valid_sources.add(RPKI_IRR_PSEUDO_SOURCE)
sources_default = set(get_setting('sources_default', []))
if 'sources' in kwargs:
query.sources(kwargs['sources'])
elif sources_default and sources_default != all_valid_sources:
query.sources(list(sources_default))
# All other parameters are generic lookup fields, like `members`
for attr, value in kwargs.items():
attr = attr.replace('_', '-')
if attr in lookup_fields:
query.lookup_attrs_in([attr], value)
ip_filters = [
'ip_exact', 'ip_less_specific', 'ip_more_specific',
'ip_less_specific_one_level', 'ip_any'
]
for ip_filter in ip_filters:
if ip_filter in kwargs:
getattr(query, ip_filter)(IP(kwargs[ip_filter]))
return _rpsl_db_query_to_graphql_out(query, info)
def suspend_for_mntner(database_handler: DatabaseHandler,
suspended_mntner: RPSLMntner) -> List[Dict[str, str]]:
"""
Suspend all RPSL objects for a mntner and return details of suspended objects.
This will move RPSL objects to the suspended table, where they are never
included in query responses. RPSL objects suspended are:
- The mntner with primary key and source of `suspended_mntner`
- Any object in the same source, that has `suspended_mntner` as its only
active mnt-by, i.e. there are no other entries in mnt-by or those mntners
do not currently exist.
Throws a ValueError if not authoritative for this source or suspended_mntner
does not exist. Returns database rows for all suspended objects.
"""
log_prelude = f'suspension {suspended_mntner.pk()}'
source = suspended_mntner.source()
if not get_setting(f'sources.{source}.authoritative'):
raise ValueError(f'Not authoritative for source {source}')
logger.info(f"{log_prelude}: Starting suspension for {suspended_mntner}")
@functools.lru_cache(maxsize=50)
def mntner_active(rpsl_pk: str):
q = RPSLDatabaseQuery(column_names=['pk']).sources(
[source]).rpsl_pk(rpsl_pk).object_classes(['mntner'])
return bool(list(database_handler.execute_query(q.first_only())))
# This runs two queries, to account for the suspension of a mntner
# who is not an mnt-by for itself. In that case, query1 will not retrieve it,
# but query2 will.
query1 = RPSLDatabaseQuery(column_names=[
'pk', 'rpsl_pk', 'object_class', 'source', 'parsed_data'
])
query1 = query1.sources([source]).lookup_attr('mnt-by',
suspended_mntner.pk())
query1_result = list(database_handler.execute_query(query1))
query2 = RPSLDatabaseQuery(column_names=[
'pk', 'rpsl_pk', 'object_class', 'source', 'parsed_data'
])
query2 = query2.sources([source]).rpsl_pk(
suspended_mntner.pk()).object_classes(['mntner'])
query2_result = list(database_handler.execute_query(query2))
if not query2_result:
msg = f"mntner {suspended_mntner.pk()} does not exist in {source} (or may already be suspended)"
logger.info(f"{log_prelude}: error: {msg}")
raise ValueError(msg)
suspended_objects = []
for row in query1_result + query2_result:
if row in suspended_objects:
continue
mntners_active = [
m for m in set(row['parsed_data']['mnt-by'])
if m != suspended_mntner.pk() and mntner_active(m)
]
if row['rpsl_pk'] != suspended_mntner.pk() and mntners_active:
logger.info(
f"{log_prelude}: Skipping suspension of {row['object_class']}/{row['rpsl_pk']} because of remaining active mntners {mntners_active}"
)
continue
logger.info(
f"{log_prelude}: Suspending {row['object_class']}/{row['rpsl_pk']}"
)
database_handler.suspend_rpsl_object(row['pk'])
suspended_objects.append(row)
return suspended_objects