def handle_irrd_database_serial_range(self, parameter: str) -> str:
"""!j query - database serial range"""
if parameter == '-*':
sources = self.sources_default if self.sources_default else self.all_valid_sources
else:
sources = [s.upper() for s in parameter.split(',')]
invalid_sources = [
s for s in sources if s not in self.all_valid_sources
]
query = DatabaseStatusQuery().sources(sources)
query_results = self.database_handler.execute_query(query)
result_txt = ''
for query_result in query_results:
source = query_result['source'].upper()
keep_journal = 'Y' if get_setting(
f'sources.{source}.keep_journal') else 'N'
serial_oldest = query_result['serial_oldest_seen']
serial_newest = query_result['serial_newest_seen']
fields = [
source,
keep_journal,
f'{serial_oldest}-{serial_newest}'
if serial_oldest and serial_newest else '-',
]
if query_result['serial_last_export']:
fields.append(str(query_result['serial_last_export']))
result_txt += ':'.join(fields) + '\n'
for invalid_source in invalid_sources:
result_txt += f'{invalid_source.upper()}:X:Database unknown\n'
return result_txt.strip()
def _export(self, export_destination):
filename_export = Path(export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(query))['serial_newest_seen']
except StopIteration:
logger.error(f'Unable to run export for {self.source}, internal database status is empty.')
return
with gzip.open(export_tmpfile, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
for obj in self.database_handler.execute_query(query):
object_bytes = remove_auth_hashes(obj['object_text']).encode('utf-8')
fh.write(object_bytes + b'\n')
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
self.database_handler.record_serial_exported(self.source, serial)
logger.info(f'Export for {self.source} complete, stored in {filename_export} / {filename_serial}')
def _status(self) -> Tuple[Optional[int], Optional[bool]]:
query = DatabaseStatusQuery().source(self.source)
result = self.database_handler.execute_query(query)
try:
status = next(result)
return status['serial_newest_seen'], status['force_reload']
except StopIteration:
return None, None
def _export(self, export_destination, remove_auth_hashes=True):
filename_export = Path(
export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(
export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(
query))['serial_newest_seen']
except StopIteration:
serial = None
with gzip.open(export_tmpfile.name, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
query = query.scopefilter_status([ScopeFilterStatus.in_scope])
for obj in self.database_handler.execute_query(query):
object_text = obj['object_text']
if remove_auth_hashes:
object_text = remove_auth_hashes_func(object_text)
object_bytes = object_text.encode('utf-8')
fh.write(object_bytes + b'\n')
fh.write(b'# EOF\n')
os.chmod(export_tmpfile.name, EXPORT_PERMISSIONS)
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
os.chmod(filename_serial, EXPORT_PERMISSIONS)
self.database_handler.record_serial_exported(self.source, serial)
logger.info(
f'Export for {self.source} complete at serial {serial}, stored in {filename_export} / {filename_serial}'
)
def generate_status(self) -> str:
"""
Generate a human-readable overview of database status.
"""
database_handler = DatabaseHandler()
statistics_query = RPSLDatabaseObjectStatisticsQuery()
self.statistics_results = list(database_handler.execute_query(statistics_query))
status_query = DatabaseStatusQuery()
self.status_results = list(database_handler.execute_query(status_query))
results = [self._generate_header(), self._generate_statistics_table(), self._generate_source_detail()]
database_handler.close()
return '\n\n'.join(results)
def database_status(
self,
sources: Optional[List[str]] = None
) -> 'OrderedDict[str, OrderedDict[str, Any]]':
"""Database status. If sources is None, return all valid sources."""
if sources is None:
sources = self.sources_default if self.sources_default else self.all_valid_sources
invalid_sources = [
s for s in sources if s not in self.all_valid_sources
]
query = DatabaseStatusQuery().sources(sources)
query_results = self._execute_query(query)
results: OrderedDict[str, OrderedDict[str, Any]] = OrderedDict()
for query_result in query_results:
source = query_result['source'].upper()
results[source] = OrderedDict()
results[source]['authoritative'] = get_setting(
f'sources.{source}.authoritative', False)
object_class_filter = get_setting(
f'sources.{source}.object_class_filter')
results[source]['object_class_filter'] = list(
object_class_filter) if object_class_filter else None
results[source]['rpki_rov_filter'] = bool(
get_setting('rpki.roa_source')
and not get_setting(f'sources.{source}.rpki_excluded'))
results[source]['scopefilter_enabled'] = bool(
get_setting('scopefilter')
) and not get_setting(f'sources.{source}.scopefilter_excluded')
results[source]['local_journal_kept'] = get_setting(
f'sources.{source}.keep_journal', False)
results[source]['serial_oldest_journal'] = query_result[
'serial_oldest_journal']
results[source]['serial_newest_journal'] = query_result[
'serial_newest_journal']
results[source]['serial_last_export'] = query_result[
'serial_last_export']
results[source]['serial_newest_mirror'] = query_result[
'serial_newest_mirror']
results[source]['last_update'] = query_result[
'updated'].astimezone(timezone('UTC')).isoformat()
results[source]['synchronised_serials'] = is_serial_synchronised(
self.database_handler, source)
for invalid_source in invalid_sources:
results[invalid_source.upper()] = OrderedDict(
{'error': 'Unknown source'})
return results
def handle_irrd_database_status(self, parameter: str) -> str:
"""!J query - database status"""
if parameter == '-*':
sources = self.sources_default if self.sources_default else self.all_valid_sources
else:
sources = [s.upper() for s in parameter.split(',')]
invalid_sources = [
s for s in sources if s not in self.all_valid_sources
]
query = DatabaseStatusQuery().sources(sources)
query_results = self.database_handler.execute_query(query)
results: OrderedDict[str, OrderedDict[str, Any]] = OrderedDict()
for query_result in query_results:
source = query_result['source'].upper()
results[source] = OrderedDict()
results[source]['authoritative'] = get_setting(
f'sources.{source}.authoritative', False)
object_class_filter = get_setting(
f'sources.{source}.object_class_filter')
results[source]['object_class_filter'] = list(
object_class_filter) if object_class_filter else None
results[source]['rpki_rov_filter'] = bool(
get_setting('rpki.roa_source')
and not get_setting(f'sources.{source}.rpki_excluded'))
results[source]['scopefilter_enabled'] = not get_setting(
f'sources.{source}.scopefilter_excluded')
results[source]['local_journal_kept'] = get_setting(
f'sources.{source}.keep_journal', False)
results[source]['serial_oldest_journal'] = query_result[
'serial_oldest_journal']
results[source]['serial_newest_journal'] = query_result[
'serial_newest_journal']
results[source]['serial_last_export'] = query_result[
'serial_last_export']
results[source]['serial_newest_mirror'] = query_result[
'serial_newest_mirror']
results[source]['last_update'] = query_result[
'updated'].astimezone(timezone('UTC')).isoformat()
results[source]['synchronised_serials'] = is_serial_synchronised(
self.database_handler, source)
for invalid_source in invalid_sources:
results[invalid_source.upper()] = OrderedDict(
{'error': 'Unknown source'})
return ujson.dumps(results, indent=4)
def handle_irrd_database_serial_range(self, parameter: str) -> str:
"""
!j query - database serial range
This query is legacy and only available in whois, so resolved
directly here instead of in the query resolver.
"""
if parameter == '-*':
sources = self.query_resolver.sources_default if self.query_resolver.sources_default else self.query_resolver.all_valid_sources
else:
sources = [s.upper() for s in parameter.split(',')]
invalid_sources = [
s for s in sources
if s not in self.query_resolver.all_valid_sources
]
query = DatabaseStatusQuery().sources(sources)
query_results = self.database_handler.execute_query(
query, refresh_on_error=True)
result_txt = ''
for query_result in query_results:
source = query_result['source'].upper()
keep_journal = 'Y' if get_setting(
f'sources.{source}.keep_journal') else 'N'
serial_newest = query_result['serial_newest_mirror']
fields = [
source,
keep_journal,
f'0-{serial_newest}' if serial_newest else '-',
]
if query_result['serial_last_export']:
fields.append(str(query_result['serial_last_export']))
result_txt += ':'.join(fields) + '\n'
for invalid_source in invalid_sources:
result_txt += f'{invalid_source.upper()}:X:Database unknown\n'
return result_txt.strip()
def generate(self, source: str, version: str, serial_start_requested: int,
serial_end_requested: Optional[int],
database_handler: DatabaseHandler) -> str:
"""
Generate an NRTM response for a particular source, serial range and
NRTM version. Raises NRTMGeneratorException for various error conditions.
For queries where the user requested NRTM updates up to LAST,
serial_end_requested is None.
"""
if not get_setting(f'sources.{source}.keep_journal'):
raise NRTMGeneratorException(
'No journal kept for this source, unable to serve NRTM queries'
)
q = DatabaseStatusQuery().source(source)
try:
status = next(database_handler.execute_query(q))
except StopIteration:
raise NRTMGeneratorException(
'There are no journal entries for this source.')
if serial_end_requested and serial_end_requested < serial_start_requested:
raise NRTMGeneratorException(
f'Start of the serial range ({serial_start_requested}) must be lower or '
f'equal to end of the serial range ({serial_end_requested})')
serial_start_available = status['serial_oldest_journal']
serial_end_available = status['serial_newest_journal']
if serial_start_available is None or serial_end_available is None:
return '% Warning: there are no updates available'
if serial_start_requested < serial_start_available:
raise NRTMGeneratorException(
f'Serials {serial_start_requested} - {serial_start_available} do not exist'
)
if serial_end_requested is not None and serial_end_requested > serial_end_available:
raise NRTMGeneratorException(
f'Serials {serial_end_available} - {serial_end_requested} do not exist'
)
if serial_end_requested is None:
if serial_start_requested == serial_end_available + 1:
# A specific message is triggered when starting from a serial
# that is the current plus one, until LAST
return '% Warning: there are no newer updates available'
elif serial_start_requested > serial_end_available:
raise NRTMGeneratorException(
f'Serials {serial_end_available} - {serial_start_requested} do not exist'
)
serial_end_display = serial_end_available if serial_end_requested is None else serial_end_requested
q = RPSLDatabaseJournalQuery().sources([source]).serial_range(
serial_start_requested, serial_end_requested)
operations = list(database_handler.execute_query(q))
output = f'%START Version: {version} {source} {serial_start_requested}-{serial_end_display}\n'
for operation in operations:
output += '\n' + operation['operation'].value
if version == '3':
output += ' ' + str(operation['serial_nrtm'])
output += '\n\n' + remove_auth_hashes(operation['object_text'])
output += f'\n%END {source}'
return output