def load(source, filename, serial) -> int:
if any([
get_setting(f'sources.{source}.import_source'),
get_setting(f'sources.{source}.import_serial_source')
]):
print(f'Error: to use this command, import_source and import_serial_source '
f'for source {source} must not be set.')
return 2
dh = DatabaseHandler()
roa_validator = BulkRouteROAValidator(dh)
dh.delete_all_rpsl_objects_with_journal(source)
dh.disable_journaling()
parser = MirrorFileImportParser(
source=source, filename=filename, serial=serial, database_handler=dh,
direct_error_return=True, roa_validator=roa_validator)
error = parser.run_import()
if error:
dh.rollback()
else:
dh.commit()
dh.close()
if error:
print(f'Error occurred while processing object:\n{error}')
return 1
return 0
def run(self,
database_handler: DatabaseHandler,
serial_newest_mirror: Optional[int] = None,
force_reload=False):
import_sources = get_setting(f'sources.{self.source}.import_source')
if isinstance(import_sources, str):
import_sources = [import_sources]
import_serial_source = get_setting(
f'sources.{self.source}.import_serial_source')
if not import_sources:
logger.info(
f'Skipping full RPSL import for {self.source}, import_source not set.'
)
return
logger.info(
f'Running full RPSL import of {self.source} from {import_sources}, serial from {import_serial_source}'
)
import_serial = None
if import_serial_source:
import_serial = int(
self._retrieve_file(import_serial_source,
return_contents=True)[0])
if not force_reload and serial_newest_mirror is not None and import_serial <= serial_newest_mirror:
logger.info(
f'Current newest serial seen from mirror for {self.source} is '
f'{serial_newest_mirror}, import_serial is {import_serial}, cancelling import.'
)
return
database_handler.delete_all_rpsl_objects_with_journal(self.source)
import_data = [
self._retrieve_file(import_source, return_contents=False)
for import_source in import_sources
]
roa_validator = None
if get_setting('rpki.roa_source'):
roa_validator = BulkRouteROAValidator(database_handler)
database_handler.disable_journaling()
for import_filename, to_delete in import_data:
p = MirrorFileImportParser(source=self.source,
filename=import_filename,
serial=None,
database_handler=database_handler,
roa_validator=roa_validator)
p.run_import()
if to_delete:
os.unlink(import_filename)
if import_serial:
database_handler.record_serial_newest_mirror(
self.source, import_serial)
def load(source, filename, serial) -> int:
dh = DatabaseHandler()
dh.delete_all_rpsl_objects_with_journal(source)
dh.disable_journaling()
parser = MirrorFileImportParser(source, filename, serial=serial, database_handler=dh, direct_error_return=True)
error = parser.run_import()
if error:
dh.rollback()
else:
dh.commit()
dh.close()
if error:
print(f'Error occurred while processing object:\n{error}')
return 1
return 0
def run(self, database_handler: DatabaseHandler):
import_sources = get_setting(f'sources.{self.source}.import_source')
if isinstance(import_sources, str):
import_sources = [import_sources]
import_serial_source = get_setting(
f'sources.{self.source}.import_serial_source')
if not import_sources:
logger.info(
f'Skipping full import for {self.source}, import_source not set.'
)
return
database_handler.delete_all_rpsl_objects_with_journal(self.source)
logger.info(
f'Running full import of {self.source} from {import_sources}, serial from {import_serial_source}'
)
import_serial = 0
if import_serial_source:
import_serial = int(
self._retrieve_file(import_serial_source,
return_contents=True)[0])
import_data = [
self._retrieve_file(import_source, return_contents=False)
for import_source in import_sources
]
database_handler.disable_journaling()
for import_filename, to_delete in import_data:
MirrorFileImportParser(source=self.source,
filename=import_filename,
serial=import_serial,
database_handler=database_handler)
if to_delete:
os.unlink(import_filename)
class ROAImportRunner(FileImportRunnerBase):
"""
This runner performs a full import of ROA objects.
The URL file for the ROA export in JSON format is provided
in the configuration.
"""
# API consistency with other importers, source is actually ignored
def __init__(self, source=None):
pass
def run(self):
self.database_handler = DatabaseHandler()
try:
self.database_handler.disable_journaling()
roa_objs = self._import_roas()
# Do an early commit to make the new ROAs available to other processes.
self.database_handler.commit()
# The ROA import does not use journaling, but updating the RPKI
# status may create journal entries.
self.database_handler.enable_journaling()
validator = BulkRouteROAValidator(self.database_handler, roa_objs)
objs_now_valid, objs_now_invalid, objs_now_not_found = validator.validate_all_routes()
self.database_handler.update_rpki_status(
rpsl_objs_now_valid=objs_now_valid,
rpsl_objs_now_invalid=objs_now_invalid,
rpsl_objs_now_not_found=objs_now_not_found,
)
self.database_handler.commit()
notified = notify_rpki_invalid_owners(self.database_handler, objs_now_invalid)
logger.info(f'RPKI status updated for all routes, {len(objs_now_valid)} newly valid, '
f'{len(objs_now_invalid)} newly invalid, '
f'{len(objs_now_not_found)} newly not_found routes, '
f'{notified} emails sent to contacts of newly invalid authoritative objects')
except OSError as ose:
# I/O errors can occur and should not log a full traceback (#177)
logger.error(f'An error occurred while attempting a ROA import: {ose}')
except ROAParserException as rpe:
logger.error(f'An exception occurred while attempting a ROA import: {rpe}')
except Exception as exc:
logger.error(f'An exception occurred while attempting a ROA import: {exc}', exc_info=exc)
finally:
self.database_handler.close()
def _import_roas(self):
roa_source = get_setting('rpki.roa_source')
slurm_source = get_setting('rpki.slurm_source')
logger.info(f'Running full ROA import from: {roa_source}, SLURM {slurm_source}')
self.database_handler.delete_all_roa_objects()
self.database_handler.delete_all_rpsl_objects_with_journal(RPKI_IRR_PSEUDO_SOURCE)
slurm_data = None
if slurm_source:
slurm_data, _ = self._retrieve_file(slurm_source, return_contents=True)
roa_filename, roa_to_delete = self._retrieve_file(roa_source, return_contents=False)
with open(roa_filename) as fh:
roa_importer = ROADataImporter(fh.read(), slurm_data, self.database_handler)
if roa_to_delete:
os.unlink(roa_filename)
logger.info(f'ROA import from {roa_source}, SLURM {slurm_source}, imported {len(roa_importer.roa_objs)} ROAs, running validator')
return roa_importer.roa_objs
