def _export(self, export_destination):
filename_export = Path(export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(query))['serial_newest_seen']
except StopIteration:
logger.error(f'Unable to run export for {self.source}, internal database status is empty.')
return
with gzip.open(export_tmpfile, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
for obj in self.database_handler.execute_query(query):
object_bytes = remove_auth_hashes(obj['object_text']).encode('utf-8')
fh.write(object_bytes + b'\n')
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
self.database_handler.record_serial_exported(self.source, serial)
logger.info(f'Export for {self.source} complete, stored in {filename_export} / {filename_serial}')
def _rpsl_db_query_to_graphql_out(query: RPSLDatabaseQuery,
info: GraphQLResolveInfo):
"""
Given an RPSL database query, execute it and clean up the output
to be suitable to return to GraphQL.
Main changes are:
- Enum handling
- Adding the asn and prefix fields if applicable
- Ensuring the right fields are returned as a list of strings or a string
"""
database_handler = info.context['request'].app.state.database_handler
if info.context.get('sql_trace'):
if 'sql_queries' not in info.context:
info.context['sql_queries'] = [repr(query)]
else:
info.context['sql_queries'].append(repr(query))
for row in database_handler.execute_query(query, refresh_on_error=True):
graphql_result = {
snake_to_camel_case(k): v
for k, v in row.items() if k != 'parsed_data'
}
if 'object_text' in row:
graphql_result['objectText'] = remove_auth_hashes(
row['object_text'])
if 'rpki_status' in row:
graphql_result['rpkiStatus'] = row['rpki_status']
if row.get('ip_first') is not None and row.get('prefix_length'):
graphql_result['prefix'] = row['ip_first'] + '/' + str(
row['prefix_length'])
if row.get('asn_first') is not None and row.get(
'asn_first') == row.get('asn_last'):
graphql_result['asn'] = row['asn_first']
object_type = resolve_rpsl_object_type(row)
for key, value in row.get('parsed_data', dict()).items():
if key == 'auth':
value = [remove_auth_hashes(v) for v in value]
graphql_type = schema.graphql_types[object_type][key]
if graphql_type == 'String' and isinstance(value, list):
value = '\n'.join(value)
graphql_result[snake_to_camel_case(key)] = value
yield graphql_result
def generate_response(self) -> str:
self.result = remove_auth_hashes(self.result)
if self.mode == WhoisQueryResponseMode.IRRD:
response = self._generate_response_irrd()
if response is not None:
return response
elif self.mode == WhoisQueryResponseMode.RIPE:
response = self._generate_response_ripe()
if response is not None:
return response
raise RuntimeError(f'Unable to formulate response for {self.response_type} / {self.mode}: {self.result}')
def test_modify_mntner(self, prepare_mocks, config_override):
validator, mock_dq, mock_dh = prepare_mocks
mntner = rpsl_object_from_text(SAMPLE_MNTNER)
mock_dh.execute_query = lambda q: [
{
'object_class': 'mntner',
'object_text': SAMPLE_MNTNER
},
]
# This counts as submitting all new hashes.
validator.passwords = [SAMPLE_MNTNER_MD5]
result = validator.process_auth(mntner, mntner)
assert result.is_valid()
assert not result.info_messages
# This counts as submitting all new hashes, but not matching any password
new_mntner = rpsl_object_from_text(
MNTNER_OBJ_CRYPT_PW.replace('CRYPT', ''))
validator.passwords = [SAMPLE_MNTNER_MD5]
result = validator.process_auth(new_mntner, mntner)
assert not result.is_valid()
assert result.error_messages == {
'Authorisation failed for the auth methods on this mntner object.'
}
# This counts as submitting all dummy hashes.
mntner_no_auth_hashes = remove_auth_hashes(SAMPLE_MNTNER)
new_mntner = rpsl_object_from_text(mntner_no_auth_hashes)
result = validator.process_auth(new_mntner, mntner)
assert result.is_valid()
assert not new_mntner.has_dummy_auth_value()
assert result.info_messages == {
'As you submitted dummy hash values, all password hashes on this '
'object were replaced with a new BCRYPT-PW hash of the password you '
'provided for authentication.'
}
# # This is a multi password submission with dummy hashes which is rejected
validator.passwords = [SAMPLE_MNTNER_MD5, SAMPLE_MNTNER_CRYPT]
new_mntner = rpsl_object_from_text(mntner_no_auth_hashes)
result = validator.process_auth(new_mntner, mntner)
assert not result.is_valid()
assert not result.info_messages
assert result.error_messages == {
'Object submitted with dummy hash values, but multiple or no passwords '
'submitted. Either submit only full hashes, or a single password.'
}
def _export(self, export_destination):
filename_export = Path(
export_destination) / f'{self.source.lower()}.db.gz'
export_tmpfile = NamedTemporaryFile(delete=False)
filename_serial = Path(
export_destination) / f'{self.source.upper()}.CURRENTSERIAL'
query = DatabaseStatusQuery().source(self.source)
try:
serial = next(self.database_handler.execute_query(
query))['serial_newest_seen']
except StopIteration:
serial = None
with gzip.open(export_tmpfile.name, 'wb') as fh:
query = RPSLDatabaseQuery().sources([self.source])
query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
query = query.scopefilter_status([ScopeFilterStatus.in_scope])
for obj in self.database_handler.execute_query(query):
object_bytes = remove_auth_hashes(
obj['object_text']).encode('utf-8')
fh.write(object_bytes + b'\n')
fh.write(b'# EOF\n')
os.chmod(export_tmpfile.name, EXPORT_PERMISSIONS)
if filename_export.exists():
os.unlink(filename_export)
if filename_serial.exists():
os.unlink(filename_serial)
shutil.move(export_tmpfile.name, filename_export)
if serial is not None:
with open(filename_serial, 'w') as fh:
fh.write(str(serial))
os.chmod(filename_serial, EXPORT_PERMISSIONS)
self.database_handler.record_serial_exported(self.source, serial)
logger.info(
f'Export for {self.source} complete at serial {serial}, stored in {filename_export} / {filename_serial}'
)
def resolve_rpsl_object_journal(rpsl_object, info: GraphQLResolveInfo):
"""
Resolve a journal subquery on an RPSL object.
"""
database_handler = info.context['request'].app.state.database_handler
access_list = f"sources.{rpsl_object['source']}.nrtm_access_list"
if not is_client_permitted(info.context['request'].client.host,
access_list):
raise GraphQLError(
f"Access to journal denied for source {rpsl_object['source']}")
query = RPSLDatabaseJournalQuery()
query.sources([rpsl_object['source']]).rpsl_pk(rpsl_object['rpslPk'])
for row in database_handler.execute_query(query, refresh_on_error=True):
response = {snake_to_camel_case(k): v for k, v in row.items()}
response['operation'] = response['operation'].name
if response['origin']:
response['origin'] = response['origin'].name
if response['objectText']:
response['objectText'] = remove_auth_hashes(response['objectText'])
yield response
def clean_response(self):
if self.remove_auth_hashes:
self.result = remove_auth_hashes(self.result)
def generate(self, source: str, version: str, serial_start_requested: int,
serial_end_requested: Optional[int],
database_handler: DatabaseHandler) -> str:
"""
Generate an NRTM response for a particular source, serial range and
NRTM version. Raises NRTMGeneratorException for various error conditions.
For queries where the user requested NRTM updates up to LAST,
serial_end_requested is None.
"""
if not get_setting(f'sources.{source}.keep_journal'):
raise NRTMGeneratorException(
'No journal kept for this source, unable to serve NRTM queries'
)
q = DatabaseStatusQuery().source(source)
try:
status = next(database_handler.execute_query(q))
except StopIteration:
raise NRTMGeneratorException(
'There are no journal entries for this source.')
if serial_end_requested and serial_end_requested < serial_start_requested:
raise NRTMGeneratorException(
f'Start of the serial range ({serial_start_requested}) must be lower or '
f'equal to end of the serial range ({serial_end_requested})')
serial_start_available = status['serial_oldest_journal']
serial_end_available = status['serial_newest_journal']
if serial_start_available is None or serial_end_available is None:
return '% Warning: there are no updates available'
if serial_start_requested < serial_start_available:
raise NRTMGeneratorException(
f'Serials {serial_start_requested} - {serial_start_available} do not exist'
)
if serial_end_requested is not None and serial_end_requested > serial_end_available:
raise NRTMGeneratorException(
f'Serials {serial_end_available} - {serial_end_requested} do not exist'
)
if serial_end_requested is None:
if serial_start_requested == serial_end_available + 1:
# A specific message is triggered when starting from a serial
# that is the current plus one, until LAST
return '% Warning: there are no newer updates available'
elif serial_start_requested > serial_end_available:
raise NRTMGeneratorException(
f'Serials {serial_end_available} - {serial_start_requested} do not exist'
)
serial_end_display = serial_end_available if serial_end_requested is None else serial_end_requested
q = RPSLDatabaseJournalQuery().sources([source]).serial_range(
serial_start_requested, serial_end_requested)
operations = list(database_handler.execute_query(q))
output = f'%START Version: {version} {source} {serial_start_requested}-{serial_end_display}\n'
for operation in operations:
output += '\n' + operation['operation'].value
if version == '3':
output += ' ' + str(operation['serial_nrtm'])
output += '\n\n' + remove_auth_hashes(operation['object_text'])
output += f'\n%END {source}'
return output