def _export(self, export_destination): filename_export = Path(export_destination) / f'{self.source.lower()}.db.gz' export_tmpfile = NamedTemporaryFile(delete=False) filename_serial = Path(export_destination) / f'{self.source.upper()}.CURRENTSERIAL' query = DatabaseStatusQuery().source(self.source) try: serial = next(self.database_handler.execute_query(query))['serial_newest_seen'] except StopIteration: logger.error(f'Unable to run export for {self.source}, internal database status is empty.') return with gzip.open(export_tmpfile, 'wb') as fh: query = RPSLDatabaseQuery().sources([self.source]) for obj in self.database_handler.execute_query(query): object_bytes = remove_auth_hashes(obj['object_text']).encode('utf-8') fh.write(object_bytes + b'\n') if filename_export.exists(): os.unlink(filename_export) if filename_serial.exists(): os.unlink(filename_serial) shutil.move(export_tmpfile.name, filename_export) if serial is not None: with open(filename_serial, 'w') as fh: fh.write(str(serial)) self.database_handler.record_serial_exported(self.source, serial) logger.info(f'Export for {self.source} complete, stored in {filename_export} / {filename_serial}')
def _rpsl_db_query_to_graphql_out(query: RPSLDatabaseQuery, info: GraphQLResolveInfo): """ Given an RPSL database query, execute it and clean up the output to be suitable to return to GraphQL. Main changes are: - Enum handling - Adding the asn and prefix fields if applicable - Ensuring the right fields are returned as a list of strings or a string """ database_handler = info.context['request'].app.state.database_handler if info.context.get('sql_trace'): if 'sql_queries' not in info.context: info.context['sql_queries'] = [repr(query)] else: info.context['sql_queries'].append(repr(query)) for row in database_handler.execute_query(query, refresh_on_error=True): graphql_result = { snake_to_camel_case(k): v for k, v in row.items() if k != 'parsed_data' } if 'object_text' in row: graphql_result['objectText'] = remove_auth_hashes( row['object_text']) if 'rpki_status' in row: graphql_result['rpkiStatus'] = row['rpki_status'] if row.get('ip_first') is not None and row.get('prefix_length'): graphql_result['prefix'] = row['ip_first'] + '/' + str( row['prefix_length']) if row.get('asn_first') is not None and row.get( 'asn_first') == row.get('asn_last'): graphql_result['asn'] = row['asn_first'] object_type = resolve_rpsl_object_type(row) for key, value in row.get('parsed_data', dict()).items(): if key == 'auth': value = [remove_auth_hashes(v) for v in value] graphql_type = schema.graphql_types[object_type][key] if graphql_type == 'String' and isinstance(value, list): value = '\n'.join(value) graphql_result[snake_to_camel_case(key)] = value yield graphql_result
def generate_response(self) -> str: self.result = remove_auth_hashes(self.result) if self.mode == WhoisQueryResponseMode.IRRD: response = self._generate_response_irrd() if response is not None: return response elif self.mode == WhoisQueryResponseMode.RIPE: response = self._generate_response_ripe() if response is not None: return response raise RuntimeError(f'Unable to formulate response for {self.response_type} / {self.mode}: {self.result}')
def test_modify_mntner(self, prepare_mocks, config_override): validator, mock_dq, mock_dh = prepare_mocks mntner = rpsl_object_from_text(SAMPLE_MNTNER) mock_dh.execute_query = lambda q: [ { 'object_class': 'mntner', 'object_text': SAMPLE_MNTNER }, ] # This counts as submitting all new hashes. validator.passwords = [SAMPLE_MNTNER_MD5] result = validator.process_auth(mntner, mntner) assert result.is_valid() assert not result.info_messages # This counts as submitting all new hashes, but not matching any password new_mntner = rpsl_object_from_text( MNTNER_OBJ_CRYPT_PW.replace('CRYPT', '')) validator.passwords = [SAMPLE_MNTNER_MD5] result = validator.process_auth(new_mntner, mntner) assert not result.is_valid() assert result.error_messages == { 'Authorisation failed for the auth methods on this mntner object.' } # This counts as submitting all dummy hashes. mntner_no_auth_hashes = remove_auth_hashes(SAMPLE_MNTNER) new_mntner = rpsl_object_from_text(mntner_no_auth_hashes) result = validator.process_auth(new_mntner, mntner) assert result.is_valid() assert not new_mntner.has_dummy_auth_value() assert result.info_messages == { 'As you submitted dummy hash values, all password hashes on this ' 'object were replaced with a new BCRYPT-PW hash of the password you ' 'provided for authentication.' } # # This is a multi password submission with dummy hashes which is rejected validator.passwords = [SAMPLE_MNTNER_MD5, SAMPLE_MNTNER_CRYPT] new_mntner = rpsl_object_from_text(mntner_no_auth_hashes) result = validator.process_auth(new_mntner, mntner) assert not result.is_valid() assert not result.info_messages assert result.error_messages == { 'Object submitted with dummy hash values, but multiple or no passwords ' 'submitted. Either submit only full hashes, or a single password.' }
def _export(self, export_destination): filename_export = Path( export_destination) / f'{self.source.lower()}.db.gz' export_tmpfile = NamedTemporaryFile(delete=False) filename_serial = Path( export_destination) / f'{self.source.upper()}.CURRENTSERIAL' query = DatabaseStatusQuery().source(self.source) try: serial = next(self.database_handler.execute_query( query))['serial_newest_seen'] except StopIteration: serial = None with gzip.open(export_tmpfile.name, 'wb') as fh: query = RPSLDatabaseQuery().sources([self.source]) query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid]) query = query.scopefilter_status([ScopeFilterStatus.in_scope]) for obj in self.database_handler.execute_query(query): object_bytes = remove_auth_hashes( obj['object_text']).encode('utf-8') fh.write(object_bytes + b'\n') fh.write(b'# EOF\n') os.chmod(export_tmpfile.name, EXPORT_PERMISSIONS) if filename_export.exists(): os.unlink(filename_export) if filename_serial.exists(): os.unlink(filename_serial) shutil.move(export_tmpfile.name, filename_export) if serial is not None: with open(filename_serial, 'w') as fh: fh.write(str(serial)) os.chmod(filename_serial, EXPORT_PERMISSIONS) self.database_handler.record_serial_exported(self.source, serial) logger.info( f'Export for {self.source} complete at serial {serial}, stored in {filename_export} / {filename_serial}' )
def resolve_rpsl_object_journal(rpsl_object, info: GraphQLResolveInfo): """ Resolve a journal subquery on an RPSL object. """ database_handler = info.context['request'].app.state.database_handler access_list = f"sources.{rpsl_object['source']}.nrtm_access_list" if not is_client_permitted(info.context['request'].client.host, access_list): raise GraphQLError( f"Access to journal denied for source {rpsl_object['source']}") query = RPSLDatabaseJournalQuery() query.sources([rpsl_object['source']]).rpsl_pk(rpsl_object['rpslPk']) for row in database_handler.execute_query(query, refresh_on_error=True): response = {snake_to_camel_case(k): v for k, v in row.items()} response['operation'] = response['operation'].name if response['origin']: response['origin'] = response['origin'].name if response['objectText']: response['objectText'] = remove_auth_hashes(response['objectText']) yield response
def clean_response(self): if self.remove_auth_hashes: self.result = remove_auth_hashes(self.result)
def generate(self, source: str, version: str, serial_start_requested: int, serial_end_requested: Optional[int], database_handler: DatabaseHandler) -> str: """ Generate an NRTM response for a particular source, serial range and NRTM version. Raises NRTMGeneratorException for various error conditions. For queries where the user requested NRTM updates up to LAST, serial_end_requested is None. """ if not get_setting(f'sources.{source}.keep_journal'): raise NRTMGeneratorException( 'No journal kept for this source, unable to serve NRTM queries' ) q = DatabaseStatusQuery().source(source) try: status = next(database_handler.execute_query(q)) except StopIteration: raise NRTMGeneratorException( 'There are no journal entries for this source.') if serial_end_requested and serial_end_requested < serial_start_requested: raise NRTMGeneratorException( f'Start of the serial range ({serial_start_requested}) must be lower or ' f'equal to end of the serial range ({serial_end_requested})') serial_start_available = status['serial_oldest_journal'] serial_end_available = status['serial_newest_journal'] if serial_start_available is None or serial_end_available is None: return '% Warning: there are no updates available' if serial_start_requested < serial_start_available: raise NRTMGeneratorException( f'Serials {serial_start_requested} - {serial_start_available} do not exist' ) if serial_end_requested is not None and serial_end_requested > serial_end_available: raise NRTMGeneratorException( f'Serials {serial_end_available} - {serial_end_requested} do not exist' ) if serial_end_requested is None: if serial_start_requested == serial_end_available + 1: # A specific message is triggered when starting from a serial # that is the current plus one, until LAST return '% Warning: there are no newer updates available' elif serial_start_requested > serial_end_available: raise NRTMGeneratorException( f'Serials {serial_end_available} - {serial_start_requested} do not exist' ) serial_end_display = serial_end_available if serial_end_requested is None else serial_end_requested q = RPSLDatabaseJournalQuery().sources([source]).serial_range( serial_start_requested, serial_end_requested) operations = list(database_handler.execute_query(q)) output = f'%START Version: {version} {source} {serial_start_requested}-{serial_end_display}\n' for operation in operations: output += '\n' + operation['operation'].value if version == '3': output += ' ' + str(operation['serial_nrtm']) output += '\n\n' + remove_auth_hashes(operation['object_text']) output += f'\n%END {source}' return output