def __init__(self, database): """Reader for the MaxMind DB file format Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database file. """ with open(database, 'rb') as db_file: self._buffer = mmap.mmap( db_file.fileno(), 0, access=mmap.ACCESS_READ) metadata_start = self._buffer.rfind(self._METADATA_START_MARKER, self._buffer.size() - 128 * 1024) if metadata_start == -1: raise InvalidDatabaseError('Error opening database file ({0}). ' 'Is this a valid MaxMind DB file?' ''.format(database)) metadata_start += len(self._METADATA_START_MARKER) metadata_decoder = Decoder(self._buffer, metadata_start) (metadata, _) = metadata_decoder.decode(metadata_start) self._metadata = Metadata(**metadata) # pylint: disable=star-args self._decoder = Decoder(self._buffer, self._metadata.search_tree_size + self._DATA_SECTION_SEPARATOR_SIZE)
def __init__(self, database, mode=MODE_AUTO): """Reader for the MaxMind DB file format Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database file, or a file descriptor in the case of MODE_FD. mode -- mode to open the database with. Valid mode are: * MODE_MMAP - read from memory map. * MODE_FILE - read database as standard file. * MODE_MEMORY - load database into memory. * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. * MODE_FD - the param passed via database is a file descriptor, not a path. This mode implies MODE_MEMORY. """ if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: with open(database, 'rb') as db_file: self._buffer = mmap.mmap(db_file.fileno(), 0, access=mmap.ACCESS_READ) self._buffer_size = self._buffer.size() filename = database elif mode in (MODE_AUTO, MODE_FILE): self._buffer = FileBuffer(database) self._buffer_size = self._buffer.size() filename = database elif mode == MODE_MEMORY: with open(database, 'rb') as db_file: self._buffer = db_file.read() self._buffer_size = len(self._buffer) filename = database elif mode == MODE_FD: self._buffer = database.read() self._buffer_size = len(self._buffer) filename = database.name else: raise ValueError( 'Unsupported open mode ({0}). Only MODE_AUTO, MODE_FILE, ' 'MODE_MEMORY and MODE_FD are supported by the pure Python ' 'Reader'.format(mode)) metadata_start = self._buffer.rfind( self._METADATA_START_MARKER, max(0, self._buffer_size - 128 * 1024)) if metadata_start == -1: self.close() raise InvalidDatabaseError('Error opening database file ({0}). ' 'Is this a valid MaxMind DB file?' ''.format(filename)) metadata_start += len(self._METADATA_START_MARKER) metadata_decoder = Decoder(self._buffer, metadata_start) (metadata, _) = metadata_decoder.decode(metadata_start) self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value self._decoder = Decoder( self._buffer, self._metadata.search_tree_size + self._DATA_SECTION_SEPARATOR_SIZE) self.closed = False
def __init__(self, filename): with open( filename, 'rb' ) as input: self.data = input.read() try: pos = self.data.rindex( MMDB.METADATA_BEGIN_MARKER ) pos += len(MMDB.METADATA_BEGIN_MARKER) except: raise Exception( "Could not find metadata marker, invalid database." ) metadata_decoder = Decoder(self.data, pos) self.metadata = metadata_decoder.decode(pos)[0] self.metadata = Metadata( **self.metadata ) self.decoder = Decoder(self.data, self.metadata.search_tree_size + MMDB.DATA_SECTION_SEPARATOR_SIZE)
def test_real_pointers(self): with open('tests/data/test-data/maps-with-pointers.raw', 'r+b') as db_file: mm = mmap.mmap(db_file.fileno(), 0) decoder = Decoder(mm, 0) self.assertEqual(({ 'long_key': 'long_value1' }, 22), decoder.decode(0)) self.assertEqual(({ 'long_key': 'long_value2' }, 37), decoder.decode(22)) self.assertEqual(({ 'long_key2': 'long_value1' }, 50), decoder.decode(37)) self.assertEqual(({ 'long_key2': 'long_value2' }, 55), decoder.decode(50)) self.assertEqual(({ 'long_key': 'long_value1' }, 57), decoder.decode(55)) self.assertEqual(({ 'long_key2': 'long_value2' }, 59), decoder.decode(57)) mm.close()
def test_real_pointers(self): with open("tests/data/test-data/maps-with-pointers.raw", "r+b") as db_file: mm = mmap.mmap(db_file.fileno(), 0) decoder = Decoder(mm, 0) self.assertEqual(({ "long_key": "long_value1" }, 22), decoder.decode(0)) self.assertEqual(({ "long_key": "long_value2" }, 37), decoder.decode(22)) self.assertEqual(({ "long_key2": "long_value1" }, 50), decoder.decode(37)) self.assertEqual(({ "long_key2": "long_value2" }, 55), decoder.decode(50)) self.assertEqual(({ "long_key": "long_value1" }, 57), decoder.decode(55)) self.assertEqual(({ "long_key2": "long_value2" }, 59), decoder.decode(57)) mm.close()
def check_decoding(self, type, input, expected, name=None): name = name or expected db = mmap.mmap(-1, len(input)) db.write(input) decoder = Decoder(db, pointer_test=True) (actual, _,) = decoder.decode(0) if type in ('float', 'double'): self.assertAlmostEqual(expected, actual, places=3, msg=type) else: self.assertEqual(expected, actual, type)
def __init__(self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO) -> None: """Reader for the MaxMind DB file format Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database file, or a file descriptor in the case of MODE_FD. mode -- mode to open the database with. Valid mode are: * MODE_MMAP - read from memory map. * MODE_FILE - read database as standard file. * MODE_MEMORY - load database into memory. * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. * MODE_FD - the param passed via database is a file descriptor, not a path. This mode implies MODE_MEMORY. """ filename: Any if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: with open(database, "rb") as db_file: # type: ignore self._buffer = mmap.mmap(db_file.fileno(), 0, access=mmap.ACCESS_READ) self._buffer_size = self._buffer.size() filename = database elif mode in (MODE_AUTO, MODE_FILE): self._buffer = FileBuffer(database) # type: ignore self._buffer_size = self._buffer.size() filename = database elif mode == MODE_MEMORY: with open(database, "rb") as db_file: # type: ignore self._buffer = db_file.read() self._buffer_size = len(self._buffer) filename = database elif mode == MODE_FD: self._buffer = database.read() # type: ignore self._buffer_size = len(self._buffer) # type: ignore filename = database.name # type: ignore else: raise ValueError( f"Unsupported open mode ({mode}). Only MODE_AUTO, MODE_FILE, " "MODE_MEMORY and MODE_FD are supported by the pure Python " "Reader") metadata_start = self._buffer.rfind( self._METADATA_START_MARKER, max(0, self._buffer_size - 128 * 1024)) if metadata_start == -1: self.close() raise InvalidDatabaseError( f"Error opening database file ({filename}). " "Is this a valid MaxMind DB file?") metadata_start += len(self._METADATA_START_MARKER) metadata_decoder = Decoder(self._buffer, metadata_start) (metadata, _) = metadata_decoder.decode(metadata_start) if not isinstance(metadata, dict): raise InvalidDatabaseError( f"Error reading metadata in database file ({filename}).") self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value self._decoder = Decoder( self._buffer, self._metadata.search_tree_size + self._DATA_SECTION_SEPARATOR_SIZE, ) self.closed = False