def __init__(self, database):
"""Reader for the MaxMind DB file format
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2
database file.
"""
with open(database, 'rb') as db_file:
self._buffer = mmap.mmap(
db_file.fileno(), 0, access=mmap.ACCESS_READ)
metadata_start = self._buffer.rfind(self._METADATA_START_MARKER,
self._buffer.size() - 128 * 1024)
if metadata_start == -1:
raise InvalidDatabaseError('Error opening database file ({0}). '
'Is this a valid MaxMind DB file?'
''.format(database))
metadata_start += len(self._METADATA_START_MARKER)
metadata_decoder = Decoder(self._buffer, metadata_start)
(metadata, _) = metadata_decoder.decode(metadata_start)
self._metadata = Metadata(**metadata) # pylint: disable=star-args
self._decoder = Decoder(self._buffer, self._metadata.search_tree_size
+ self._DATA_SECTION_SEPARATOR_SIZE)
def __init__(self, database, mode=MODE_AUTO):
"""Reader for the MaxMind DB file format
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2 database
file, or a file descriptor in the case of MODE_FD.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP - read from memory map.
* MODE_FILE - read database as standard file.
* MODE_MEMORY - load database into memory.
* MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default.
* MODE_FD - the param passed via database is a file descriptor, not
a path. This mode implies MODE_MEMORY.
"""
if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP:
with open(database, 'rb') as db_file:
self._buffer = mmap.mmap(db_file.fileno(),
0,
access=mmap.ACCESS_READ)
self._buffer_size = self._buffer.size()
filename = database
elif mode in (MODE_AUTO, MODE_FILE):
self._buffer = FileBuffer(database)
self._buffer_size = self._buffer.size()
filename = database
elif mode == MODE_MEMORY:
with open(database, 'rb') as db_file:
self._buffer = db_file.read()
self._buffer_size = len(self._buffer)
filename = database
elif mode == MODE_FD:
self._buffer = database.read()
self._buffer_size = len(self._buffer)
filename = database.name
else:
raise ValueError(
'Unsupported open mode ({0}). Only MODE_AUTO, MODE_FILE, '
'MODE_MEMORY and MODE_FD are supported by the pure Python '
'Reader'.format(mode))
metadata_start = self._buffer.rfind(
self._METADATA_START_MARKER, max(0,
self._buffer_size - 128 * 1024))
if metadata_start == -1:
self.close()
raise InvalidDatabaseError('Error opening database file ({0}). '
'Is this a valid MaxMind DB file?'
''.format(filename))
metadata_start += len(self._METADATA_START_MARKER)
metadata_decoder = Decoder(self._buffer, metadata_start)
(metadata, _) = metadata_decoder.decode(metadata_start)
self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value
self._decoder = Decoder(
self._buffer, self._metadata.search_tree_size +
self._DATA_SECTION_SEPARATOR_SIZE)
self.closed = False
def __init__(self, filename):
with open( filename, 'rb' ) as input:
self.data = input.read()
try:
pos = self.data.rindex( MMDB.METADATA_BEGIN_MARKER )
pos += len(MMDB.METADATA_BEGIN_MARKER)
except:
raise Exception( "Could not find metadata marker, invalid database." )
metadata_decoder = Decoder(self.data, pos)
self.metadata = metadata_decoder.decode(pos)[0]
self.metadata = Metadata( **self.metadata )
self.decoder = Decoder(self.data, self.metadata.search_tree_size + MMDB.DATA_SECTION_SEPARATOR_SIZE)
def test_real_pointers(self):
with open('tests/data/test-data/maps-with-pointers.raw',
'r+b') as db_file:
mm = mmap.mmap(db_file.fileno(), 0)
decoder = Decoder(mm, 0)
self.assertEqual(({
'long_key': 'long_value1'
}, 22), decoder.decode(0))
self.assertEqual(({
'long_key': 'long_value2'
}, 37), decoder.decode(22))
self.assertEqual(({
'long_key2': 'long_value1'
}, 50), decoder.decode(37))
self.assertEqual(({
'long_key2': 'long_value2'
}, 55), decoder.decode(50))
self.assertEqual(({
'long_key': 'long_value1'
}, 57), decoder.decode(55))
self.assertEqual(({
'long_key2': 'long_value2'
}, 59), decoder.decode(57))
mm.close()
def test_real_pointers(self):
with open("tests/data/test-data/maps-with-pointers.raw",
"r+b") as db_file:
mm = mmap.mmap(db_file.fileno(), 0)
decoder = Decoder(mm, 0)
self.assertEqual(({
"long_key": "long_value1"
}, 22), decoder.decode(0))
self.assertEqual(({
"long_key": "long_value2"
}, 37), decoder.decode(22))
self.assertEqual(({
"long_key2": "long_value1"
}, 50), decoder.decode(37))
self.assertEqual(({
"long_key2": "long_value2"
}, 55), decoder.decode(50))
self.assertEqual(({
"long_key": "long_value1"
}, 57), decoder.decode(55))
self.assertEqual(({
"long_key2": "long_value2"
}, 59), decoder.decode(57))
mm.close()
def check_decoding(self, type, input, expected, name=None):
name = name or expected
db = mmap.mmap(-1, len(input))
db.write(input)
decoder = Decoder(db, pointer_test=True)
(actual, _,) = decoder.decode(0)
if type in ('float', 'double'):
self.assertAlmostEqual(expected, actual, places=3, msg=type)
else:
self.assertEqual(expected, actual, type)
def __init__(self,
database: Union[AnyStr, int, PathLike, IO],
mode: int = MODE_AUTO) -> None:
"""Reader for the MaxMind DB file format
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2 database
file, or a file descriptor in the case of MODE_FD.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP - read from memory map.
* MODE_FILE - read database as standard file.
* MODE_MEMORY - load database into memory.
* MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default.
* MODE_FD - the param passed via database is a file descriptor, not
a path. This mode implies MODE_MEMORY.
"""
filename: Any
if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP:
with open(database, "rb") as db_file: # type: ignore
self._buffer = mmap.mmap(db_file.fileno(),
0,
access=mmap.ACCESS_READ)
self._buffer_size = self._buffer.size()
filename = database
elif mode in (MODE_AUTO, MODE_FILE):
self._buffer = FileBuffer(database) # type: ignore
self._buffer_size = self._buffer.size()
filename = database
elif mode == MODE_MEMORY:
with open(database, "rb") as db_file: # type: ignore
self._buffer = db_file.read()
self._buffer_size = len(self._buffer)
filename = database
elif mode == MODE_FD:
self._buffer = database.read() # type: ignore
self._buffer_size = len(self._buffer) # type: ignore
filename = database.name # type: ignore
else:
raise ValueError(
f"Unsupported open mode ({mode}). Only MODE_AUTO, MODE_FILE, "
"MODE_MEMORY and MODE_FD are supported by the pure Python "
"Reader")
metadata_start = self._buffer.rfind(
self._METADATA_START_MARKER, max(0,
self._buffer_size - 128 * 1024))
if metadata_start == -1:
self.close()
raise InvalidDatabaseError(
f"Error opening database file ({filename}). "
"Is this a valid MaxMind DB file?")
metadata_start += len(self._METADATA_START_MARKER)
metadata_decoder = Decoder(self._buffer, metadata_start)
(metadata, _) = metadata_decoder.decode(metadata_start)
if not isinstance(metadata, dict):
raise InvalidDatabaseError(
f"Error reading metadata in database file ({filename}).")
self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value
self._decoder = Decoder(
self._buffer,
self._metadata.search_tree_size +
self._DATA_SECTION_SEPARATOR_SIZE,
)
self.closed = False
