def test_fraction_position_conversion(self):
start_int = 0
stop_int = 2**96 - 1
start = _ObjectIdHelper.int_to_id(start_int)
stop = _ObjectIdHelper.int_to_id(stop_int)
test_cases = ([start_int, stop_int, 2**32, 2**32 - 1, 2**64, 2**64 - 1] +
[random.randint(start_int, stop_int) for _ in range(100)])
tracker = _ObjectIdRangeTracker()
for pos in test_cases:
id = _ObjectIdHelper.int_to_id(pos - start_int)
desired_fraction = (pos - start_int) / (stop_int - start_int)
self.assertAlmostEqual(
tracker.position_to_fraction(id, start, stop),
desired_fraction,
places=20)
convert_id = tracker.fraction_to_position(
(pos - start_int) / (stop_int - start_int), start, stop)
# due to precision loss, the convert fraction is only gonna be close to
# original fraction.
convert_fraction = tracker.position_to_fraction(convert_id, start, stop)
self.assertGreater(convert_id, start)
self.assertLess(convert_id, stop)
self.assertAlmostEqual(convert_fraction, desired_fraction, places=20)
def test_conversion(self):
test_cases = [
(objectid.ObjectId('000000000000000000000000'), 0),
(objectid.ObjectId('000000000000000100000000'), 2**32),
(objectid.ObjectId('0000000000000000ffffffff'), 2**32 - 1),
(objectid.ObjectId('000000010000000000000000'), 2**64),
(objectid.ObjectId('00000000ffffffffffffffff'), 2**64 - 1),
(objectid.ObjectId('ffffffffffffffffffffffff'), 2**96 - 1),
]
for (_id, number) in test_cases:
self.assertEqual(_id, _ObjectIdHelper.int_to_id(number))
self.assertEqual(number, _ObjectIdHelper.id_to_int(_id))
# random tests
for _ in range(100):
_id = objectid.ObjectId()
number = int(_id.binary.hex(), 16)
self.assertEqual(_id, _ObjectIdHelper.int_to_id(number))
self.assertEqual(number, _ObjectIdHelper.id_to_int(_id))
def test_read(self, mock_client):
mock_tracker = mock.MagicMock()
test_cases = [
{
# range covers the first(inclusive) to third(exclusive) documents
'start': self._ids[0],
'stop': self._ids[2],
'expected': self._docs[0:2]
},
{
# range covers from the first to the third documents
'start': _ObjectIdHelper.int_to_id(0), # smallest possible id
'stop': self._ids[2],
'expected': self._docs[0:2]
},
{
# range covers from the third to last documents
'start': self._ids[2],
'stop':
_ObjectIdHelper.int_to_id(2**96 - 1), # largest possible id
'expected': self._docs[2:]
},
{
# range covers all documents
'start': _ObjectIdHelper.int_to_id(0),
'stop': _ObjectIdHelper.int_to_id(2**96 - 1),
'expected': self._docs
},
{
# range doesn't include any document
'start': _ObjectIdHelper.increment_id(self._ids[2], 1),
'stop': _ObjectIdHelper.increment_id(self._ids[3], -1),
'expected': []
},
]
mock_client.return_value = _MockMongoClient(self._docs)
for case in test_cases:
mock_tracker.start_position.return_value = case['start']
mock_tracker.stop_position.return_value = case['stop']
result = list(self.mongo_source.read(mock_tracker))
self.assertListEqual(case['expected'], result)
def test_range_is_not_splittable(self):
self.assertTrue(
self.mongo_source._range_is_not_splittable(
_ObjectIdHelper.int_to_id(1),
_ObjectIdHelper.int_to_id(1),
))
self.assertTrue(
self.mongo_source._range_is_not_splittable(
_ObjectIdHelper.int_to_id(1),
_ObjectIdHelper.int_to_id(2),
))
self.assertFalse(
self.mongo_source._range_is_not_splittable(
_ObjectIdHelper.int_to_id(1),
_ObjectIdHelper.int_to_id(3),
))
self.assertTrue(self.mongo_source._range_is_not_splittable(0, 0))
self.assertTrue(self.mongo_source._range_is_not_splittable(0, 1))
self.assertFalse(self.mongo_source._range_is_not_splittable(0, 2))
self.assertTrue(
self.mongo_source._range_is_not_splittable("AAA", "AAA"))
self.assertFalse(
self.mongo_source._range_is_not_splittable("AAA", "AAA\x00"))
self.assertFalse(
self.mongo_source._range_is_not_splittable("AAA", "AAB"))
def test_conversion(self):
test_cases = [
(objectid.ObjectId('000000000000000000000000'), 0),
(objectid.ObjectId('000000000000000100000000'), 2**32),
(objectid.ObjectId('0000000000000000ffffffff'), 2**32 - 1),
(objectid.ObjectId('000000010000000000000000'), 2**64),
(objectid.ObjectId('00000000ffffffffffffffff'), 2**64 - 1),
(objectid.ObjectId('ffffffffffffffffffffffff'), 2**96 - 1),
]
for (id, number) in test_cases:
self.assertEqual(id, _ObjectIdHelper.int_to_id(number))
self.assertEqual(number, _ObjectIdHelper.id_to_int(id))
# random tests
for _ in range(100):
id = objectid.ObjectId()
if sys.version_info[0] < 3:
number = int(id.binary.encode('hex'), 16)
else: # PY3
number = int(id.binary.hex(), 16)
self.assertEqual(id, _ObjectIdHelper.int_to_id(number))
self.assertEqual(number, _ObjectIdHelper.id_to_int(id))
datetime.datetime(year=2020, month=i + 1, day=i + 1)) for i in range(5)
]
INT_IDS = [n for n in range(5)] # [0, 1, 2, 3, 4]
STR_IDS_1 = [str(n) for n in range(5)] # ['0', '1', '2', '3', '4']
# ['aaaaa', 'bbbbb', 'ccccc', 'ddddd', 'eeeee']
STR_IDS_2 = [chr(97 + n) * 5 for n in range(5)]
# ['AAAAAAAAAAAAAAAAAAAA', 'BBBBBBBBBBBBBBBBBBBB', ..., 'EEEEEEEEEEEEEEEEEEEE']
STR_IDS_3 = [chr(65 + n) * 20 for n in range(5)]
@parameterized_class(('bucket_auto', '_ids', 'min_id', 'max_id'), [
(None, OBJECT_IDS, _ObjectIdHelper.int_to_id(0),
_ObjectIdHelper.int_to_id(2**96 - 1)),
(True, OBJECT_IDS, _ObjectIdHelper.int_to_id(0),
_ObjectIdHelper.int_to_id(2**96 - 1)),
(
None,
INT_IDS,
0,
2**96 - 1,
),
(
True,
INT_IDS,
0,
2**96 - 1,
),
