def get_range_tracker(
self,
start_position: Union[int, str, ObjectId] = None,
stop_position: Union[int, str, ObjectId] = None,
) -> Union[_ObjectIdRangeTracker, OffsetRangeTracker,
LexicographicKeyRangeTracker]:
"""Returns a RangeTracker for a given position range depending on type.
Args:
start_position: starting position of the range. If 'None' default start
position of the source must be used.
stop_position: ending position of the range. If 'None' default stop
position of the source must be used.
Returns:
a ``_ObjectIdRangeTracker``, ``OffsetRangeTracker``
or ``LexicographicKeyRangeTracker`` depending on the given position range.
"""
start_position, stop_position = self._replace_none_positions(
start_position, stop_position)
if isinstance(start_position, ObjectId):
return _ObjectIdRangeTracker(start_position, stop_position)
if isinstance(start_position, int):
return OffsetRangeTracker(start_position, stop_position)
if isinstance(start_position, str):
return LexicographicKeyRangeTracker(start_position, stop_position)
raise NotImplementedError(
f"RangeTracker for {type(start_position)} not implemented!")
def split_range_size(self, desired_size, sample_row_keys, range_):
''' This method split the row_set ranges using the desired_bundle_size
you get.
:param desired_size(int): The size you need to split the ranges.
:param sample_row_keys(list): A list of row keys with a end size.
:param range_: A Row Range Element, to split if it is necessary.
'''
start = None
end = None
last_offset = 0
for sample_row in sample_row_keys:
current = sample_row.offset_bytes - last_offset
# if the sample_row is the first or last element in the sample_row
# keys parameter, it avoids this sample row element.
if sample_row.row_key == b'':
continue
if (range_.start_key <= sample_row.row_key
and range_.end_key >= sample_row.row_key):
if start is not None:
end = sample_row.row_key
ranges = LexicographicKeyRangeTracker(start, end)
for fraction in self.split_range_subranges(
current, desired_size, ranges):
yield fraction
start = sample_row.row_key
last_offset = sample_row.offset_bytes
def get_range_tracker(self, start_position, stop_position):
if self._counts == 0:
self._counts = self.source.client.counts_estimator(
self.source.query)
if start_position is None:
start_position = 0
if stop_position is None:
stop_position = self._counts
return LexicographicKeyRangeTracker(start_position, stop_position)
def fraction_to_position(self, position, range_start, range_stop):
''' We use the ``fraction_to_position`` method in
``LexicographicKeyRangeTracker`` class to split a
range into two chunks.
:param position:
:param range_start:
:param range_stop:
:return:
'''
return LexicographicKeyRangeTracker.fraction_to_position(
position, range_start, range_stop)
def get_range_tracker(self, start_position, stop_position):
self._validate_query()
return LexicographicKeyRangeTracker(start_position, stop_position)
def get_range_tracker(self, start_position, stop_position):
if stop_position == b'':
return LexicographicKeyRangeTracker(start_position)
else:
return LexicographicKeyRangeTracker(start_position, stop_position)
def get_range_tracker(self, start_position, stop_position):
logging.info("ReadFromBigtable get_range_tracker")
return LexicographicKeyRangeTracker(start_position, stop_position)
def range_split_fraction(self, current_size, desired_bundle_size,
start_key, end_key):
range_tracker = LexicographicKeyRangeTracker(start_key, end_key)
return self.split_range_subranges(current_size, desired_bundle_size,
range_tracker)