def get_range_tracker( self, start_position: Union[int, str, ObjectId] = None, stop_position: Union[int, str, ObjectId] = None, ) -> Union[_ObjectIdRangeTracker, OffsetRangeTracker, LexicographicKeyRangeTracker]: """Returns a RangeTracker for a given position range depending on type. Args: start_position: starting position of the range. If 'None' default start position of the source must be used. stop_position: ending position of the range. If 'None' default stop position of the source must be used. Returns: a ``_ObjectIdRangeTracker``, ``OffsetRangeTracker`` or ``LexicographicKeyRangeTracker`` depending on the given position range. """ start_position, stop_position = self._replace_none_positions( start_position, stop_position) if isinstance(start_position, ObjectId): return _ObjectIdRangeTracker(start_position, stop_position) if isinstance(start_position, int): return OffsetRangeTracker(start_position, stop_position) if isinstance(start_position, str): return LexicographicKeyRangeTracker(start_position, stop_position) raise NotImplementedError( f"RangeTracker for {type(start_position)} not implemented!")
def split_range_size(self, desired_size, sample_row_keys, range_): ''' This method split the row_set ranges using the desired_bundle_size you get. :param desired_size(int): The size you need to split the ranges. :param sample_row_keys(list): A list of row keys with a end size. :param range_: A Row Range Element, to split if it is necessary. ''' start = None end = None last_offset = 0 for sample_row in sample_row_keys: current = sample_row.offset_bytes - last_offset # if the sample_row is the first or last element in the sample_row # keys parameter, it avoids this sample row element. if sample_row.row_key == b'': continue if (range_.start_key <= sample_row.row_key and range_.end_key >= sample_row.row_key): if start is not None: end = sample_row.row_key ranges = LexicographicKeyRangeTracker(start, end) for fraction in self.split_range_subranges( current, desired_size, ranges): yield fraction start = sample_row.row_key last_offset = sample_row.offset_bytes
def get_range_tracker(self, start_position, stop_position): if self._counts == 0: self._counts = self.source.client.counts_estimator( self.source.query) if start_position is None: start_position = 0 if stop_position is None: stop_position = self._counts return LexicographicKeyRangeTracker(start_position, stop_position)
def fraction_to_position(self, position, range_start, range_stop): ''' We use the ``fraction_to_position`` method in ``LexicographicKeyRangeTracker`` class to split a range into two chunks. :param position: :param range_start: :param range_stop: :return: ''' return LexicographicKeyRangeTracker.fraction_to_position( position, range_start, range_stop)
def get_range_tracker(self, start_position, stop_position): self._validate_query() return LexicographicKeyRangeTracker(start_position, stop_position)
def get_range_tracker(self, start_position, stop_position): if stop_position == b'': return LexicographicKeyRangeTracker(start_position) else: return LexicographicKeyRangeTracker(start_position, stop_position)
def get_range_tracker(self, start_position, stop_position): logging.info("ReadFromBigtable get_range_tracker") return LexicographicKeyRangeTracker(start_position, stop_position)
def range_split_fraction(self, current_size, desired_bundle_size, start_key, end_key): range_tracker = LexicographicKeyRangeTracker(start_key, end_key) return self.split_range_subranges(current_size, desired_bundle_size, range_tracker)