def read(self, range_tracker: OffsetRangeTracker) -> Optional[Generator]:
"""
Returns the content of the next file
"""
for i in range(range_tracker.start_position(),
range_tracker.stop_position()):
if not range_tracker.try_claim(i):
return
path = self.file_paths[i]
content = self.dataset.read_file(path)
yield content
def get_range_tracker(
self,
start_position, # type: Optional[Any]
stop_position, # type: Optional[Any]
): # type: (...) -> RangeTracker
if start_position is None:
start_position = 0
if stop_position is None:
stop_position = self.count()
return OffsetRangeTracker(start_position, stop_position)
def get_range_tracker(self, start_position: Optional[int],
stop_position: Optional[int]) -> OffsetRangeTracker:
"""
Creates and returns an OffsetRangeTracker
"""
if start_position is None:
start_position = 0
if stop_position is None:
stop_position = len(self.file_paths)
return OffsetRangeTracker(start_position, stop_position)
def test_read(self): tracker = OffsetRangeTracker(3, 6) res = list(self.source.read(tracker)) self.assertEqual([3, 4, 5], res)