def __init__(self, method, remote_path, local_path, keep=False, parts=4, speed_limit=None, timeout=20, restart=False, tracker=None, limiter=None): self.method = method self.remote_path = remote_path self.local_path = local_path self.keep = keep self.parts = parts or 4 self.restart = restart # Control variables self.speed_limit = speed_limit self.timeout = timeout self.streamers = [] self.tracker = ProgressTracker(parent=tracker) self.limiter = SpeedLimiter(parent=limiter) self.counter = Counter() if self.speed_limit: self.set_speed_limit(*self.speed_limit)
def test_add(self): tracker = ProgressTracker() tracker.add(5) key = list(tracker.time_total.keys())[0] assert tracker.total == 5 assert len(tracker.times) == 1 assert tracker.time_total[key] == 5
def __init__(self, concurrent=4, parts=4, speed_limit=None, timeout=20, restart=False): self.concurrent = concurrent self.parts = parts self.restart = restart # Control variables self.is_running = False self.is_paused = False self.speed_limit = speed_limit self.timeout = timeout self.tracker = ProgressTracker() self.limiter = SpeedLimiter() self.unfinished = deque() self.workers = deque() self.finished = [] self.errors = [] if self.speed_limit: self.set_speed_limit(*self.speed_limit)
def test_defaults(self): tracker = ProgressTracker() assert tracker.size == 0 assert tracker.window == 10 assert tracker.is_finished == False assert len(tracker.times) == 0 assert len(tracker.time_total) == 0
def test_progress_purges_outside_window(self): tracker = ProgressTracker(10, 1) tracker.add(3) time.sleep(0.5) tracker.add(5) time.sleep(0.6) assert tracker.get_progress() == (0, 0, 8, 10) assert len(tracker.times) == len(tracker.time_total) == 1
def test_progress_correct(self): tracker = ProgressTracker(10) tracker.add(3) time.sleep(0.1) tracker.add(5) seconds = tracker.times[-1] - tracker.times[0] ups = sum(tracker.time_total.values()) / seconds eta = (tracker._size - tracker.total) / ups assert tracker.get_progress() == (ups, eta, 8, 10)
def test_remove(self): tracker = ProgressTracker() tracker.add(10) tracker.remove(3) key = list(tracker.time_total.keys())[0] assert tracker.total == 7 assert tracker.time_total[key] == 10
def test_progress_no_eta_when_no_size(self): tracker = ProgressTracker() tracker.add(5) tracker.add(5) assert tracker.get_progress()[1] == 0
class Session: def __init__(self, concurrent=4, parts=4, speed_limit=None, timeout=20, restart=False): self.concurrent = concurrent self.parts = parts self.restart = restart # Control variables self.is_running = False self.is_paused = False self.speed_limit = speed_limit self.timeout = timeout self.tracker = ProgressTracker() self.limiter = SpeedLimiter() self.unfinished = deque() self.workers = deque() self.finished = [] self.errors = [] if self.speed_limit: self.set_speed_limit(*self.speed_limit) def get(self, *args, **kwargs): raise NotImplementedError def send(self, *args, **kwargs): raise NotImplementedError def _run(self, forever=False): self.is_running = True while True: time.sleep(STATE_CHECK) if not self.is_running: break elif self.is_paused: continue # Remove finished workers from queue for _ in range(len(self.workers)): worker = self.workers[0] if not worker.is_alive(): if worker.success(): self.finished.append(self.workers.popleft()) else: self.errors.append(self.workers.popleft()) else: self.workers.rotate(-1) # Repopulate worker queue while len(self.workers) < self.concurrent: if self.unfinished: self.workers.append(self.unfinished.popleft()) continue break for worker in self.workers: if not worker.is_alive(): worker.run() if not forever and not self.unfinished and not self.workers: break self.is_running = False def run(self, *args, **kwargs): if not self.is_running: threading.Thread(target=self._run, args=args, kwargs=kwargs).start() def stop(self): self.is_running = False def pause(self): self.is_paused = True def resume(self): self.is_paused = False def get_progress(self): return self.tracker.get_progress() def set_speed_limit(self, value, unit='KiB'): if value: self.speed_limit = (value, unit) self.limiter.set_limit( unit_pair_to_bytes(self.speed_limit) ) else: self.speed_limit = None self.limiter.set_limit(0) @property def done(self): return not self.unfinished
def test_progress_when_no_time_elapsed(self): tracker = ProgressTracker(5) tracker.add(3) assert tracker.get_progress() == (0, 0, 3, 5)
def test_progress_when_empty(self): tracker = ProgressTracker(5) assert tracker.get_progress() == (0, 0, 0, 5)
def test_done_false_if_no_size_and_not_is_finished(self): tracker = ProgressTracker() assert tracker.done == False
def test_done_false_if_size_and_not_complete(self): tracker = ProgressTracker(1) assert tracker.done == False
def test_done_true_if_size_and_complete(self): tracker = ProgressTracker(1) tracker.add(1) assert tracker.done == True
def test_done_true_if_is_finished(self): tracker = ProgressTracker() tracker.is_finished = True assert tracker.done == True
def test_shrink(self): tracker = ProgressTracker(5) tracker.shrink(3) assert tracker._size == 2
def test_add_same_time(self): tracker = ProgressTracker() num_adds = 20 for i in range(num_adds): tracker.add(i) assert len(tracker.times) == len(tracker.time_total) < num_adds
class FileSync: def __init__(self, method, remote_path, local_path, keep=False, parts=4, speed_limit=None, timeout=20, restart=False, tracker=None, limiter=None): self.method = method self.remote_path = remote_path self.local_path = local_path self.keep = keep self.parts = parts or 4 self.restart = restart # Control variables self.speed_limit = speed_limit self.timeout = timeout self.streamers = [] self.tracker = ProgressTracker(parent=tracker) self.limiter = SpeedLimiter(parent=limiter) self.counter = Counter() if self.speed_limit: self.set_speed_limit(*self.speed_limit) def _spawn(self, *args, **kwargs): raise NotImplementedError def run(self, *args, **kwargs): if not self.is_alive(): self._spawn(*args, **kwargs) def is_alive(self): for streamer in self.streamers: if streamer.is_alive: return True return False def success(self): for streamer in self.streamers: if not streamer.is_done: return False return True def stop(self): for streamer in self.streamers: streamer.stop() def pause(self): for streamer in self.streamers: streamer.pause() def resume(self): for streamer in self.streamers: streamer.resume() def get_progress(self): return self.tracker.get_progress() def set_speed_limit(self, value, unit='KiB'): if value: self.speed_limit = (value, unit) self.limiter.set_limit(unit_pair_to_bytes(self.speed_limit)) else: self.speed_limit = None self.limiter.set_limit(0) @property def done(self): return self.tracker.done def _reset(self): self.streamers.clear() self.tracker.clear() self.limiter.reset() self.counter.set(0)
def test_with_args(self): tracker = ProgressTracker(50, 5) assert tracker.size == 50 assert tracker.window == 5
class Session: def __init__(self, concurrent=4, parts=4, speed_limit=None, timeout=20, restart=False): self.concurrent = concurrent self.parts = parts self.restart = restart # Control variables self.is_running = False self.is_paused = False self.speed_limit = speed_limit self.timeout = timeout self.tracker = ProgressTracker() self.limiter = SpeedLimiter() self.unfinished = deque() self.workers = deque() self.finished = [] self.errors = [] if self.speed_limit: self.set_speed_limit(*self.speed_limit) def get(self, *args, **kwargs): raise NotImplementedError def send(self, *args, **kwargs): raise NotImplementedError def _run(self, forever=False): self.is_running = True while True: time.sleep(STATE_CHECK) if not self.is_running: break elif self.is_paused: continue # Remove finished workers from queue for _ in range(len(self.workers)): worker = self.workers[0] if not worker.is_alive(): if worker.success(): self.finished.append(self.workers.popleft()) else: self.errors.append(self.workers.popleft()) else: self.workers.rotate(-1) # Repopulate worker queue while len(self.workers) < self.concurrent: if self.unfinished: self.workers.append(self.unfinished.popleft()) continue break for worker in self.workers: if not worker.is_alive(): worker.run() if not forever and not self.unfinished and not self.workers: break self.is_running = False def run(self, *args, **kwargs): if not self.is_running: threading.Thread(target=self._run, args=args, kwargs=kwargs).start() def stop(self): self.is_running = False def pause(self): self.is_paused = True def resume(self): self.is_paused = False def get_progress(self): return self.tracker.get_progress() def set_speed_limit(self, value, unit='KiB'): if value: self.speed_limit = (value, unit) self.limiter.set_limit(unit_pair_to_bytes(self.speed_limit)) else: self.speed_limit = None self.limiter.set_limit(0) @property def done(self): return not self.unfinished
def test_grow(self): tracker = ProgressTracker() tracker.grow(5) assert tracker._size == 5
class FileSync: def __init__(self, method, remote_path, local_path, keep=False, parts=4, speed_limit=None, timeout=20, restart=False, tracker=None, limiter=None): self.method = method self.remote_path = remote_path self.local_path = local_path self.keep = keep self.parts = parts or 4 self.restart = restart # Control variables self.speed_limit = speed_limit self.timeout = timeout self.streamers = [] self.tracker = ProgressTracker(parent=tracker) self.limiter = SpeedLimiter(parent=limiter) self.counter = Counter() if self.speed_limit: self.set_speed_limit(*self.speed_limit) def _spawn(self, *args, **kwargs): raise NotImplementedError def run(self, *args, **kwargs): if not self.is_alive(): self._spawn(*args, **kwargs) def is_alive(self): for streamer in self.streamers: if streamer.is_alive: return True return False def success(self): for streamer in self.streamers: if not streamer.is_done: return False return True def stop(self): for streamer in self.streamers: streamer.stop() def pause(self): for streamer in self.streamers: streamer.pause() def resume(self): for streamer in self.streamers: streamer.resume() def get_progress(self): return self.tracker.get_progress() def set_speed_limit(self, value, unit='KiB'): if value: self.speed_limit = (value, unit) self.limiter.set_limit( unit_pair_to_bytes(self.speed_limit) ) else: self.speed_limit = None self.limiter.set_limit(0) @property def done(self): return self.tracker.done def _reset(self): self.streamers.clear() self.tracker.clear() self.limiter.reset() self.counter.set(0)