def __init__(self,
method,
remote_path,
local_path,
keep=False,
parts=4,
speed_limit=None,
timeout=20,
restart=False,
tracker=None,
limiter=None):
self.method = method
self.remote_path = remote_path
self.local_path = local_path
self.keep = keep
self.parts = parts or 4
self.restart = restart
# Control variables
self.speed_limit = speed_limit
self.timeout = timeout
self.streamers = []
self.tracker = ProgressTracker(parent=tracker)
self.limiter = SpeedLimiter(parent=limiter)
self.counter = Counter()
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def test_add(self):
tracker = ProgressTracker()
tracker.add(5)
key = list(tracker.time_total.keys())[0]
assert tracker.total == 5
assert len(tracker.times) == 1
assert tracker.time_total[key] == 5
def __init__(self,
concurrent=4,
parts=4,
speed_limit=None,
timeout=20,
restart=False):
self.concurrent = concurrent
self.parts = parts
self.restart = restart
# Control variables
self.is_running = False
self.is_paused = False
self.speed_limit = speed_limit
self.timeout = timeout
self.tracker = ProgressTracker()
self.limiter = SpeedLimiter()
self.unfinished = deque()
self.workers = deque()
self.finished = []
self.errors = []
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def test_defaults(self):
tracker = ProgressTracker()
assert tracker.size == 0
assert tracker.window == 10
assert tracker.is_finished == False
assert len(tracker.times) == 0
assert len(tracker.time_total) == 0
def test_progress_purges_outside_window(self):
tracker = ProgressTracker(10, 1)
tracker.add(3)
time.sleep(0.5)
tracker.add(5)
time.sleep(0.6)
assert tracker.get_progress() == (0, 0, 8, 10)
assert len(tracker.times) == len(tracker.time_total) == 1
def test_progress_correct(self):
tracker = ProgressTracker(10)
tracker.add(3)
time.sleep(0.1)
tracker.add(5)
seconds = tracker.times[-1] - tracker.times[0]
ups = sum(tracker.time_total.values()) / seconds
eta = (tracker._size - tracker.total) / ups
assert tracker.get_progress() == (ups, eta, 8, 10)
def test_remove(self):
tracker = ProgressTracker()
tracker.add(10)
tracker.remove(3)
key = list(tracker.time_total.keys())[0]
assert tracker.total == 7
assert tracker.time_total[key] == 10
def __init__(self, concurrent=4, parts=4, speed_limit=None, timeout=20, restart=False):
self.concurrent = concurrent
self.parts = parts
self.restart = restart
# Control variables
self.is_running = False
self.is_paused = False
self.speed_limit = speed_limit
self.timeout = timeout
self.tracker = ProgressTracker()
self.limiter = SpeedLimiter()
self.unfinished = deque()
self.workers = deque()
self.finished = []
self.errors = []
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def __init__(self, method, remote_path, local_path, keep=False,
parts=4, speed_limit=None, timeout=20, restart=False,
tracker=None, limiter=None):
self.method = method
self.remote_path = remote_path
self.local_path = local_path
self.keep = keep
self.parts = parts or 4
self.restart = restart
# Control variables
self.speed_limit = speed_limit
self.timeout = timeout
self.streamers = []
self.tracker = ProgressTracker(parent=tracker)
self.limiter = SpeedLimiter(parent=limiter)
self.counter = Counter()
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def test_progress_no_eta_when_no_size(self):
tracker = ProgressTracker()
tracker.add(5)
tracker.add(5)
assert tracker.get_progress()[1] == 0
class Session:
def __init__(self, concurrent=4, parts=4, speed_limit=None, timeout=20, restart=False):
self.concurrent = concurrent
self.parts = parts
self.restart = restart
# Control variables
self.is_running = False
self.is_paused = False
self.speed_limit = speed_limit
self.timeout = timeout
self.tracker = ProgressTracker()
self.limiter = SpeedLimiter()
self.unfinished = deque()
self.workers = deque()
self.finished = []
self.errors = []
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def get(self, *args, **kwargs):
raise NotImplementedError
def send(self, *args, **kwargs):
raise NotImplementedError
def _run(self, forever=False):
self.is_running = True
while True:
time.sleep(STATE_CHECK)
if not self.is_running:
break
elif self.is_paused:
continue
# Remove finished workers from queue
for _ in range(len(self.workers)):
worker = self.workers[0]
if not worker.is_alive():
if worker.success():
self.finished.append(self.workers.popleft())
else:
self.errors.append(self.workers.popleft())
else:
self.workers.rotate(-1)
# Repopulate worker queue
while len(self.workers) < self.concurrent:
if self.unfinished:
self.workers.append(self.unfinished.popleft())
continue
break
for worker in self.workers:
if not worker.is_alive():
worker.run()
if not forever and not self.unfinished and not self.workers:
break
self.is_running = False
def run(self, *args, **kwargs):
if not self.is_running:
threading.Thread(target=self._run, args=args, kwargs=kwargs).start()
def stop(self):
self.is_running = False
def pause(self):
self.is_paused = True
def resume(self):
self.is_paused = False
def get_progress(self):
return self.tracker.get_progress()
def set_speed_limit(self, value, unit='KiB'):
if value:
self.speed_limit = (value, unit)
self.limiter.set_limit(
unit_pair_to_bytes(self.speed_limit)
)
else:
self.speed_limit = None
self.limiter.set_limit(0)
@property
def done(self):
return not self.unfinished
def test_progress_when_no_time_elapsed(self):
tracker = ProgressTracker(5)
tracker.add(3)
assert tracker.get_progress() == (0, 0, 3, 5)
def test_progress_when_empty(self):
tracker = ProgressTracker(5)
assert tracker.get_progress() == (0, 0, 0, 5)
def test_done_false_if_no_size_and_not_is_finished(self):
tracker = ProgressTracker()
assert tracker.done == False
def test_done_false_if_size_and_not_complete(self):
tracker = ProgressTracker(1)
assert tracker.done == False
def test_done_true_if_size_and_complete(self):
tracker = ProgressTracker(1)
tracker.add(1)
assert tracker.done == True
def test_done_true_if_is_finished(self):
tracker = ProgressTracker()
tracker.is_finished = True
assert tracker.done == True
def test_shrink(self):
tracker = ProgressTracker(5)
tracker.shrink(3)
assert tracker._size == 2
def test_add_same_time(self):
tracker = ProgressTracker()
num_adds = 20
for i in range(num_adds):
tracker.add(i)
assert len(tracker.times) == len(tracker.time_total) < num_adds
class FileSync:
def __init__(self,
method,
remote_path,
local_path,
keep=False,
parts=4,
speed_limit=None,
timeout=20,
restart=False,
tracker=None,
limiter=None):
self.method = method
self.remote_path = remote_path
self.local_path = local_path
self.keep = keep
self.parts = parts or 4
self.restart = restart
# Control variables
self.speed_limit = speed_limit
self.timeout = timeout
self.streamers = []
self.tracker = ProgressTracker(parent=tracker)
self.limiter = SpeedLimiter(parent=limiter)
self.counter = Counter()
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def _spawn(self, *args, **kwargs):
raise NotImplementedError
def run(self, *args, **kwargs):
if not self.is_alive():
self._spawn(*args, **kwargs)
def is_alive(self):
for streamer in self.streamers:
if streamer.is_alive:
return True
return False
def success(self):
for streamer in self.streamers:
if not streamer.is_done:
return False
return True
def stop(self):
for streamer in self.streamers:
streamer.stop()
def pause(self):
for streamer in self.streamers:
streamer.pause()
def resume(self):
for streamer in self.streamers:
streamer.resume()
def get_progress(self):
return self.tracker.get_progress()
def set_speed_limit(self, value, unit='KiB'):
if value:
self.speed_limit = (value, unit)
self.limiter.set_limit(unit_pair_to_bytes(self.speed_limit))
else:
self.speed_limit = None
self.limiter.set_limit(0)
@property
def done(self):
return self.tracker.done
def _reset(self):
self.streamers.clear()
self.tracker.clear()
self.limiter.reset()
self.counter.set(0)
def test_with_args(self):
tracker = ProgressTracker(50, 5)
assert tracker.size == 50
assert tracker.window == 5
class Session:
def __init__(self,
concurrent=4,
parts=4,
speed_limit=None,
timeout=20,
restart=False):
self.concurrent = concurrent
self.parts = parts
self.restart = restart
# Control variables
self.is_running = False
self.is_paused = False
self.speed_limit = speed_limit
self.timeout = timeout
self.tracker = ProgressTracker()
self.limiter = SpeedLimiter()
self.unfinished = deque()
self.workers = deque()
self.finished = []
self.errors = []
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def get(self, *args, **kwargs):
raise NotImplementedError
def send(self, *args, **kwargs):
raise NotImplementedError
def _run(self, forever=False):
self.is_running = True
while True:
time.sleep(STATE_CHECK)
if not self.is_running:
break
elif self.is_paused:
continue
# Remove finished workers from queue
for _ in range(len(self.workers)):
worker = self.workers[0]
if not worker.is_alive():
if worker.success():
self.finished.append(self.workers.popleft())
else:
self.errors.append(self.workers.popleft())
else:
self.workers.rotate(-1)
# Repopulate worker queue
while len(self.workers) < self.concurrent:
if self.unfinished:
self.workers.append(self.unfinished.popleft())
continue
break
for worker in self.workers:
if not worker.is_alive():
worker.run()
if not forever and not self.unfinished and not self.workers:
break
self.is_running = False
def run(self, *args, **kwargs):
if not self.is_running:
threading.Thread(target=self._run, args=args,
kwargs=kwargs).start()
def stop(self):
self.is_running = False
def pause(self):
self.is_paused = True
def resume(self):
self.is_paused = False
def get_progress(self):
return self.tracker.get_progress()
def set_speed_limit(self, value, unit='KiB'):
if value:
self.speed_limit = (value, unit)
self.limiter.set_limit(unit_pair_to_bytes(self.speed_limit))
else:
self.speed_limit = None
self.limiter.set_limit(0)
@property
def done(self):
return not self.unfinished
def test_grow(self):
tracker = ProgressTracker()
tracker.grow(5)
assert tracker._size == 5
class FileSync:
def __init__(self, method, remote_path, local_path, keep=False,
parts=4, speed_limit=None, timeout=20, restart=False,
tracker=None, limiter=None):
self.method = method
self.remote_path = remote_path
self.local_path = local_path
self.keep = keep
self.parts = parts or 4
self.restart = restart
# Control variables
self.speed_limit = speed_limit
self.timeout = timeout
self.streamers = []
self.tracker = ProgressTracker(parent=tracker)
self.limiter = SpeedLimiter(parent=limiter)
self.counter = Counter()
if self.speed_limit:
self.set_speed_limit(*self.speed_limit)
def _spawn(self, *args, **kwargs):
raise NotImplementedError
def run(self, *args, **kwargs):
if not self.is_alive():
self._spawn(*args, **kwargs)
def is_alive(self):
for streamer in self.streamers:
if streamer.is_alive:
return True
return False
def success(self):
for streamer in self.streamers:
if not streamer.is_done:
return False
return True
def stop(self):
for streamer in self.streamers:
streamer.stop()
def pause(self):
for streamer in self.streamers:
streamer.pause()
def resume(self):
for streamer in self.streamers:
streamer.resume()
def get_progress(self):
return self.tracker.get_progress()
def set_speed_limit(self, value, unit='KiB'):
if value:
self.speed_limit = (value, unit)
self.limiter.set_limit(
unit_pair_to_bytes(self.speed_limit)
)
else:
self.speed_limit = None
self.limiter.set_limit(0)
@property
def done(self):
return self.tracker.done
def _reset(self):
self.streamers.clear()
self.tracker.clear()
self.limiter.reset()
self.counter.set(0)