def test_many_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
for key, value in selects.items():
path = test_urls[key]["path"]
dirname = FileManager.get_dirname(path)
basename = FileManager.get_basename(path)
for i in range(value):
filepath = os.path.join(dirname, "{}_{}".format(i, basename)) if i != 0 else path
self.assertTrue(os.path.exists(filepath))
with open(filepath, "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(filepath)
def test_sftp_download(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": test_urls["sftp_small"]["url"], "dest": test_dest}))
q.put((1, {"url": test_urls["sftp_pkey_small"]["url"], "dest": test_dest,
"key_filename": test_urls["sftp_pkey_small"]["key_filename"],
"passphrase": test_urls["sftp_pkey_small"]["key_filename"]}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertTrue(os.path.exists(test_urls["sftp_small"]["path"]))
self.assertTrue(os.path.exists(test_urls["sftp_pkey_small"]["path"]))
with open(test_urls["sftp_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_small"]["md5"])
with open(test_urls["sftp_pkey_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_pkey_small"]["md5"])
FileManager.remove_file(test_urls["sftp_small"]["path"])
FileManager.remove_file(test_urls["sftp_pkey_small"]["path"])
def remove_incomplete(self, dest):
"""
Remove partial file.
dest: destination path
"""
# Notify failed work
self.progress["state"] = "Failed"
self.progresses.put((self.i, self.progress.copy()))
# Retry loop if file cannot be removed
for i in range(self.config.get("max_retry", 3)):
try:
FileManager.remove_file(dest)
break
except:
time.sleep(3)
def test_success_download(self):
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
for i, key in enumerate(test_urls):
works.put((i + 1, {
"url": test_urls[key]["url"],
"dest": test_dest
}))
for i in range(4):
worker = Worker({
"wait_task": 1,
"wait_retry": 0,
"max_retry": 1
},
works,
progresses,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
visualizer = Visualizer(4, progresses, name="visualizer")
visualizer.start()
works.join()
visualizer.join()
self.assertTrue(works.empty())
self.assertTrue(progresses.empty())
self.assertEqual(visualizer.success, 4)
self.assertEqual(visualizer.success, visualizer.task)
self.assertTrue(not visualizer.results)
for key in test_urls:
self.assertTrue(os.path.exists(test_urls[key]["path"]))
with open(test_urls[key]["path"], "rb") as f:
data = f.read()
self.assertEqual(
hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(test_urls[key]["path"])