def test_many_download(self): q = Queue() q2 = Queue() selects = {} for i in range(10): choice = random.choice(list(test_urls.keys())) if choice in selects.keys(): selects[choice] += 1 else: selects[choice] = 1 q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest})) worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2) worker.start() worker.join() self.assertTrue(q.empty()) for key, value in selects.items(): path = test_urls[key]["path"] dirname = FileManager.get_dirname(path) basename = FileManager.get_basename(path) for i in range(value): filepath = os.path.join(dirname, "{}_{}".format(i, basename)) if i != 0 else path self.assertTrue(os.path.exists(filepath)) with open(filepath, "rb") as f: data = f.read() self.assertEqual(hashlib.md5(data).hexdigest(), test_urls[key]["md5"]) FileManager.remove_file(filepath)
def test_sftp_download(self): q = Queue() q2 = Queue() q.put((0, {"url": test_urls["sftp_small"]["url"], "dest": test_dest})) q.put((1, {"url": test_urls["sftp_pkey_small"]["url"], "dest": test_dest, "key_filename": test_urls["sftp_pkey_small"]["key_filename"], "passphrase": test_urls["sftp_pkey_small"]["key_filename"]})) worker = Worker({"wait_task": 0}, q, q2) worker.start() worker.join() self.assertTrue(q.empty()) self.assertTrue(os.path.exists(test_urls["sftp_small"]["path"])) self.assertTrue(os.path.exists(test_urls["sftp_pkey_small"]["path"])) with open(test_urls["sftp_small"]["path"], "rb") as f: data = f.read() self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_small"]["md5"]) with open(test_urls["sftp_pkey_small"]["path"], "rb") as f: data = f.read() self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_pkey_small"]["md5"]) FileManager.remove_file(test_urls["sftp_small"]["path"]) FileManager.remove_file(test_urls["sftp_pkey_small"]["path"])
def remove_incomplete(self, dest): """ Remove partial file. dest: destination path """ # Notify failed work self.progress["state"] = "Failed" self.progresses.put((self.i, self.progress.copy())) # Retry loop if file cannot be removed for i in range(self.config.get("max_retry", 3)): try: FileManager.remove_file(dest) break except: time.sleep(3)
def test_success_download(self): works = Queue(maxsize=0) progresses = Queue(maxsize=0) for i, key in enumerate(test_urls): works.put((i + 1, { "url": test_urls[key]["url"], "dest": test_dest })) for i in range(4): worker = Worker({ "wait_task": 1, "wait_retry": 0, "max_retry": 1 }, works, progresses, name="worker{}".format(i)) worker.setDaemon(True) worker.start() visualizer = Visualizer(4, progresses, name="visualizer") visualizer.start() works.join() visualizer.join() self.assertTrue(works.empty()) self.assertTrue(progresses.empty()) self.assertEqual(visualizer.success, 4) self.assertEqual(visualizer.success, visualizer.task) self.assertTrue(not visualizer.results) for key in test_urls: self.assertTrue(os.path.exists(test_urls[key]["path"])) with open(test_urls[key]["path"], "rb") as f: data = f.read() self.assertEqual( hashlib.md5(data).hexdigest(), test_urls[key]["md5"]) FileManager.remove_file(test_urls[key]["path"])