def test_fail_download(self):
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
for i, key in enumerate(test_urls):
works.put((i + 1, {
"url": test_urls[key]["url"],
"dest": test_dest
}))
for i in range(4):
worker = Worker({
"wait_task": 1,
"wait_retry": 0,
"max_retry": 1
},
works,
progresses,
test_net=True,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
visualizer = Visualizer(4, progresses, name="visualizer")
visualizer.start()
works.join()
visualizer.join()
self.assertTrue(works.empty())
self.assertTrue(progresses.empty())
self.assertEqual(visualizer.fail, 4)
self.assertEqual(visualizer.fail, visualizer.task)
self.assertEqual(len(visualizer.results), 4)
self.assertTrue(len(os.listdir(test_dest)) == 0)
def test_fail_progress(self):
q = Queue()
q.put((0, {
"filename": "name",
"filepath": "path",
"state": "Failed",
"error": "error"
}))
v = Visualizer(1, q)
v.start()
v.join()
self.assertTrue(q.empty())
self.assertEqual(v.fail, 1)
self.assertEqual(v.fail, v.task)
self.assertEqual(len(v.results), 1)
def test_success_progress(self):
q = Queue()
q.put((0, {
"filename": "name",
"filepath": "path",
"state": "Success",
"error": "error"
}))
v = Visualizer(1, q)
v.start()
v.join()
self.assertTrue(q.empty())
self.assertEqual(v.success, 1)
self.assertEqual(v.success, v.task)
self.assertTrue(not v.results)
def test_success_download(self):
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
for i, key in enumerate(test_urls):
works.put((i + 1, {
"url": test_urls[key]["url"],
"dest": test_dest
}))
for i in range(4):
worker = Worker({
"wait_task": 1,
"wait_retry": 0,
"max_retry": 1
},
works,
progresses,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
visualizer = Visualizer(4, progresses, name="visualizer")
visualizer.start()
works.join()
visualizer.join()
self.assertTrue(works.empty())
self.assertTrue(progresses.empty())
self.assertEqual(visualizer.success, 4)
self.assertEqual(visualizer.success, visualizer.task)
self.assertTrue(not visualizer.results)
for key in test_urls:
self.assertTrue(os.path.exists(test_urls[key]["path"]))
with open(test_urls[key]["path"], "rb") as f:
data = f.read()
self.assertEqual(
hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(test_urls[key]["path"])
raise Exception("No inputs given")
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
# Put work to Queue
for i, info in enumerate(inputs.values()):
works.put((i + 1, info))
# Setup workers
num_threads = min(config.get("max_worker", 5), len(inputs))
for i in range(num_threads):
worker = Worker(config,
works,
progresses,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
# Setup visualizer
visualizer = Visualizer(len(inputs), progresses, name="visualizer")
visualizer.start()
# Wait until works Queue and visualizer finished
works.join()
visualizer.join()
except FileNotFoundError as errf:
print(errf)
except Exception as e:
print(e)
