def test_many_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
for key, value in selects.items():
path = test_urls[key]["path"]
dirname = FileManager.get_dirname(path)
basename = FileManager.get_basename(path)
for i in range(value):
filepath = os.path.join(dirname, "{}_{}".format(i, basename)) if i != 0 else path
self.assertTrue(os.path.exists(filepath))
with open(filepath, "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(filepath)
def test_sftp_download(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": test_urls["sftp_small"]["url"], "dest": test_dest}))
q.put((1, {"url": test_urls["sftp_pkey_small"]["url"], "dest": test_dest,
"key_filename": test_urls["sftp_pkey_small"]["key_filename"],
"passphrase": test_urls["sftp_pkey_small"]["key_filename"]}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertTrue(os.path.exists(test_urls["sftp_small"]["path"]))
self.assertTrue(os.path.exists(test_urls["sftp_pkey_small"]["path"]))
with open(test_urls["sftp_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_small"]["md5"])
with open(test_urls["sftp_pkey_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_pkey_small"]["md5"])
FileManager.remove_file(test_urls["sftp_small"]["path"])
FileManager.remove_file(test_urls["sftp_pkey_small"]["path"])
def test_404_download(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "http://google.com/blah", "dest": test_dest}))
worker = Worker({"wait_task": 0, "wait_retry": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
q2.get()
self.assertIsInstance(q2.get()[1]["error"], requests.exceptions.HTTPError)
def test_no_dest(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "url"}))
q.put((1, {"url": "url", "dest": ""}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertIsInstance(q2.get()[1]["error"], NoDestinationPathException)
self.assertIsInstance(q2.get()[1]["error"], NoDestinationPathException)
def test_unsupport_protocol(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "url", "dest": test_dest}))
q.put((1, {"url": "abc://path", "dest": test_dest}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
q2.get()
self.assertIsInstance(q2.get()[1]["error"], UnsupportedProtocolException)
q2.get()
self.assertIsInstance(q2.get()[1]["error"], UnsupportedProtocolException)
def test_fail_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2, test_net=True)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertTrue(len(os.listdir(test_dest)) == 0)