def test_many_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
for key, value in selects.items():
path = test_urls[key]["path"]
dirname = FileManager.get_dirname(path)
basename = FileManager.get_basename(path)
for i in range(value):
filepath = os.path.join(dirname, "{}_{}".format(i, basename)) if i != 0 else path
self.assertTrue(os.path.exists(filepath))
with open(filepath, "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(filepath)
def run(self):
while not self.works.empty():
# Get work
work = self.works.get()
info = work[1]
self.i = work[0]
self.progress = {}
try:
url = info.get("url")
directory = info.get("dest")
if not url:
raise NoURLException("file {} does not have url".format(self.i))
if not directory:
raise NoDestinationPathException("file {} does not have dest".format(self.i))
# Get filename and protocol from url
split = urllib.parse.urlparse(url)
protocol = split.scheme
filename = FileManager.get_basename(split.path)
# Create directory if the directory does not exist
# Check whether directory has write permission
if FileManager.is_path_creatable(directory):
FileManager.create_directory(directory)
else:
import os
raise OSError(errno.EACCES, os.strerror(errno.EACCES), directory)
# Download file with random filename to prevent overwritten file with same name
filepath = FileManager.random_filepath(directory)
self.progress["filepath"] = filepath
self.progress["filename"] = filename
if protocol in ["http", "https"]:
self.http_download(url, filepath)
elif protocol in ["ftp", "ftps"]:
self.ftp_download(split, filepath)
elif protocol in ["sftp"]:
self.sftp_download(split, info.get("key_filename"), info.get("passphrase"), filepath)
else:
raise UnsupportedProtocolException("file {}({}) uses unsupported protocol".format(self.i, filename))
except Exception as e:
# Notify failed work
self.progress["state"] = "Failed"
self.progress["error"] = e
self.progresses.put((self.i, self.progress.copy()))
self.works.task_done()
time.sleep(self.config.get("wait_task", 3))
return
def rename_file(self, dest):
"""
Rename downloaded file from random name to name identified in url.
Filename will have incremental number at the front if filename exist.
dest: destination path
"""
filename = self.progress.get("filename")
dirname = FileManager.get_dirname(dest)
# Get filename to rename the file
# Increment number at the front of filename if filename exist
new_dest = FileManager.join_path(dirname, filename)
if FileManager.is_path_exists(new_dest):
new_dest = FileManager.generate_filepath(new_dest)
FileManager.rename_file(dest, new_dest)
# Notify success work
self.progress["filename"] = FileManager.get_basename(new_dest)
self.progress["state"] = "Success"
self.progresses.put((self.i, self.progress.copy()))