def _downloader(self):
while (not self.empty()) and (not self.project.shutdown_signal):
t = threading.current_thread()
Common.check_for_pause(self.project)
slip = self.get()
if callable(slip.url):
file_url = slip.url()
else:
file_url = slip.url
t.name = "Downloading: " + slip.item[slip.filename_key]
self.project.log("transaction", "Downloading " + slip.item[slip.filename_key], "info", True)
try:
data = Common.webrequest(
file_url, self.headers(), self.http_callback, None, False, True
) # Response object gets passed to shutil.copyfileobj
self.storage_callback(data, slip)
except urllib.error.HTTPError as err:
self.project.log(
"exception",
"{} failed to download - HTTPError {}".format(slip.item[slip.filename_key], err.code),
"warning",
)
if self.project.shutdown_signal:
self.project.log(
"exception",
"{} received shutdown signal. Stopping...".format(threading.current_thread().name),
"warning",
)
else:
self.project.log("transaction", "{} has completed.".format(threading.current_thread().name), "info")
def _downloader(self):
while (not self.empty()) and (not self.project.shutdown_signal):
t = threading.current_thread()
Common.check_for_pause(self.project)
slip = self.get()
if callable(slip.url):
file_url = slip.url()
else:
file_url = slip.url
t.name = 'Downloading: ' + slip.item[slip.filename_key]
self.project.log("transaction",
"Downloading " + slip.item[slip.filename_key],
"info", True)
try:
data = Common.webrequest(
file_url, self.headers(), self.http_callback, None, False,
True) # Response object gets passed to shutil.copyfileobj
self.storage_callback(data, slip)
except urllib.error.HTTPError as err:
self.project.log(
"exception", "{} failed to download - HTTPError {}".format(
slip.item[slip.filename_key], err.code), "warning")
if self.project.shutdown_signal:
self.project.log(
"exception", "{} received shutdown signal. Stopping...".format(
threading.current_thread().name), "warning")
else:
self.project.log(
"transaction",
"{} has completed.".format(threading.current_thread().name),
"info")
def _save_file(self, data, slip, stream=True):
Common.check_for_pause(self.project)
savepath = slip.savepath
path_to_create = os.path.dirname(savepath) # Just the directory not the filename
if not os.path.isdir(path_to_create):
os.makedirs(path_to_create, exist_ok=True)
self.project.savedata(data, savepath, stream)
self.project.log("transaction", "Saved file to " + savepath, "info", True)
def _save_file(self, data, slip, stream=True):
Common.check_for_pause(self.project)
savepath = slip.savepath
path_to_create = os.path.dirname(
savepath) # Just the directory not the filename
if not os.path.isdir(path_to_create):
os.makedirs(path_to_create, exist_ok=True)
self.project.savedata(data, savepath, stream)
self.project.log("transaction", "Saved file to " + savepath, "info",
True)