class _DownloadAPI(QObject):
"""Download API based on QNetworkAccessManager."""
def __init__(self, chunk_size=1024, load_rc_func=None):
"""Download API based on QNetworkAccessManager."""
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._load_rc_func = load_rc_func
self._manager = QNetworkAccessManager(self)
self._proxy_factory = NetworkProxyFactory(load_rc_func=load_rc_func)
self._timer = QTimer()
# Setup
self._manager.setProxyFactory(self._proxy_factory)
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
self._manager.proxyAuthenticationRequired.connect(
self._handle_proxy_auth)
@staticmethod
def _handle_ssl_errors(reply, errors):
"""Callback for ssl_errors."""
logger.error(str(('SSL Errors', errors, reply)))
@staticmethod
def _handle_proxy_auth(proxy, authenticator):
"""Callback for ssl_errors."""
# authenticator.setUser('1')`
# authenticator.setPassword('1')
logger.error(str(('Proxy authentication Error. '
'Enter credentials in condarc',
proxy,
authenticator)))
def _clean(self):
"""Check for inactive workers and remove their references."""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
"""Callback for download once the request has finished."""
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
error = reply.error()
# print(url, error)
if error:
logger.error(str(('Head Reply Error:', error)))
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, error)
return
self._head_requests.pop(url)
start_download = not bool(error)
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file or error?
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""Save `data` of downloaded `url` in `path`."""
worker = self._workers[url]
path = self._paths[url]
if len(data):
try:
with open(path, 'wb') as f:
f.write(data)
except Exception:
logger.error((url, path))
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
@staticmethod
def _progress(bytes_received, bytes_total, worker):
"""Return download progress."""
worker.sig_download_progress.emit(
worker.url, worker.path, bytes_received, bytes_total)
def download(self, url, path):
"""Download url and save data to path."""
# original_url = url
# print(url)
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
"""Terminate all download workers and threads."""
pass
class _DownloadAPI(QObject):
"""
Download API based on QNetworkAccessManager
"""
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def _handle_ssl_errors(self, reply, errors):
logger.error(str(('SSL Errors', errors)))
def _clean(self):
"""
Periodically check for inactive workers and remove their references.
"""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
self._head_requests.pop(url)
start_download = True
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""
"""
worker = self._workers[url]
path = self._paths[url]
if len(data):
with open(path, 'wb') as f:
f.write(data)
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
def _progress(self, bytes_received, bytes_total, worker):
"""
"""
worker.sig_download_progress.emit(
worker.url, worker.path, bytes_received, bytes_total)
def download(self, url, path):
"""
"""
# original_url = url
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
pass
class _DownloadAPI(QObject):
"""
Download API based on QNetworkAccessManager
"""
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def _handle_ssl_errors(self, reply, errors):
logger.error(str(('SSL Errors', errors)))
def _clean(self):
"""
Periodically check for inactive workers and remove their references.
"""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
self._head_requests.pop(url)
start_download = True
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""
"""
worker = self._workers[url]
path = self._paths[url]
if len(data):
with open(path, 'wb') as f:
f.write(data)
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
def _progress(self, bytes_received, bytes_total, worker):
"""
"""
worker.sig_download_progress.emit(worker.url, worker.path,
bytes_received, bytes_total)
def download(self, url, path):
"""
"""
# original_url = url
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
pass
class DownloadManager(QObject):
"""Synchronous download manager.
http://qt-project.org/doc/qt-4.8/network-downloadmanager-downloadmanager-cpp.html
as inspiration.
"""
def __init__(self, parent, on_finished_func, on_progress_func, save_path):
super(DownloadManager, self).__init__(parent)
self._parent = parent
self._on_finished_func = on_finished_func
self._on_progress_func = on_progress_func
self._manager = QNetworkAccessManager(self)
self._request = None
self._reply = None
self._queue = None # [['filename', 'uri'], ...]
self._url = None # current url in process
self._filename = None # current filename in process
self._save_path = None # current defined save path
self._error = None # error number
self._free = True # lock process flag
self.set_save_path(save_path)
def _start_next_download(self):
""" """
if self._free:
if len(self._queue) != 0:
self._free = False
self._filename, self._url = self._queue.pop(0)
full_path = osp.join(self._save_path, self._filename)
if osp.isfile(full_path):
# compare file versions by getting headers first
self._get(header_only=True)
else:
# file does not exists, first download
self._get()
# print(full_path)
else:
self._on_finished_func()
def _get(self, header_only=False):
"""Download file specified by uri"""
self._request = QNetworkRequest(QUrl(self._url))
self._reply = None
self._error = None
if header_only:
self._reply = self._manager.head(self._request)
self._reply.finished.connect(self._on_downloaded_headers)
else:
self._reply = self._manager.get(self._request)
self._reply.finished.connect(self._on_downloaded)
self._reply.downloadProgress.connect(self._on_progress)
def _on_downloaded_headers(self):
"""On header from uri downloaded"""
# handle error for headers...
error_code = self._reply.error()
if error_code > 0:
self._on_errors(error_code)
return None
fullpath = osp.join(self._save_path, self._filename)
headers = {}
data = self._reply.rawHeaderPairs()
for d in data:
if isinstance(d[0], QByteArray):
d = [d[0].data(), d[1].data()]
key = to_text_string(d[0], encoding='ascii')
value = to_text_string(d[1], encoding='ascii')
headers[key.lower()] = value
if len(headers) != 0:
header_filesize = int(headers['content-length'])
local_filesize = int(osp.getsize(fullpath))
if header_filesize == local_filesize:
self._free = True
self._start_next_download()
else:
self._get()
def _on_downloaded(self):
"""On file downloaded"""
# check if errors
error_code = self._reply.error()
if error_code > 0:
self._on_errors(error_code)
return None
# process data if no errors
data = self._reply.readAll()
self._save_file(data)
def _on_errors(self, e):
"""On download errors"""
self._free = True # otherwise update button cannot work!
self._error = e
self._on_finished_func()
def _on_progress(self, downloaded_size, total_size):
"""On Partial progress"""
self._on_progress_func([downloaded_size, total_size])
def _save_file(self, data):
""" """
if not osp.isdir(self._save_path):
os.mkdir(self._save_path)
fullpath = osp.join(self._save_path, self._filename)
if isinstance(data, QByteArray):
data = data.data()
with open(fullpath, 'wb') as f:
f.write(data)
self._free = True
self._start_next_download()
# public api
# ----------
def set_save_path(self, path):
""" """
self._save_path = path
def set_queue(self, queue):
"""[['filename', 'uri'], ['filename', 'uri'], ...]"""
self._queue = queue
def get_errors(self):
""" """
return self._error
def start_download(self):
""" """
self._start_next_download()
def stop_download(self):
""" """
pass
