def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.manager = QNetworkAccessManager()
self.url = 'http://localhost:9998/jpg/image.jpg'
self.request = QNetworkRequest()
self.request.setUrl(QUrl(self.url))
self.buffer = QByteArray()
self.reply = None
class Downloader(QObject):
imageReady = Signal(QByteArray)
def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.manager = QNetworkAccessManager()
self.url = 'http://localhost:9998/jpg/image.jpg'
self.request = QNetworkRequest()
self.request.setUrl(QUrl(self.url))
self.buffer = QByteArray()
self.reply = None
def setUrl(self, url):
self.url = url
self.request.setUrl(QUrl(self.url))
def downloadData(self):
""" Only request a new image if this is the first/last completed. """
if self.reply is None:
self.reply = self.manager.get(self.request)
self.reply.finished.connect(self.finished)
def finished(self):
""" Read the buffer, emit a signal with the new image in it. """
self.buffer = self.reply.readAll()
self.imageReady.emit(self.buffer)
self.reply.deleteLater()
self.reply = None
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def loadTile(self, x, y, zoom, url):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
cache = QNetworkDiskCache()
cacheDir = getCacheFolder()
cache.setCacheDirectory(cacheDir)
cache.setMaximumCacheSize(self._cacheSize)
self._manager.setCache(cache)
key = (x, y, zoom)
url = QUrl(url)
if key not in self._tileInDownload:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, key)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
def loadTile(self, x, y, zoom, url, cache_dir):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
self._cache = './tiles'
key = (x, y, zoom)
url = QUrl(url)
#base = parsed_url.netloc
if key not in self._tileInDownload:
path = os.path.join(self._cache, str(cache_dir), str(zoom), str(x),
str(y) + '.png')
if os.path.exists(path):
self.tileLoaded.emit(x, y, zoom, open(path, 'rb').read())
else:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, [key, cache_dir])
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
def __init__(self, chunk_size=1024, load_rc_func=None):
"""Download API based on QNetworkAccessManager."""
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._load_rc_func = load_rc_func
self._manager = QNetworkAccessManager(self)
self._proxy_factory = NetworkProxyFactory(load_rc_func=load_rc_func)
self._timer = QTimer()
# Setup
self._manager.setProxyFactory(self._proxy_factory)
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
self._manager.proxyAuthenticationRequired.connect(
self._handle_proxy_auth)
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def __init__(self, parent, on_finished_func, on_progress_func, save_path):
super(DownloadManager, self).__init__(parent)
self._parent = parent
self._on_finished_func = on_finished_func
self._on_progress_func = on_progress_func
self._manager = QNetworkAccessManager(self)
self._request = None
self._reply = None
self._queue = None # [['filename', 'uri'], ...]
self._url = None # current url in process
self._filename = None # current filename in process
self._save_path = None # current defined save path
self._error = None # error number
self._free = True # lock process flag
self.set_save_path(save_path)
def loadTile(self, x, y, zoom, url):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
cache = QNetworkDiskCache()
cacheDir = getCacheFolder()
cache.setCacheDirectory(cacheDir)
cache.setMaximumCacheSize(self._cacheSize)
self._manager.setCache(cache)
key = (x, y, zoom)
url = QUrl(url)
if key not in self._tileInDownload:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, key)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
class _DownloadAPI(QObject):
"""Download API based on QNetworkAccessManager."""
def __init__(self, chunk_size=1024, load_rc_func=None):
"""Download API based on QNetworkAccessManager."""
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._load_rc_func = load_rc_func
self._manager = QNetworkAccessManager(self)
self._proxy_factory = NetworkProxyFactory(load_rc_func=load_rc_func)
self._timer = QTimer()
# Setup
self._manager.setProxyFactory(self._proxy_factory)
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
self._manager.proxyAuthenticationRequired.connect(
self._handle_proxy_auth)
@staticmethod
def _handle_ssl_errors(reply, errors):
"""Callback for ssl_errors."""
logger.error(str(('SSL Errors', errors, reply)))
@staticmethod
def _handle_proxy_auth(proxy, authenticator):
"""Callback for ssl_errors."""
# authenticator.setUser('1')`
# authenticator.setPassword('1')
logger.error(str(('Proxy authentication Error. '
'Enter credentials in condarc',
proxy,
authenticator)))
def _clean(self):
"""Check for inactive workers and remove their references."""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
"""Callback for download once the request has finished."""
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
error = reply.error()
# print(url, error)
if error:
logger.error(str(('Head Reply Error:', error)))
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, error)
return
self._head_requests.pop(url)
start_download = not bool(error)
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file or error?
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""Save `data` of downloaded `url` in `path`."""
worker = self._workers[url]
path = self._paths[url]
if len(data):
try:
with open(path, 'wb') as f:
f.write(data)
except Exception:
logger.error((url, path))
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
@staticmethod
def _progress(bytes_received, bytes_total, worker):
"""Return download progress."""
worker.sig_download_progress.emit(
worker.url, worker.path, bytes_received, bytes_total)
def download(self, url, path):
"""Download url and save data to path."""
# original_url = url
# print(url)
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
"""Terminate all download workers and threads."""
pass
class MapTileHTTPLoader(QObject):
tileLoaded = Signal(int, int, int, QByteArray)
def __init__(self, cacheSize=DEFAULT_CACHE_SIZE, userAgent='(PyQt) TileMap 1.0', parent=None):
QObject.__init__(self, parent=parent)
self._manager = None
self._cache = None
self._cacheSize = cacheSize
try:
# Convert user agent to bytes
userAgent = userAgent.encode()
except:
# no encode method exists. This hsould be the Python 2 case
pass
self._userAgent = userAgent
self._tileInDownload = dict()
@Slot(int, int, int, str)
def loadTile(self, x, y, zoom, url):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
cache = QNetworkDiskCache()
cacheDir = getCacheFolder()
cache.setCacheDirectory(cacheDir)
cache.setMaximumCacheSize(self._cacheSize)
self._manager.setCache(cache)
key = (x, y, zoom)
url = QUrl(url)
if key not in self._tileInDownload:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, key)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
@Slot(QNetworkReply)
def handleNetworkData(self, reply):
tp = getQVariantValue(reply.request().attribute(QNetworkRequest.User))
if tp in self._tileInDownload:
del self._tileInDownload[tp]
if not reply.error():
data = reply.readAll()
self.tileLoaded.emit(tp[0], tp[1], tp[2], data)
reply.close()
reply.deleteLater()
@Slot()
def abortRequest(self, x, y, zoom):
p = (x, y, zoom)
if p in self._tileInDownload:
reply = self._tileInDownload[p]
del self._tileInDownload[p]
reply.close()
reply.deleteLater()
@Slot()
def abortAllRequests(self):
for x, y, zoom in list(self._tileInDownload.keys()):
self.abortRequest(x, y, zoom)
class _DownloadAPI(QObject):
"""
Download API based on QNetworkAccessManager
"""
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def _handle_ssl_errors(self, reply, errors):
logger.error(str(('SSL Errors', errors)))
def _clean(self):
"""
Periodically check for inactive workers and remove their references.
"""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
self._head_requests.pop(url)
start_download = True
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""
"""
worker = self._workers[url]
path = self._paths[url]
if len(data):
with open(path, 'wb') as f:
f.write(data)
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
def _progress(self, bytes_received, bytes_total, worker):
"""
"""
worker.sig_download_progress.emit(worker.url, worker.path,
bytes_received, bytes_total)
def download(self, url, path):
"""
"""
# original_url = url
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
pass
class MapTileHTTPLoader(QObject):
tileLoaded = Signal(int, int, int, QByteArray)
def __init__(self,
cacheSize=DEFAULT_CACHE_SIZE,
userAgent='(PyQt) TileMap 1.0',
parent=None):
QObject.__init__(self, parent=parent)
self._manager = None
self._cache = None
self._cacheSize = cacheSize
try:
# Convert user agent to bytes
userAgent = userAgent.encode()
except:
# no encode method exists. This hsould be the Python 2 case
pass
self._userAgent = userAgent
self._tileInDownload = dict()
@Slot(int, int, int, str)
def loadTile(self, x, y, zoom, url):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
cache = QNetworkDiskCache()
cacheDir = getCacheFolder()
cache.setCacheDirectory(cacheDir)
cache.setMaximumCacheSize(self._cacheSize)
self._manager.setCache(cache)
key = (x, y, zoom)
url = QUrl(url)
if key not in self._tileInDownload:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, key)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
@Slot(QNetworkReply)
def handleNetworkData(self, reply):
tp = getQVariantValue(reply.request().attribute(QNetworkRequest.User))
if tp in self._tileInDownload:
del self._tileInDownload[tp]
if not reply.error():
data = reply.readAll()
self.tileLoaded.emit(tp[0], tp[1], tp[2], data)
reply.close()
reply.deleteLater()
@Slot()
def abortRequest(self, x, y, zoom):
p = (x, y, zoom)
if p in self._tileInDownload:
reply = self._tileInDownload[p]
del self._tileInDownload[p]
reply.close()
reply.deleteLater()
@Slot()
def abortAllRequests(self):
for x, y, zoom in list(self._tileInDownload.keys()):
self.abortRequest(x, y, zoom)
class _DownloadAPI(QObject):
"""
Download API based on QNetworkAccessManager
"""
def __init__(self, chunk_size=1024):
super(_DownloadAPI, self).__init__()
self._chunk_size = chunk_size
self._head_requests = {}
self._get_requests = {}
self._paths = {}
self._workers = {}
self._manager = QNetworkAccessManager(self)
self._timer = QTimer()
# Setup
self._timer.setInterval(1000)
self._timer.timeout.connect(self._clean)
# Signals
self._manager.finished.connect(self._request_finished)
self._manager.sslErrors.connect(self._handle_ssl_errors)
def _handle_ssl_errors(self, reply, errors):
logger.error(str(('SSL Errors', errors)))
def _clean(self):
"""
Periodically check for inactive workers and remove their references.
"""
if self._workers:
for url in self._workers.copy():
w = self._workers[url]
if w.is_finished():
self._workers.pop(url)
self._paths.pop(url)
if url in self._get_requests:
self._get_requests.pop(url)
else:
self._timer.stop()
def _request_finished(self, reply):
url = to_text_string(reply.url().toEncoded(), encoding='utf-8')
if url in self._paths:
path = self._paths[url]
if url in self._workers:
worker = self._workers[url]
if url in self._head_requests:
self._head_requests.pop(url)
start_download = True
header_pairs = reply.rawHeaderPairs()
headers = {}
for hp in header_pairs:
headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])
total_size = int(headers.get('content-length', 0))
# Check if file exists
if os.path.isfile(path):
file_size = os.path.getsize(path)
# Check if existing file matches size of requested file
start_download = file_size != total_size
if start_download:
# File sizes dont match, hence download file
qurl = QUrl(url)
request = QNetworkRequest(qurl)
self._get_requests[url] = request
reply = self._manager.get(request)
error = reply.error()
if error:
logger.error(str(('Reply Error:', error)))
reply.downloadProgress.connect(
lambda r, t, w=worker: self._progress(r, t, w))
else:
# File sizes match, dont download file
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
elif url in self._get_requests:
data = reply.readAll()
self._save(url, path, data)
def _save(self, url, path, data):
"""
"""
worker = self._workers[url]
path = self._paths[url]
if len(data):
with open(path, 'wb') as f:
f.write(data)
# Clean up
worker.finished = True
worker.sig_download_finished.emit(url, path)
worker.sig_finished.emit(worker, path, None)
self._get_requests.pop(url)
self._workers.pop(url)
self._paths.pop(url)
def _progress(self, bytes_received, bytes_total, worker):
"""
"""
worker.sig_download_progress.emit(
worker.url, worker.path, bytes_received, bytes_total)
def download(self, url, path):
"""
"""
# original_url = url
qurl = QUrl(url)
url = to_text_string(qurl.toEncoded(), encoding='utf-8')
logger.debug(str((url, path)))
if url in self._workers:
while not self._workers[url].finished:
return self._workers[url]
worker = DownloadWorker(url, path)
# Check download folder exists
folder = os.path.dirname(os.path.abspath(path))
if not os.path.isdir(folder):
os.makedirs(folder)
request = QNetworkRequest(qurl)
self._head_requests[url] = request
self._paths[url] = path
self._workers[url] = worker
self._manager.head(request)
self._timer.start()
return worker
def terminate(self):
pass
class MapTileHTTPLoader(QObject):
tileLoaded = Signal(int, int, int, QByteArray)
def __init__(self,
cacheSize=DEFAULT_CACHE_SIZE,
userAgent='(PyQt) TileMap 1.0',
parent=None):
QObject.__init__(self, parent=parent)
self._manager = None
self._cache = None
self._cacheSize = cacheSize
try:
# Convert user agent to bytes
userAgent = userAgent.encode()
except:
# no encode method exists. This hsould be the Python 2 case
pass
self._userAgent = userAgent
self._tileInDownload = dict()
@Slot(int, int, int, str, str)
def loadTile(self, x, y, zoom, url, cache_dir):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
self._cache = './tiles'
key = (x, y, zoom)
url = QUrl(url)
#base = parsed_url.netloc
if key not in self._tileInDownload:
path = os.path.join(self._cache, str(cache_dir), str(zoom), str(x),
str(y) + '.png')
if os.path.exists(path):
self.tileLoaded.emit(x, y, zoom, open(path, 'rb').read())
else:
# Request the image to the map service
request = QNetworkRequest(url=url)
request.setRawHeader(b'User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, [key, cache_dir])
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
self._tileInDownload[key] = self._manager.get(request)
@Slot(QNetworkReply)
def handleNetworkData(self, reply):
[tp, cache_dir
] = getQVariantValue(reply.request().attribute(QNetworkRequest.User))
if tp in self._tileInDownload:
del self._tileInDownload[tp]
zoom = tp[2]
x = tp[0]
y = tp[1]
base = cache_dir
if not reply.error():
data = reply.readAll()
if not os.path.isdir(
os.path.join(self._cache, base, str(zoom), str(x))):
os.makedirs(os.path.join(self._cache, base, str(zoom), str(x)))
fout = open(
os.path.join(self._cache, base, str(zoom), str(x),
str(y) + '.png'), 'wb')
fout.write(data)
fout.close()
self.tileLoaded.emit(tp[0], tp[1], tp[2], data)
reply.close()
reply.deleteLater()
@Slot()
def abortRequest(self, x, y, zoom):
p = (x, y, zoom)
if p in self._tileInDownload:
reply = self._tileInDownload[p]
del self._tileInDownload[p]
reply.close()
reply.deleteLater()
@Slot()
def abortAllRequests(self):
for x, y, zoom in list(self._tileInDownload.keys()):
self.abortRequest(x, y, zoom)
class DownloadManager(QObject):
"""Synchronous download manager.
http://qt-project.org/doc/qt-4.8/network-downloadmanager-downloadmanager-cpp.html
as inspiration.
"""
def __init__(self, parent, on_finished_func, on_progress_func, save_path):
super(DownloadManager, self).__init__(parent)
self._parent = parent
self._on_finished_func = on_finished_func
self._on_progress_func = on_progress_func
self._manager = QNetworkAccessManager(self)
self._request = None
self._reply = None
self._queue = None # [['filename', 'uri'], ...]
self._url = None # current url in process
self._filename = None # current filename in process
self._save_path = None # current defined save path
self._error = None # error number
self._free = True # lock process flag
self.set_save_path(save_path)
def _start_next_download(self):
""" """
if self._free:
if len(self._queue) != 0:
self._free = False
self._filename, self._url = self._queue.pop(0)
full_path = osp.join(self._save_path, self._filename)
if osp.isfile(full_path):
# compare file versions by getting headers first
self._get(header_only=True)
else:
# file does not exists, first download
self._get()
# print(full_path)
else:
self._on_finished_func()
def _get(self, header_only=False):
"""Download file specified by uri"""
self._request = QNetworkRequest(QUrl(self._url))
self._reply = None
self._error = None
if header_only:
self._reply = self._manager.head(self._request)
self._reply.finished.connect(self._on_downloaded_headers)
else:
self._reply = self._manager.get(self._request)
self._reply.finished.connect(self._on_downloaded)
self._reply.downloadProgress.connect(self._on_progress)
def _on_downloaded_headers(self):
"""On header from uri downloaded"""
# handle error for headers...
error_code = self._reply.error()
if error_code > 0:
self._on_errors(error_code)
return None
fullpath = osp.join(self._save_path, self._filename)
headers = {}
data = self._reply.rawHeaderPairs()
for d in data:
if isinstance(d[0], QByteArray):
d = [d[0].data(), d[1].data()]
key = to_text_string(d[0], encoding='ascii')
value = to_text_string(d[1], encoding='ascii')
headers[key.lower()] = value
if len(headers) != 0:
header_filesize = int(headers['content-length'])
local_filesize = int(osp.getsize(fullpath))
if header_filesize == local_filesize:
self._free = True
self._start_next_download()
else:
self._get()
def _on_downloaded(self):
"""On file downloaded"""
# check if errors
error_code = self._reply.error()
if error_code > 0:
self._on_errors(error_code)
return None
# process data if no errors
data = self._reply.readAll()
self._save_file(data)
def _on_errors(self, e):
"""On download errors"""
self._free = True # otherwise update button cannot work!
self._error = e
self._on_finished_func()
def _on_progress(self, downloaded_size, total_size):
"""On Partial progress"""
self._on_progress_func([downloaded_size, total_size])
def _save_file(self, data):
""" """
if not osp.isdir(self._save_path):
os.mkdir(self._save_path)
fullpath = osp.join(self._save_path, self._filename)
if isinstance(data, QByteArray):
data = data.data()
with open(fullpath, 'wb') as f:
f.write(data)
self._free = True
self._start_next_download()
# public api
# ----------
def set_save_path(self, path):
""" """
self._save_path = path
def set_queue(self, queue):
"""[['filename', 'uri'], ['filename', 'uri'], ...]"""
self._queue = queue
def get_errors(self):
""" """
return self._error
def start_download(self):
""" """
self._start_next_download()
def stop_download(self):
""" """
pass