def __init__(self, parent=None): super(Downloader, self).__init__(parent) self.manager = QNetworkAccessManager() self.url = 'http://localhost:9998/jpg/image.jpg' self.request = QNetworkRequest() self.request.setUrl(QUrl(self.url)) self.buffer = QByteArray() self.reply = None
class Downloader(QObject): imageReady = Signal(QByteArray) def __init__(self, parent=None): super(Downloader, self).__init__(parent) self.manager = QNetworkAccessManager() self.url = 'http://localhost:9998/jpg/image.jpg' self.request = QNetworkRequest() self.request.setUrl(QUrl(self.url)) self.buffer = QByteArray() self.reply = None def setUrl(self, url): self.url = url self.request.setUrl(QUrl(self.url)) def downloadData(self): """ Only request a new image if this is the first/last completed. """ if self.reply is None: self.reply = self.manager.get(self.request) self.reply.finished.connect(self.finished) def finished(self): """ Read the buffer, emit a signal with the new image in it. """ self.buffer = self.reply.readAll() self.imageReady.emit(self.buffer) self.reply.deleteLater() self.reply = None
def __init__(self, chunk_size=1024): super(_DownloadAPI, self).__init__() self._chunk_size = chunk_size self._head_requests = {} self._get_requests = {} self._paths = {} self._workers = {} self._manager = QNetworkAccessManager(self) self._timer = QTimer() # Setup self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) # Signals self._manager.finished.connect(self._request_finished) self._manager.sslErrors.connect(self._handle_ssl_errors)
def loadTile(self, x, y, zoom, url): if self._manager is None: self._manager = QNetworkAccessManager(parent=self) self._manager.finished.connect(self.handleNetworkData) cache = QNetworkDiskCache() cacheDir = getCacheFolder() cache.setCacheDirectory(cacheDir) cache.setMaximumCacheSize(self._cacheSize) self._manager.setCache(cache) key = (x, y, zoom) url = QUrl(url) if key not in self._tileInDownload: # Request the image to the map service request = QNetworkRequest(url=url) request.setRawHeader(b'User-Agent', self._userAgent) request.setAttribute(QNetworkRequest.User, key) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) self._tileInDownload[key] = self._manager.get(request)
def loadTile(self, x, y, zoom, url, cache_dir): if self._manager is None: self._manager = QNetworkAccessManager(parent=self) self._manager.finished.connect(self.handleNetworkData) self._cache = './tiles' key = (x, y, zoom) url = QUrl(url) #base = parsed_url.netloc if key not in self._tileInDownload: path = os.path.join(self._cache, str(cache_dir), str(zoom), str(x), str(y) + '.png') if os.path.exists(path): self.tileLoaded.emit(x, y, zoom, open(path, 'rb').read()) else: # Request the image to the map service request = QNetworkRequest(url=url) request.setRawHeader(b'User-Agent', self._userAgent) request.setAttribute(QNetworkRequest.User, [key, cache_dir]) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) self._tileInDownload[key] = self._manager.get(request)
def __init__(self, chunk_size=1024, load_rc_func=None): """Download API based on QNetworkAccessManager.""" super(_DownloadAPI, self).__init__() self._chunk_size = chunk_size self._head_requests = {} self._get_requests = {} self._paths = {} self._workers = {} self._load_rc_func = load_rc_func self._manager = QNetworkAccessManager(self) self._proxy_factory = NetworkProxyFactory(load_rc_func=load_rc_func) self._timer = QTimer() # Setup self._manager.setProxyFactory(self._proxy_factory) self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) # Signals self._manager.finished.connect(self._request_finished) self._manager.sslErrors.connect(self._handle_ssl_errors) self._manager.proxyAuthenticationRequired.connect( self._handle_proxy_auth)
def __init__(self, parent, on_finished_func, on_progress_func, save_path): super(DownloadManager, self).__init__(parent) self._parent = parent self._on_finished_func = on_finished_func self._on_progress_func = on_progress_func self._manager = QNetworkAccessManager(self) self._request = None self._reply = None self._queue = None # [['filename', 'uri'], ...] self._url = None # current url in process self._filename = None # current filename in process self._save_path = None # current defined save path self._error = None # error number self._free = True # lock process flag self.set_save_path(save_path)
class _DownloadAPI(QObject): """Download API based on QNetworkAccessManager.""" def __init__(self, chunk_size=1024, load_rc_func=None): """Download API based on QNetworkAccessManager.""" super(_DownloadAPI, self).__init__() self._chunk_size = chunk_size self._head_requests = {} self._get_requests = {} self._paths = {} self._workers = {} self._load_rc_func = load_rc_func self._manager = QNetworkAccessManager(self) self._proxy_factory = NetworkProxyFactory(load_rc_func=load_rc_func) self._timer = QTimer() # Setup self._manager.setProxyFactory(self._proxy_factory) self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) # Signals self._manager.finished.connect(self._request_finished) self._manager.sslErrors.connect(self._handle_ssl_errors) self._manager.proxyAuthenticationRequired.connect( self._handle_proxy_auth) @staticmethod def _handle_ssl_errors(reply, errors): """Callback for ssl_errors.""" logger.error(str(('SSL Errors', errors, reply))) @staticmethod def _handle_proxy_auth(proxy, authenticator): """Callback for ssl_errors.""" # authenticator.setUser('1')` # authenticator.setPassword('1') logger.error(str(('Proxy authentication Error. ' 'Enter credentials in condarc', proxy, authenticator))) def _clean(self): """Check for inactive workers and remove their references.""" if self._workers: for url in self._workers.copy(): w = self._workers[url] if w.is_finished(): self._workers.pop(url) self._paths.pop(url) if url in self._get_requests: self._get_requests.pop(url) else: self._timer.stop() def _request_finished(self, reply): """Callback for download once the request has finished.""" url = to_text_string(reply.url().toEncoded(), encoding='utf-8') if url in self._paths: path = self._paths[url] if url in self._workers: worker = self._workers[url] if url in self._head_requests: error = reply.error() # print(url, error) if error: logger.error(str(('Head Reply Error:', error))) worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, error) return self._head_requests.pop(url) start_download = not bool(error) header_pairs = reply.rawHeaderPairs() headers = {} for hp in header_pairs: headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1]) total_size = int(headers.get('content-length', 0)) # Check if file exists if os.path.isfile(path): file_size = os.path.getsize(path) # Check if existing file matches size of requested file start_download = file_size != total_size if start_download: # File sizes dont match, hence download file qurl = QUrl(url) request = QNetworkRequest(qurl) self._get_requests[url] = request reply = self._manager.get(request) error = reply.error() if error: logger.error(str(('Reply Error:', error))) reply.downloadProgress.connect( lambda r, t, w=worker: self._progress(r, t, w)) else: # File sizes match, dont download file or error? worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) elif url in self._get_requests: data = reply.readAll() self._save(url, path, data) def _save(self, url, path, data): """Save `data` of downloaded `url` in `path`.""" worker = self._workers[url] path = self._paths[url] if len(data): try: with open(path, 'wb') as f: f.write(data) except Exception: logger.error((url, path)) # Clean up worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) self._get_requests.pop(url) self._workers.pop(url) self._paths.pop(url) @staticmethod def _progress(bytes_received, bytes_total, worker): """Return download progress.""" worker.sig_download_progress.emit( worker.url, worker.path, bytes_received, bytes_total) def download(self, url, path): """Download url and save data to path.""" # original_url = url # print(url) qurl = QUrl(url) url = to_text_string(qurl.toEncoded(), encoding='utf-8') logger.debug(str((url, path))) if url in self._workers: while not self._workers[url].finished: return self._workers[url] worker = DownloadWorker(url, path) # Check download folder exists folder = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(folder): os.makedirs(folder) request = QNetworkRequest(qurl) self._head_requests[url] = request self._paths[url] = path self._workers[url] = worker self._manager.head(request) self._timer.start() return worker def terminate(self): """Terminate all download workers and threads.""" pass
class MapTileHTTPLoader(QObject): tileLoaded = Signal(int, int, int, QByteArray) def __init__(self, cacheSize=DEFAULT_CACHE_SIZE, userAgent='(PyQt) TileMap 1.0', parent=None): QObject.__init__(self, parent=parent) self._manager = None self._cache = None self._cacheSize = cacheSize try: # Convert user agent to bytes userAgent = userAgent.encode() except: # no encode method exists. This hsould be the Python 2 case pass self._userAgent = userAgent self._tileInDownload = dict() @Slot(int, int, int, str) def loadTile(self, x, y, zoom, url): if self._manager is None: self._manager = QNetworkAccessManager(parent=self) self._manager.finished.connect(self.handleNetworkData) cache = QNetworkDiskCache() cacheDir = getCacheFolder() cache.setCacheDirectory(cacheDir) cache.setMaximumCacheSize(self._cacheSize) self._manager.setCache(cache) key = (x, y, zoom) url = QUrl(url) if key not in self._tileInDownload: # Request the image to the map service request = QNetworkRequest(url=url) request.setRawHeader(b'User-Agent', self._userAgent) request.setAttribute(QNetworkRequest.User, key) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) self._tileInDownload[key] = self._manager.get(request) @Slot(QNetworkReply) def handleNetworkData(self, reply): tp = getQVariantValue(reply.request().attribute(QNetworkRequest.User)) if tp in self._tileInDownload: del self._tileInDownload[tp] if not reply.error(): data = reply.readAll() self.tileLoaded.emit(tp[0], tp[1], tp[2], data) reply.close() reply.deleteLater() @Slot() def abortRequest(self, x, y, zoom): p = (x, y, zoom) if p in self._tileInDownload: reply = self._tileInDownload[p] del self._tileInDownload[p] reply.close() reply.deleteLater() @Slot() def abortAllRequests(self): for x, y, zoom in list(self._tileInDownload.keys()): self.abortRequest(x, y, zoom)
class _DownloadAPI(QObject): """ Download API based on QNetworkAccessManager """ def __init__(self, chunk_size=1024): super(_DownloadAPI, self).__init__() self._chunk_size = chunk_size self._head_requests = {} self._get_requests = {} self._paths = {} self._workers = {} self._manager = QNetworkAccessManager(self) self._timer = QTimer() # Setup self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) # Signals self._manager.finished.connect(self._request_finished) self._manager.sslErrors.connect(self._handle_ssl_errors) def _handle_ssl_errors(self, reply, errors): logger.error(str(('SSL Errors', errors))) def _clean(self): """ Periodically check for inactive workers and remove their references. """ if self._workers: for url in self._workers.copy(): w = self._workers[url] if w.is_finished(): self._workers.pop(url) self._paths.pop(url) if url in self._get_requests: self._get_requests.pop(url) else: self._timer.stop() def _request_finished(self, reply): url = to_text_string(reply.url().toEncoded(), encoding='utf-8') if url in self._paths: path = self._paths[url] if url in self._workers: worker = self._workers[url] if url in self._head_requests: self._head_requests.pop(url) start_download = True header_pairs = reply.rawHeaderPairs() headers = {} for hp in header_pairs: headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1]) total_size = int(headers.get('content-length', 0)) # Check if file exists if os.path.isfile(path): file_size = os.path.getsize(path) # Check if existing file matches size of requested file start_download = file_size != total_size if start_download: # File sizes dont match, hence download file qurl = QUrl(url) request = QNetworkRequest(qurl) self._get_requests[url] = request reply = self._manager.get(request) error = reply.error() if error: logger.error(str(('Reply Error:', error))) reply.downloadProgress.connect( lambda r, t, w=worker: self._progress(r, t, w)) else: # File sizes match, dont download file worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) elif url in self._get_requests: data = reply.readAll() self._save(url, path, data) def _save(self, url, path, data): """ """ worker = self._workers[url] path = self._paths[url] if len(data): with open(path, 'wb') as f: f.write(data) # Clean up worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) self._get_requests.pop(url) self._workers.pop(url) self._paths.pop(url) def _progress(self, bytes_received, bytes_total, worker): """ """ worker.sig_download_progress.emit(worker.url, worker.path, bytes_received, bytes_total) def download(self, url, path): """ """ # original_url = url qurl = QUrl(url) url = to_text_string(qurl.toEncoded(), encoding='utf-8') logger.debug(str((url, path))) if url in self._workers: while not self._workers[url].finished: return self._workers[url] worker = DownloadWorker(url, path) # Check download folder exists folder = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(folder): os.makedirs(folder) request = QNetworkRequest(qurl) self._head_requests[url] = request self._paths[url] = path self._workers[url] = worker self._manager.head(request) self._timer.start() return worker def terminate(self): pass
class _DownloadAPI(QObject): """ Download API based on QNetworkAccessManager """ def __init__(self, chunk_size=1024): super(_DownloadAPI, self).__init__() self._chunk_size = chunk_size self._head_requests = {} self._get_requests = {} self._paths = {} self._workers = {} self._manager = QNetworkAccessManager(self) self._timer = QTimer() # Setup self._timer.setInterval(1000) self._timer.timeout.connect(self._clean) # Signals self._manager.finished.connect(self._request_finished) self._manager.sslErrors.connect(self._handle_ssl_errors) def _handle_ssl_errors(self, reply, errors): logger.error(str(('SSL Errors', errors))) def _clean(self): """ Periodically check for inactive workers and remove their references. """ if self._workers: for url in self._workers.copy(): w = self._workers[url] if w.is_finished(): self._workers.pop(url) self._paths.pop(url) if url in self._get_requests: self._get_requests.pop(url) else: self._timer.stop() def _request_finished(self, reply): url = to_text_string(reply.url().toEncoded(), encoding='utf-8') if url in self._paths: path = self._paths[url] if url in self._workers: worker = self._workers[url] if url in self._head_requests: self._head_requests.pop(url) start_download = True header_pairs = reply.rawHeaderPairs() headers = {} for hp in header_pairs: headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1]) total_size = int(headers.get('content-length', 0)) # Check if file exists if os.path.isfile(path): file_size = os.path.getsize(path) # Check if existing file matches size of requested file start_download = file_size != total_size if start_download: # File sizes dont match, hence download file qurl = QUrl(url) request = QNetworkRequest(qurl) self._get_requests[url] = request reply = self._manager.get(request) error = reply.error() if error: logger.error(str(('Reply Error:', error))) reply.downloadProgress.connect( lambda r, t, w=worker: self._progress(r, t, w)) else: # File sizes match, dont download file worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) elif url in self._get_requests: data = reply.readAll() self._save(url, path, data) def _save(self, url, path, data): """ """ worker = self._workers[url] path = self._paths[url] if len(data): with open(path, 'wb') as f: f.write(data) # Clean up worker.finished = True worker.sig_download_finished.emit(url, path) worker.sig_finished.emit(worker, path, None) self._get_requests.pop(url) self._workers.pop(url) self._paths.pop(url) def _progress(self, bytes_received, bytes_total, worker): """ """ worker.sig_download_progress.emit( worker.url, worker.path, bytes_received, bytes_total) def download(self, url, path): """ """ # original_url = url qurl = QUrl(url) url = to_text_string(qurl.toEncoded(), encoding='utf-8') logger.debug(str((url, path))) if url in self._workers: while not self._workers[url].finished: return self._workers[url] worker = DownloadWorker(url, path) # Check download folder exists folder = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(folder): os.makedirs(folder) request = QNetworkRequest(qurl) self._head_requests[url] = request self._paths[url] = path self._workers[url] = worker self._manager.head(request) self._timer.start() return worker def terminate(self): pass
class MapTileHTTPLoader(QObject): tileLoaded = Signal(int, int, int, QByteArray) def __init__(self, cacheSize=DEFAULT_CACHE_SIZE, userAgent='(PyQt) TileMap 1.0', parent=None): QObject.__init__(self, parent=parent) self._manager = None self._cache = None self._cacheSize = cacheSize try: # Convert user agent to bytes userAgent = userAgent.encode() except: # no encode method exists. This hsould be the Python 2 case pass self._userAgent = userAgent self._tileInDownload = dict() @Slot(int, int, int, str, str) def loadTile(self, x, y, zoom, url, cache_dir): if self._manager is None: self._manager = QNetworkAccessManager(parent=self) self._manager.finished.connect(self.handleNetworkData) self._cache = './tiles' key = (x, y, zoom) url = QUrl(url) #base = parsed_url.netloc if key not in self._tileInDownload: path = os.path.join(self._cache, str(cache_dir), str(zoom), str(x), str(y) + '.png') if os.path.exists(path): self.tileLoaded.emit(x, y, zoom, open(path, 'rb').read()) else: # Request the image to the map service request = QNetworkRequest(url=url) request.setRawHeader(b'User-Agent', self._userAgent) request.setAttribute(QNetworkRequest.User, [key, cache_dir]) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) self._tileInDownload[key] = self._manager.get(request) @Slot(QNetworkReply) def handleNetworkData(self, reply): [tp, cache_dir ] = getQVariantValue(reply.request().attribute(QNetworkRequest.User)) if tp in self._tileInDownload: del self._tileInDownload[tp] zoom = tp[2] x = tp[0] y = tp[1] base = cache_dir if not reply.error(): data = reply.readAll() if not os.path.isdir( os.path.join(self._cache, base, str(zoom), str(x))): os.makedirs(os.path.join(self._cache, base, str(zoom), str(x))) fout = open( os.path.join(self._cache, base, str(zoom), str(x), str(y) + '.png'), 'wb') fout.write(data) fout.close() self.tileLoaded.emit(tp[0], tp[1], tp[2], data) reply.close() reply.deleteLater() @Slot() def abortRequest(self, x, y, zoom): p = (x, y, zoom) if p in self._tileInDownload: reply = self._tileInDownload[p] del self._tileInDownload[p] reply.close() reply.deleteLater() @Slot() def abortAllRequests(self): for x, y, zoom in list(self._tileInDownload.keys()): self.abortRequest(x, y, zoom)
class DownloadManager(QObject): """Synchronous download manager. http://qt-project.org/doc/qt-4.8/network-downloadmanager-downloadmanager-cpp.html as inspiration. """ def __init__(self, parent, on_finished_func, on_progress_func, save_path): super(DownloadManager, self).__init__(parent) self._parent = parent self._on_finished_func = on_finished_func self._on_progress_func = on_progress_func self._manager = QNetworkAccessManager(self) self._request = None self._reply = None self._queue = None # [['filename', 'uri'], ...] self._url = None # current url in process self._filename = None # current filename in process self._save_path = None # current defined save path self._error = None # error number self._free = True # lock process flag self.set_save_path(save_path) def _start_next_download(self): """ """ if self._free: if len(self._queue) != 0: self._free = False self._filename, self._url = self._queue.pop(0) full_path = osp.join(self._save_path, self._filename) if osp.isfile(full_path): # compare file versions by getting headers first self._get(header_only=True) else: # file does not exists, first download self._get() # print(full_path) else: self._on_finished_func() def _get(self, header_only=False): """Download file specified by uri""" self._request = QNetworkRequest(QUrl(self._url)) self._reply = None self._error = None if header_only: self._reply = self._manager.head(self._request) self._reply.finished.connect(self._on_downloaded_headers) else: self._reply = self._manager.get(self._request) self._reply.finished.connect(self._on_downloaded) self._reply.downloadProgress.connect(self._on_progress) def _on_downloaded_headers(self): """On header from uri downloaded""" # handle error for headers... error_code = self._reply.error() if error_code > 0: self._on_errors(error_code) return None fullpath = osp.join(self._save_path, self._filename) headers = {} data = self._reply.rawHeaderPairs() for d in data: if isinstance(d[0], QByteArray): d = [d[0].data(), d[1].data()] key = to_text_string(d[0], encoding='ascii') value = to_text_string(d[1], encoding='ascii') headers[key.lower()] = value if len(headers) != 0: header_filesize = int(headers['content-length']) local_filesize = int(osp.getsize(fullpath)) if header_filesize == local_filesize: self._free = True self._start_next_download() else: self._get() def _on_downloaded(self): """On file downloaded""" # check if errors error_code = self._reply.error() if error_code > 0: self._on_errors(error_code) return None # process data if no errors data = self._reply.readAll() self._save_file(data) def _on_errors(self, e): """On download errors""" self._free = True # otherwise update button cannot work! self._error = e self._on_finished_func() def _on_progress(self, downloaded_size, total_size): """On Partial progress""" self._on_progress_func([downloaded_size, total_size]) def _save_file(self, data): """ """ if not osp.isdir(self._save_path): os.mkdir(self._save_path) fullpath = osp.join(self._save_path, self._filename) if isinstance(data, QByteArray): data = data.data() with open(fullpath, 'wb') as f: f.write(data) self._free = True self._start_next_download() # public api # ---------- def set_save_path(self, path): """ """ self._save_path = path def set_queue(self, queue): """[['filename', 'uri'], ['filename', 'uri'], ...]""" self._queue = queue def get_errors(self): """ """ return self._error def start_download(self): """ """ self._start_next_download() def stop_download(self): """ """ pass