class Thumbnailer(QObject): compatibleType = re.compile( "(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff|video).*", re.IGNORECASE) def __init__(self): QObject.__init__(self) self.thumbnailManager = ThumbnailManager() self.thumbnailManager.register(self) self.requests = set() @staticmethod def isThumbnailable(node): if Thumbnailer.compatibleType.search(str(node.dataType())) != None: return True return False def generate(self, node, iconSize=128, percent=10, frames=1, blocking=False): if blocking: self.blockingLoop = QEventLoop() self.blockingLoop.connect(self, SIGNAL("ThumbnailUpdate"), self.blockingUpdate) config = ScaleConfig(node, iconSize, percent, frames) self.requests.add(config) pixmap = self.thumbnailManager.generate(config) if pixmap: return pixmap if blocking: self.blockingLoop.exec_() return self.pixmap def blockingUpdate(self, node, pixmap): self.pixmap = pixmap self.blockingLoop.quit() def requestRemove(self, config): try: self.requests.remove(config) except: pass def request(self, config): if config in self.requests: return True else: return False def unregister(self): self.thumbnailManager.unregister(self)
class Thumbnailer(QObject): compatibleType = re.compile( "(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff|video).*", re.IGNORECASE, ) def __init__(self): QObject.__init__(self) self.thumbnailManager = ThumbnailManager() self.thumbnailManager.register(self) self.requests = set() @staticmethod def isThumbnailable(node): if Thumbnailer.compatibleType.search(str(node.dataType())) != None: return True return False def generate(self, node, iconSize=128, percent=10, frames=1, blocking=False): if blocking: self.blockingLoop = QEventLoop() self.blockingLoop.connect(self, SIGNAL("ThumbnailUpdate"), self.blockingUpdate) config = ScaleConfig(node, iconSize, percent, frames) self.requests.add(config) pixmap = self.thumbnailManager.generate(config) if pixmap: return pixmap if blocking: self.blockingLoop.exec_() return self.pixmap def blockingUpdate(self, node, pixmap): self.pixmap = pixmap self.blockingLoop.quit() def requestRemove(self, config): try: self.requests.remove(config) except: pass def request(self, config): if config in self.requests: return True else: return False def unregister(self): self.thumbnailManager.unregister(self)
def waitForSignal(signal, message="", timeout=0): """Waits (max timeout msecs if given) for a signal to be emitted. It the waiting lasts more than 2 seconds, a progress dialog is displayed with the message. Returns True if the signal was emitted. Return False if the wait timed out or the dialog was canceled by the user. """ loop = QEventLoop() dlg = QProgressDialog(minimum=0, maximum=0, labelText=message) dlg.setWindowTitle(appinfo.appname) dlg.setWindowModality(Qt.ApplicationModal) QTimer.singleShot(2000, dlg.show) dlg.canceled.connect(loop.quit) if timeout: QTimer.singleShot(timeout, dlg.cancel) stop = lambda: loop.quit() signal.connect(stop) loop.exec_() signal.disconnect(stop) dlg.hide() dlg.deleteLater() return not dlg.wasCanceled()
class _RunnableAsyncCall(_RunnableTask): def run(self): self.eventLoop = QEventLoop() self.eventLoop.processEvents() QObject.connect(self._call, SIGNAL("finished(QString)"), lambda str: self.eventLoop.quit()) QMetaObject.invokeMethod( self._call, "moveToAndInit", Qt.QueuedConnection, Q_ARG("PyQt_PyObject", QThread.currentThread())) self.eventLoop.processEvents() self.eventLoop.exec_()
class _RunnableAsyncCall(_RunnableTask): def run(self): self.eventLoop = QEventLoop() self.eventLoop.processEvents() QObject.connect(self._call, SIGNAL("finished(QString)"), lambda str: self.eventLoop.quit()) QMetaObject.invokeMethod(self._call, "moveToAndInit", Qt.QueuedConnection, Q_ARG("PyQt_PyObject", QThread.currentThread())) self.eventLoop.processEvents() self.eventLoop.exec_()
class Waiter(): def __init__(self): self.timer = QTimer() self.timer.setSingleShot(True) self.loop = QEventLoop() self.timer.timeout.connect(self.ontimeout) def onsignal(self): self.timer.stop() self.loop.quit() self.waited = True def ontimeout(self): self.loop.quit() self.waited = False def wait(self, signal, ms): signal.connect(self.onsignal) self.timer.start(ms) self.loop.exec_() return self.waited
class _RunnableTask(QRunnable): """ Wrapper for an AsyncCall """ def __init__(self, call): QRunnable.__init__(self) self.setAutoDelete(False) self._call = call def run(self): if isinstance(self._call, AsyncCall): self.eventLoop = QEventLoop() self.eventLoop.processEvents() QObject.connect(self._call, SIGNAL("finished(QString)"), lambda str: self.eventLoop.quit()) QMetaObject.invokeMethod( self._call, "moveToAndInit", Qt.QueuedConnection, Q_ARG("PyQt_PyObject", QThread.currentThread())) self.eventLoop.processEvents() self.eventLoop.exec_() else: self._return = self._call()
class _RunnableTask(QRunnable): """ Wrapper for an AsyncCall """ def __init__(self, call): QRunnable.__init__(self) self.setAutoDelete(False) self._call = call def run(self): if isinstance(self._call, AsyncCall): self.eventLoop = QEventLoop() self.eventLoop.processEvents() QObject.connect(self._call, SIGNAL("finished(QString)"), lambda str: self.eventLoop.quit()) QMetaObject.invokeMethod(self._call, "moveToAndInit", Qt.QueuedConnection, Q_ARG("PyQt_PyObject", QThread.currentThread())) self.eventLoop.processEvents() self.eventLoop.exec_() else: self._return = self._call()
class Downloader(QObject): NOT_FOUND = 0 NO_ERROR = 0 TIMEOUT_ERROR = 4 UNKNOWN_ERROR = -1 def __init__(self, parent=None): QObject.__init__(self, parent) self.queue = [] self.redirected_urls = {} self.requestingUrls = [] self.replies = [] self.eventLoop = QEventLoop() self.sync = False self.fetchedFiles = {} self.clearCounts() self.timer = QTimer() self.timer.setSingleShot(True) self.timer.timeout.connect(self.fetchTimedOut) # network settings self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)" self.max_connection = 4 self.default_cache_expiration = 24 self.errorStatus = Downloader.NO_ERROR def clearCounts(self): self.fetchSuccesses = 0 self.fetchErrors = 0 self.cacheHits = 0 def fetchTimedOut(self): self.log("Downloader.timeOut()") self.abort() self.errorStatus = Downloader.TIMEOUT_ERROR def abort(self): # clear queue and abort sent requests self.queue = [] self.timer.stop() for reply in self.replies: reply.abort() self.errorStatus = Downloader.UNKNOWN_ERROR def replyFinished(self): reply = self.sender() url = reply.request().url().toString() self.log("replyFinished: %s" % url) if not url in self.fetchedFiles: self.fetchedFiles[url] = None self.requestingUrls.remove(url) self.replies.remove(reply) isFromCache = 0 httpStatusCode = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute) if reply.error() == QNetworkReply.NoError: if httpStatusCode == 301: new_url = str(reply.rawHeader("Location")) self.addToQueue(new_url, url) else: self.fetchSuccesses += 1 if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute): self.cacheHits += 1 isFromCache = 1 elif not reply.hasRawHeader("Cache-Control"): cache = QgsNetworkAccessManager.instance().cache() if cache: metadata = cache.metaData(reply.request().url()) # self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8")) if metadata.expirationDate().isNull(): metadata.setExpirationDate( QDateTime.currentDateTime().addSecs( self.default_cache_expiration * 60 * 60)) cache.updateMetaData(metadata) self.log( "Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration)) if reply.isReadable(): data = reply.readAll() if self.redirected_urls.has_key(url): url = self.redirected_urls[url] self.fetchedFiles[url] = data else: qDebug("http status code: " + str(httpStatusCode)) self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache) else: if self.sync and httpStatusCode == 404: self.fetchedFiles[url] = self.NOT_FOUND self.fetchErrors += 1 if self.errorStatus == self.NO_ERROR: self.errorStatus = self.UNKNOWN_ERROR reply.deleteLater() if debug_mode: qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls))) if len(self.queue) + len(self.requestingUrls) == 0: # all replies have been received if self.sync: self.logT("eventLoop.quit()") self.eventLoop.quit() else: self.timer.stop() elif len(self.queue) > 0: # start fetching the next file self.fetchNext() self.log("replyFinished End: %s" % url) def fetchNext(self): if len(self.queue) == 0: return url = self.queue.pop(0) self.log("fetchNext: %s" % url) request = QNetworkRequest(QUrl(url)) request.setRawHeader("User-Agent", self.userAgent) reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self.replyFinished) self.requestingUrls.append(url) self.replies.append(reply) return reply def fetchFiles(self, urlList, timeout_ms=0): self.log("fetchFiles()") self.sync = True self.queue = [] self.redirected_urls = {} self.clearCounts() self.errorStatus = Downloader.NO_ERROR self.fetchedFiles = {} if len(urlList) == 0: return self.fetchedFiles for url in urlList: self.addToQueue(url) for i in range(self.max_connection): self.fetchNext() if timeout_ms > 0: self.timer.setInterval(timeout_ms) self.timer.start() self.logT("eventLoop.exec_(): " + str(self.eventLoop)) self.eventLoop.exec_() self.log("fetchFiles() End: %d" % self.errorStatus) if timeout_ms > 0: self.timer.stop() return self.fetchedFiles def addToQueue(self, url, redirected_from=None): if url in self.queue: return False self.queue.append(url) if redirected_from is not None: self.redirected_urls[url] = redirected_from return True def queueCount(self): return len(self.queue) def finishedCount(self): return len(self.fetchedFiles) def unfinishedCount(self): return len(self.queue) + len(self.requestingUrls) def log(self, msg): if debug_mode: qDebug(msg) def logT(self, msg): if debug_mode: qDebug("%s: %s" % (str(threading.current_thread()), msg)) def fetchFilesAsync(self, urlList, timeout_ms=0): self.log("fetchFilesAsync()") self.sync = False self.queue = [] self.clearCounts() self.errorStatus = Downloader.NO_ERROR self.fetchedFiles = {} if len(urlList) == 0: return self.fetchedFiles for url in urlList: self.addToQueue(url) for i in range(self.max_connection): self.fetchNext() if timeout_ms > 0: self.timer.setInterval(timeout_ms) self.timer.start()
class Downloader(QObject): MAX_CONNECTION = 2 DEFAULT_CACHE_EXPIRATION = 24 # hours NOT_FOUND = 0 NO_ERROR = 0 TIMEOUT_ERROR = 4 UNKNOWN_ERROR = -1 def __init__(self, parent=None): QObject.__init__(self, parent) self.queue = [] self.requestingUrls = [] self.replies = [] self.eventLoop = QEventLoop() self.sync = False self.fetchedFiles = {} self.clearCounts() self.timer = QTimer() self.timer.setSingleShot(True) self.timer.timeout.connect(self.fetchTimedOut) self.userAgent = "Mozilla/5.0" self.errorStatus = Downloader.NO_ERROR def clearCounts(self): self.fetchSuccesses = 0 self.fetchErrors = 0 self.cacheHits = 0 def fetchTimedOut(self): self.log("Downloader.timeOut()") self.abort() self.errorStatus = Downloader.TIMEOUT_ERROR def abort(self): # clear queue and abort sent requests self.queue = [] self.timer.stop() for reply in self.replies: reply.abort() self.errorStatus = Downloader.UNKNOWN_ERROR def replyFinished(self): reply = self.sender() url = reply.request().url().toString() self.log("replyFinished: %s" % url) if not url in self.fetchedFiles: self.fetchedFiles[url] = None self.requestingUrls.remove(url) self.replies.remove(reply) isFromCache = 0 httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if reply.error() == QNetworkReply.NoError: self.fetchSuccesses += 1 if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute): self.cacheHits += 1 isFromCache = 1 elif not reply.hasRawHeader("Cache-Control"): cache = QgsNetworkAccessManager.instance().cache() if cache: metadata = cache.metaData(reply.request().url()) # self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8")) if metadata.expirationDate().isNull(): metadata.setExpirationDate( QDateTime.currentDateTime().addSecs(self.DEFAULT_CACHE_EXPIRATION * 60 * 60) ) cache.updateMetaData(metadata) self.log( "Default expiration date has been set: %s (%d h)" % (url, self.DEFAULT_CACHE_EXPIRATION) ) if reply.isReadable(): data = reply.readAll() self.fetchedFiles[url] = data else: qDebug("http status code: " + str(httpStatusCode)) else: if self.sync and httpStatusCode == 404: self.fetchedFiles[url] = self.NOT_FOUND self.fetchErrors += 1 if self.errorStatus == self.NO_ERROR: self.errorStatus = self.UNKNOWN_ERROR self.emit(SIGNAL("replyFinished(QString, int, int)"), url, reply.error(), isFromCache) reply.deleteLater() if debug_mode: qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls))) if len(self.queue) + len(self.requestingUrls) == 0: # all replies have been received if self.sync: self.logT("eventLoop.quit()") self.eventLoop.quit() else: self.timer.stop() elif len(self.queue) > 0: # start fetching the next file self.fetchNext() self.log("replyFinished End: %s" % url) def fetchNext(self): if len(self.queue) == 0: return url = self.queue.pop(0) self.log("fetchNext: %s" % url) request = QNetworkRequest(QUrl(url)) request.setRawHeader("User-Agent", self.userAgent) reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self.replyFinished) self.requestingUrls.append(url) self.replies.append(reply) return reply def fetchFiles(self, urlList, timeoutSec=0): self.log("fetchFiles()") self.sync = True self.queue = [] self.clearCounts() self.errorStatus = Downloader.NO_ERROR self.fetchedFiles = {} if len(urlList) == 0: return self.fetchedFiles for url in urlList: self.addToQueue(url) for i in range(self.MAX_CONNECTION): self.fetchNext() if timeoutSec > 0: self.timer.setInterval(timeoutSec * 1000) self.timer.start() self.logT("eventLoop.exec_(): " + str(self.eventLoop)) self.eventLoop.exec_() self.log("fetchFiles() End: %d" % self.errorStatus) if timeoutSec > 0: self.timer.stop() return self.fetchedFiles def addToQueue(self, url): if url in self.queue: return False self.queue.append(url) return True def queueCount(self): return len(self.queue) def finishedCount(self): return len(self.fetchedFiles) def unfinishedCount(self): return len(self.queue) + len(self.requestingUrls) def log(self, msg): if debug_mode: qDebug(msg) def logT(self, msg): if debug_mode: qDebug("%s: %s" % (str(threading.current_thread()), msg)) def fetchFilesAsync(self, urlList, timeoutSec=0): self.log("fetchFilesAsync()") self.sync = False self.queue = [] self.clearCounts() self.errorStatus = Downloader.NO_ERROR self.fetchedFiles = {} if len(urlList) == 0: return self.fetchedFiles for url in urlList: self.addToQueue(url) for i in range(self.MAX_CONNECTION): self.fetchNext() if timeoutSec > 0: self.timer.setInterval(timeoutSec * 1000) self.timer.start()
class ConnexionOAPI(object): """ Manage connexion to the overpass API """ def __init__(self, url="http://overpass-api.de/api/", output=None): """ Constructor @param url:URL of OverPass @type url:str @param output:Output desired (XML or JSON) @type output:str """ if not url: url = "http://overpass-api.de/api/" self.__url = url self.data = None if output not in (None, "json", "xml"): raise OutPutFormatException self.__output = output self.network = QNetworkAccessManager() self.network_reply = None self.loop = None def query(self, query): """ Make a query to the overpass @param query:Query to execute @type query:str @raise OverpassBadRequestException,NetWorkErrorException, OverpassTimeoutException @return: the result of the query @rtype: str """ url_query = QUrl(self.__url + 'interpreter') # The output format can be forced (JSON or XML) if self.__output: query = re.sub( r'output="[a-z]*"', 'output="' + self.__output + '"', query) query = re.sub( r'\[out:[a-z]*', '[out:' + self.__output, query) # noinspection PyCallByClass encoded_query = QUrl.toPercentEncoding(query) url_query.addEncodedQueryItem('data', encoded_query) url_query.addQueryItem('info', 'QgisQuickOSMPlugin') url_query.setPort(80) proxy = get_proxy() if proxy: self.network.setProxy(proxy) request = QNetworkRequest(url_query) request.setRawHeader("User-Agent", "QuickOSM") self.network_reply = self.network.get(request) self.loop = QEventLoop() self.network.finished.connect(self._end_of_request) self.loop.exec_() if self.network_reply.error() == QNetworkReply.NoError: timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \ 'at line [\d]+ after ([\d]+) seconds. </remark>' if re.search(timeout, self.data): raise OverpassTimeoutException else: return self.data elif self.network_reply.error() == QNetworkReply.UnknownContentError: raise OverpassBadRequestException else: raise NetWorkErrorException(suffix="Overpass API") def _end_of_request(self): self.data = self.network_reply.readAll() self.loop.quit() def get_file_from_query(self, query): """ Make a query to the overpass and put the result in a temp file @param query:Query to execute @type query:str @return: temporary file path @rtype: str """ query = self.query(query) tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm") tf.write(query) name_file = tf.name tf.flush() tf.close() return name_file def get_timestamp(self): """ Get the timestamp of the OSM data on the server @return: Timestamp @rtype: str """ url_query = self.__url + 'timestamp' try: return urllib2.urlopen(url=url_query).read() except urllib2.HTTPError as e: if e.code == 400: raise OverpassBadRequestException def is_valid(self): """ Try if the url is valid, NOT TESTED YET """ url_query = self.__url + 'interpreter' try: urllib2.urlopen(url=url_query) return True except urllib2.HTTPError: return False
class ConnexionOAPI(object): """ Manage connexion to the overpass API """ def __init__(self, url="http://overpass-api.de/api/", output=None): """ Constructor @param url:URL of OverPass @type url:str @param output:Output desired (XML or JSON) @type output:str """ if not url: url = "http://overpass-api.de/api/" self.__url = url self.data = None if output not in (None, "json", "xml"): raise OutPutFormatException self.__output = output self.network = QNetworkAccessManager() self.network_reply = None self.loop = None def query(self, query): """ Make a query to the overpass @param query:Query to execute @type query:str @raise OverpassBadRequestException,NetWorkErrorException, OverpassTimeoutException @return: the result of the query @rtype: str """ url_query = QUrl(self.__url + 'interpreter') # The output format can be forced (JSON or XML) if self.__output: query = re.sub(r'output="[a-z]*"', 'output="' + self.__output + '"', query) query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query) # noinspection PyCallByClass encoded_query = QUrl.toPercentEncoding(query) url_query.addEncodedQueryItem('data', encoded_query) url_query.addQueryItem('info', 'QgisQuickOSMPlugin') url_query.setPort(80) proxy = get_proxy() if proxy: self.network.setProxy(proxy) request = QNetworkRequest(url_query) request.setRawHeader("User-Agent", "QuickOSM") self.network_reply = self.network.get(request) self.loop = QEventLoop() self.network.finished.connect(self._end_of_request) self.loop.exec_() if self.network_reply.error() == QNetworkReply.NoError: timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \ 'at line [\d]+ after ([\d]+) seconds. </remark>' if re.search(timeout, self.data): raise OverpassTimeoutException else: return self.data elif self.network_reply.error() == QNetworkReply.UnknownContentError: raise OverpassBadRequestException else: raise NetWorkErrorException(suffix="Overpass API") def _end_of_request(self): self.data = self.network_reply.readAll() self.loop.quit() def get_file_from_query(self, query): """ Make a query to the overpass and put the result in a temp file @param query:Query to execute @type query:str @return: temporary file path @rtype: str """ query = self.query(query) tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm") tf.write(query) name_file = tf.name tf.flush() tf.close() return name_file def get_timestamp(self): """ Get the timestamp of the OSM data on the server @return: Timestamp @rtype: str """ url_query = self.__url + 'timestamp' try: return urllib2.urlopen(url=url_query).read() except urllib2.HTTPError as e: if e.code == 400: raise OverpassBadRequestException def is_valid(self): """ Try if the url is valid, NOT TESTED YET """ url_query = self.__url + 'interpreter' try: urllib2.urlopen(url=url_query) return True except urllib2.HTTPError: return False
class Downloader(QObject): # error status NO_ERROR = 0 TIMEOUT_ERROR = 4 UNKNOWN_ERROR = -1 # PyQt signals replyFinished = pyqtSignal(str) allRepliesFinished = pyqtSignal() def __init__(self, parent=None, maxConnections=2, defaultCacheExpiration=24, userAgent=""): QObject.__init__(self, parent) self.maxConnections = maxConnections self.defaultCacheExpiration = defaultCacheExpiration # hours self.userAgent = userAgent # initialize variables self.clear() self.sync = False self.eventLoop = QEventLoop() self.timer = QTimer() self.timer.setSingleShot(True) self.timer.timeout.connect(self.timeOut) def clear(self): self.queue = [] self.requestingReplies = {} self.fetchedFiles = {} self._successes = 0 self._errors = 0 self._cacheHits = 0 self.errorStatus = Downloader.NO_ERROR def _replyFinished(self): reply = self.sender() url = reply.request().url().toString() if url not in self.fetchedFiles: self.fetchedFiles[url] = None if url in self.requestingReplies: del self.requestingReplies[url] httpStatusCode = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute) if reply.error() == QNetworkReply.NoError: self._successes += 1 if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute): self._cacheHits += 1 elif not reply.hasRawHeader("Cache-Control"): cache = QgsNetworkAccessManager.instance().cache() if cache: metadata = cache.metaData(reply.request().url()) if metadata.expirationDate().isNull(): metadata.setExpirationDate( QDateTime.currentDateTime().addSecs( self.defaultCacheExpiration * 3600)) cache.updateMetaData(metadata) self.log( "Default expiration date has been set: %s (%d h)" % (url, self.defaultCacheExpiration)) if reply.isReadable(): data = reply.readAll() self.fetchedFiles[url] = data else: qDebug("http status code: " + str(httpStatusCode)) else: self._errors += 1 if self.errorStatus == self.NO_ERROR: self.errorStatus = self.UNKNOWN_ERROR self.replyFinished.emit(url) reply.deleteLater() if len(self.queue) + len(self.requestingReplies) == 0: # all replies have been received if self.sync: self.logT("eventLoop.quit()") self.eventLoop.quit() else: self.timer.stop() self.allRepliesFinished.emit() elif len(self.queue) > 0: # start fetching the next file self.fetchNext() def timeOut(self): self.log("Downloader.timeOut()") self.abort() self.errorStatus = Downloader.TIMEOUT_ERROR @pyqtSlot() def abort(self, stopTimer=True): # clear queue and abort requests self.queue = [] for reply in self.requestingReplies.itervalues(): url = reply.url().toString() reply.abort() reply.deleteLater() self.log("request aborted: {0}".format(url)) self.errorStatus = Downloader.UNKNOWN_ERROR self.requestingReplies = {} if stopTimer: self.timer.stop() def fetchNext(self): if len(self.queue) == 0: return url = self.queue.pop(0) self.log("fetchNext: %s" % url) # create request request = QNetworkRequest(QUrl(url)) if self.userAgent: request.setRawHeader( "User-Agent", self.userAgent ) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2 # send request reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self._replyFinished) self.requestingReplies[url] = reply return reply def fetchFiles(self, urlList, timeoutSec=0): self.log("fetchFiles()") files = self._fetch(True, urlList, timeoutSec) self.log("fetchFiles() End: %d" % self.errorStatus) return files @pyqtSlot(list, int) def fetchFilesAsync(self, urlList, timeoutSec=0): self.log("fetchFilesAsync()") self._fetch(False, urlList, timeoutSec) def _fetch(self, sync, urlList, timeoutSec): self.clear() self.sync = sync if not urlList: return {} for url in urlList: if url not in self.queue: self.queue.append(url) for i in range(self.maxConnections): self.fetchNext() if timeoutSec > 0: self.timer.setInterval(timeoutSec * 1000) self.timer.start() if sync: self.logT("eventLoop.exec_(): " + str(self.eventLoop)) self.eventLoop.exec_() if timeoutSec > 0: self.timer.stop() return self.fetchedFiles def log(self, msg): if debug_mode: qDebug(msg) def logT(self, msg): if debug_mode: qDebug("%s: %s" % (str(threading.current_thread()), msg)) def finishedCount(self): return len(self.fetchedFiles) def unfinishedCount(self): return len(self.queue) + len(self.requestingReplies) def stats(self): finished = self.finishedCount() unfinished = self.unfinishedCount() return { "total": finished + unfinished, "finished": finished, "unfinished": unfinished, "successed": self._successes, "errors": self._errors, "cacheHits": self._cacheHits, "downloaded": self._successes - self._cacheHits }
def quit(self, node, pixmap): QEventLoop.quit(self) return pixmap
class Downloader(QObject): # error status NO_ERROR = 0 TIMEOUT_ERROR = 4 UNKNOWN_ERROR = -1 # PyQt signals replyFinished = pyqtSignal(str) allRepliesFinished = pyqtSignal() def __init__(self, parent=None, maxConnections=2, defaultCacheExpiration=24, userAgent=""): QObject.__init__(self, parent) self.maxConnections = maxConnections self.defaultCacheExpiration = defaultCacheExpiration # hours self.userAgent = userAgent # initialize variables self.clear() self.sync = False self.eventLoop = QEventLoop() self.timer = QTimer() self.timer.setSingleShot(True) self.timer.timeout.connect(self.timeOut) def clear(self): self.queue = [] self.requests = Requests() self.errorStatus = Downloader.NO_ERROR def _replyFinished(self): reply = self.sender() url = reply.request().url().toString() status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if status_code in [301, 302]: # print "Redirection" redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute) # create request request = QNetworkRequest(redirect) if self.userAgent: request.setRawHeader("User-Agent", self.userAgent) reply.deleteLater() # send request reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self._replyFinished) self.requests.update(url, URL, redirect.toString()) self.requests.update(url, REPLY, reply) self.replyFinished.emit(url) return httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) #test = reply.header(QNetworkRequest.ContentTypeHeader) #t2 = reply.rawHeaderList() self.requests.update(url,STATE, COMPLETE) if reply.error() == QNetworkReply.NoError: if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute): self.requests.update(url, EVENT, FROM_CACHE) else: self.requests.update(url, EVENT, DOWNLOADED) if not reply.hasRawHeader("Cache-Control"): cache = QgsNetworkAccessManager.instance().cache() if cache: metadata = cache.metaData(reply.request().url()) if metadata.expirationDate().isNull(): metadata.setExpirationDate( QDateTime.currentDateTime().addSecs(self.defaultCacheExpiration * 3600)) cache.updateMetaData(metadata) self.log("Default expiration date has been set: %s (%d h)" % (url, self.defaultCacheExpiration)) if reply.isReadable(): self.requests.update(url, PDATA, reply.readAll()) else: qDebug("http status code: " + str(httpStatusCode)) else: self.requests.update(url, EVENT, FAILED) if self.errorStatus == self.NO_ERROR: self.errorStatus = self.UNKNOWN_ERROR self.replyFinished.emit(url) reply.deleteLater() self.next_step() def next_step(self): if len(self.queue) + self.requests.count_of(STATE, INCOMPLETE) == 0: # all replies have been received if self.sync: self.logT("eventLoop.quit()") self.eventLoop.quit() else: self.timer.stop() self.allRepliesFinished.emit() elif len(self.queue) > 0: # start fetching the next file self.fetchNext() def timeOut(self): self.log("Downloader.timeOut()") self.abort() self.errorStatus = Downloader.TIMEOUT_ERROR def abort(self): # clear queue and abort sent requests self.queue = [] self.timer.stop() self.requests.abort() def fetchNext(self): if len(self.queue) == 0: return url = self.queue.pop(0) self.log("fetchNext: %s" % url) # create request request = QNetworkRequest(QUrl(url)) if self.userAgent: request.setRawHeader("User-Agent", self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2 # send request reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self._replyFinished) self.requests.append_if_new(url) self.requests.update(url, REPLY, reply) def fetchFiles(self, urlList, timeoutSec=0): self.log("processedRequests()") processed_requests = self._fetch(True, urlList, timeoutSec) self.log("processedRequests() End: %d" % self.errorStatus) return processed_requests def fetchFilesAsync(self, urlList, timeoutSec=0): self.log("fetchFilesAsync()") self._fetch(False, urlList, timeoutSec) def _fetch(self, sync, urlList, timeoutSec): self.clear() self.sync = sync if not urlList: return {} for url in urlList: if url not in self.queue: self.queue.append(url) for i in range(self.maxConnections): self.fetchNext() if timeoutSec > 0: self.timer.setInterval(timeoutSec * 1000) self.timer.start() if sync: self.logT("eventLoop.exec_(): " + str(self.eventLoop)) self.eventLoop.exec_() if timeoutSec > 0: self.timer.stop() return self.requests def log(self, msg): if debug_mode: qDebug(msg) def logT(self, msg): if debug_mode: qDebug("%s: %s" % (str(threading.current_thread()), msg)) def unfinishedCount(self): return self.requests.count_of(STATE, INCOMPLETE)+len(self.queue) def stats(self): stats = self.requests.stats() stats["pending"] = len(self.queue) stats["total"] = stats["pending"] + self.requests.size() return stats
class Nominatim(object): """Manage connexion to Nominatim.""" def __init__(self, url="http://nominatim.openstreetmap.org/search?format=json"): """ Constructor @param url:URL of Nominatim @type url:str """ self.__url = url self.network = QNetworkAccessManager() self.data = None self.network_reply = None self.loop = None def query(self, query): """ Perform a nominatim query @param query: Query to execute @type query: str @raise NetWorkErrorException @return: the result of the query @rtype: str """ url_query = QUrl(self.__url) query = QUrl.toPercentEncoding(query) url_query.addEncodedQueryItem('q', query) url_query.addQueryItem('info', 'QgisQuickOSMPlugin') url_query.setPort(80) proxy = get_proxy() if proxy: self.network.setProxy(proxy) request = QNetworkRequest(url_query) request.setRawHeader("User-Agent", "QuickOSM") self.network_reply = self.network.get(request) self.loop = QEventLoop() self.network.finished.connect(self._end_of_request) self.loop.exec_() if self.network_reply.error() == QNetworkReply.NoError: return json.loads(self.data) else: raise NetWorkErrorException(suffix="Nominatim API") def _end_of_request(self): self.data = self.network_reply.readAll().data().decode('utf-8') self.loop.quit() def get_first_polygon_from_query(self, query): """ Get first OSM_ID of a Nominatim area @param query: Query to execute @type query: str @raise NominatimAreaException: @return: First relation's osm_id @rtype: str """ data = self.query(query) for result in data: if result['osm_type'] == "relation": return result['osm_id'] # If no result has been return raise NominatimAreaException def get_first_point_from_query(self, query): """ Get first longitude, latitude of a Nominatim point @param query: Query to execute @type query: str @raise NominatimAreaException: @return: First relation's osm_id @rtype: str """ data = self.query(query) for result in data: if result['osm_type'] == "node": return result['lon'], result['lat'] # If no result has been return raise NominatimAreaException
class Downloader(QObject): # error status NO_ERROR = 0 TIMEOUT_ERROR = 4 UNKNOWN_ERROR = -1 # PyQt signals replyFinished = pyqtSignal(str) allRepliesFinished = pyqtSignal() def __init__(self, parent=None, maxConnections=2, defaultCacheExpiration=24, userAgent=""): QObject.__init__(self, parent) self.maxConnections = maxConnections self.defaultCacheExpiration = defaultCacheExpiration # hours self.userAgent = userAgent # initialize variables self.clear() self.sync = False self.eventLoop = QEventLoop() self.timer = QTimer() self.timer.setSingleShot(True) self.timer.timeout.connect(self.timeOut) def clear(self): self.queue = [] self.requestingReplies = {} self.fetchedFiles = {} self._successes = 0 self._errors = 0 self._cacheHits = 0 self.errorStatus = Downloader.NO_ERROR def _replyFinished(self): reply = self.sender() url = reply.request().url().toString() if url not in self.fetchedFiles: self.fetchedFiles[url] = None if url in self.requestingReplies: del self.requestingReplies[url] httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if reply.error() == QNetworkReply.NoError: self._successes += 1 if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute): self._cacheHits += 1 elif not reply.hasRawHeader("Cache-Control"): cache = QgsNetworkAccessManager.instance().cache() if cache: metadata = cache.metaData(reply.request().url()) if metadata.expirationDate().isNull(): metadata.setExpirationDate(QDateTime.currentDateTime().addSecs(self.defaultCacheExpiration * 3600)) cache.updateMetaData(metadata) self.log("Default expiration date has been set: %s (%d h)" % (url, self.defaultCacheExpiration)) if reply.isReadable(): data = reply.readAll() self.fetchedFiles[url] = data else: qDebug("http status code: " + str(httpStatusCode)) else: self._errors += 1 if self.errorStatus == self.NO_ERROR: self.errorStatus = self.UNKNOWN_ERROR self.replyFinished.emit(url) reply.deleteLater() if len(self.queue) + len(self.requestingReplies) == 0: # all replies have been received if self.sync: self.logT("eventLoop.quit()") self.eventLoop.quit() else: self.timer.stop() self.allRepliesFinished.emit() elif len(self.queue) > 0: # start fetching the next file self.fetchNext() def timeOut(self): self.log("Downloader.timeOut()") self.abort() self.errorStatus = Downloader.TIMEOUT_ERROR @pyqtSlot() def abort(self, stopTimer=True): # clear queue and abort requests self.queue = [] for reply in self.requestingReplies.itervalues(): url = reply.url().toString() reply.abort() reply.deleteLater() self.log("request aborted: {0}".format(url)) self.errorStatus = Downloader.UNKNOWN_ERROR self.requestingReplies = {} if stopTimer: self.timer.stop() def fetchNext(self): if len(self.queue) == 0: return url = self.queue.pop(0) self.log("fetchNext: %s" % url) # create request request = QNetworkRequest(QUrl(url)) if self.userAgent: request.setRawHeader("User-Agent", self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2 # send request reply = QgsNetworkAccessManager.instance().get(request) reply.finished.connect(self._replyFinished) self.requestingReplies[url] = reply return reply def fetchFiles(self, urlList, timeoutSec=0): self.log("fetchFiles()") files = self._fetch(True, urlList, timeoutSec) self.log("fetchFiles() End: %d" % self.errorStatus) return files @pyqtSlot(list, int) def fetchFilesAsync(self, urlList, timeoutSec=0): self.log("fetchFilesAsync()") self._fetch(False, urlList, timeoutSec) def _fetch(self, sync, urlList, timeoutSec): self.clear() self.sync = sync if not urlList: return {} for url in urlList: if url not in self.queue: self.queue.append(url) for i in range(self.maxConnections): self.fetchNext() if timeoutSec > 0: self.timer.setInterval(timeoutSec * 1000) self.timer.start() if sync: self.logT("eventLoop.exec_(): " + str(self.eventLoop)) self.eventLoop.exec_() if timeoutSec > 0: self.timer.stop() return self.fetchedFiles def log(self, msg): if debug_mode: qDebug(msg) def logT(self, msg): if debug_mode: qDebug("%s: %s" % (str(threading.current_thread()), msg)) def finishedCount(self): return len(self.fetchedFiles) def unfinishedCount(self): return len(self.queue) + len(self.requestingReplies) def stats(self): finished = self.finishedCount() unfinished = self.unfinishedCount() return {"total": finished + unfinished, "finished": finished, "unfinished": unfinished, "successed": self._successes, "errors": self._errors, "cacheHits": self._cacheHits, "downloaded": self._successes - self._cacheHits}
class Nominatim(object): """Manage connexion to Nominatim.""" def __init__(self, url="http://nominatim.openstreetmap.org/search?format=json"): """ Constructor @param url:URL of Nominatim @type url:str """ self.__url = url self.network = QgsNetworkAccessManager.instance() self.data = None self.network_reply = None self.loop = None def query(self, query): """ Perform a nominatim query @param query: Query to execute @type query: str @raise NetWorkErrorException @return: the result of the query @rtype: str """ url_query = QUrl(self.__url) query = QUrl.toPercentEncoding(query) url_query.addEncodedQueryItem('q', query) url_query.addQueryItem('info', 'QgisQuickOSMPlugin') request = QNetworkRequest(url_query) request.setRawHeader("User-Agent", "QuickOSM") self.network_reply = self.network.get(request) self.loop = QEventLoop() self.network.finished.connect(self._end_of_request) self.loop.exec_() if self.network_reply.error() == QNetworkReply.NoError: return json.loads(self.data) else: raise NetWorkErrorException(suffix="Nominatim API") def _end_of_request(self): self.data = self.network_reply.readAll().data().decode('utf-8') self.loop.quit() def get_first_polygon_from_query(self, query): """ Get first OSM_ID of a Nominatim area @param query: Query to execute @type query: str @raise NominatimAreaException: @return: First relation's osm_id @rtype: str """ data = self.query(query) for result in data: if result['osm_type'] == "relation": return result['osm_id'] # If no result has been return raise NominatimAreaException def get_first_point_from_query(self, query): """ Get first longitude, latitude of a Nominatim point @param query: Query to execute @type query: str @raise NominatimAreaException: @return: First relation's osm_id @rtype: str """ data = self.query(query) for result in data: if result['osm_type'] == "node": return result['lon'], result['lat'] # If no result has been return raise NominatimAreaException