class Thumbnailer(QObject):
compatibleType = re.compile(
"(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff|video).*",
re.IGNORECASE)
def __init__(self):
QObject.__init__(self)
self.thumbnailManager = ThumbnailManager()
self.thumbnailManager.register(self)
self.requests = set()
@staticmethod
def isThumbnailable(node):
if Thumbnailer.compatibleType.search(str(node.dataType())) != None:
return True
return False
def generate(self,
node,
iconSize=128,
percent=10,
frames=1,
blocking=False):
if blocking:
self.blockingLoop = QEventLoop()
self.blockingLoop.connect(self, SIGNAL("ThumbnailUpdate"),
self.blockingUpdate)
config = ScaleConfig(node, iconSize, percent, frames)
self.requests.add(config)
pixmap = self.thumbnailManager.generate(config)
if pixmap:
return pixmap
if blocking:
self.blockingLoop.exec_()
return self.pixmap
def blockingUpdate(self, node, pixmap):
self.pixmap = pixmap
self.blockingLoop.quit()
def requestRemove(self, config):
try:
self.requests.remove(config)
except:
pass
def request(self, config):
if config in self.requests:
return True
else:
return False
def unregister(self):
self.thumbnailManager.unregister(self)
class Thumbnailer(QObject):
compatibleType = re.compile(
"(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff|video).*",
re.IGNORECASE,
)
def __init__(self):
QObject.__init__(self)
self.thumbnailManager = ThumbnailManager()
self.thumbnailManager.register(self)
self.requests = set()
@staticmethod
def isThumbnailable(node):
if Thumbnailer.compatibleType.search(str(node.dataType())) != None:
return True
return False
def generate(self, node, iconSize=128, percent=10, frames=1, blocking=False):
if blocking:
self.blockingLoop = QEventLoop()
self.blockingLoop.connect(self, SIGNAL("ThumbnailUpdate"), self.blockingUpdate)
config = ScaleConfig(node, iconSize, percent, frames)
self.requests.add(config)
pixmap = self.thumbnailManager.generate(config)
if pixmap:
return pixmap
if blocking:
self.blockingLoop.exec_()
return self.pixmap
def blockingUpdate(self, node, pixmap):
self.pixmap = pixmap
self.blockingLoop.quit()
def requestRemove(self, config):
try:
self.requests.remove(config)
except:
pass
def request(self, config):
if config in self.requests:
return True
else:
return False
def unregister(self):
self.thumbnailManager.unregister(self)
def waitForSignal(signal, message="", timeout=0):
"""Waits (max timeout msecs if given) for a signal to be emitted.
It the waiting lasts more than 2 seconds, a progress dialog is displayed
with the message.
Returns True if the signal was emitted.
Return False if the wait timed out or the dialog was canceled by the user.
"""
loop = QEventLoop()
dlg = QProgressDialog(minimum=0, maximum=0, labelText=message)
dlg.setWindowTitle(appinfo.appname)
dlg.setWindowModality(Qt.ApplicationModal)
QTimer.singleShot(2000, dlg.show)
dlg.canceled.connect(loop.quit)
if timeout:
QTimer.singleShot(timeout, dlg.cancel)
stop = lambda: loop.quit()
signal.connect(stop)
loop.exec_()
signal.disconnect(stop)
dlg.hide()
dlg.deleteLater()
return not dlg.wasCanceled()
class _RunnableAsyncCall(_RunnableTask):
def run(self):
self.eventLoop = QEventLoop()
self.eventLoop.processEvents()
QObject.connect(self._call, SIGNAL("finished(QString)"),
lambda str: self.eventLoop.quit())
QMetaObject.invokeMethod(
self._call, "moveToAndInit", Qt.QueuedConnection,
Q_ARG("PyQt_PyObject", QThread.currentThread()))
self.eventLoop.processEvents()
self.eventLoop.exec_()
class _RunnableAsyncCall(_RunnableTask):
def run(self):
self.eventLoop = QEventLoop()
self.eventLoop.processEvents()
QObject.connect(self._call, SIGNAL("finished(QString)"),
lambda str: self.eventLoop.quit())
QMetaObject.invokeMethod(self._call, "moveToAndInit",
Qt.QueuedConnection,
Q_ARG("PyQt_PyObject",
QThread.currentThread()))
self.eventLoop.processEvents()
self.eventLoop.exec_()
class Waiter():
def __init__(self):
self.timer = QTimer()
self.timer.setSingleShot(True)
self.loop = QEventLoop()
self.timer.timeout.connect(self.ontimeout)
def onsignal(self):
self.timer.stop()
self.loop.quit()
self.waited = True
def ontimeout(self):
self.loop.quit()
self.waited = False
def wait(self, signal, ms):
signal.connect(self.onsignal)
self.timer.start(ms)
self.loop.exec_()
return self.waited
class _RunnableTask(QRunnable):
""" Wrapper for an AsyncCall
"""
def __init__(self, call):
QRunnable.__init__(self)
self.setAutoDelete(False)
self._call = call
def run(self):
if isinstance(self._call, AsyncCall):
self.eventLoop = QEventLoop()
self.eventLoop.processEvents()
QObject.connect(self._call, SIGNAL("finished(QString)"),
lambda str: self.eventLoop.quit())
QMetaObject.invokeMethod(
self._call, "moveToAndInit", Qt.QueuedConnection,
Q_ARG("PyQt_PyObject", QThread.currentThread()))
self.eventLoop.processEvents()
self.eventLoop.exec_()
else:
self._return = self._call()
class _RunnableTask(QRunnable):
""" Wrapper for an AsyncCall
"""
def __init__(self, call):
QRunnable.__init__(self)
self.setAutoDelete(False)
self._call = call
def run(self):
if isinstance(self._call, AsyncCall):
self.eventLoop = QEventLoop()
self.eventLoop.processEvents()
QObject.connect(self._call, SIGNAL("finished(QString)"),
lambda str: self.eventLoop.quit())
QMetaObject.invokeMethod(self._call, "moveToAndInit",
Qt.QueuedConnection,
Q_ARG("PyQt_PyObject",
QThread.currentThread()))
self.eventLoop.processEvents()
self.eventLoop.exec_()
else:
self._return = self._call()
class Downloader(QObject):
NOT_FOUND = 0
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def clearCounts(self):
self.fetchSuccesses = 0
self.fetchErrors = 0
self.cacheHits = 0
def fetchTimedOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
for reply in self.replies:
reply.abort()
self.errorStatus = Downloader.UNKNOWN_ERROR
def replyFinished(self):
reply = self.sender()
url = reply.request().url().toString()
self.log("replyFinished: %s" % url)
if not url in self.fetchedFiles:
self.fetchedFiles[url] = None
self.requestingUrls.remove(url)
self.replies.remove(reply)
isFromCache = 0
httpStatusCode = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
if httpStatusCode == 301:
new_url = str(reply.rawHeader("Location"))
self.addToQueue(new_url, url)
else:
self.fetchSuccesses += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.cacheHits += 1
isFromCache = 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
# self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(
self.default_cache_expiration * 60 * 60))
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)"
% (url, self.default_cache_expiration))
if reply.isReadable():
data = reply.readAll()
if self.redirected_urls.has_key(url):
url = self.redirected_urls[url]
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
self.emit(SIGNAL('replyFinished(QString, int, int)'), url,
reply.error(), isFromCache)
else:
if self.sync and httpStatusCode == 404:
self.fetchedFiles[url] = self.NOT_FOUND
self.fetchErrors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
reply.deleteLater()
if debug_mode:
qDebug("queue: %d, requesting: %d" %
(len(self.queue), len(self.requestingUrls)))
if len(self.queue) + len(self.requestingUrls) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
self.log("replyFinished End: %s" % url)
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinished)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def fetchFiles(self, urlList, timeout_ms=0):
self.log("fetchFiles()")
self.sync = True
self.queue = []
self.redirected_urls = {}
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
self.log("fetchFiles() End: %d" % self.errorStatus)
if timeout_ms > 0:
self.timer.stop()
return self.fetchedFiles
def addToQueue(self, url, redirected_from=None):
if url in self.queue:
return False
self.queue.append(url)
if redirected_from is not None:
self.redirected_urls[url] = redirected_from
return True
def queueCount(self):
return len(self.queue)
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingUrls)
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def fetchFilesAsync(self, urlList, timeout_ms=0):
self.log("fetchFilesAsync()")
self.sync = False
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
class Downloader(QObject):
MAX_CONNECTION = 2
DEFAULT_CACHE_EXPIRATION = 24 # hours
NOT_FOUND = 0
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
self.userAgent = "Mozilla/5.0"
self.errorStatus = Downloader.NO_ERROR
def clearCounts(self):
self.fetchSuccesses = 0
self.fetchErrors = 0
self.cacheHits = 0
def fetchTimedOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
for reply in self.replies:
reply.abort()
self.errorStatus = Downloader.UNKNOWN_ERROR
def replyFinished(self):
reply = self.sender()
url = reply.request().url().toString()
self.log("replyFinished: %s" % url)
if not url in self.fetchedFiles:
self.fetchedFiles[url] = None
self.requestingUrls.remove(url)
self.replies.remove(reply)
isFromCache = 0
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
self.fetchSuccesses += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.cacheHits += 1
isFromCache = 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
# self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(self.DEFAULT_CACHE_EXPIRATION * 60 * 60)
)
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)" % (url, self.DEFAULT_CACHE_EXPIRATION)
)
if reply.isReadable():
data = reply.readAll()
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
else:
if self.sync and httpStatusCode == 404:
self.fetchedFiles[url] = self.NOT_FOUND
self.fetchErrors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
self.emit(SIGNAL("replyFinished(QString, int, int)"), url, reply.error(), isFromCache)
reply.deleteLater()
if debug_mode:
qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))
if len(self.queue) + len(self.requestingUrls) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
self.log("replyFinished End: %s" % url)
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinished)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def fetchFiles(self, urlList, timeoutSec=0):
self.log("fetchFiles()")
self.sync = True
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.MAX_CONNECTION):
self.fetchNext()
if timeoutSec > 0:
self.timer.setInterval(timeoutSec * 1000)
self.timer.start()
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
self.log("fetchFiles() End: %d" % self.errorStatus)
if timeoutSec > 0:
self.timer.stop()
return self.fetchedFiles
def addToQueue(self, url):
if url in self.queue:
return False
self.queue.append(url)
return True
def queueCount(self):
return len(self.queue)
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingUrls)
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def fetchFilesAsync(self, urlList, timeoutSec=0):
self.log("fetchFilesAsync()")
self.sync = False
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.MAX_CONNECTION):
self.fetchNext()
if timeoutSec > 0:
self.timer.setInterval(timeoutSec * 1000)
self.timer.start()
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.data = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QNetworkAccessManager()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(
r'output="[a-z]*"', 'output="' + self.__output + '"', query)
query = re.sub(
r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def _end_of_request(self):
self.data = self.network_reply.readAll()
self.loop.quit()
def get_file_from_query(self, query):
"""
Make a query to the overpass and put the result in a temp file
@param query:Query to execute
@type query:str
@return: temporary file path
@rtype: str
"""
query = self.query(query)
tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm")
tf.write(query)
name_file = tf.name
tf.flush()
tf.close()
return name_file
def get_timestamp(self):
"""
Get the timestamp of the OSM data on the server
@return: Timestamp
@rtype: str
"""
url_query = self.__url + 'timestamp'
try:
return urllib2.urlopen(url=url_query).read()
except urllib2.HTTPError as e:
if e.code == 400:
raise OverpassBadRequestException
def is_valid(self):
"""
Try if the url is valid, NOT TESTED YET
"""
url_query = self.__url + 'interpreter'
try:
urllib2.urlopen(url=url_query)
return True
except urllib2.HTTPError:
return False
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.data = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QNetworkAccessManager()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(r'output="[a-z]*"',
'output="' + self.__output + '"', query)
query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def _end_of_request(self):
self.data = self.network_reply.readAll()
self.loop.quit()
def get_file_from_query(self, query):
"""
Make a query to the overpass and put the result in a temp file
@param query:Query to execute
@type query:str
@return: temporary file path
@rtype: str
"""
query = self.query(query)
tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm")
tf.write(query)
name_file = tf.name
tf.flush()
tf.close()
return name_file
def get_timestamp(self):
"""
Get the timestamp of the OSM data on the server
@return: Timestamp
@rtype: str
"""
url_query = self.__url + 'timestamp'
try:
return urllib2.urlopen(url=url_query).read()
except urllib2.HTTPError as e:
if e.code == 400:
raise OverpassBadRequestException
def is_valid(self):
"""
Try if the url is valid, NOT TESTED YET
"""
url_query = self.__url + 'interpreter'
try:
urllib2.urlopen(url=url_query)
return True
except urllib2.HTTPError:
return False
class Downloader(QObject):
# error status
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
# PyQt signals
replyFinished = pyqtSignal(str)
allRepliesFinished = pyqtSignal()
def __init__(self,
parent=None,
maxConnections=2,
defaultCacheExpiration=24,
userAgent=""):
QObject.__init__(self, parent)
self.maxConnections = maxConnections
self.defaultCacheExpiration = defaultCacheExpiration # hours
self.userAgent = userAgent
# initialize variables
self.clear()
self.sync = False
self.eventLoop = QEventLoop()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.timeOut)
def clear(self):
self.queue = []
self.requestingReplies = {}
self.fetchedFiles = {}
self._successes = 0
self._errors = 0
self._cacheHits = 0
self.errorStatus = Downloader.NO_ERROR
def _replyFinished(self):
reply = self.sender()
url = reply.request().url().toString()
if url not in self.fetchedFiles:
self.fetchedFiles[url] = None
if url in self.requestingReplies:
del self.requestingReplies[url]
httpStatusCode = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
self._successes += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self._cacheHits += 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(
self.defaultCacheExpiration * 3600))
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)" %
(url, self.defaultCacheExpiration))
if reply.isReadable():
data = reply.readAll()
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
else:
self._errors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
self.replyFinished.emit(url)
reply.deleteLater()
if len(self.queue) + len(self.requestingReplies) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
self.allRepliesFinished.emit()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
def timeOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
@pyqtSlot()
def abort(self, stopTimer=True):
# clear queue and abort requests
self.queue = []
for reply in self.requestingReplies.itervalues():
url = reply.url().toString()
reply.abort()
reply.deleteLater()
self.log("request aborted: {0}".format(url))
self.errorStatus = Downloader.UNKNOWN_ERROR
self.requestingReplies = {}
if stopTimer:
self.timer.stop()
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
# create request
request = QNetworkRequest(QUrl(url))
if self.userAgent:
request.setRawHeader(
"User-Agent", self.userAgent
) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requestingReplies[url] = reply
return reply
def fetchFiles(self, urlList, timeoutSec=0):
self.log("fetchFiles()")
files = self._fetch(True, urlList, timeoutSec)
self.log("fetchFiles() End: %d" % self.errorStatus)
return files
@pyqtSlot(list, int)
def fetchFilesAsync(self, urlList, timeoutSec=0):
self.log("fetchFilesAsync()")
self._fetch(False, urlList, timeoutSec)
def _fetch(self, sync, urlList, timeoutSec):
self.clear()
self.sync = sync
if not urlList:
return {}
for url in urlList:
if url not in self.queue:
self.queue.append(url)
for i in range(self.maxConnections):
self.fetchNext()
if timeoutSec > 0:
self.timer.setInterval(timeoutSec * 1000)
self.timer.start()
if sync:
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
if timeoutSec > 0:
self.timer.stop()
return self.fetchedFiles
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingReplies)
def stats(self):
finished = self.finishedCount()
unfinished = self.unfinishedCount()
return {
"total": finished + unfinished,
"finished": finished,
"unfinished": unfinished,
"successed": self._successes,
"errors": self._errors,
"cacheHits": self._cacheHits,
"downloaded": self._successes - self._cacheHits
}
def quit(self, node, pixmap):
QEventLoop.quit(self)
return pixmap
class Downloader(QObject):
# error status
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
# PyQt signals
replyFinished = pyqtSignal(str)
allRepliesFinished = pyqtSignal()
def __init__(self, parent=None, maxConnections=2, defaultCacheExpiration=24, userAgent=""):
QObject.__init__(self, parent)
self.maxConnections = maxConnections
self.defaultCacheExpiration = defaultCacheExpiration # hours
self.userAgent = userAgent
# initialize variables
self.clear()
self.sync = False
self.eventLoop = QEventLoop()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.timeOut)
def clear(self):
self.queue = []
self.requests = Requests()
self.errorStatus = Downloader.NO_ERROR
def _replyFinished(self):
reply = self.sender()
url = reply.request().url().toString()
status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if status_code in [301, 302]:
# print "Redirection"
redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
# create request
request = QNetworkRequest(redirect)
if self.userAgent:
request.setRawHeader("User-Agent", self.userAgent)
reply.deleteLater()
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requests.update(url, URL, redirect.toString())
self.requests.update(url, REPLY, reply)
self.replyFinished.emit(url)
return
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
#test = reply.header(QNetworkRequest.ContentTypeHeader)
#t2 = reply.rawHeaderList()
self.requests.update(url,STATE, COMPLETE)
if reply.error() == QNetworkReply.NoError:
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.requests.update(url, EVENT, FROM_CACHE)
else:
self.requests.update(url, EVENT, DOWNLOADED)
if not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(self.defaultCacheExpiration * 3600))
cache.updateMetaData(metadata)
self.log("Default expiration date has been set: %s (%d h)" % (url, self.defaultCacheExpiration))
if reply.isReadable():
self.requests.update(url, PDATA, reply.readAll())
else:
qDebug("http status code: " + str(httpStatusCode))
else:
self.requests.update(url, EVENT, FAILED)
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
self.replyFinished.emit(url)
reply.deleteLater()
self.next_step()
def next_step(self):
if len(self.queue) + self.requests.count_of(STATE, INCOMPLETE) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
self.allRepliesFinished.emit()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
def timeOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
self.requests.abort()
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
# create request
request = QNetworkRequest(QUrl(url))
if self.userAgent:
request.setRawHeader("User-Agent",
self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requests.append_if_new(url)
self.requests.update(url, REPLY, reply)
def fetchFiles(self, urlList, timeoutSec=0):
self.log("processedRequests()")
processed_requests = self._fetch(True, urlList, timeoutSec)
self.log("processedRequests() End: %d" % self.errorStatus)
return processed_requests
def fetchFilesAsync(self, urlList, timeoutSec=0):
self.log("fetchFilesAsync()")
self._fetch(False, urlList, timeoutSec)
def _fetch(self, sync, urlList, timeoutSec):
self.clear()
self.sync = sync
if not urlList:
return {}
for url in urlList:
if url not in self.queue:
self.queue.append(url)
for i in range(self.maxConnections):
self.fetchNext()
if timeoutSec > 0:
self.timer.setInterval(timeoutSec * 1000)
self.timer.start()
if sync:
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
if timeoutSec > 0:
self.timer.stop()
return self.requests
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def unfinishedCount(self):
return self.requests.count_of(STATE, INCOMPLETE)+len(self.queue)
def stats(self):
stats = self.requests.stats()
stats["pending"] = len(self.queue)
stats["total"] = stats["pending"] + self.requests.size()
return stats
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="http://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QNetworkAccessManager()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
class Downloader(QObject):
# error status
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
# PyQt signals
replyFinished = pyqtSignal(str)
allRepliesFinished = pyqtSignal()
def __init__(self, parent=None, maxConnections=2, defaultCacheExpiration=24, userAgent=""):
QObject.__init__(self, parent)
self.maxConnections = maxConnections
self.defaultCacheExpiration = defaultCacheExpiration # hours
self.userAgent = userAgent
# initialize variables
self.clear()
self.sync = False
self.eventLoop = QEventLoop()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.timeOut)
def clear(self):
self.queue = []
self.requestingReplies = {}
self.fetchedFiles = {}
self._successes = 0
self._errors = 0
self._cacheHits = 0
self.errorStatus = Downloader.NO_ERROR
def _replyFinished(self):
reply = self.sender()
url = reply.request().url().toString()
if url not in self.fetchedFiles:
self.fetchedFiles[url] = None
if url in self.requestingReplies:
del self.requestingReplies[url]
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
self._successes += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self._cacheHits += 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
if metadata.expirationDate().isNull():
metadata.setExpirationDate(QDateTime.currentDateTime().addSecs(self.defaultCacheExpiration * 3600))
cache.updateMetaData(metadata)
self.log("Default expiration date has been set: %s (%d h)" % (url, self.defaultCacheExpiration))
if reply.isReadable():
data = reply.readAll()
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
else:
self._errors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
self.replyFinished.emit(url)
reply.deleteLater()
if len(self.queue) + len(self.requestingReplies) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
self.allRepliesFinished.emit()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
def timeOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
@pyqtSlot()
def abort(self, stopTimer=True):
# clear queue and abort requests
self.queue = []
for reply in self.requestingReplies.itervalues():
url = reply.url().toString()
reply.abort()
reply.deleteLater()
self.log("request aborted: {0}".format(url))
self.errorStatus = Downloader.UNKNOWN_ERROR
self.requestingReplies = {}
if stopTimer:
self.timer.stop()
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
# create request
request = QNetworkRequest(QUrl(url))
if self.userAgent:
request.setRawHeader("User-Agent", self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requestingReplies[url] = reply
return reply
def fetchFiles(self, urlList, timeoutSec=0):
self.log("fetchFiles()")
files = self._fetch(True, urlList, timeoutSec)
self.log("fetchFiles() End: %d" % self.errorStatus)
return files
@pyqtSlot(list, int)
def fetchFilesAsync(self, urlList, timeoutSec=0):
self.log("fetchFilesAsync()")
self._fetch(False, urlList, timeoutSec)
def _fetch(self, sync, urlList, timeoutSec):
self.clear()
self.sync = sync
if not urlList:
return {}
for url in urlList:
if url not in self.queue:
self.queue.append(url)
for i in range(self.maxConnections):
self.fetchNext()
if timeoutSec > 0:
self.timer.setInterval(timeoutSec * 1000)
self.timer.start()
if sync:
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
if timeoutSec > 0:
self.timer.stop()
return self.fetchedFiles
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingReplies)
def stats(self):
finished = self.finishedCount()
unfinished = self.unfinishedCount()
return {"total": finished + unfinished,
"finished": finished,
"unfinished": unfinished,
"successed": self._successes,
"errors": self._errors,
"cacheHits": self._cacheHits,
"downloaded": self._successes - self._cacheHits}
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="http://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
def quit(self, node, pixmap):
QEventLoop.quit(self)
return pixmap
