def update_referecedata_cache_model(self, filter_models, type):
# updates the model and waits for the end
loop = QEventLoop()
self.ilireferencedatacache.model_refreshed.connect(lambda: loop.quit())
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.quit())
timer.start(10000)
self.refresh_referencedata_cache(filter_models, type)
loop.exec()
return self.ilireferencedatacache.model
def _run(self):
loop = QEventLoop()
for session_widget in self.session_widget_list:
session_widget.on_done_or_skipped.connect(lambda: loop.quit())
# fall in a loop on fail untill the user skipped it or it has been successful
if not session_widget.run():
loop.exec()
def get_topping_file_model(self, id_list):
topping_file_cache = IliToppingFileCache(
self.import_schema_configuration.base_configuration, id_list)
# we wait for the download or we timeout after 30 seconds and we apply what we have
loop = QEventLoop()
topping_file_cache.download_finished.connect(lambda: loop.quit())
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.quit())
timer.start(30000)
topping_file_cache.refresh()
self.log_panel.print_info(self.tr("- - Downloading…"),
LogColor.COLOR_TOPPING)
if len(topping_file_cache.downloaded_files) != len(id_list):
loop.exec()
if len(topping_file_cache.downloaded_files) == len(id_list):
self.log_panel.print_info(
self.tr("- - All topping files successfully downloaded"),
LogColor.COLOR_TOPPING,
)
else:
missing_file_ids = id_list
for downloaded_file_id in topping_file_cache.downloaded_files:
if downloaded_file_id in missing_file_ids:
missing_file_ids.remove(downloaded_file_id)
self.log_panel.print_info(
self.
tr("- - Some topping files where not successfully downloaded: {}"
).format(" ".join(missing_file_ids)),
LogColor.COLOR_TOPPING,
)
return topping_file_cache.model
class Downloader(QObject):
NOT_FOUND = 0
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
replyFinished = pyqtSignal(str, int, int)
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def clearCounts(self):
self.fetchSuccesses = 0
self.fetchErrors = 0
self.cacheHits = 0
def fetchTimedOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
for reply in self.replies:
reply.abort()
self.errorStatus = Downloader.UNKNOWN_ERROR
def replyFinishedSlot(self):
reply = self.sender()
url = reply.request().url().toString()
self.log("replyFinishedSlot: %s" % url)
if not url in self.fetchedFiles:
self.fetchedFiles[url] = None
self.requestingUrls.remove(url)
self.replies.remove(reply)
isFromCache = 0
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
if httpStatusCode == 301:
new_url = str(reply.rawHeader("Location"))
self.addToQueue(new_url, url)
else:
self.fetchSuccesses += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.cacheHits += 1
isFromCache = 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
# self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))
if reply.isReadable():
data = reply.readAll()
if self.redirected_urls.has_key(url):
url = self.redirected_urls[url]
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
# self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
self.replyFinished.emit(url, reply.error(), isFromCache)
else:
if self.sync and httpStatusCode == 404:
self.fetchedFiles[url] = self.NOT_FOUND
self.fetchErrors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
reply.deleteLater()
if debug_mode:
qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))
if len(self.queue) + len(self.requestingUrls) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
self.log("replyFinishedSlot End: %s" % url)
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinishedSlot)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def fetchFiles(self, urlList, timeout_ms=0):
self.log("fetchFiles()")
self.sync = True
self.queue = []
self.redirected_urls = {}
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
self.log("fetchFiles() End: %d" % self.errorStatus)
if timeout_ms > 0:
self.timer.stop()
return self.fetchedFiles
def addToQueue(self, url, redirected_from=None):
if url in self.queue:
return False
self.queue.append(url)
if redirected_from is not None:
self.redirected_urls[url] = redirected_from
return True
def queueCount(self):
return len(self.queue)
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingUrls)
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def fetchFilesAsync(self, urlList, timeout_ms=0):
self.log("fetchFilesAsync()")
self.sync = False
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.result_path = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QgsNetworkAccessManager.instance()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(r'output="[a-z]*"',
'output="' + self.__output + '"', query)
query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
# encoded_query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('data', query)
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass OSM API")
def _end_of_request(self):
tf = QTemporaryFile(os.path.join(QDir.tempPath(),
'request-XXXXXX.osm'))
tf.setAutoRemove(False)
tf.open(QIODevice.WriteOnly | QIODevice.Text)
tf.write(self.network_reply.readAll().simplified())
tf.close()
self.result_path = tf.fileName()
self.loop.quit()
class BaseTestAsync(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.app = start_app()
@classmethod
def tearDownClass(cls):
pass
# cls.app.quit() # useless
# stop_app() # NameError: name 'QGISAPP' is not defined
# cls.app.exitQgis() # crash when run unittest (all module)
# del cls.app
def setUp(self):
if isinstance(super(), BaseTestAsync) and hasattr(
super(), self._testMethodName):
self.skipTest("duplicated test")
# self.loop = self.app
self.loop = QEventLoop()
self._output = list()
self._idx = 0
self._lst_error = list()
self.startTime = time.time()
self._log_debug("Test start. ###################")
def tearDown(self):
self._stop_async() # useless ?
self._log_debug("Test ended. ################### \n")
self._log_info("%s: %.3fs" %
(self._id(), time.time() - self.startTime))
def _add_output(self, output):
self._output.append(output)
def output(self, idx=None):
if idx is None:
idx = self._idx
self._idx = min(idx + 1, len(self._output))
return self._output[idx] if idx < len(self._output) else None
def _stop_async(self):
self.loop.quit()
self._log_debug("Stop Async. ###################")
def _handle_error(self, e):
pretty_print_error(e)
self._stop_async()
self._lst_error.append(e)
# raise ErrorDuringTest(e)
def _process_async(self):
self.loop.processEvents()
def _wait_async(self):
t0 = time.time()
self.loop.exec_()
self._log_info("%s: wait_async end: %.3fs" %
(self._id(), time.time() - t0))
if self._lst_error:
raise AllErrorsDuringTest(self._lst_error)
def _make_async_fun(self, fun):
return AsyncFun(fun)
def _log_error(self, *a, **kw):
print(*a, file=sys.stderr, **kw)
def _log_info(self, *a, **kw):
log_truncate(*a, **kw)
def _log_debug(self, *a, **kw):
fn_name = "{}:".format(self._id())
# fn_name = sys._getframe(1).f_code.co_name
log_truncate(fn_name, *a, **kw)
def _id(self):
return self._subtest.id() if self._subtest else self.id()
def assertMultiInput(self, expected, lst_input, msg="multi input"):
for i, actual in enumerate(lst_input):
self.assertEqual(expected, actual, "{} [{}]".format(msg, i))
#unused
def assertPairEqual(self, *a):
pairs = [a[i:i + 2] for i in range(0, len(a), 2)]
return self.assertEqual(*zip(*pairs))
class Nominatim:
"""Manage connexion to Nominatim."""
def __init__(self,
url="https://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="https://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
class BaseTestAsync(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.app = start_app()
@classmethod
def tearDownClass(cls):
pass
# cls.app.quit() # useless
# stop_app() # NameError: name 'QGISAPP' is not defined
# cls.app.exitQgis() # crash when run unittest (all module)
# del cls.app
def setUp(self):
if isinstance(super(), BaseTestAsync) and hasattr(
super(), self._testMethodName):
self.skipTest("duplicated test")
# self.loop = self.app
self.loop = QEventLoop()
self._output = list()
self._idx = 0
self.flag_error = False
self.startTime = time.time()
def tearDown(self):
self._stop_async() # useless ?
self._log_debug("Test ended. ################### \n")
self._log_info("%s: %.3fs" % (self.id(), time.time() - self.startTime))
def _add_output(self, output):
self._output.append(output)
def output(self, idx=None):
if idx is None:
idx = self._idx
self._idx += 1
return self._output[idx] if idx < len(self._output) else None
def _stop_async(self):
self.loop.quit()
self._log_debug("Stop Async. ################### \n")
def _handle_error(self, e):
pretty_print_error(e)
self._stop_async()
self.flag_error = True
def _wait_async(self):
t0 = time.time()
self.loop.exec_()
self._log_info("%s: wait_async end: %.3fs" %
(self.id(), time.time() - t0))
self.assertFalse(self.flag_error, "error")
def _make_async_fun(self, fun):
return AsyncFun(fun)
def _log_info(self, *a, **kw):
print(*a, file=sys.stderr, **kw)
def _log_debug(self, *a, **kw):
log_debug(str(self.id()), *a, **kw)
#unused
def assertPairEqual(self, *a):
pairs = [a[i:i + 2] for i in range(0, len(a), 2)]
return self.assertEqual(*zip(*pairs))