Example #1
0
 def update_referecedata_cache_model(self, filter_models, type):
     # updates the model and waits for the end
     loop = QEventLoop()
     self.ilireferencedatacache.model_refreshed.connect(lambda: loop.quit())
     timer = QTimer()
     timer.setSingleShot(True)
     timer.timeout.connect(lambda: loop.quit())
     timer.start(10000)
     self.refresh_referencedata_cache(filter_models, type)
     loop.exec()
     return self.ilireferencedatacache.model
Example #2
0
 def _run(self):
     loop = QEventLoop()
     for session_widget in self.session_widget_list:
         session_widget.on_done_or_skipped.connect(lambda: loop.quit())
         # fall in a loop on fail untill the user skipped it or it has been successful
         if not session_widget.run():
             loop.exec()
Example #3
0
    def get_topping_file_model(self, id_list):
        topping_file_cache = IliToppingFileCache(
            self.import_schema_configuration.base_configuration, id_list)

        # we wait for the download or we timeout after 30 seconds and we apply what we have
        loop = QEventLoop()
        topping_file_cache.download_finished.connect(lambda: loop.quit())
        timer = QTimer()
        timer.setSingleShot(True)
        timer.timeout.connect(lambda: loop.quit())
        timer.start(30000)

        topping_file_cache.refresh()
        self.log_panel.print_info(self.tr("- - Downloading…"),
                                  LogColor.COLOR_TOPPING)

        if len(topping_file_cache.downloaded_files) != len(id_list):
            loop.exec()

        if len(topping_file_cache.downloaded_files) == len(id_list):
            self.log_panel.print_info(
                self.tr("- - All topping files successfully downloaded"),
                LogColor.COLOR_TOPPING,
            )
        else:
            missing_file_ids = id_list
            for downloaded_file_id in topping_file_cache.downloaded_files:
                if downloaded_file_id in missing_file_ids:
                    missing_file_ids.remove(downloaded_file_id)
            self.log_panel.print_info(
                self.
                tr("- - Some topping files where not successfully downloaded: {}"
                   ).format(" ".join(missing_file_ids)),
                LogColor.COLOR_TOPPING,
            )

        return topping_file_cache.model
class Downloader(QObject):

    NOT_FOUND = 0
    NO_ERROR = 0
    TIMEOUT_ERROR = 4
    UNKNOWN_ERROR = -1

    replyFinished = pyqtSignal(str, int, int)

    def __init__(self, parent=None):
        QObject.__init__(self, parent)
        self.queue = []
        self.redirected_urls = {}
        self.requestingUrls = []
        self.replies = []

        self.eventLoop = QEventLoop()
        self.sync = False
        self.fetchedFiles = {}
        self.clearCounts()

        self.timer = QTimer()
        self.timer.setSingleShot(True)
        self.timer.timeout.connect(self.fetchTimedOut)

        # network settings
        self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
        self.max_connection = 4
        self.default_cache_expiration = 24
        self.errorStatus = Downloader.NO_ERROR

    def clearCounts(self):
        self.fetchSuccesses = 0
        self.fetchErrors = 0
        self.cacheHits = 0

    def fetchTimedOut(self):
        self.log("Downloader.timeOut()")
        self.abort()
        self.errorStatus = Downloader.TIMEOUT_ERROR

    def abort(self):
        # clear queue and abort sent requests
        self.queue = []
        self.timer.stop()
        for reply in self.replies:
            reply.abort()
        self.errorStatus = Downloader.UNKNOWN_ERROR

    def replyFinishedSlot(self):
        reply = self.sender()
        url = reply.request().url().toString()
        self.log("replyFinishedSlot: %s" % url)
        if not url in self.fetchedFiles:
            self.fetchedFiles[url] = None
        self.requestingUrls.remove(url)
        self.replies.remove(reply)
        isFromCache = 0
        httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
        if reply.error() == QNetworkReply.NoError:
            if httpStatusCode == 301:
                new_url = str(reply.rawHeader("Location"))
                self.addToQueue(new_url, url)
            else:
                self.fetchSuccesses += 1
                if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
                    self.cacheHits += 1
                    isFromCache = 1
                elif not reply.hasRawHeader("Cache-Control"):
                    cache = QgsNetworkAccessManager.instance().cache()
                    if cache:
                        metadata = cache.metaData(reply.request().url())
                        # self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
                        if metadata.expirationDate().isNull():
                            metadata.setExpirationDate(
                                QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
                            cache.updateMetaData(metadata)
                            self.log(
                                "Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))

                if reply.isReadable():
                    data = reply.readAll()
                    if self.redirected_urls.has_key(url):
                        url = self.redirected_urls[url]

                    self.fetchedFiles[url] = data
                else:
                    qDebug("http status code: " + str(httpStatusCode))

                # self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
                self.replyFinished.emit(url, reply.error(), isFromCache)
        else:
            if self.sync and httpStatusCode == 404:
                self.fetchedFiles[url] = self.NOT_FOUND
            self.fetchErrors += 1
            if self.errorStatus == self.NO_ERROR:
                self.errorStatus = self.UNKNOWN_ERROR

        reply.deleteLater()

        if debug_mode:
            qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))

        if len(self.queue) + len(self.requestingUrls) == 0:
            # all replies have been received
            if self.sync:
                self.logT("eventLoop.quit()")
                self.eventLoop.quit()
            else:
                self.timer.stop()
        elif len(self.queue) > 0:
            # start fetching the next file
            self.fetchNext()
        self.log("replyFinishedSlot End: %s" % url)

    def fetchNext(self):
        if len(self.queue) == 0:
            return
        url = self.queue.pop(0)
        self.log("fetchNext: %s" % url)

        request = QNetworkRequest(QUrl(url))
        request.setRawHeader("User-Agent", self.userAgent)
        reply = QgsNetworkAccessManager.instance().get(request)
        reply.finished.connect(self.replyFinishedSlot)
        self.requestingUrls.append(url)
        self.replies.append(reply)
        return reply

    def fetchFiles(self, urlList, timeout_ms=0):
        self.log("fetchFiles()")
        self.sync = True
        self.queue = []
        self.redirected_urls = {} 
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()

        self.logT("eventLoop.exec_(): " + str(self.eventLoop))
        self.eventLoop.exec_()
        self.log("fetchFiles() End: %d" % self.errorStatus)
        if timeout_ms > 0:
            self.timer.stop()
        return self.fetchedFiles

    def addToQueue(self, url, redirected_from=None):
        if url in self.queue:
            return False
        self.queue.append(url)
        if redirected_from is not None:
            self.redirected_urls[url] = redirected_from
        return True

    def queueCount(self):
        return len(self.queue)

    def finishedCount(self):
        return len(self.fetchedFiles)

    def unfinishedCount(self):
        return len(self.queue) + len(self.requestingUrls)

    def log(self, msg):
        if debug_mode:
            qDebug(msg)

    def logT(self, msg):
        if debug_mode:
            qDebug("%s: %s" % (str(threading.current_thread()), msg))

    def fetchFilesAsync(self, urlList, timeout_ms=0):
        self.log("fetchFilesAsync()")
        self.sync = False
        self.queue = []
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()
class ConnexionOAPI(object):
    """
    Manage connexion to the overpass API
    """
    def __init__(self, url="http://overpass-api.de/api/", output=None):
        """
        Constructor

        @param url:URL of OverPass
        @type url:str

        @param output:Output desired (XML or JSON)
        @type output:str
        """

        if not url:
            url = "http://overpass-api.de/api/"

        self.__url = url
        self.result_path = None

        if output not in (None, "json", "xml"):
            raise OutPutFormatException

        self.__output = output
        self.network = QgsNetworkAccessManager.instance()
        self.network_reply = None
        self.loop = None

    def query(self, query):
        """
        Make a query to the overpass

        @param query:Query to execute
        @type query:str

        @raise OverpassBadRequestException,NetWorkErrorException,
        OverpassTimeoutException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url + 'interpreter')

        # The output format can be forced (JSON or XML)
        if self.__output:
            query = re.sub(r'output="[a-z]*"',
                           'output="' + self.__output + '"', query)
            query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query)

        # noinspection PyCallByClass
        # encoded_query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('data', query)
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            file_obj = codecs.open(self.result_path, 'r', 'utf-8')
            file_obj.seek(0, 2)
            fsize = file_obj.tell()
            file_obj.seek(max(fsize - 1024, 0), 0)
            lines = file_obj.readlines()

            lines = lines[-10:]  # Get last 10 lines
            timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
                      'at line [\d]+ after ([\d]+) seconds. </remark>'
            if re.search(timeout, ''.join(lines)):
                raise OverpassTimeoutException
            else:
                return self.result_path

        elif self.network_reply.error() == QNetworkReply.UnknownContentError:
            raise OverpassBadRequestException
        else:
            raise NetWorkErrorException(suffix="Overpass OSM API")

    def _end_of_request(self):
        tf = QTemporaryFile(os.path.join(QDir.tempPath(),
                                         'request-XXXXXX.osm'))
        tf.setAutoRemove(False)
        tf.open(QIODevice.WriteOnly | QIODevice.Text)
        tf.write(self.network_reply.readAll().simplified())
        tf.close()
        self.result_path = tf.fileName()
        self.loop.quit()
Example #6
0
class BaseTestAsync(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.app = start_app()

    @classmethod
    def tearDownClass(cls):
        pass
        # cls.app.quit() # useless
        # stop_app() # NameError: name 'QGISAPP' is not defined
        # cls.app.exitQgis() # crash when run unittest (all module)
        # del cls.app
    def setUp(self):
        if isinstance(super(), BaseTestAsync) and hasattr(
                super(), self._testMethodName):
            self.skipTest("duplicated test")

        # self.loop = self.app
        self.loop = QEventLoop()
        self._output = list()
        self._idx = 0
        self._lst_error = list()
        self.startTime = time.time()

        self._log_debug("Test start. ###################")

    def tearDown(self):
        self._stop_async()  # useless ?
        self._log_debug("Test ended. ################### \n")
        self._log_info("%s: %.3fs" %
                       (self._id(), time.time() - self.startTime))

    def _add_output(self, output):
        self._output.append(output)

    def output(self, idx=None):
        if idx is None:
            idx = self._idx
            self._idx = min(idx + 1, len(self._output))
        return self._output[idx] if idx < len(self._output) else None

    def _stop_async(self):
        self.loop.quit()
        self._log_debug("Stop Async. ###################")

    def _handle_error(self, e):
        pretty_print_error(e)
        self._stop_async()
        self._lst_error.append(e)
        # raise ErrorDuringTest(e)

    def _process_async(self):
        self.loop.processEvents()

    def _wait_async(self):
        t0 = time.time()
        self.loop.exec_()

        self._log_info("%s: wait_async end: %.3fs" %
                       (self._id(), time.time() - t0))
        if self._lst_error:
            raise AllErrorsDuringTest(self._lst_error)

    def _make_async_fun(self, fun):
        return AsyncFun(fun)

    def _log_error(self, *a, **kw):
        print(*a, file=sys.stderr, **kw)

    def _log_info(self, *a, **kw):
        log_truncate(*a, **kw)

    def _log_debug(self, *a, **kw):
        fn_name = "{}:".format(self._id())
        # fn_name = sys._getframe(1).f_code.co_name
        log_truncate(fn_name, *a, **kw)

    def _id(self):
        return self._subtest.id() if self._subtest else self.id()

    def assertMultiInput(self, expected, lst_input, msg="multi input"):
        for i, actual in enumerate(lst_input):
            self.assertEqual(expected, actual, "{} [{}]".format(msg, i))

    #unused
    def assertPairEqual(self, *a):
        pairs = [a[i:i + 2] for i in range(0, len(a), 2)]
        return self.assertEqual(*zip(*pairs))
Example #7
0
class Nominatim:
    """Manage connexion to Nominatim."""
    def __init__(self,
                 url="https://nominatim.openstreetmap.org/search?format=json"):
        """
        Constructor
        @param url:URL of Nominatim
        @type url:str
        """

        self.__url = url
        self.network = QgsNetworkAccessManager.instance()
        self.data = None
        self.network_reply = None
        self.loop = None

    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")

    def _end_of_request(self):
        self.data = self.network_reply.readAll().data().decode('utf-8')
        self.loop.quit()

    def get_first_polygon_from_query(self, query):
        """
        Get first OSM_ID of a Nominatim area

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "relation":
                return result['osm_id']

        # If no result has been return
        raise NominatimAreaException

    def get_first_point_from_query(self, query):
        """
        Get first longitude, latitude of a Nominatim point

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "node":
                return result['lon'], result['lat']

        # If no result has been return
        raise NominatimAreaException
Example #8
0
class Nominatim(object):
    """Manage connexion to Nominatim."""

    def __init__(self,
                 url="https://nominatim.openstreetmap.org/search?format=json"):
        """
        Constructor
        @param url:URL of Nominatim
        @type url:str
        """

        self.__url = url
        self.network = QgsNetworkAccessManager.instance()
        self.data = None
        self.network_reply = None
        self.loop = None

    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")

    def _end_of_request(self):
        self.data = self.network_reply.readAll().data().decode('utf-8')
        self.loop.quit()

    def get_first_polygon_from_query(self, query):
        """
        Get first OSM_ID of a Nominatim area

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "relation":
                return result['osm_id']

        # If no result has been return
        raise NominatimAreaException

    def get_first_point_from_query(self, query):
        """
        Get first longitude, latitude of a Nominatim point

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "node":
                return result['lon'], result['lat']

        # If no result has been return
        raise NominatimAreaException
Example #9
0
class BaseTestAsync(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.app = start_app()

    @classmethod
    def tearDownClass(cls):
        pass
        # cls.app.quit() # useless
        # stop_app() # NameError: name 'QGISAPP' is not defined
        # cls.app.exitQgis() # crash when run unittest (all module)
        # del cls.app
    def setUp(self):
        if isinstance(super(), BaseTestAsync) and hasattr(
                super(), self._testMethodName):
            self.skipTest("duplicated test")

        # self.loop = self.app
        self.loop = QEventLoop()
        self._output = list()
        self._idx = 0
        self.flag_error = False
        self.startTime = time.time()

    def tearDown(self):
        self._stop_async()  # useless ?
        self._log_debug("Test ended. ################### \n")
        self._log_info("%s: %.3fs" % (self.id(), time.time() - self.startTime))

    def _add_output(self, output):
        self._output.append(output)

    def output(self, idx=None):
        if idx is None:
            idx = self._idx
            self._idx += 1
        return self._output[idx] if idx < len(self._output) else None

    def _stop_async(self):
        self.loop.quit()
        self._log_debug("Stop Async. ################### \n")

    def _handle_error(self, e):
        pretty_print_error(e)
        self._stop_async()
        self.flag_error = True

    def _wait_async(self):
        t0 = time.time()
        self.loop.exec_()

        self._log_info("%s: wait_async end: %.3fs" %
                       (self.id(), time.time() - t0))
        self.assertFalse(self.flag_error, "error")

    def _make_async_fun(self, fun):
        return AsyncFun(fun)

    def _log_info(self, *a, **kw):
        print(*a, file=sys.stderr, **kw)

    def _log_debug(self, *a, **kw):
        log_debug(str(self.id()), *a, **kw)

    #unused
    def assertPairEqual(self, *a):
        pairs = [a[i:i + 2] for i in range(0, len(a), 2)]
        return self.assertEqual(*zip(*pairs))