def testInvalidAuthFileDownload(self):
        """
        Download a protected map tile without authcfg
        """
        qs = "?" + "&".join(["%s=%s" % i for i in list({
            "MAP": urllib.parse.quote(self.project_path),
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetMap",
            "LAYERS": "testlayer_èé".replace('_', '%20'),
            "STYLES": "",
            "FORMAT": "image/png",
            "BBOX": "-16817707,-4710778,5696513,14587125",
            "HEIGHT": "500",
            "WIDTH": "500",
            "CRS": "EPSG:3857"
        }.items())])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, None, False)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        self.assertTrue(self.error_was_called)
        self.assertTrue("Download failed: Host requires authentication" in str(self.error_args), "Error args is: %s" % str(self.error_args))
 def wait_object_destruction(self, my_object):
     loop = QEventLoop()
     name = my_object.objectName()
     my_object.destroyed.connect(loop.quit)
     loop.exec_()
     self.assertIsNone(find_window(name))
     return None
    def __sync_request(self, url):
        _url = QUrl(url)
        _request = QNetworkRequest(_url)
        self.__replies.append(_request)

        QgsNetworkAccessManager.instance().sslErrors.connect(self.__supress_ssl_errors)

        _reply = QgsNetworkAccessManager.instance().get(_request)

        # wait
        loop = QEventLoop()
        _reply.finished.connect(loop.quit)
        loop.exec_()
        _reply.finished.disconnect(loop.quit)
        QgsNetworkAccessManager.instance().sslErrors.disconnect(self.__supress_ssl_errors)
        loop = None

        error = _reply.error()
        if error != QNetworkReply.NoError:
            raise Exception(error)

        result_code = _reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)

        result = _reply.readAll()
        self.__replies.append(_reply)
        _reply.deleteLater()

        if result_code in [301, 302, 307]:
            redirect_url = _reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
            return self.__sync_request(redirect_url)
        else:
            return result
Exemple #4
0
    def run(self):
        """Run the query.

        @raise OverpassBadRequestException,NetWorkErrorException,
        OverpassTimeoutException

        @return: The result of the query.
        @rtype: str
        """
        loop = QEventLoop()
        downloader = QgsFileDownloader(
            self._url, self.result_path, delayStart=True)
        downloader.downloadExited.connect(loop.quit)
        downloader.downloadError.connect(self.error)
        downloader.downloadCanceled.connect(self.canceled)
        downloader.downloadCompleted.connect(self.completed)
        downloader.startDownload()
        loop.exec_()

        file_obj = codecs.open(self.result_path, 'r', 'utf-8')
        file_obj.seek(0, 2)
        fsize = file_obj.tell()
        file_obj.seek(max(fsize - 1024, 0), 0)
        lines = file_obj.readlines()
        file_obj.close()

        lines = lines[-10:]  # Get last 10 lines
        timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
                  'at line [\d]+ after ([\d]+) seconds. </remark>'
        if re.search(timeout, ''.join(lines)):
            raise OverpassTimeoutException
        else:
            return self.result_path
Exemple #5
0
    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")
    def fetchFiles(self, urls):
        self.logT("TileLayer.fetchFiles() starts")
        # create a QEventLoop object that belongs to the current thread (if ver. > 2.1, it is render thread)
        eventLoop = QEventLoop()
        self.logT("Create event loop: " + str(eventLoop))  # DEBUG
        # QObject.connect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
        self.allRepliesFinished.connect(eventLoop.quit)

        # create a timer to watch whether rendering is stopped
        watchTimer = QTimer()
        watchTimer.timeout.connect(eventLoop.quit)

        # send a fetch request to the main thread
        # self.emit(SIGNAL("fetchRequest(QStringList)"), urls)
        self.fetchRequest.emit(urls)

        # wait for the fetch to finish
        tick = 0
        interval = 500
        timeoutTick = self.downloadTimeout / interval
        watchTimer.start(interval)
        while tick < timeoutTick:
            # run event loop for 0.5 seconds at maximum
            eventLoop.exec_()

            if debug_mode:
                qDebug("watchTimerTick: %d" % tick)
                qDebug("unfinished downloads: %d" % self.downloader.unfinishedCount())

            if self.downloader.unfinishedCount() == 0 or self.renderContext.renderingStopped():
                break
            tick += 1
        watchTimer.stop()

        if tick == timeoutTick and self.downloader.unfinishedCount() > 0:
            self.log("fetchFiles timeout")
            # self.emitShowBarMessage("fetchFiles timeout", duration=5)   #DEBUG
            self.downloader.abort()
            self.downloader.errorStatus = Downloader.TIMEOUT_ERROR
        files = self.downloader.fetchedFiles

        watchTimer.timeout.disconnect(eventLoop.quit)  #
        # QObject.disconnect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
        self.allRepliesFinished.disconnect(eventLoop.quit)

        self.logT("TileLayer.fetchFiles() ends")
        return files
Exemple #7
0
    def run(self):
        ili2db_bin = get_ili2db_bin(self.tool, self.stdout, self.stderr)
        if not ili2db_bin:
            return

        ili2db_jar_arg = ["-jar", ili2db_bin]

        self.configuration.tool = self.tool
        db_simple_factory = DbSimpleFactory()
        db_factory = db_simple_factory.create_factory(self.tool)

        config_manager = db_factory.get_db_command_config_manager(self.configuration)

        args = config_manager.get_ili2db_args(False)
        args_hide_password = config_manager.get_ili2db_args(True)

        java_path = get_java_path(self.configuration.base_configuration)

        proc = QProcess()
        proc.readyReadStandardError.connect(
            functools.partial(self.stderr_ready, proc=proc))
        proc.readyReadStandardOutput.connect(
            functools.partial(self.stdout_ready, proc=proc))

        proc.start(java_path, ili2db_jar_arg + args)

        if not proc.waitForStarted():
            proc = None

        if not proc:
            raise JavaNotFoundError()

        safe_args = ili2db_jar_arg + args_hide_password
        safe_command = java_path + ' ' + ' '.join(safe_args)
        self.process_started.emit(safe_command)

        self.__result = Exporter.ERROR

        loop = QEventLoop()
        proc.finished.connect(loop.exit)
        loop.exec()

        self.process_finished.emit(proc.exitCode(), self.__result)
        return self.__result
    def download_image(self, url):
        res = False
        img = None
        msg = {'text': '', 'level': Qgis.Warning}
        if url:
            self.logger.info(__name__, "Downloading file from {}".format(url))
            msg_status_bar = "Downloading image from document repository (this might take a while)..."
            with ProcessWithStatus(msg_status_bar):
                if is_connected(TEST_SERVER):

                    nam = QNetworkAccessManager()
                    request = QNetworkRequest(QUrl(url))
                    reply = nam.get(request)

                    loop = QEventLoop()
                    reply.finished.connect(loop.quit)
                    loop.exec_()

                    status = reply.attribute(
                        QNetworkRequest.HttpStatusCodeAttribute)
                    if status == 200:
                        res = True
                        img = reply.readAll()
                    else:
                        res = False
                        msg['text'] = QCoreApplication.translate(
                            "SettingsDialog",
                            "There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
                        )
                else:
                    res = False
                    msg['text'] = QCoreApplication.translate(
                        "SettingsDialog",
                        "There was a problem connecting to Internet.")

        else:
            res = False
            msg['text'] = QCoreApplication.translate("SettingsDialog",
                                                     "Not valid URL")

        if not res:
            self.logger.log_message(__name__, msg['text'], msg['level'])

        return res, img
Exemple #9
0
    def testInvalidAuthFileDownload(self):
        """
        Download a protected map tile without authcfg
        """
        qs = "?" + "&".join([
            "%s=%s" % i
            for i in list({
                "MAP": urllib.parse.quote(self.project_path),
                "SERVICE": "WMS",
                "VERSION": "1.1.1",
                "REQUEST": "GetMap",
                "LAYERS": "testlayer_èé".replace('_', '%20'),
                "STYLES": "",
                "FORMAT": "image/png",
                "BBOX": "-16817707,-4710778,5696513,14587125",
                "HEIGHT": "500",
                "WIDTH": "500",
                "CRS": "EPSG:3857"
            }.items())
        ])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, None, False)
        downloader.downloadCompleted.connect(
            partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot,
                                                    'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot,
                                                    'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        self.assertTrue(self.error_was_called)
        self.assertTrue(
            "Download failed: Host requires authentication"
            in str(self.error_args),
            "Error args is: %s" % str(self.error_args))
Exemple #10
0
    def test_api_usage_async(self):
        """Test standard async API usage"""
        api = NzElectoralApi('http://localhost:%s' % self.port)
        nam = api.status()
        self.last_result = ''
        expected = {
            "version": self.API_VERSION,
            "gmsVersion": "LINZ_Output_20180108_2018_V1_00"
        }
        el = QEventLoop()

        def f(nam):
            """Wrapper"""
            self.last_result = api.parse_async(nam)['content']

        nam.reply.finished.connect(partial(f, nam))
        nam.reply.finished.connect(el.quit)
        el.exec_(QEventLoop.ExcludeUserInputEvents)
        self.assertEqual(self.last_result, expected)
def _sync_get(url):
    global __network_manager
    if __network_manager is None:
        __network_manager = QNetworkAccessManager()
        __network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
    pause = QEventLoop()
    req = QNetworkRequest(url)
    req.setRawHeader(b"Accept", b"application/xml")
    req.setRawHeader(b"Accept-Language", bytes(settings.value("default_language", "fr"), "utf8"))
    req.setRawHeader(b"User-Agent", bytes(settings.value('http_user_agent', plugin_name()), "utf8"))
    reply = __network_manager.get(req)
    reply.finished.connect(pause.quit)
    is_ok = [True]
    def onError(self):
        is_ok[0] = False
        pause.quit()
    reply.error.connect(onError)
    pause.exec_()
    return reply, is_ok[0]
Exemple #12
0
    def testStoreWithoutFileName(self):
        """
        Test file storing and fetching
        """

        f = self.getNewFile(b"New content")

        # store
        storedContent = self.storage.store(f.name, self.url + "/",
                                           self.auth_config.id())
        self.assertTrue(storedContent)
        self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted)

        spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
        spyProgressChanged = QSignalSpy(storedContent.progressChanged)

        loop = QEventLoop()
        storedContent.stored.connect(loop.quit)
        storedContent.errorOccurred.connect(loop.quit)
        QTimer.singleShot(1, lambda: storedContent.store())
        loop.exec()

        self.assertEqual(len(spyErrorOccurred), 0)
        self.assertFalse(storedContent.errorString())
        self.assertEqual(storedContent.url(),
                         self.url + "/" + os.path.basename(f.name))
        self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
        self.assertTrue(len(spyProgressChanged) > 0)
        self.assertEqual(spyProgressChanged[-1][0], 100)

        # fetch
        fetchedContent = self.storage.fetch(
            self.url + "/" + os.path.basename(f.name), self.auth_config.id())
        self.assertTrue(fetchedContent)
        self.assertEqual(fetchedContent.status(),
                         Qgis.ContentStatus.NotStarted)

        spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)

        loop = QEventLoop()
        fetchedContent.fetched.connect(loop.quit)
        fetchedContent.errorOccurred.connect(loop.quit)
        QTimer.singleShot(1, lambda: fetchedContent.fetch())
        loop.exec()

        self.assertEqual(len(spyErrorOccurred), 0)
        self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
        self.assertFalse(fetchedContent.errorString())
        self.assertTrue(fetchedContent.filePath())
        self.checkContent(fetchedContent.filePath(), b"New content")
        self.assertEqual(
            os.path.splitext(fetchedContent.filePath())[1], '.txt')
def _sync_get(url):
    global __network_manager
    if __network_manager is None:
        __network_manager = QNetworkAccessManager()
        __network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
    pause = QEventLoop()
    req = QNetworkRequest(url)
    req.setRawHeader(b"Accept", b"application/xml")
    req.setRawHeader(b"Accept-Language", b"fr")
    reply = __network_manager.get(req)
    reply.finished.connect(pause.quit)
    is_ok = [True]

    def onError(self):
        is_ok[0] = False
        pause.quit()

    reply.error.connect(onError)
    pause.exec_()
    return reply, is_ok[0]
Exemple #14
0
    def run(self):
        """Run the query.

        @raise OverpassBadRequestException,NetWorkErrorException,
        OverpassTimeoutException

        @return: The result of the query.
        @rtype: str
        """
        loop = QEventLoop()
        downloader = QgsFileDownloader(self._url,
                                       self.result_path,
                                       delayStart=True)
        downloader.downloadExited.connect(loop.quit)
        downloader.downloadError.connect(self.error)
        downloader.downloadCanceled.connect(self.canceled)
        downloader.downloadCompleted.connect(self.completed)
        downloader.startDownload()
        loop.exec_()

        osm_file = QFileInfo(self.result_path)
        if not osm_file.exists() and not osm_file.isFile():
            raise OverpassTimeoutException

        # The download is done, checking for not complete OSM file.
        # Overpass might aborted the request with HTTP 200.
        file_obj = codecs.open(self.result_path, 'r', 'utf-8')
        file_obj.seek(0, 2)
        fsize = file_obj.tell()
        file_obj.seek(max(fsize - 1024, 0), 0)
        lines = file_obj.readlines()
        file_obj.close()
        lines = lines[-10:]  # Get last 10 lines
        timeout = (
            '<remark> runtime error: Query timed out in "[a-z]+" at line '
            '[\d]+ after ([\d]+) seconds. </remark>')
        if re.search(timeout, ''.join(lines)):
            raise OverpassTimeoutException

        # Everything went fine
        return self.result_path
Exemple #15
0
def get_available_providers(providers_uri):
    """Fetch the list of available providers from BCS endpoint,
    apparently this API method does not require auth"""
    # For testing purposes, we can also access to a json file directly
    if not providers_uri.startswith('http'):
        try:
            j = json.load(open(providers_uri, encoding='utf-8'))
        except:
            j = json.load(open(providers_uri))
    else:
        t = mktemp()
        q = QgsFileDownloader(QUrl(providers_uri), t)
        loop = QEventLoop()
        q.downloadExited.connect(loop.quit)
        loop.exec_()
        if not os.path.isfile(t):
            return []
        with open(t) as f:
            j = json.load(f)
        os.unlink(t)
    return j
    def __init__(self, parent=None):
        QObject.__init__(self, parent)
        self.queue = []
        self.redirected_urls = {}
        self.requestingUrls = []
        self.replies = []

        self.eventLoop = QEventLoop()
        self.sync = False
        self.fetchedFiles = {}
        self.clearCounts()

        self.timer = QTimer()
        self.timer.setSingleShot(True)
        self.timer.timeout.connect(self.fetchTimedOut)

        # network settings
        self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
        self.max_connection = 4
        self.default_cache_expiration = 24
        self.errorStatus = Downloader.NO_ERROR
Exemple #17
0
    def _get_sync(self, qurl: QUrl, timeout: int = 20000) -> Reply:
        '''
        synchronous GET-request
        '''
        request = QNetworkRequest(qurl)
        ## newer versions of QGIS (3.6+) support synchronous requests
        #if hasattr(self._manager, 'blockingGet'):
        #reply = self._manager.blockingGet(request, forceRefresh=True)
        ## use blocking event loop for older versions
        #else:
        loop = QEventLoop()
        timer = QTimer()
        timer.setSingleShot(True)
        # reply or timeout break event loop, whoever comes first
        timer.timeout.connect(loop.quit)
        reply = self._manager.get(request)
        reply.finished.connect(loop.quit)

        timer.start(timeout)

        # start blocking loop
        loop.exec()
        loop.deleteLater()
        if not timer.isActive():
            reply.deleteLater()
            raise ConnectionError('Timeout')

        timer.stop()
        #if reply.error():
        #self.error.emit(reply.errorString())
        #raise ConnectionError(reply.errorString())
        res = Reply(reply)
        self.finished.emit(res)
        return res
class OpenlayersRenderer(QgsMapLayerRenderer):
    def __init__(self, layer, context, webPage, layerType):
        """ Initialize the object. This function is still run in the GUI thread.
            Should refrain from doing any heavy work.
        """
        QgsMapLayerRenderer.__init__(self, layer.id())
        self.context = context
        self.controller = OpenlayersController(None,
                                               context, webPage, layerType)
        self.loop = None

    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True

    def onTimeout(self):
        """ periodically check whether the rendering should not be stopped """
        if self.context.renderingStopped():
            debug("[WORKER THREAD] Cancelling rendering", 3)
            self.loop.exit()
class OpenlayersRenderer(QgsMapLayerRenderer):
    def __init__(self, layer, context, webPage, layerType):
        """ Initialize the object. This function is still run in the GUI thread.
            Should refrain from doing any heavy work.
        """
        QgsMapLayerRenderer.__init__(self, layer.id())
        self.context = context
        self.controller = OpenlayersController(None, context, webPage,
                                               layerType)
        self.loop = None

    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True

    def onTimeout(self):
        """ periodically check whether the rendering should not be stopped """
        if self.context.renderingStopped():
            debug("[WORKER THREAD] Cancelling rendering", 3)
            self.loop.exit()
    def process_delete_call(self, url, report_url=True):
        """
        Run a DELETE request and return reply data
        :param url: url for request
        :param report_url: True if URL should be reported to feedback
        :return: response or error message in json format
        """

        if self.connection.read_only:
            return {
                "error": {
                    "msg": "Graphium connection is set to read-only!"
                }
            }

        url_query = QUrl(url)
        if report_url:
            self.report_info('DELETE ' + url_query.toString())

        request = QNetworkRequest(url_query)
        if self.connection.auth_cfg != '':
            self.auth = 0
            config = QgsAuthMethodConfig()
            QgsApplication.authManager().loadAuthenticationConfig(
                self.connection.auth_cfg, config, True)
            concatenated = config.configMap(
            )['username'] + ":" + config.configMap()['password']

            data = base64.encodebytes(concatenated.encode("utf-8")).replace(
                '\n'.encode("utf-8"), ''.encode("utf-8"))
            request.setRawHeader("Authorization".encode("utf-8"),
                                 ("Basic %s" % data).encode("utf-8"))
            request.setRawHeader("Accept".encode("utf-8"),
                                 "*/*".encode("utf-8"))
        loop = QEventLoop()
        reply = self.network_access_manager.deleteResource(request)
        reply.finished.connect(loop.quit)
        loop.exec_()

        return self.process_q_reply(reply)
Exemple #21
0
    def testPythonCreateChildrenCalledFromCplusplus(self):
        """ test createChildren() method implemented in Python, called from C++ """

        loop = QEventLoop()
        NUM_ITERS = 10  # set more to detect memory leaks
        for i in range(NUM_ITERS):
            tabSetDestroyedFlag = [False]

            item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
            item.tabSetDestroyedFlag = tabSetDestroyedFlag

            # Causes PyQgsDataConnectionItem.createChildren() to be called
            item.populate()

            # wait for populate() to have done its job
            item.stateChanged.connect(loop.quit)
            loop.exec_()

            # Python object PyQgsLayerItem should still be alive
            self.assertFalse(tabSetDestroyedFlag[0])

            self.assertEqual(len(item.children()), 2)
            self.assertEqual(item.children()[0].name(), "name")
            self.assertEqual(item.children()[1].name(), "name2")

            # Delete the object and make sure all deferred deletions are processed
            item.destroyed.connect(loop.quit)
            item.deleteLater()
            loop.exec_()

            # Check that the PyQgsLayerItem Python object is now destroyed
            self.assertTrue(tabSetDestroyedFlag[0])
            tabSetDestroyedFlag[0] = False
    def _make_download(self, url, destination, cancel=False):
        self.completed_was_called = False
        self.error_was_called = False
        self.canceled_was_called = False
        self.progress_was_called = False
        self.exited_was_called = False

        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        if cancel:
            downloader.downloadProgress.connect(downloader.cancelDownload)

        loop.exec_()
Exemple #23
0
    def _make_download(self, url, destination, cancel=False):
        self.completed_was_called = False
        self.error_was_called = False
        self.canceled_was_called = False
        self.progress_was_called = False
        self.exited_was_called = False

        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        if cancel:
            downloader.downloadProgress.connect(downloader.cancelDownload)

        loop.exec_()
Exemple #24
0
    def testValidAuthFileDownload(self):
        """
        Download a map tile with valid authcfg
        """
        qs = "?" + "&".join(["%s=%s" % i for i in list({
            "MAP": urllib.parse.quote(self.project_path),
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetMap",
            "LAYERS": "testlayer_èé".replace('_', '%20'),
            "STYLES": "",
            "FORMAT": "image/png",
            "BBOX": "-16817707,-4710778,5696513,14587125",
            "HEIGHT": "500",
            "WIDTH": "500",
            "CRS": "EPSG:3857"
        }.items())])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        # Check the we've got a likely PNG image
        self.assertTrue(self.completed_was_called)
        self.assertTrue(os.path.getsize(destination) > 2000, "Image size: %s" % os.path.getsize(destination))  # > 1MB
        with open(destination, 'rb') as f:
            self.assertTrue(b'PNG' in f.read())  # is a PNG
Exemple #25
0
    def run(self):
        """Run the query.

        @raise OverpassBadRequestException,NetWorkErrorException,
        OverpassTimeoutException

        @return: The result of the query.
        @rtype: str
        """
        loop = QEventLoop()
        downloader = QgsFileDownloader(self._url,
                                       self.result_path,
                                       delayStart=True)
        downloader.downloadExited.connect(loop.quit)
        downloader.downloadError.connect(self.error)
        downloader.downloadCanceled.connect(self.canceled)
        downloader.downloadCompleted.connect(self.completed)
        downloader.startDownload()
        loop.exec_()

        for message in self.errors:
            self.is_query_timed_out(message)
            self.is_bad_request(message)
            LOGGER.error(message)

        if len(self.errors):
            raise NetWorkErrorException('Overpass API', ', '.join(self.errors))

        osm_file = QFileInfo(self.result_path)
        if not osm_file.exists() and not osm_file.isFile():
            # Do not raise a QuickOSM exception here
            # It must be a bug from QuickOSM
            raise FileNotFoundError

        self.check_file(self.result_path)

        # Everything went fine
        return self.result_path
Exemple #26
0
    def loadModel(self):
        imd = None
        try:
            loader = ModelLoader(self.ui.mDataLineEdit.text())
            models = loader.detect_models()
            model_names = map(lambda m: m.name, models)
            self._log_output("Looking up models: " + ', '.join(model_names))
            ili = loader.gen_lookup_ili()
            qDebug(ili)
            wpsreq = self._create_wps_request(ili)
            url = self.ui.mIlisMetaUrlLineEdit.text()
            req = QNetworkRequest(QUrl(url))
            req.setHeader(QNetworkRequest.ContentTypeHeader, 'application/xml')
            reply = QgsNetworkAccessManager.instance().post(req, wpsreq)

            # Wait for reply or timeout
            loop = QEventLoop()
            reply.finished.connect(loop.quit)
            QTimer.singleShot(15000, reply.abort)
            loop.exec_()

            if reply.isFinished() and reply.error() == QNetworkReply.NoError:
                result = reply.readAll()
                imd = self._parse_wps_response(result)
        except:
            qDebug("Exception during IlisModel download")
        if imd is None:
            self._show_log_window()
            QgsApplication.messageLog().logMessage(
                "Couldn't download Ilismeta model", "Interlis",
                Qgis.MessageLevel(1))
            self.ui.mModelLineEdit.setText("")
        else:
            fh, imdfn = tempfile.mkstemp(suffix='.imd')
            os.close(fh)
            with codecs.open(imdfn, "w", encoding='utf-8') as file:
                file.write(imd)
            self.ui.mModelLineEdit.setText(imdfn)
    def testValidAuthFileDownload(self):
        """
        Download a map tile with valid authcfg
        """
        qs = "?" + "&".join(["%s=%s" % i for i in list({
            "MAP": urllib.parse.quote(self.project_path),
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetMap",
            "LAYERS": "testlayer_èé".replace('_', '%20'),
            "STYLES": "",
            "FORMAT": "image/png",
            "BBOX": "-16817707,-4710778,5696513,14587125",
            "HEIGHT": "500",
            "WIDTH": "500",
            "CRS": "EPSG:3857"
        }.items())])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        # Check the we've got a likely PNG image
        self.assertTrue(self.completed_was_called)
        self.assertTrue(os.path.getsize(destination) > 700000, "Image size: %s" % os.path.getsize(destination))  # > 1MB
        with open(destination, 'rb') as f:
            self.assertTrue(b'PNG' in f.read())  # is a PNG
    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True
Exemple #29
0
    def testPythonCreateChildrenCalledFromCplusplus(self):
        """
        test createChildren() method implemented in Python, called from C++

        This test was originally working under Qt 5.12, but is broken on newer Qt or sip
        versions. The test currently segfaults, as the children created by the python QgsDataCollectionItem
        subclass PyQgsDataConnectionItem are immediately garbage collected.

        The SIP SIP_VIRTUAL_CATCHER_CODE in qgsdataitem.h is supposed to fix this situation by
        adding an extra reference to the returned python objects, but the lines

          // pyItem is given an extra reference which is removed when the C++ instance’s destructor is called.
          sipTransferTo( pyItem, Py_None );

        no longer have any effect and the object is still immediately deleted.

        Attempted solutions include:
        - all combinations of the existing VirtualCatcherCode with the different Factory/TransferBack annotations
        - removing the VirtualCatcherCode and replacing with raw Factory/TransferBack annotations
        - disabling the python garbage collection of the object entirely with sipTransferTo( pyItem, NULL )

        When fixed this test should be moved back to test_qgsdataitem.py
        """

        loop = QEventLoop()
        NUM_ITERS = 10  # set more to detect memory leaks
        for i in range(NUM_ITERS):
            tabSetDestroyedFlag = [False]

            item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
            item.tabSetDestroyedFlag = tabSetDestroyedFlag

            # Causes PyQgsDataConnectionItem.createChildren() to be called
            item.populate()

            # wait for populate() to have done its job
            item.stateChanged.connect(loop.quit)
            loop.exec_()

            # Python object PyQgsLayerItem should still be alive
            self.assertFalse(tabSetDestroyedFlag[0])

            children = item.children()
            self.assertEqual(len(children), 2)
            self.assertEqual(children[0].name(), "name")
            self.assertEqual(children[1].name(), "name2")

            del children

            # Delete the object and make sure all deferred deletions are processed
            item.destroyed.connect(loop.quit)
            item.deleteLater()
            loop.exec_()

            # Check that the PyQgsLayerItem Python object is now destroyed
            self.assertTrue(tabSetDestroyedFlag[0])
            tabSetDestroyedFlag[0] = False
def _sync_get(url):
    global __network_manager
    if __network_manager is None:
        __network_manager = QNetworkAccessManager()
        __network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
    pause = QEventLoop()
    req = QNetworkRequest(url)
    req.setRawHeader(b"Accept", b"application/xml")
    req.setRawHeader(b"Accept-Language",
                     bytes(settings.value("default_language", "fr"), "utf8"))
    req.setRawHeader(
        b"User-Agent",
        bytes(settings.value("http_user_agent", plugin_name()), "utf8"))
    reply = __network_manager.get(req)
    reply.finished.connect(pause.quit)
    is_ok = [True]

    def onError(self):
        is_ok[0] = False
        pause.quit()

    reply.error.connect(onError)
    pause.exec_()
    return reply, is_ok[0]
Exemple #31
0
 def update_referecedata_cache_model(self, filter_models, type):
     # updates the model and waits for the end
     loop = QEventLoop()
     self.ilireferencedatacache.model_refreshed.connect(lambda: loop.quit())
     timer = QTimer()
     timer.setSingleShot(True)
     timer.timeout.connect(lambda: loop.quit())
     timer.start(10000)
     self.refresh_referencedata_cache(filter_models, type)
     loop.exec()
     return self.ilireferencedatacache.model
    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True
Exemple #33
0
    def _post_sync(self,
                   qurl: QUrl,
                   timeout: int = 20000,
                   data: bytes = b'',
                   content_type=None):
        '''
        synchronous POST-request
        '''
        request = QNetworkRequest(qurl)
        if content_type:
            request.setHeader(QNetworkRequest.ContentTypeHeader, content_type)
        # newer versions of QGIS (3.6+) support synchronous requests
        if hasattr(self._manager, 'blockingPost'):
            reply = self._manager.blockingPost(request,
                                               data,
                                               forceRefresh=True)
        # use blocking event loop for older versions
        else:
            loop = QEventLoop()
            timer = QTimer()
            timer.setSingleShot(True)
            # reply or timeout break event loop, whoever comes first
            timer.timeout.connect(loop.quit)
            reply = self._manager.post(request, data)
            reply.finished.connect(loop.quit)

            timer.start(timeout)

            # start blocking loop
            loop.exec()
            loop.deleteLater()
            if not timer.isActive():
                reply.deleteLater()
                raise ConnectionError('Timeout')

            timer.stop()
        if reply.error():
            self.error.emit(reply.errorString())
            raise ConnectionError(reply.errorString())
        res = Reply(reply)
        self.finished.emit(res)
        return res
Exemple #34
0
    def get_topping_file_model(self, id_list):
        topping_file_cache = IliToppingFileCache(
            self.import_schema_configuration.base_configuration, id_list)

        # we wait for the download or we timeout after 30 seconds and we apply what we have
        loop = QEventLoop()
        topping_file_cache.download_finished.connect(lambda: loop.quit())
        timer = QTimer()
        timer.setSingleShot(True)
        timer.timeout.connect(lambda: loop.quit())
        timer.start(30000)

        topping_file_cache.refresh()
        self.log_panel.print_info(self.tr("- - Downloading…"),
                                  LogColor.COLOR_TOPPING)

        if len(topping_file_cache.downloaded_files) != len(id_list):
            loop.exec()

        if len(topping_file_cache.downloaded_files) == len(id_list):
            self.log_panel.print_info(
                self.tr("- - All topping files successfully downloaded"),
                LogColor.COLOR_TOPPING,
            )
        else:
            missing_file_ids = id_list
            for downloaded_file_id in topping_file_cache.downloaded_files:
                if downloaded_file_id in missing_file_ids:
                    missing_file_ids.remove(downloaded_file_id)
            self.log_panel.print_info(
                self.
                tr("- - Some topping files where not successfully downloaded: {}"
                   ).format(" ".join(missing_file_ids)),
                LogColor.COLOR_TOPPING,
            )

        return topping_file_cache.model
Exemple #35
0
class NetworkAccessManager(object):
    """
    This class mimicks httplib2 by using QgsNetworkAccessManager for all
    network calls.

    The return value is a tuple of (response, content), the first being and
    instance of the Response class, the second being a string that contains
    the response entity body.

    Parameters
    ----------
    debug : bool
        verbose logging if True
    exception_class : Exception
        Custom exception class

    Usage 1 (blocking mode)
    -----
    ::
        nam = NetworkAccessManager(authcgf)
        try:
            (response, content) = nam.request('http://www.example.com')
        except RequestsException as e:
            # Handle exception
            pass

    Usage 2 (Non blocking mode)
    -------------------------
    ::
        NOTE! if blocking mode returns immediatly
              it's up to the caller to manage listeners in case
              of non blocking mode

        nam = NetworkAccessManager(authcgf)
        try:
            nam.request('http://www.example.com', blocking=False)
            nam.reply.finished.connect(a_signal_listener)
        except RequestsException as e:
            # Handle exception
            pass

        Get response using method:
        nam.httpResult() that return a dictionary with keys:
            'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
            'content' - bytearray returned from reply
            'ok' - request success [True, False]
            'headers' - Dicionary containing the reply header
            'reason' - fomatted message string with reply.errorString()
            'exception' - the exception returne dduring execution
    """
    def __init__(self,
                 authid=None,
                 disable_ssl_certificate_validation=False,
                 exception_class=None,
                 debug=True,
                 timeout=None):
        self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
        self.authid = authid
        self.reply = None
        self.debug = debug
        self.exception_class = exception_class
        self.on_abort = False
        self.blocking_mode = False
        self.http_call_result = Response({
            'status': 0,
            'status_code': 0,
            'status_message': '',
            'content': '',
            'ok': False,
            'headers': {},
            'reason': '',
            'exception': None,
        })
        if timeout is None:
            timeout = 60
        self.timeout = timeout

    def msg_log(self, msg):
        if self.debug:
            QgsMessageLog.logMessage(msg, "NetworkAccessManager")

    def httpResult(self):
        return self.http_call_result

    def auth_manager(self):
        return QgsApplication.authManager()

    def request(self,
                url,
                method="GET",
                body=None,
                headers=None,
                redirections=DEFAULT_MAX_REDIRECTS,
                connection_type=None,
                blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                if k and v:
                    req.setRawHeader(k.encode(), v.encode())
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            self.auth_manager().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(),
                           'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" %
                     (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, io.IOBase):
                body = body.read()
            if isinstance(body, str):
                body = body.encode()
            if isinstance(body, dict):
                body = str(json.dumps(body)).encode(encoding='utf-8')
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            self.auth_manager().updateNetworkReply(self.reply, self.authid)

        QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)

        # necessary to trap local timeout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(
            self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)

    def downloadProgress(self, bytesReceived, bytesTotal):
        """Keep track of the download progress"""
        #self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
        pass

    def requestTimedOut(self, reply):
        """Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
        # adapt http_call_result basing on receiving qgs timer timout signal
        self.exception_class = RequestsExceptionTimeout
        self.http_call_result.exception = RequestsExceptionTimeout(
            "Timeout error")

    def replyFinished(self):
        err = self.reply.error()
        httpStatus = self.reply.attribute(
            QNetworkRequest.HttpStatusCodeAttribute)
        httpStatusMessage = self.reply.attribute(
            QNetworkRequest.HttpReasonPhraseAttribute)
        self.http_call_result.status_code = httpStatus
        self.http_call_result.status = httpStatus
        self.http_call_result.status_message = httpStatusMessage
        for k, v in self.reply.rawHeaderPairs():
            self.http_call_result.headers[str(
                k.data(), encoding='utf-8')] = str(v.data(), encoding='utf-8')
            self.http_call_result.headers[str(
                k.data(), encoding='utf-8').lower()] = str(v.data(),
                                                           encoding='utf-8')

        if err != QNetworkReply.NoError:
            # handle error
            # check if errorString is empty, if so, then set err string as
            # reply dump
            if re.match('(.)*server replied: $', self.reply.errorString()):
                errString = self.reply.errorString(
                ) + self.http_call_result.content
            else:
                errString = self.reply.errorString()
            # check if self.http_call_result.status_code is available (client abort
            # does not produce http.status_code)
            if self.http_call_result.status_code:
                msg = "Network error #{0}: {1}".format(
                    self.http_call_result.status_code, errString)
            else:
                msg = "Network error: {0}".format(errString)

            self.http_call_result.reason = msg
            self.http_call_result.text = str(self.reply.readAll().data(),
                                             encoding='utf-8')
            self.http_call_result.ok = False
            self.msg_log(msg)
            # set return exception
            if err == QNetworkReply.TimeoutError:
                self.http_call_result.exception = RequestsExceptionTimeout(msg)

            elif err == QNetworkReply.ConnectionRefusedError:
                self.http_call_result.exception = RequestsExceptionConnectionError(
                    msg)

            elif err == QNetworkReply.OperationCanceledError:
                # request abort by calling NAM.abort() => cancelled by the user
                if self.on_abort:
                    self.http_call_result.exception = RequestsExceptionUserAbort(
                        msg)
                else:
                    self.http_call_result.exception = RequestsException(msg)

            else:
                self.http_call_result.exception = RequestsException(msg)

            # overload exception to the custom exception if available
            if self.exception_class:
                self.http_call_result.exception = self.exception_class(msg)

        else:
            # Handle redirections
            redirectionUrl = self.reply.attribute(
                QNetworkRequest.RedirectionTargetAttribute)
            if redirectionUrl is not None and redirectionUrl != self.reply.url(
            ):
                if redirectionUrl.isRelative():
                    redirectionUrl = self.reply.url().resolved(redirectionUrl)

                msg = "Redirected from '{}' to '{}'".format(
                    self.reply.url().toString(), redirectionUrl.toString())
                self.msg_log(msg)

                self.reply.deleteLater()
                self.reply = None
                self.request(redirectionUrl.toString())

            # really end request
            else:
                msg = "Network success #{0}".format(self.reply.error())
                self.http_call_result.reason = msg
                self.msg_log(msg)

                ba = self.reply.readAll()
                self.http_call_result.content = bytes(ba)
                self.http_call_result.text = str(ba.data(), encoding='utf-8')
                self.http_call_result.ok = True

        # Let's log the whole response for debugging purposes:
        self.msg_log("Got response %s %s from %s" % \
                    (self.http_call_result.status_code,
                     self.http_call_result.status_message,
                     self.reply.url().toString()))
        for k, v in list(self.http_call_result.headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if len(self.http_call_result.content) < 1024:
            self.msg_log("Payload :\n%s" % self.http_call_result.text)
        else:
            self.msg_log("Payload is > 1 KB ...")

        # clean reply
        if self.reply is not None:
            if self.reply.isRunning():
                self.reply.close()
            self.msg_log("Deleting reply ...")
            # Disconnect all slots
            self.reply.sslErrors.disconnect(self.sslErrors)
            self.reply.finished.disconnect(self.replyFinished)
            self.reply.downloadProgress.disconnect(self.downloadProgress)
            self.reply.deleteLater()
            self.reply = None
        else:
            self.msg_log("Reply was already deleted ...")

    def sslErrors(self, ssl_errors):
        """
        Handle SSL errors, logging them if debug is on and ignoring them
        if disable_ssl_certificate_validation is set.
        """
        if ssl_errors:
            for v in ssl_errors:
                self.msg_log("SSL Error: %s" % v.errorString())
        if self.disable_ssl_certificate_validation:
            self.reply.ignoreSslErrors()

    def abort(self):
        """
        Handle request to cancel HTTP call
        """
        if (self.reply and self.reply.isRunning()):
            self.on_abort = True
            self.reply.abort()
Exemple #36
0
    def request(self,
                url,
                method="GET",
                body=None,
                headers=None,
                redirections=DEFAULT_MAX_REDIRECTS,
                connection_type=None,
                blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                if k and v:
                    req.setRawHeader(k.encode(), v.encode())
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            self.auth_manager().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(),
                           'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" %
                     (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, io.IOBase):
                body = body.read()
            if isinstance(body, str):
                body = body.encode()
            if isinstance(body, dict):
                body = str(json.dumps(body)).encode(encoding='utf-8')
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            self.auth_manager().updateNetworkReply(self.reply, self.authid)

        QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)

        # necessary to trap local timeout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(
            self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)
Exemple #37
0
class Nominatim(object):
    """Manage connexion to Nominatim."""

    def __init__(self,
                 url="https://nominatim.openstreetmap.org/search?format=json"):
        """
        Constructor
        @param url:URL of Nominatim
        @type url:str
        """

        self.__url = url
        self.network = QgsNetworkAccessManager.instance()
        self.data = None
        self.network_reply = None
        self.loop = None

    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")

    def _end_of_request(self):
        self.data = self.network_reply.readAll().data().decode('utf-8')
        self.loop.quit()

    def get_first_polygon_from_query(self, query):
        """
        Get first OSM_ID of a Nominatim area

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "relation":
                return result['osm_id']

        # If no result has been return
        raise NominatimAreaException

    def get_first_point_from_query(self, query):
        """
        Get first longitude, latitude of a Nominatim point

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "node":
                return result['lon'], result['lat']

        # If no result has been return
        raise NominatimAreaException
    def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                req.setRawHeader(k, v)
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, file):
                body = body.read()
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)

        # necessary to trap local timout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)
class NetworkAccessManager(object):
    """
    This class mimicks httplib2 by using QgsNetworkAccessManager for all
    network calls.
    The return value is a tuple of (response, content), the first being and
    instance of the Response class, the second being a string that contains
    the response entity body.
    Parameters
    ----------
    debug : bool
        verbose logging if True
    exception_class : Exception
        Custom exception class
    Usage 1 (blocking mode)
    -----
    ::
        nam = NetworkAccessManager(authcgf)
        try:
            (response, content) = nam.request('http://www.example.com')
        except RequestsException as e:
            # Handle exception
            pass
    Usage 2 (Non blocking mode)
    -------------------------
    ::
        NOTE! if blocking mode returns immediatly
              it's up to the caller to manage listeners in case
              of non blocking mode
        nam = NetworkAccessManager(authcgf)
        try:
            nam.request('http://www.example.com', blocking=False)
            nam.reply.finished.connect(a_signal_listener)
        except RequestsException as e:
            # Handle exception
            pass
        Get response using method:
        nam.httpResult() that return a dictionary with keys:
            'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
            'content' - bytearray returned from reply
            'ok' - request success [True, False]
            'headers' - Dicionary containing the reply header
            'reason' - fomatted message string with reply.errorString()
            'exception' - the exception returne dduring execution
    """

    def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=False):
        self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
        self.authid = authid
        self.reply = None
        self.debug = debug
        self.exception_class = exception_class
        self.on_abort = False
        self.blocking_mode = False
        self.http_call_result = Response({
            'status': 0,
            'status_code': 0,
            'status_message': '',
            'content' : '',
            'ok': False,
            'headers': {},
            'reason': '',
            'exception': None,
        })

    def msg_log(self, msg):
        if self.debug:
            QgsMessageLog.logMessage(msg, "NetworkAccessManager")

    def httpResult(self):
        return self.http_call_result

    def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                req.setRawHeader(k, v)
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, file):
                body = body.read()
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)

        # necessary to trap local timout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)

    #@pyqtSlot()
    def downloadProgress(self, bytesReceived, bytesTotal):
        """Keep track of the download progress"""
        #self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
        pass

    #@pyqtSlot(QNetworkReply)
    def requestTimedOut(self, QNetworkReply):
        """Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
        # adapt http_call_result basing on receiving qgs timer timout signal
        self.exception_class = RequestsExceptionTimeout
        self.http_call_result.exception = RequestsExceptionTimeout("Timeout error")

    #@pyqtSlot(QObject)
    def replyFinished(self):
        err = self.reply.error()
        httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
        httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
        self.http_call_result.status_code = httpStatus
        self.http_call_result.status = httpStatus
        self.http_call_result.status_message = httpStatusMessage
        for k, v in self.reply.rawHeaderPairs():
            self.http_call_result.headers[str(k)] = str(v)
            self.http_call_result.headers[str(k).lower()] = str(v)

        if err != QNetworkReply.NoError:
            # handle error
            # check if errorString is empty, if so, then set err string as
            # reply dump
            if re.match('(.)*server replied: $', self.reply.errorString()):
                errString = self.reply.errorString() + self.http_call_result.content
            else:
                errString = self.reply.errorString()
            # check if self.http_call_result.status_code is available (client abort
            # does not produce http.status_code)
            if self.http_call_result.status_code:
                msg = "Network error #{0}: {1}".format(
                    self.http_call_result.status_code, errString)
            else:
                msg = "Network error: {0}".format(errString)

            self.http_call_result.reason = msg
            self.http_call_result.ok = False
            self.msg_log(msg)
            # set return exception
            if err == QNetworkReply.TimeoutError:
                self.http_call_result.exception = RequestsExceptionTimeout(msg)

            elif err == QNetworkReply.ConnectionRefusedError:
                self.http_call_result.exception = RequestsExceptionConnectionError(msg)

            elif err == QNetworkReply.OperationCanceledError:
                # request abort by calling NAM.abort() => cancelled by the user
                if self.on_abort:
                    self.http_call_result.exception = RequestsExceptionUserAbort(msg)
                else:
                    self.http_call_result.exception = RequestsException(msg)

            else:
                self.http_call_result.exception = RequestsException(msg)

            # overload exception to the custom exception if available
            if self.exception_class:
                self.http_call_result.exception = self.exception_class(msg)

        else:
            # Handle redirections
            redirectionUrl = self.reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
            if redirectionUrl is not None and redirectionUrl != self.reply.url():
                if redirectionUrl.isRelative():
                    redirectionUrl = self.reply.url().resolved(redirectionUrl)

                msg = "Redirected from '{}' to '{}'".format(
                    self.reply.url().toString(), redirectionUrl.toString())
                self.msg_log(msg)

                self.reply.deleteLater()
                self.reply = None
                self.request(redirectionUrl.toString())

            # really end request
            else:
                msg = "Network success #{0}".format(self.reply.error())
                self.http_call_result.reason = msg
                self.msg_log(msg)

                ba = self.reply.readAll()
                self.http_call_result.content = bytes(ba)
                self.http_call_result.ok = True

        # Let's log the whole response for debugging purposes:
        self.msg_log("Got response %s %s from %s" % \
                    (self.http_call_result.status_code,
                     self.http_call_result.status_message,
                     self.reply.url().toString()))
        for k, v in list(self.http_call_result.headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if len(self.http_call_result.content) < 1024:
            self.msg_log("Payload :\n%s" % self.http_call_result.content)
        else:
            self.msg_log("Payload is > 1 KB ...")

        # clean reply
        if self.reply is not None:
            if self.reply.isRunning():
                self.reply.close()
            self.msg_log("Deleting reply ...")
            # Disconnect all slots
            self.reply.sslErrors.disconnect(self.sslErrors)
            self.reply.finished.disconnect(self.replyFinished)
            self.reply.downloadProgress.disconnect(self.downloadProgress)
            self.reply.deleteLater()
            self.reply = None
        else:
            self.msg_log("Reply was already deleted ...")

    #@pyqtSlot()
    def sslErrors(self, ssl_errors):
        """
        Handle SSL errors, logging them if debug is on and ignoring them
        if disable_ssl_certificate_validation is set.
        """
        if ssl_errors:
            for v in ssl_errors:
                self.msg_log("SSL Error: %s" % v.errorString())
        if self.disable_ssl_certificate_validation:
            self.reply.ignoreSslErrors()

    #@pyqtSlot()
    def abort(self):
        """
        Handle request to cancel HTTP call
        """
        if (self.reply and self.reply.isRunning()):
            self.on_abort = True
            self.reply.abort()
    def test(self):

        # This test is quite fragile as it depends on windows manager behaviour
        # regarding focus, so not surprising it doesn't pass
        # on other platforms than Linux.
        #if 'TRAVIS_OS_NAME' in os.environ and os.environ['TRAVIS_OS_NAME'] == 'osx':
        #    return

        main_dialog = QgsProviderRegistry.instance().selectWidget("WFS")
        main_dialog.setProperty("hideDialogs", True)

        self.assertIsNotNone(main_dialog)

        # Create new connection
        btnNew = main_dialog.findChild(QWidget, "btnNew")
        self.assertIsNotNone(btnNew)
        QTest.mouseClick(btnNew, Qt.LeftButton)
        new_conn = find_window('QgsNewHttpConnectionBase')
        self.assertIsNotNone(new_conn)
        txtName = new_conn.findChild(QLineEdit, "txtName")
        self.assertIsNotNone(txtName)
        txtName.setText("test_connection")
        txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
        self.assertIsNotNone(txtUrl)
        txtUrl.setText("test_url")
        new_conn.accept()

        # Wait for object to be destroyed
        new_conn = self.wait_object_destruction(new_conn)

        # Try to connect
        btnConnect = main_dialog.findChild(QWidget, "btnConnect")
        self.assertIsNotNone(btnConnect)
        QTest.mouseClick(btnConnect, Qt.LeftButton)
        # Depends on asynchronous signal
        QApplication.processEvents()
        error_box = find_window('WFSCapabilitiesErrorBox')
        self.assertIsNotNone(error_box)
        # Close error box
        error_box.accept()

        # Wait for object to be destroyed
        error_box = self.wait_object_destruction(error_box)

        # Edit connection
        btnEdit = main_dialog.findChild(QWidget, "btnEdit")
        self.assertIsNotNone(btnEdit)
        QTest.mouseClick(btnEdit, Qt.LeftButton)
        new_conn = find_window('QgsNewHttpConnectionBase',)
        self.assertIsNotNone(new_conn)
        txtName = new_conn.findChild(QLineEdit, "txtName")
        self.assertIsNotNone(txtName)
        txtName.setText("test_connection")
        txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
        self.assertIsNotNone(txtUrl)

        endpoint = self.basetestpath + '/fake_qgis_http_endpoint'
        expected_endpoint = endpoint
        if sys.platform == 'win32' and expected_endpoint[1] == ':':
            expected_endpoint = expected_endpoint[0] + expected_endpoint[2:]
        with open(sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'), 'wb') as f:
            f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
  <FeatureTypeList>
    <FeatureType>
      <Name>my:typename</Name>
      <Title>Title</Title>
      <Abstract>Abstract</Abstract>
      <DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
      <ows:WGS84BoundingBox>
        <ows:LowerCorner>-71.123 66.33</ows:LowerCorner>
        <ows:UpperCorner>-65.32 78.3</ows:UpperCorner>
      </ows:WGS84BoundingBox>
    </FeatureType>
  </FeatureTypeList>
  <fes:Filter_Capabilities>
    <fes:Spatial_Capabilities>
      <fes:GeometryOperands>
        <fes:GeometryOperand name="gml:Envelope"/>
        <fes:GeometryOperand name="gml:Point"/>
        <fes:GeometryOperand name="gml:MultiPoint"/>
        <fes:GeometryOperand name="gml:LineString"/>
        <fes:GeometryOperand name="gml:MultiLineString"/>
        <fes:GeometryOperand name="gml:Polygon"/>
        <fes:GeometryOperand name="gml:MultiPolygon"/>
        <fes:GeometryOperand name="gml:MultiGeometry"/>
      </fes:GeometryOperands>
      <fes:SpatialOperators>
        <fes:SpatialOperator name="Disjoint"/>
        <fes:SpatialOperator name="Equals"/>
        <fes:SpatialOperator name="DWithin"/>
        <fes:SpatialOperator name="Beyond"/>
        <fes:SpatialOperator name="Intersects"/>
        <fes:SpatialOperator name="Touches"/>
        <fes:SpatialOperator name="Crosses"/>
        <fes:SpatialOperator name="Within"/>
        <fes:SpatialOperator name="Contains"/>
        <fes:SpatialOperator name="Overlaps"/>
        <fes:SpatialOperator name="BBOX"/>
      </fes:SpatialOperators>
    </fes:Spatial_Capabilities>
    <fes:Functions>
      <fes:Function name="abs">
        <fes:Returns>xs:int</fes:Returns>
        <fes:Arguments>
          <fes:Argument name="param">
            <fes:Type>xs:int</fes:Type>
          </fes:Argument>
        </fes:Arguments>
      </fes:Function>
    </fes:Functions>
  </fes:Filter_Capabilities>
</wfs:WFS_Capabilities>""".encode('UTF-8'))

        txtUrl.setText("http://" + endpoint)
        new_conn.accept()

        # Wait for object to be destroyed
        new_conn = self.wait_object_destruction(new_conn)

        # Try to connect
        btnConnect = main_dialog.findChild(QWidget, "btnConnect")
        self.assertIsNotNone(btnConnect)
        QTest.mouseClick(btnConnect, Qt.LeftButton)

        # We need to show (and raise for Mac) the dialog so that the focus changes
        self.loop = QEventLoop()
        treeView = main_dialog.findChild(QTreeView, "treeView")
        treeView.selectionModel().currentRowChanged.connect(main_dialog.hide)
        treeView.selectionModel().currentRowChanged.connect(self.loop.quit)
        main_dialog.show()
        main_dialog.raise_()
        self.loop.exec_()

        # Add layer
        buttonAdd = self.get_button_add(main_dialog)
        self.assertTrue(buttonAdd.isEnabled())

        self.addWfsLayer_uri = None
        self.addWfsLayer_layer_name = None
        main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
        QTest.mouseClick(buttonAdd, Qt.LeftButton)
        self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=')
        self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')

        # Click on Build Query
        buttonBuildQuery = self.get_button_build_query(main_dialog)
        self.assertTrue(buttonBuildQuery.isEnabled())
        QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
        error_box = find_window('WFSFeatureTypeErrorBox')
        self.assertIsNotNone(error_box)
        # Close error box
        error_box.accept()
        # Wait for object to be destroyed
        error_box = self.wait_object_destruction(error_box)

        # Click again but with valid DescribeFeatureType

        with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'), 'wb') as f:
            f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
  <xsd:import namespace="http://www.opengis.net/gml/3.2"/>
  <xsd:complexType name="typenameType">
    <xsd:complexContent>
      <xsd:extension base="gml:AbstractFeatureType">
        <xsd:sequence>
          <xsd:element maxOccurs="1" minOccurs="0" name="intfield" nillable="true" type="xsd:int"/>
          <xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PolygonPropertyType"/>
        </xsd:sequence>
      </xsd:extension>
    </xsd:complexContent>
  </xsd:complexType>
  <xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
        QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)

        # Check that the combos are properly initialized
        dialog = find_window('QgsSQLComposerDialogBase')
        self.assertIsNotNone(dialog)

        mTablesCombo = dialog.findChild(QComboBox, "mTablesCombo")
        self.assertIsNotNone(mTablesCombo)
        self.assertEqual(mTablesCombo.itemText(1), 'typename (Title)')

        mColumnsCombo = dialog.findChild(QComboBox, "mColumnsCombo")
        self.assertIsNotNone(mColumnsCombo)
        self.assertEqual(mColumnsCombo.itemText(1), 'intfield (int)')
        self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 2), 'geometryProperty (geometry)')
        self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 1), '*')

        mFunctionsCombo = dialog.findChild(QComboBox, "mFunctionsCombo")
        self.assertIsNotNone(mFunctionsCombo)
        self.assertEqual(mFunctionsCombo.itemText(1), 'abs(param: int): int')

        mSpatialPredicatesCombo = dialog.findChild(QComboBox, "mSpatialPredicatesCombo")
        self.assertIsNotNone(mSpatialPredicatesCombo)
        self.assertEqual(mSpatialPredicatesCombo.itemText(1), 'ST_Disjoint(geometry, geometry): boolean')

        mWhereEditor = dialog.findChild(QTextEdit, "mWhereEditor")
        self.assertIsNotNone(mWhereEditor)
        mWhereEditor.setText('1 = 1')

        dialog.accept()
        # Wait for object to be destroyed
        dialog = self.wait_object_destruction(dialog)

        # Add layer
        buttonAdd = self.get_button_add(main_dialog)
        self.assertTrue(buttonAdd.isEnabled())

        self.addWfsLayer_uri = None
        self.addWfsLayer_layer_name = None
        main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
        QTest.mouseClick(buttonAdd, Qt.LeftButton)
        self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=SELECT * FROM typename WHERE 1 = 1')
        self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
Exemple #41
0
class NetworkAccessManager():
    """
    This class mimicks httplib2 by using QgsNetworkAccessManager for all
    network calls.

    The return value is a tuple of (response, content), the first being and
    instance of the Response class, the second being a string that contains
    the response entity body.

    Parameters
    ----------
    debug : bool
        verbose logging if True
    exception_class : Exception
        Custom exception class

    Usage
    -----
    ::
        nam = NetworkAccessManager(authcgf)
        try:
            (response, content) = nam.request('http://www.example.com')
        except RequestsException, e:
            # Handle exception
            pass


    """
    def __init__(self,
                 authid=None,
                 disable_ssl_certificate_validation=False,
                 exception_class=None,
                 debug=True):
        self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
        self.authid = authid
        self.reply = None
        self.debug = debug
        self.exception_class = exception_class
        self.cookie = None
        self.basicauth = None

    def setBasicauth(self, encodedString):
        self.basicauth = encodedString

    def setCookie(self, cookie):
        self.cookie = cookie

    def msg_log(self, msg):
        if self.debug:
            QgsMessageLog.logMessage(msg, "NetworkAccessManager")

    def request(self,
                url,
                method="GET",
                body=None,
                headers=None,
                redirections=DEFAULT_MAX_REDIRECTS,
                connection_type=None):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))
        self.http_call_result = Response({
            'status': 0,
            'status_code': 0,
            'status_message': '',
            'text': '',
            'ok': False,
            'headers': {},
            'reason': '',
            'exception': None,
        })
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        if PYTHON_VERSION >= 3:
            url = urllib.parse.unquote(url)
        else:
            url = urllib2.unquote(url)
        req.setUrl(QUrl(url))

        if self.cookie is not None:
            if headers is not None:
                headers['Cookie'] = self.cookie
            else:
                headers = {'Cookie': self.cookie}

        if self.basicauth is not None:
            if headers is not None:
                headers['Authorization'] = self.basicauth
            else:
                headers = {'Authorization': self.basicauth}

        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in headers.items():
                if PYTHON_VERSION >= 3:
                    if isinstance(k, str):
                        k = k.encode('utf-8')
                    if isinstance(v, str):
                        v = v.encode('utf-8')
                req.setRawHeader(k, v)

        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(),
                           'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" %
                     (method.upper(), req.url().toString()))
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in headers.items():
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if PYTHON_VERSION >= 3:
                if isinstance(body, str):
                    body = body.encode('utf-8')
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(
                self.reply, self.authid)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_()
            # Let's log the whole response for debugging purposes:
            self.msg_log("Got response %s %s from %s" % \
                        (self.http_call_result.status_code,
                         self.http_call_result.status_message,
                         self.reply.url().toString()))
            headers = {
                str(h): str(self.reply.rawHeader(h))
                for h in self.reply.rawHeaderList()
            }
            for k, v in headers.items():
                self.msg_log("%s: %s" % (k, v))
            if len(self.http_call_result.text) < 1024:
                self.msg_log("Payload :\n%s" % self.http_call_result.text)
            else:
                self.msg_log("Payload is > 1 KB ...")
        except Exception as e:
            raise e
        finally:
            if self.reply is not None:
                if self.reply.isRunning():
                    self.reply.close()
                self.msg_log("Deleting reply ...")
                self.reply.deleteLater()
                self.reply = None
            else:
                self.msg_log("Reply was already deleted ...")
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)
        return (self.http_call_result, self.http_call_result.text)

    #@pyqtSlot()
    def downloadProgress(self, bytesReceived, bytesTotal):
        """Keep track of the download progress"""
        #self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
        pass

    #@pyqtSlot()
    def replyFinished(self):
        err = self.reply.error()
        httpStatus = self.reply.attribute(
            QNetworkRequest.HttpStatusCodeAttribute)
        httpStatusMessage = self.reply.attribute(
            QNetworkRequest.HttpReasonPhraseAttribute)
        self.http_call_result.status_code = httpStatus
        self.http_call_result.status = httpStatus
        self.http_call_result.status_message = httpStatusMessage
        for k, v in self.reply.rawHeaderPairs():
            self.http_call_result.headers[str(k)] = str(v)
            self.http_call_result.headers[str(k).lower()] = str(v)
        if err != QNetworkReply.NoError:
            msg = "Network error #{0}: {1}".format(self.reply.error(),
                                                   self.reply.errorString())
            self.http_call_result.reason = msg
            self.http_call_result.ok = False
            self.msg_log(msg)
            if err == QNetworkReply.TimeoutError:
                self.http_call_result.exception = RequestsExceptionTimeout(msg)
            elif err == QNetworkReply.ConnectionRefusedError:
                self.http_call_result.exception = RequestsExceptionConnectionError(
                    msg)
            else:
                self.http_call_result.exception = RequestsException(msg)
        else:
            # since Python 3 readAll() returns a PyQt5.QByteArray, we
            # want only the data
            if PYTHON_VERSION >= 3:
                self.http_call_result.text = self.reply.readAll().data(
                ).decode('utf-8')
            else:
                self.http_call_result.text = str(self.reply.readAll())
            self.http_call_result.ok = True
        self.reply.deleteLater()

    #@pyqtSlot()
    def sslErrors(self, reply, ssl_errors):
        """
        Handle SSL errors, logging them if debug is on and ignoring them
        if disable_ssl_certificate_validation is set.
        """
        if ssl_errors:
            for v in ssl_errors:
                self.msg_log("SSL Error: %s" % v)
        if self.disable_ssl_certificate_validation:
            reply.ignoreSslErrors()
Exemple #42
0
    def request(self,
                url,
                method="GET",
                body=None,
                headers=None,
                redirections=DEFAULT_MAX_REDIRECTS,
                connection_type=None):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))
        self.http_call_result = Response({
            'status': 0,
            'status_code': 0,
            'status_message': '',
            'text': '',
            'ok': False,
            'headers': {},
            'reason': '',
            'exception': None,
        })
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        if PYTHON_VERSION >= 3:
            url = urllib.parse.unquote(url)
        else:
            url = urllib2.unquote(url)
        req.setUrl(QUrl(url))

        if self.cookie is not None:
            if headers is not None:
                headers['Cookie'] = self.cookie
            else:
                headers = {'Cookie': self.cookie}

        if self.basicauth is not None:
            if headers is not None:
                headers['Authorization'] = self.basicauth
            else:
                headers = {'Authorization': self.basicauth}

        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in headers.items():
                if PYTHON_VERSION >= 3:
                    if isinstance(k, str):
                        k = k.encode('utf-8')
                    if isinstance(v, str):
                        v = v.encode('utf-8')
                req.setRawHeader(k, v)

        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(),
                           'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" %
                     (method.upper(), req.url().toString()))
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in headers.items():
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if PYTHON_VERSION >= 3:
                if isinstance(body, str):
                    body = body.encode('utf-8')
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(
                self.reply, self.authid)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_()
            # Let's log the whole response for debugging purposes:
            self.msg_log("Got response %s %s from %s" % \
                        (self.http_call_result.status_code,
                         self.http_call_result.status_message,
                         self.reply.url().toString()))
            headers = {
                str(h): str(self.reply.rawHeader(h))
                for h in self.reply.rawHeaderList()
            }
            for k, v in headers.items():
                self.msg_log("%s: %s" % (k, v))
            if len(self.http_call_result.text) < 1024:
                self.msg_log("Payload :\n%s" % self.http_call_result.text)
            else:
                self.msg_log("Payload is > 1 KB ...")
        except Exception as e:
            raise e
        finally:
            if self.reply is not None:
                if self.reply.isRunning():
                    self.reply.close()
                self.msg_log("Deleting reply ...")
                self.reply.deleteLater()
                self.reply = None
            else:
                self.msg_log("Reply was already deleted ...")
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)
        return (self.http_call_result, self.http_call_result.text)
Exemple #43
0
    def testStoreFetchFileImmediately(self):
        """
        Test file storing and fetching (Immediately mode)
        """

        f = self.getNewFile(b"New content")

        # store
        url = self.url + "/" + os.path.basename(f.name)
        storedContent = self.storage.store(f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate)
        self.assertTrue(storedContent)
        self.assertEqual(storedContent.status(), Qgis.ContentStatus.Running)

        spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
        spyProgressChanged = QSignalSpy(storedContent.progressChanged)

        loop = QEventLoop()
        storedContent.stored.connect(loop.quit)
        storedContent.errorOccurred.connect(loop.quit)
        loop.exec()

        self.assertEqual(len(spyErrorOccurred), 0)
        self.assertEqual(storedContent.url(), url)
        self.assertFalse(storedContent.errorString())
        self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
        self.assertTrue(len(spyProgressChanged) > 0)
        self.assertEqual(spyProgressChanged[-1][0], 100)

        # fetch
        fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
        self.assertTrue(fetchedContent)

        # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
        self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Finished or
                        fetchedContent.status() == Qgis.ContentStatus.Running)

        if (fetchedContent.status() == Qgis.ContentStatus.Running):

            spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)

            loop = QEventLoop()
            fetchedContent.fetched.connect(loop.quit)
            fetchedContent.errorOccurred.connect(loop.quit)
            loop.exec()

            self.assertEqual(len(spyErrorOccurred), 0)

        self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
        self.assertFalse(fetchedContent.errorString())
        self.assertTrue(fetchedContent.filePath())
        self.checkContent(fetchedContent.filePath(), b"New content")
        self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')

        # fetch again, should be cached
        fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
        self.assertTrue(fetchedContent)
        self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)

        self.assertTrue(not fetchedContent.errorString())
        self.assertTrue(fetchedContent.filePath())
        self.checkContent(fetchedContent.filePath(), b"New content")
        self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')

        # fetch bad url
        fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate)
        self.assertTrue(fetchedContent)

        # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
        self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Failed or
                        fetchedContent.status() == Qgis.ContentStatus.Running)

        if (fetchedContent.status() == Qgis.ContentStatus.Running):
            spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)

            loop = QEventLoop()
            fetchedContent.errorOccurred.connect(loop.quit)
            fetchedContent.fetched.connect(loop.quit)
            loop.exec()

            self.assertEqual(len(spyErrorOccurred), 1)

        self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Failed)
        self.assertTrue(fetchedContent.errorString())
        self.assertFalse(fetchedContent.filePath())
Exemple #44
0
def getVideoLocationInfo(videoPath, dataFile=False):
    """ Get basic location info about the video """
    location = []

    try:
        if dataFile == False:
            p = _spawn([
                '-i', videoPath, '-ss', '00:00:00', '-to', '00:00:01', '-map',
                'data-re', '-f', 'data', '-'
            ])

            stdout_data, _ = p.communicate()

        ################
        else:
            global listOfMetadata
            stdout_data = listOfMetadata[0]
        ################

        if stdout_data == b'':
            return
        for packet in StreamParser(stdout_data):
            if isinstance(packet, UnknownElement):
                qgsu.showUserAndLogMessage(
                    "Error interpreting klv data, metadata cannot be read.",
                    "the parser did not recognize KLV data",
                    level=QGis.Warning)
                continue
            packet.MetadataList()
            frameCenterLat = packet.FrameCenterLatitude
            frameCenterLon = packet.FrameCenterLongitude
            loc = "-"

            if Reverse_geocoding_url != "":
                try:
                    url = QUrl(
                        Reverse_geocoding_url.format(str(frameCenterLat),
                                                     str(frameCenterLon)))
                    request = QNetworkRequest(url)
                    reply = QgsNetworkAccessManager.instance().get(request)
                    loop = QEventLoop()
                    reply.finished.connect(loop.quit)
                    loop.exec_()
                    reply.finished.disconnect(loop.quit)
                    loop = None
                    result = reply.readAll()
                    data = json.loads(result.data())

                    if "village" in data["address"] and "state" in data[
                            "address"]:
                        loc = data["address"]["village"] + \
                            ", " + data["address"]["state"]
                    elif "town" in data["address"] and "state" in data[
                            "address"]:
                        loc = data["address"]["town"] + \
                            ", " + data["address"]["state"]
                    else:
                        loc = data["display_name"]

                except Exception:
                    qgsu.showUserAndLogMessage(
                        "",
                        "getVideoLocationInfo: failed to get address from reverse geocoding service.",
                        onlyLog=True)

            location = [frameCenterLat, frameCenterLon, loc]

            qgsu.showUserAndLogMessage(
                "",
                "Got Location: lon: " + str(frameCenterLon) + " lat: " +
                str(frameCenterLat) + " location: " + str(loc),
                onlyLog=True)

            break
        else:

            qgsu.showUserAndLogMessage(
                QCoreApplication.translate(
                    "QgsFmvUtils", "This video doesn't have Metadata ! : "))

    except Exception as e:
        qgsu.showUserAndLogMessage(
            QCoreApplication.translate("QgsFmvUtils",
                                       "Video info callback failed! : "),
            str(e))

    return location
class Downloader(QObject):

    NOT_FOUND = 0
    NO_ERROR = 0
    TIMEOUT_ERROR = 4
    UNKNOWN_ERROR = -1

    replyFinished = pyqtSignal(str, int, int)

    def __init__(self, parent=None):
        QObject.__init__(self, parent)
        self.queue = []
        self.redirected_urls = {}
        self.requestingUrls = []
        self.replies = []

        self.eventLoop = QEventLoop()
        self.sync = False
        self.fetchedFiles = {}
        self.clearCounts()

        self.timer = QTimer()
        self.timer.setSingleShot(True)
        self.timer.timeout.connect(self.fetchTimedOut)

        # network settings
        self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
        self.max_connection = 4
        self.default_cache_expiration = 24
        self.errorStatus = Downloader.NO_ERROR

    def clearCounts(self):
        self.fetchSuccesses = 0
        self.fetchErrors = 0
        self.cacheHits = 0

    def fetchTimedOut(self):
        self.log("Downloader.timeOut()")
        self.abort()
        self.errorStatus = Downloader.TIMEOUT_ERROR

    def abort(self):
        # clear queue and abort sent requests
        self.queue = []
        self.timer.stop()
        for reply in self.replies:
            reply.abort()
        self.errorStatus = Downloader.UNKNOWN_ERROR

    def replyFinishedSlot(self):
        reply = self.sender()
        url = reply.request().url().toString()
        self.log("replyFinishedSlot: %s" % url)
        if not url in self.fetchedFiles:
            self.fetchedFiles[url] = None
        self.requestingUrls.remove(url)
        self.replies.remove(reply)
        isFromCache = 0
        httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
        if reply.error() == QNetworkReply.NoError:
            if httpStatusCode == 301:
                new_url = str(reply.rawHeader("Location"))
                self.addToQueue(new_url, url)
            else:
                self.fetchSuccesses += 1
                if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
                    self.cacheHits += 1
                    isFromCache = 1
                elif not reply.hasRawHeader("Cache-Control"):
                    cache = QgsNetworkAccessManager.instance().cache()
                    if cache:
                        metadata = cache.metaData(reply.request().url())
                        # self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
                        if metadata.expirationDate().isNull():
                            metadata.setExpirationDate(
                                QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
                            cache.updateMetaData(metadata)
                            self.log(
                                "Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))

                if reply.isReadable():
                    data = reply.readAll()
                    if self.redirected_urls.has_key(url):
                        url = self.redirected_urls[url]

                    self.fetchedFiles[url] = data
                else:
                    qDebug("http status code: " + str(httpStatusCode))

                # self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
                self.replyFinished.emit(url, reply.error(), isFromCache)
        else:
            if self.sync and httpStatusCode == 404:
                self.fetchedFiles[url] = self.NOT_FOUND
            self.fetchErrors += 1
            if self.errorStatus == self.NO_ERROR:
                self.errorStatus = self.UNKNOWN_ERROR

        reply.deleteLater()

        if debug_mode:
            qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))

        if len(self.queue) + len(self.requestingUrls) == 0:
            # all replies have been received
            if self.sync:
                self.logT("eventLoop.quit()")
                self.eventLoop.quit()
            else:
                self.timer.stop()
        elif len(self.queue) > 0:
            # start fetching the next file
            self.fetchNext()
        self.log("replyFinishedSlot End: %s" % url)

    def fetchNext(self):
        if len(self.queue) == 0:
            return
        url = self.queue.pop(0)
        self.log("fetchNext: %s" % url)

        request = QNetworkRequest(QUrl(url))
        request.setRawHeader("User-Agent", self.userAgent)
        reply = QgsNetworkAccessManager.instance().get(request)
        reply.finished.connect(self.replyFinishedSlot)
        self.requestingUrls.append(url)
        self.replies.append(reply)
        return reply

    def fetchFiles(self, urlList, timeout_ms=0):
        self.log("fetchFiles()")
        self.sync = True
        self.queue = []
        self.redirected_urls = {} 
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()

        self.logT("eventLoop.exec_(): " + str(self.eventLoop))
        self.eventLoop.exec_()
        self.log("fetchFiles() End: %d" % self.errorStatus)
        if timeout_ms > 0:
            self.timer.stop()
        return self.fetchedFiles

    def addToQueue(self, url, redirected_from=None):
        if url in self.queue:
            return False
        self.queue.append(url)
        if redirected_from is not None:
            self.redirected_urls[url] = redirected_from
        return True

    def queueCount(self):
        return len(self.queue)

    def finishedCount(self):
        return len(self.fetchedFiles)

    def unfinishedCount(self):
        return len(self.queue) + len(self.requestingUrls)

    def log(self, msg):
        if debug_mode:
            qDebug(msg)

    def logT(self, msg):
        if debug_mode:
            qDebug("%s: %s" % (str(threading.current_thread()), msg))

    def fetchFilesAsync(self, urlList, timeout_ms=0):
        self.log("fetchFilesAsync()")
        self.sync = False
        self.queue = []
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()