def wait_object_destruction(self, my_object):
     loop = QEventLoop()
     name = my_object.objectName()
     my_object.destroyed.connect(loop.quit)
     loop.exec_()
     self.assertIsNone(find_window(name))
     return None
    def testInvalidAuthFileDownload(self):
        """
        Download a protected map tile without authcfg
        """
        qs = "?" + "&".join(["%s=%s" % i for i in list({
            "MAP": urllib.parse.quote(self.project_path),
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetMap",
            "LAYERS": "testlayer_èé".replace('_', '%20'),
            "STYLES": "",
            "FORMAT": "image/png",
            "BBOX": "-16817707,-4710778,5696513,14587125",
            "HEIGHT": "500",
            "WIDTH": "500",
            "CRS": "EPSG:3857"
        }.items())])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, None, False)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        self.assertTrue(self.error_was_called)
        self.assertTrue("Download failed: Host requires authentication" in str(self.error_args), "Error args is: %s" % str(self.error_args))
    def __sync_request(self, url):
        _url = QUrl(url)
        _request = QNetworkRequest(_url)
        self.__replies.append(_request)

        QgsNetworkAccessManager.instance().sslErrors.connect(self.__supress_ssl_errors)

        _reply = QgsNetworkAccessManager.instance().get(_request)

        # wait
        loop = QEventLoop()
        _reply.finished.connect(loop.quit)
        loop.exec_()
        _reply.finished.disconnect(loop.quit)
        QgsNetworkAccessManager.instance().sslErrors.disconnect(self.__supress_ssl_errors)
        loop = None

        error = _reply.error()
        if error != QNetworkReply.NoError:
            raise Exception(error)

        result_code = _reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)

        result = _reply.readAll()
        self.__replies.append(_reply)
        _reply.deleteLater()

        if result_code in [301, 302, 307]:
            redirect_url = _reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
            return self.__sync_request(redirect_url)
        else:
            return result
Beispiel #4
0
    def run(self):
        """Run the query.

        @raise OverpassBadRequestException,NetWorkErrorException,
        OverpassTimeoutException

        @return: The result of the query.
        @rtype: str
        """
        loop = QEventLoop()
        downloader = QgsFileDownloader(
            self._url, self.result_path, delayStart=True)
        downloader.downloadExited.connect(loop.quit)
        downloader.downloadError.connect(self.error)
        downloader.downloadCanceled.connect(self.canceled)
        downloader.downloadCompleted.connect(self.completed)
        downloader.startDownload()
        loop.exec_()

        file_obj = codecs.open(self.result_path, 'r', 'utf-8')
        file_obj.seek(0, 2)
        fsize = file_obj.tell()
        file_obj.seek(max(fsize - 1024, 0), 0)
        lines = file_obj.readlines()
        file_obj.close()

        lines = lines[-10:]  # Get last 10 lines
        timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
                  'at line [\d]+ after ([\d]+) seconds. </remark>'
        if re.search(timeout, ''.join(lines)):
            raise OverpassTimeoutException
        else:
            return self.result_path
Beispiel #5
0
    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")
Beispiel #6
0
    def fetchFiles(self, urls):
        self.logT("TileLayer.fetchFiles() starts")
        # create a QEventLoop object that belongs to the current thread (if ver. > 2.1, it is render thread)
        eventLoop = QEventLoop()
        self.logT("Create event loop: " + str(eventLoop))  # DEBUG
        # QObject.connect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
        self.allRepliesFinished.connect(eventLoop.quit)

        # create a timer to watch whether rendering is stopped
        watchTimer = QTimer()
        watchTimer.timeout.connect(eventLoop.quit)

        # send a fetch request to the main thread
        # self.emit(SIGNAL("fetchRequest(QStringList)"), urls)
        self.fetchRequest.emit(urls)

        # wait for the fetch to finish
        tick = 0
        interval = 500
        timeoutTick = self.downloadTimeout / interval
        watchTimer.start(interval)
        while tick < timeoutTick:
            # run event loop for 0.5 seconds at maximum
            eventLoop.exec_()

            if debug_mode:
                qDebug("watchTimerTick: %d" % tick)
                qDebug("unfinished downloads: %d" % self.downloader.unfinishedCount())

            if self.downloader.unfinishedCount() == 0 or self.renderContext.renderingStopped():
                break
            tick += 1
        watchTimer.stop()

        if tick == timeoutTick and self.downloader.unfinishedCount() > 0:
            self.log("fetchFiles timeout")
            # self.emitShowBarMessage("fetchFiles timeout", duration=5)   #DEBUG
            self.downloader.abort()
            self.downloader.errorStatus = Downloader.TIMEOUT_ERROR
        files = self.downloader.fetchedFiles

        watchTimer.timeout.disconnect(eventLoop.quit)  #
        # QObject.disconnect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
        self.allRepliesFinished.disconnect(eventLoop.quit)

        self.logT("TileLayer.fetchFiles() ends")
        return files
def _sync_get(url):
    global __network_manager
    if __network_manager is None:
        __network_manager = QNetworkAccessManager()
        __network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
    pause = QEventLoop()
    req = QNetworkRequest(url)
    req.setRawHeader(b"Accept", b"application/xml")
    req.setRawHeader(b"Accept-Language", bytes(settings.value("default_language", "fr"), "utf8"))
    req.setRawHeader(b"User-Agent", bytes(settings.value('http_user_agent', plugin_name()), "utf8"))
    reply = __network_manager.get(req)
    reply.finished.connect(pause.quit)
    is_ok = [True]
    def onError(self):
        is_ok[0] = False
        pause.quit()
    reply.error.connect(onError)
    pause.exec_()
    return reply, is_ok[0]
class OpenlayersRenderer(QgsMapLayerRenderer):
    def __init__(self, layer, context, webPage, layerType):
        """ Initialize the object. This function is still run in the GUI thread.
            Should refrain from doing any heavy work.
        """
        QgsMapLayerRenderer.__init__(self, layer.id())
        self.context = context
        self.controller = OpenlayersController(None,
                                               context, webPage, layerType)
        self.loop = None

    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True

    def onTimeout(self):
        """ periodically check whether the rendering should not be stopped """
        if self.context.renderingStopped():
            debug("[WORKER THREAD] Cancelling rendering", 3)
            self.loop.exit()
    def _make_download(self, url, destination, cancel=False):
        self.completed_was_called = False
        self.error_was_called = False
        self.canceled_was_called = False
        self.progress_was_called = False
        self.exited_was_called = False

        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        if cancel:
            downloader.downloadProgress.connect(downloader.cancelDownload)

        loop.exec_()
    def testValidAuthFileDownload(self):
        """
        Download a map tile with valid authcfg
        """
        qs = "?" + "&".join(["%s=%s" % i for i in list({
            "MAP": urllib.parse.quote(self.project_path),
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetMap",
            "LAYERS": "testlayer_èé".replace('_', '%20'),
            "STYLES": "",
            "FORMAT": "image/png",
            "BBOX": "-16817707,-4710778,5696513,14587125",
            "HEIGHT": "500",
            "WIDTH": "500",
            "CRS": "EPSG:3857"
        }.items())])
        url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)

        destination = tempfile.mktemp()
        loop = QEventLoop()

        downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
        downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
        downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
        downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
        downloader.downloadError.connect(partial(self._set_slot, 'error'))
        downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))

        downloader.downloadExited.connect(loop.quit)

        loop.exec_()

        # Check the we've got a likely PNG image
        self.assertTrue(self.completed_was_called)
        self.assertTrue(os.path.getsize(destination) > 700000, "Image size: %s" % os.path.getsize(destination))  # > 1MB
        with open(destination, 'rb') as f:
            self.assertTrue(b'PNG' in f.read())  # is a PNG
Beispiel #11
0
    def __init__(self, parent=None):
        QObject.__init__(self, parent)
        self.queue = []
        self.redirected_urls = {}
        self.requestingUrls = []
        self.replies = []

        self.eventLoop = QEventLoop()
        self.sync = False
        self.fetchedFiles = {}
        self.clearCounts()

        self.timer = QTimer()
        self.timer.setSingleShot(True)
        self.timer.timeout.connect(self.fetchTimedOut)

        # network settings
        self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
        self.max_connection = 4
        self.default_cache_expiration = 24
        self.errorStatus = Downloader.NO_ERROR
    def render(self):
        """ do the rendering. This function is called in the worker thread """

        debug("[WORKER THREAD] Calling request() asynchronously", 3)
        QMetaObject.invokeMethod(self.controller, "request")

        # setup a timer that checks whether the rendering has not been stopped
        # in the meanwhile
        timer = QTimer()
        timer.setInterval(50)
        timer.timeout.connect(self.onTimeout)
        timer.start()

        debug("[WORKER THREAD] Waiting for the async request to complete", 3)
        self.loop = QEventLoop()
        self.controller.finished.connect(self.loop.exit)
        self.loop.exec_()

        debug("[WORKER THREAD] Async request finished", 3)

        painter = self.context.painter()
        painter.drawImage(0, 0, self.controller.img)
        return True
    def test(self):

        # This test is quite fragile as it depends on windows manager behaviour
        # regarding focus, so not surprising it doesn't pass
        # on other platforms than Linux.
        #if 'TRAVIS_OS_NAME' in os.environ and os.environ['TRAVIS_OS_NAME'] == 'osx':
        #    return

        main_dialog = QgsProviderRegistry.instance().selectWidget("WFS")
        main_dialog.setProperty("hideDialogs", True)

        self.assertIsNotNone(main_dialog)

        # Create new connection
        btnNew = main_dialog.findChild(QWidget, "btnNew")
        self.assertIsNotNone(btnNew)
        QTest.mouseClick(btnNew, Qt.LeftButton)
        new_conn = find_window('QgsNewHttpConnectionBase')
        self.assertIsNotNone(new_conn)
        txtName = new_conn.findChild(QLineEdit, "txtName")
        self.assertIsNotNone(txtName)
        txtName.setText("test_connection")
        txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
        self.assertIsNotNone(txtUrl)
        txtUrl.setText("test_url")
        new_conn.accept()

        # Wait for object to be destroyed
        new_conn = self.wait_object_destruction(new_conn)

        # Try to connect
        btnConnect = main_dialog.findChild(QWidget, "btnConnect")
        self.assertIsNotNone(btnConnect)
        QTest.mouseClick(btnConnect, Qt.LeftButton)
        # Depends on asynchronous signal
        QApplication.processEvents()
        error_box = find_window('WFSCapabilitiesErrorBox')
        self.assertIsNotNone(error_box)
        # Close error box
        error_box.accept()

        # Wait for object to be destroyed
        error_box = self.wait_object_destruction(error_box)

        # Edit connection
        btnEdit = main_dialog.findChild(QWidget, "btnEdit")
        self.assertIsNotNone(btnEdit)
        QTest.mouseClick(btnEdit, Qt.LeftButton)
        new_conn = find_window('QgsNewHttpConnectionBase',)
        self.assertIsNotNone(new_conn)
        txtName = new_conn.findChild(QLineEdit, "txtName")
        self.assertIsNotNone(txtName)
        txtName.setText("test_connection")
        txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
        self.assertIsNotNone(txtUrl)

        endpoint = self.basetestpath + '/fake_qgis_http_endpoint'
        expected_endpoint = endpoint
        if sys.platform == 'win32' and expected_endpoint[1] == ':':
            expected_endpoint = expected_endpoint[0] + expected_endpoint[2:]
        with open(sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'), 'wb') as f:
            f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
  <FeatureTypeList>
    <FeatureType>
      <Name>my:typename</Name>
      <Title>Title</Title>
      <Abstract>Abstract</Abstract>
      <DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
      <ows:WGS84BoundingBox>
        <ows:LowerCorner>-71.123 66.33</ows:LowerCorner>
        <ows:UpperCorner>-65.32 78.3</ows:UpperCorner>
      </ows:WGS84BoundingBox>
    </FeatureType>
  </FeatureTypeList>
  <fes:Filter_Capabilities>
    <fes:Spatial_Capabilities>
      <fes:GeometryOperands>
        <fes:GeometryOperand name="gml:Envelope"/>
        <fes:GeometryOperand name="gml:Point"/>
        <fes:GeometryOperand name="gml:MultiPoint"/>
        <fes:GeometryOperand name="gml:LineString"/>
        <fes:GeometryOperand name="gml:MultiLineString"/>
        <fes:GeometryOperand name="gml:Polygon"/>
        <fes:GeometryOperand name="gml:MultiPolygon"/>
        <fes:GeometryOperand name="gml:MultiGeometry"/>
      </fes:GeometryOperands>
      <fes:SpatialOperators>
        <fes:SpatialOperator name="Disjoint"/>
        <fes:SpatialOperator name="Equals"/>
        <fes:SpatialOperator name="DWithin"/>
        <fes:SpatialOperator name="Beyond"/>
        <fes:SpatialOperator name="Intersects"/>
        <fes:SpatialOperator name="Touches"/>
        <fes:SpatialOperator name="Crosses"/>
        <fes:SpatialOperator name="Within"/>
        <fes:SpatialOperator name="Contains"/>
        <fes:SpatialOperator name="Overlaps"/>
        <fes:SpatialOperator name="BBOX"/>
      </fes:SpatialOperators>
    </fes:Spatial_Capabilities>
    <fes:Functions>
      <fes:Function name="abs">
        <fes:Returns>xs:int</fes:Returns>
        <fes:Arguments>
          <fes:Argument name="param">
            <fes:Type>xs:int</fes:Type>
          </fes:Argument>
        </fes:Arguments>
      </fes:Function>
    </fes:Functions>
  </fes:Filter_Capabilities>
</wfs:WFS_Capabilities>""".encode('UTF-8'))

        txtUrl.setText("http://" + endpoint)
        new_conn.accept()

        # Wait for object to be destroyed
        new_conn = self.wait_object_destruction(new_conn)

        # Try to connect
        btnConnect = main_dialog.findChild(QWidget, "btnConnect")
        self.assertIsNotNone(btnConnect)
        QTest.mouseClick(btnConnect, Qt.LeftButton)

        # We need to show (and raise for Mac) the dialog so that the focus changes
        self.loop = QEventLoop()
        treeView = main_dialog.findChild(QTreeView, "treeView")
        treeView.selectionModel().currentRowChanged.connect(main_dialog.hide)
        treeView.selectionModel().currentRowChanged.connect(self.loop.quit)
        main_dialog.show()
        main_dialog.raise_()
        self.loop.exec_()

        # Add layer
        buttonAdd = self.get_button_add(main_dialog)
        self.assertTrue(buttonAdd.isEnabled())

        self.addWfsLayer_uri = None
        self.addWfsLayer_layer_name = None
        main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
        QTest.mouseClick(buttonAdd, Qt.LeftButton)
        self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=')
        self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')

        # Click on Build Query
        buttonBuildQuery = self.get_button_build_query(main_dialog)
        self.assertTrue(buttonBuildQuery.isEnabled())
        QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
        error_box = find_window('WFSFeatureTypeErrorBox')
        self.assertIsNotNone(error_box)
        # Close error box
        error_box.accept()
        # Wait for object to be destroyed
        error_box = self.wait_object_destruction(error_box)

        # Click again but with valid DescribeFeatureType

        with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'), 'wb') as f:
            f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
  <xsd:import namespace="http://www.opengis.net/gml/3.2"/>
  <xsd:complexType name="typenameType">
    <xsd:complexContent>
      <xsd:extension base="gml:AbstractFeatureType">
        <xsd:sequence>
          <xsd:element maxOccurs="1" minOccurs="0" name="intfield" nillable="true" type="xsd:int"/>
          <xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PolygonPropertyType"/>
        </xsd:sequence>
      </xsd:extension>
    </xsd:complexContent>
  </xsd:complexType>
  <xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
        QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)

        # Check that the combos are properly initialized
        dialog = find_window('QgsSQLComposerDialogBase')
        self.assertIsNotNone(dialog)

        mTablesCombo = dialog.findChild(QComboBox, "mTablesCombo")
        self.assertIsNotNone(mTablesCombo)
        self.assertEqual(mTablesCombo.itemText(1), 'typename (Title)')

        mColumnsCombo = dialog.findChild(QComboBox, "mColumnsCombo")
        self.assertIsNotNone(mColumnsCombo)
        self.assertEqual(mColumnsCombo.itemText(1), 'intfield (int)')
        self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 2), 'geometryProperty (geometry)')
        self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 1), '*')

        mFunctionsCombo = dialog.findChild(QComboBox, "mFunctionsCombo")
        self.assertIsNotNone(mFunctionsCombo)
        self.assertEqual(mFunctionsCombo.itemText(1), 'abs(param: int): int')

        mSpatialPredicatesCombo = dialog.findChild(QComboBox, "mSpatialPredicatesCombo")
        self.assertIsNotNone(mSpatialPredicatesCombo)
        self.assertEqual(mSpatialPredicatesCombo.itemText(1), 'ST_Disjoint(geometry, geometry): boolean')

        mWhereEditor = dialog.findChild(QTextEdit, "mWhereEditor")
        self.assertIsNotNone(mWhereEditor)
        mWhereEditor.setText('1 = 1')

        dialog.accept()
        # Wait for object to be destroyed
        dialog = self.wait_object_destruction(dialog)

        # Add layer
        buttonAdd = self.get_button_add(main_dialog)
        self.assertTrue(buttonAdd.isEnabled())

        self.addWfsLayer_uri = None
        self.addWfsLayer_layer_name = None
        main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
        QTest.mouseClick(buttonAdd, Qt.LeftButton)
        self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=SELECT * FROM typename WHERE 1 = 1')
        self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
Beispiel #14
0
class Nominatim(object):
    """Manage connexion to Nominatim."""

    def __init__(self,
                 url="https://nominatim.openstreetmap.org/search?format=json"):
        """
        Constructor
        @param url:URL of Nominatim
        @type url:str
        """

        self.__url = url
        self.network = QgsNetworkAccessManager.instance()
        self.data = None
        self.network_reply = None
        self.loop = None

    def query(self, query):
        """
        Perform a nominatim query

        @param query: Query to execute
        @type query: str

        @raise NetWorkErrorException

        @return: the result of the query
        @rtype: str
        """

        url_query = QUrl(self.__url)

        # query = QUrl.toPercentEncoding(query)
        query_string = QUrlQuery()
        query_string.addQueryItem('q', query)
        query_string.addQueryItem('format', 'json')
        query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
        url_query.setQuery(query_string)

        request = QNetworkRequest(url_query)
        # request.setRawHeader("User-Agent", "QuickOSM")
        self.network_reply = self.network.get(request)
        self.loop = QEventLoop()
        self.network.finished.connect(self._end_of_request)
        self.loop.exec_()

        if self.network_reply.error() == QNetworkReply.NoError:
            return json.loads(self.data)
        else:
            raise NetWorkErrorException(suffix="Nominatim API")

    def _end_of_request(self):
        self.data = self.network_reply.readAll().data().decode('utf-8')
        self.loop.quit()

    def get_first_polygon_from_query(self, query):
        """
        Get first OSM_ID of a Nominatim area

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "relation":
                return result['osm_id']

        # If no result has been return
        raise NominatimAreaException

    def get_first_point_from_query(self, query):
        """
        Get first longitude, latitude of a Nominatim point

        @param query: Query to execute
        @type query: str

        @raise NominatimAreaException:

        @return: First relation's osm_id
        @rtype: str
        """
        data = self.query(query)
        for result in data:
            if result['osm_type'] == "node":
                return result['lon'], result['lat']

        # If no result has been return
        raise NominatimAreaException
Beispiel #15
0
class Downloader(QObject):

    NOT_FOUND = 0
    NO_ERROR = 0
    TIMEOUT_ERROR = 4
    UNKNOWN_ERROR = -1

    replyFinished = pyqtSignal(str, int, int)

    def __init__(self, parent=None):
        QObject.__init__(self, parent)
        self.queue = []
        self.redirected_urls = {}
        self.requestingUrls = []
        self.replies = []

        self.eventLoop = QEventLoop()
        self.sync = False
        self.fetchedFiles = {}
        self.clearCounts()

        self.timer = QTimer()
        self.timer.setSingleShot(True)
        self.timer.timeout.connect(self.fetchTimedOut)

        # network settings
        self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
        self.max_connection = 4
        self.default_cache_expiration = 24
        self.errorStatus = Downloader.NO_ERROR

    def clearCounts(self):
        self.fetchSuccesses = 0
        self.fetchErrors = 0
        self.cacheHits = 0

    def fetchTimedOut(self):
        self.log("Downloader.timeOut()")
        self.abort()
        self.errorStatus = Downloader.TIMEOUT_ERROR

    def abort(self):
        # clear queue and abort sent requests
        self.queue = []
        self.timer.stop()
        for reply in self.replies:
            reply.abort()
        self.errorStatus = Downloader.UNKNOWN_ERROR

    def replyFinishedSlot(self):
        reply = self.sender()
        url = reply.request().url().toString()
        self.log("replyFinishedSlot: %s" % url)
        if not url in self.fetchedFiles:
            self.fetchedFiles[url] = None
        self.requestingUrls.remove(url)
        self.replies.remove(reply)
        isFromCache = 0
        httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
        if reply.error() == QNetworkReply.NoError:
            if httpStatusCode == 301:
                new_url = str(reply.rawHeader("Location"))
                self.addToQueue(new_url, url)
            else:
                self.fetchSuccesses += 1
                if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
                    self.cacheHits += 1
                    isFromCache = 1
                elif not reply.hasRawHeader("Cache-Control"):
                    cache = QgsNetworkAccessManager.instance().cache()
                    if cache:
                        metadata = cache.metaData(reply.request().url())
                        # self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
                        if metadata.expirationDate().isNull():
                            metadata.setExpirationDate(
                                QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
                            cache.updateMetaData(metadata)
                            self.log(
                                "Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))

                if reply.isReadable():
                    data = reply.readAll()
                    if self.redirected_urls.has_key(url):
                        url = self.redirected_urls[url]

                    self.fetchedFiles[url] = data
                else:
                    qDebug("http status code: " + str(httpStatusCode))

                # self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
                self.replyFinished.emit(url, reply.error(), isFromCache)
        else:
            if self.sync and httpStatusCode == 404:
                self.fetchedFiles[url] = self.NOT_FOUND
            self.fetchErrors += 1
            if self.errorStatus == self.NO_ERROR:
                self.errorStatus = self.UNKNOWN_ERROR

        reply.deleteLater()

        if debug_mode:
            qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))

        if len(self.queue) + len(self.requestingUrls) == 0:
            # all replies have been received
            if self.sync:
                self.logT("eventLoop.quit()")
                self.eventLoop.quit()
            else:
                self.timer.stop()
        elif len(self.queue) > 0:
            # start fetching the next file
            self.fetchNext()
        self.log("replyFinishedSlot End: %s" % url)

    def fetchNext(self):
        if len(self.queue) == 0:
            return
        url = self.queue.pop(0)
        self.log("fetchNext: %s" % url)

        request = QNetworkRequest(QUrl(url))
        request.setRawHeader("User-Agent", self.userAgent)
        reply = QgsNetworkAccessManager.instance().get(request)
        reply.finished.connect(self.replyFinishedSlot)
        self.requestingUrls.append(url)
        self.replies.append(reply)
        return reply

    def fetchFiles(self, urlList, timeout_ms=0):
        self.log("fetchFiles()")
        self.sync = True
        self.queue = []
        self.redirected_urls = {} 
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()

        self.logT("eventLoop.exec_(): " + str(self.eventLoop))
        self.eventLoop.exec_()
        self.log("fetchFiles() End: %d" % self.errorStatus)
        if timeout_ms > 0:
            self.timer.stop()
        return self.fetchedFiles

    def addToQueue(self, url, redirected_from=None):
        if url in self.queue:
            return False
        self.queue.append(url)
        if redirected_from is not None:
            self.redirected_urls[url] = redirected_from
        return True

    def queueCount(self):
        return len(self.queue)

    def finishedCount(self):
        return len(self.fetchedFiles)

    def unfinishedCount(self):
        return len(self.queue) + len(self.requestingUrls)

    def log(self, msg):
        if debug_mode:
            qDebug(msg)

    def logT(self, msg):
        if debug_mode:
            qDebug("%s: %s" % (str(threading.current_thread()), msg))

    def fetchFilesAsync(self, urlList, timeout_ms=0):
        self.log("fetchFilesAsync()")
        self.sync = False
        self.queue = []
        self.clearCounts()
        self.errorStatus = Downloader.NO_ERROR
        self.fetchedFiles = {}

        if len(urlList) == 0:
            return self.fetchedFiles

        for url in urlList:
            self.addToQueue(url)

        for i in range(self.max_connection):
            self.fetchNext()

        if timeout_ms > 0:
            self.timer.setInterval(timeout_ms)
            self.timer.start()
    def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                req.setRawHeader(k, v)
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, file):
                body = body.read()
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)

        # necessary to trap local timout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)
class NetworkAccessManager(object):
    """
    This class mimicks httplib2 by using QgsNetworkAccessManager for all
    network calls.
    The return value is a tuple of (response, content), the first being and
    instance of the Response class, the second being a string that contains
    the response entity body.
    Parameters
    ----------
    debug : bool
        verbose logging if True
    exception_class : Exception
        Custom exception class
    Usage 1 (blocking mode)
    -----
    ::
        nam = NetworkAccessManager(authcgf)
        try:
            (response, content) = nam.request('http://www.example.com')
        except RequestsException as e:
            # Handle exception
            pass
    Usage 2 (Non blocking mode)
    -------------------------
    ::
        NOTE! if blocking mode returns immediatly
              it's up to the caller to manage listeners in case
              of non blocking mode
        nam = NetworkAccessManager(authcgf)
        try:
            nam.request('http://www.example.com', blocking=False)
            nam.reply.finished.connect(a_signal_listener)
        except RequestsException as e:
            # Handle exception
            pass
        Get response using method:
        nam.httpResult() that return a dictionary with keys:
            'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
            'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
            'content' - bytearray returned from reply
            'ok' - request success [True, False]
            'headers' - Dicionary containing the reply header
            'reason' - fomatted message string with reply.errorString()
            'exception' - the exception returne dduring execution
    """

    def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=False):
        self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
        self.authid = authid
        self.reply = None
        self.debug = debug
        self.exception_class = exception_class
        self.on_abort = False
        self.blocking_mode = False
        self.http_call_result = Response({
            'status': 0,
            'status_code': 0,
            'status_message': '',
            'content' : '',
            'ok': False,
            'headers': {},
            'reason': '',
            'exception': None,
        })

    def msg_log(self, msg):
        if self.debug:
            QgsMessageLog.logMessage(msg, "NetworkAccessManager")

    def httpResult(self):
        return self.http_call_result

    def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
        """
        Make a network request by calling QgsNetworkAccessManager.
        redirections argument is ignored and is here only for httplib2 compatibility.
        """
        self.msg_log(u'http_call request: {0}'.format(url))

        self.blocking_mode = blocking
        req = QNetworkRequest()
        # Avoid double quoting form QUrl
        url = urllib.parse.unquote(url)
        req.setUrl(QUrl(url))
        if headers is not None:
            # This fixes a wierd error with compressed content not being correctly
            # inflated.
            # If you set the header on the QNetworkRequest you are basically telling
            # QNetworkAccessManager "I know what I'm doing, please don't do any content
            # encoding processing".
            # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
            try:
                del headers['Accept-Encoding']
            except KeyError:
                pass
            for k, v in list(headers.items()):
                self.msg_log("Setting header %s to %s" % (k, v))
                req.setRawHeader(k, v)
        if self.authid:
            self.msg_log("Update request w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
        if self.reply is not None and self.reply.isRunning():
            self.reply.close()
        if method.lower() == 'delete':
            func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
        else:
            func = getattr(QgsNetworkAccessManager.instance(), method.lower())
        # Calling the server ...
        # Let's log the whole call for debugging purposes:
        self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
        self.on_abort = False
        headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
        for k, v in list(headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if method.lower() in ['post', 'put']:
            if isinstance(body, file):
                body = body.read()
            self.reply = func(req, body)
        else:
            self.reply = func(req)
        if self.authid:
            self.msg_log("Update reply w/ authid: {0}".format(self.authid))
            QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)

        # necessary to trap local timout manage by QgsNetworkAccessManager
        # calling QgsNetworkAccessManager::abortRequest
        QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)

        self.reply.sslErrors.connect(self.sslErrors)
        self.reply.finished.connect(self.replyFinished)
        self.reply.downloadProgress.connect(self.downloadProgress)

        # block if blocking mode otherwise return immediatly
        # it's up to the caller to manage listeners in case of no blocking mode
        if not self.blocking_mode:
            return (None, None)

        # Call and block
        self.el = QEventLoop()
        self.reply.finished.connect(self.el.quit)

        # Catch all exceptions (and clean up requests)
        try:
            self.el.exec_(QEventLoop.ExcludeUserInputEvents)
        except Exception as e:
            raise e

        if self.reply:
            self.reply.finished.disconnect(self.el.quit)

        # emit exception in case of error
        if not self.http_call_result.ok:
            if self.http_call_result.exception and not self.exception_class:
                raise self.http_call_result.exception
            else:
                raise self.exception_class(self.http_call_result.reason)

        return (self.http_call_result, self.http_call_result.content)

    #@pyqtSlot()
    def downloadProgress(self, bytesReceived, bytesTotal):
        """Keep track of the download progress"""
        #self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
        pass

    #@pyqtSlot(QNetworkReply)
    def requestTimedOut(self, QNetworkReply):
        """Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
        # adapt http_call_result basing on receiving qgs timer timout signal
        self.exception_class = RequestsExceptionTimeout
        self.http_call_result.exception = RequestsExceptionTimeout("Timeout error")

    #@pyqtSlot(QObject)
    def replyFinished(self):
        err = self.reply.error()
        httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
        httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
        self.http_call_result.status_code = httpStatus
        self.http_call_result.status = httpStatus
        self.http_call_result.status_message = httpStatusMessage
        for k, v in self.reply.rawHeaderPairs():
            self.http_call_result.headers[str(k)] = str(v)
            self.http_call_result.headers[str(k).lower()] = str(v)

        if err != QNetworkReply.NoError:
            # handle error
            # check if errorString is empty, if so, then set err string as
            # reply dump
            if re.match('(.)*server replied: $', self.reply.errorString()):
                errString = self.reply.errorString() + self.http_call_result.content
            else:
                errString = self.reply.errorString()
            # check if self.http_call_result.status_code is available (client abort
            # does not produce http.status_code)
            if self.http_call_result.status_code:
                msg = "Network error #{0}: {1}".format(
                    self.http_call_result.status_code, errString)
            else:
                msg = "Network error: {0}".format(errString)

            self.http_call_result.reason = msg
            self.http_call_result.ok = False
            self.msg_log(msg)
            # set return exception
            if err == QNetworkReply.TimeoutError:
                self.http_call_result.exception = RequestsExceptionTimeout(msg)

            elif err == QNetworkReply.ConnectionRefusedError:
                self.http_call_result.exception = RequestsExceptionConnectionError(msg)

            elif err == QNetworkReply.OperationCanceledError:
                # request abort by calling NAM.abort() => cancelled by the user
                if self.on_abort:
                    self.http_call_result.exception = RequestsExceptionUserAbort(msg)
                else:
                    self.http_call_result.exception = RequestsException(msg)

            else:
                self.http_call_result.exception = RequestsException(msg)

            # overload exception to the custom exception if available
            if self.exception_class:
                self.http_call_result.exception = self.exception_class(msg)

        else:
            # Handle redirections
            redirectionUrl = self.reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
            if redirectionUrl is not None and redirectionUrl != self.reply.url():
                if redirectionUrl.isRelative():
                    redirectionUrl = self.reply.url().resolved(redirectionUrl)

                msg = "Redirected from '{}' to '{}'".format(
                    self.reply.url().toString(), redirectionUrl.toString())
                self.msg_log(msg)

                self.reply.deleteLater()
                self.reply = None
                self.request(redirectionUrl.toString())

            # really end request
            else:
                msg = "Network success #{0}".format(self.reply.error())
                self.http_call_result.reason = msg
                self.msg_log(msg)

                ba = self.reply.readAll()
                self.http_call_result.content = bytes(ba)
                self.http_call_result.ok = True

        # Let's log the whole response for debugging purposes:
        self.msg_log("Got response %s %s from %s" % \
                    (self.http_call_result.status_code,
                     self.http_call_result.status_message,
                     self.reply.url().toString()))
        for k, v in list(self.http_call_result.headers.items()):
            self.msg_log("%s: %s" % (k, v))
        if len(self.http_call_result.content) < 1024:
            self.msg_log("Payload :\n%s" % self.http_call_result.content)
        else:
            self.msg_log("Payload is > 1 KB ...")

        # clean reply
        if self.reply is not None:
            if self.reply.isRunning():
                self.reply.close()
            self.msg_log("Deleting reply ...")
            # Disconnect all slots
            self.reply.sslErrors.disconnect(self.sslErrors)
            self.reply.finished.disconnect(self.replyFinished)
            self.reply.downloadProgress.disconnect(self.downloadProgress)
            self.reply.deleteLater()
            self.reply = None
        else:
            self.msg_log("Reply was already deleted ...")

    #@pyqtSlot()
    def sslErrors(self, ssl_errors):
        """
        Handle SSL errors, logging them if debug is on and ignoring them
        if disable_ssl_certificate_validation is set.
        """
        if ssl_errors:
            for v in ssl_errors:
                self.msg_log("SSL Error: %s" % v.errorString())
        if self.disable_ssl_certificate_validation:
            self.reply.ignoreSslErrors()

    #@pyqtSlot()
    def abort(self):
        """
        Handle request to cancel HTTP call
        """
        if (self.reply and self.reply.isRunning()):
            self.on_abort = True
            self.reply.abort()