def query(self, query):
"""Perform a nominatim query.
:param query: Query to execute on the nominatim server.
:type query: basestring
:return: The result of the query as a dictionary.
:rtype: dict
:raise NetWorkErrorException
"""
url_query = QUrl(self.__url)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
loop = QEventLoop()
downloader = QgsFileDownloader(url_query,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
with open(self.result_path) as json_file:
data = json.load(json_file)
return data
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(
self._url, self.result_path, delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
def read_from_http(uri: str, cache_folder: Path):
"""Read a QGIS project stored into on a remote web server accessible through HTTP.
:param uri: web URL to the QGIS project
:type uri: str
:return: a tuple with XML document and the filepath.
:rtype: Tuple[QtXml.QDomDocument, str]
"""
# get filename from URL parts
parsed = urlparse(uri)
if not parsed.path.rpartition("/")[2].endswith((".qgs", ".qgz")):
raise ValueError(
"URI doesn't ends with QGIS project extension (.qgs or .qgz): {}".
format(uri))
cached_filepath = cache_folder / parsed.path.rpartition("/")[2]
# download it
loop = QEventLoop()
project_download = QgsFileDownloader(url=QUrl(uri),
outputFileName=str(cached_filepath),
delayStart=True)
project_download.downloadExited.connect(loop.quit)
project_download.startDownload()
loop.exec_()
return read_from_file(str(cached_filepath))
def download(self):
"""Download the data"""
loop = QEventLoop()
downloader = QgsFileDownloader(
self._url, self.result_path, delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
def run(self):
# Run file download in separate event loop
loop = QEventLoop()
downloader = QgsFileDownloader(self.url,
self.filePath,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
def gbif_GET(url, args, **kwargs):
handle, output_path = mkstemp()
QgsMessageLog.logMessage('gbif_GET outfile: %s' % output_path,
'SpeciesExplorer', 0)
QgsMessageLog.logMessage('gbif_GET URL: %s' % url, 'SpeciesExplorer', 0)
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), output_path, delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.startDownload()
loop.exec_()
file = open(output_path, 'rt', encoding='utf-8')
data = file.read()
file.close()
return json.loads(data, encoding='utf-8')
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue("Download failed: Host requires authentication" in str(self.error_args), "Error args is: %s" % str(self.error_args))
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
raise OverpassTimeoutException
# The download is done, checking for not complete OSM file.
# Overpass might aborted the request with HTTP 200.
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = (
'<remark> runtime error: Query timed out in "[a-z]+" at line '
'[\d]+ after ([\d]+) seconds. </remark>')
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
# Everything went fine
return self.result_path
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
for message in self.errors:
self.is_query_timed_out(message)
self.is_bad_request(message)
LOGGER.error(message)
if len(self.errors):
raise NetWorkErrorException('Overpass API', ', '.join(self.errors))
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
# Do not raise a QuickOSM exception here
# It must be a bug from QuickOSM
raise FileNotFoundError
self.check_file(self.result_path)
# Everything went fine
return self.result_path
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join([
"%s=%s" % i
for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())
])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination,
self.auth_config.id(), False)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(
os.path.getsize(destination) > 2000,
"Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def _downloadAndUnzip(self, itemid):
self.progress_bar = QProgressBar()
self.progress_bar.setMinimum(0)
self.progress_bar.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.progess_message_bar = self.iface.messageBar().createMessage(
tr("Downloading..."))
self.progess_message_bar.layout().addWidget(self.progress_bar)
self.iface.messageBar().pushWidget(self.progess_message_bar, Qgis.Info)
# FIXME: the signal is always emitted
# self.iface.messageBar().widgetRemoved.connect(self.widget_removed)
def extract_data(tmpPath, itemid):
LOG.debug("Extract data")
try:
targetFolder = DataCatalogueClient.folderForDataItem(itemid)
QDir(targetFolder).removeRecursively()
unzip.unzip(tmpPath, targetFolder)
QFile(tmpPath).remove()
except Exception as e:
LOG.debug("Error on extracting data %s" % e)
url = f"{self.url}/content/items/{itemid}/data"
tmpDir = QDir.tempPath()
filename = f"{itemid}.zip"
tmpPath = QDir.cleanPath(os.path.join(tmpDir, filename))
loop = QEventLoop()
self.downloader = QgsFileDownloader(QUrl(url), tmpPath)
self.downloader.downloadProgress.connect(self.update_progress)
self.downloader.downloadCompleted.connect(self.download_finished)
self.downloader.downloadCompleted.connect(
partial(extract_data, tmpPath, itemid))
self.downloader.downloadCanceled.connect(self.download_canceled)
self.downloader.downloadExited.connect(loop.quit)
loop.exec_()
