class ThreeDiResultSelection(QObject):
"""QGIS Plugin Implementation."""
# TODO: Reinout suggests to use requests library and get rid e.g. QNetworkRequest,
# QgsNetworkAccessManager
state_changed = pyqtSignal([str, str, list])
tool_name = "result_selection"
def __init__(self, iface, ts_datasources):
"""Constructor.
:param iface: An interface instance that will be passed to this class
which provides the hook by which you can manipulate the QGIS
application at run time.
:type iface: QgsInterface
"""
super().__init__()
# Save reference to the QGIS interface
self.iface = iface
self.ts_datasources = ts_datasources
# TODO: unsure if this is the right place for initializing this model
self.downloadable_results = models.DownloadableResultModel()
# TODO: fix this fugly shizzle
self.download_directory = None
self.username = None
self.password = None
# download administration
self.network_manager = QgsNetworkAccessManager(self)
self.icon_path = ":/plugins/ThreeDiToolbox/icons/icon_add_datasource.png"
self.menu_text = u"Select 3Di results"
self.is_active = False
self.dialog = None
self.ts_datasources.model_schematisation_change.connect(
self.on_state_changed)
self.ts_datasources.results_change.connect(self.on_state_changed)
def on_unload(self):
"""Cleanup necessary items here when dialog is closed"""
# disconnects
if self.dialog:
self.dialog.close()
def on_close_dialog(self):
"""Cleanup necessary items here when dialog is closed"""
self.dialog.closingDialog.disconnect(self.on_close_dialog)
self.dialog = None
self.is_active = False
def run(self):
"""Run method that loads and starts the plugin"""
if not self.is_active:
self.is_active = True
if self.dialog is None:
# Create the dialog (after translation) and keep reference
self.dialog = result_selection_view.ThreeDiResultSelectionWidget(
parent=None,
iface=self.iface,
ts_datasources=self.ts_datasources,
downloadable_results=self.downloadable_results,
tool=self,
)
# download signals; signal connections should persist after
# closing the dialog.
self.dialog.downloadResultButton.clicked.connect(
self.handle_download)
# connect to provide cleanup on closing of dockwidget
self.dialog.closingDialog.connect(self.on_close_dialog)
# show the widget
self.dialog.show()
else:
self.dialog.setWindowState(self.dialog.windowState()
& ~Qt.WindowMinimized | Qt.WindowActive)
self.dialog.raise_()
def on_state_changed(self, setting_key, value):
if setting_key == "result_directories":
output = []
for result in value:
output.append(json.JSONEncoder().encode({
"active":
result.active.value,
"name":
result.name.value,
"file_path":
result.file_path.value,
"type":
result.type.value,
}))
else:
output = value
self.state_changed.emit(self.tool_name, setting_key, [output])
def set_state(self, setting_dict):
self.ts_datasources.reset()
self.ts_datasources.model_spatialite_filepath = setting_dict.get(
"model_schematisation", None)
result_list = setting_dict.get("result_directories", None)
if result_list is not None:
for result_json in result_list:
result = json.JSONDecoder().decode(result_json)
self.ts_datasources.insertRows([result])
def get_state_description(self):
return (
self.tool_name,
{
"model_schematisation": IOBase,
"result_directories": list
}, # file,
)
@property
def logged_in(self):
return self.username is not None and self.password is not None
def handle_download(self):
result_type_codes_download = [
"logfiles",
# non-groundwater codes
"subgrid_map",
"flow-aggregate",
"id-mapping",
# groundwater codes
"results-3di",
"aggregate-results-3di",
"grid-admin",
]
selection_model = self.dialog.downloadResultTableView.selectionModel()
proxy_indexes = selection_model.selectedIndexes()
if len(proxy_indexes) != 1:
pop_up_info("Please select one result.")
return
proxy_selection_index = proxy_indexes[0]
selection_index = self.dialog.download_proxy_model.mapToSource(
proxy_selection_index)
item = self.downloadable_results.rows[selection_index.row()]
to_download = [
r for r in item.results.value
if r["result_type"]["code"] in result_type_codes_download
]
to_download_urls = [dl["attachment_url"] for dl in to_download]
logger.debug(item.name.value)
# ask user where to store download
directory = QFileDialog.getExistingDirectory(None,
"Choose a directory",
os.path.expanduser("~"))
if not directory:
return
dir_name = get_valid_filename(item.name.value)
self.download_directory = os.path.join(directory, dir_name)
# For now, only work with empty directories that we create ourselves.
# Because the files are downloaded and processed in chunks, we cannot
# guarantee data integrity with existing files.
if os.path.exists(self.download_directory):
pop_up_info("The directory %s already exists." %
self.download_directory)
return
logger.info("Creating download directory.")
os.mkdir(self.download_directory)
logger.debug(self.download_directory)
CHUNK_SIZE = 16 * 1024
# Important note: QNetworkAccessManager is asynchronous, which means
# the downloads are processed asynchronous using callbacks.
for url in to_download_urls:
request = QNetworkRequest(QUrl(url))
request.setRawHeader(b"username", bytes(self.username, "utf-8"))
request.setRawHeader(b"password", bytes(self.password, "utf-8"))
request.setAttribute(USER_DOWNLOAD_DIRECTORY,
self.download_directory)
reply = self.network_manager.get(request)
# Get replies in chunks, and process them
reply.setReadBufferSize(CHUNK_SIZE)
reply.readyRead.connect(
self.on_single_download_ready_to_read_chunk)
reply.finished.connect(self.on_single_download_finished)
pop_up_info("Download started.")
def on_single_download_ready_to_read_chunk(self):
"""Process a chunk of the downloaded data."""
# TODO: do some exception handling if the download did not succeed
reply = self.sender()
raw_chunk = reply.readAll() # QByteArray
url = urlparse(reply.url().toString())
filename = url.path.split("/")[-1]
download_directory = reply.request().attribute(USER_DOWNLOAD_DIRECTORY)
if not download_directory:
raise RuntimeError(
"Request is not set up properly, USER_DOWNLOAD_DIRECTORY is "
"required to locate the download directory.")
with open(os.path.join(download_directory, filename), "ab") as f:
f.write(raw_chunk)
def on_single_download_finished(self):
"""Usage: mostly for notifying the user the download has finished."""
reply = self.sender()
filename = reply.url().toString().split("/")[-1]
reply.close()
messagebar_message("Done", "Finished downloading %s" % filename)
class Streaming(QObject):
""" Class for keeping track of stream chunks and
providing methods for handling and visualizing them
"""
# Define SIGNALS/SLOTS
playlistHandled = pyqtSignal(dict)
urlReady = pyqtSignal(str, int, str)
dataReady = pyqtSignal(str, int)
def __init__(self, parent, iface, chunks, playlistUrl, mimeType, encoding):
super(Streaming, self).__init__()
self.DEBUG = True
# Variables from other classes
self.parent = parent # For GUI access
self.iface = iface
self.chunks = chunks
self.playlistUrl = playlistUrl
self.mimeType = mimeType
self.encoding = encoding
# Internal variables
self.__endTag = "#PLAYLIST-END"
self.__exceptionTag = "#EXCEPTION"
self.__exceptionUrl = ""
self.__exceptionFound = False
self.__playlistFinished = False # Did the end tag appeared?
self.__bytesInlastReply = 0 # To compare last and current reply sizes
self.__loadedChunks = 0 # For keeping track of # of loaded (to local vars) chunks
self.__deliveredChunks = 0 # For keeping track of # of loaded (to the map) chunks
self.__bFirstChunk = True
self.__features = {} # {0:[f0,f1,f2], 1:[f0,f1]}
self.__bGeomMulti = False # Is the geometry multi{point|line|polygon}
self.__geometryType = "" # Values: "Point","LineString","Polygon","Unknown", "NoGeometry"
self.__tmpGeometry = {
} # For visualization purposes {chunkId1: rb1, chunkId2: rb2 }
self.__memoryLayer = None # The whole merged data
# For rasters only
self.__legend = self.iface.legendInterface()
self.__groupIndex = 0
self.__chunksDir = None
self.__virtualFile = "" # Virtual raster file path
if isMimeTypeRaster(self.mimeType, True) != None:
self.__chunksDir = tempfile.mkdtemp(prefix="tmpChunks")
# Other objects
self.timer = QTimer()
self.timer.setInterval(1 * 1000) # 1 second
self.QNAM4Playlist = QgsNetworkAccessManager()
self.QNAM4Chunks = QgsNetworkAccessManager()
self.QNAM4Exception = QgsNetworkAccessManager()
# SIGNAL/SLOT connections
self.playlistHandled.connect(self.fetchChunks)
self.urlReady.connect(self.fetchResult)
self.dataReady.connect(self.loadData)
self.timer.timeout.connect(
partial(self.fetchPlaylist, self.playlistUrl))
self.QNAM4Playlist.finished.connect(self.handlePlaylist)
self.QNAM4Chunks.finished.connect(self.handleChunk)
self.QNAM4Exception.finished.connect(self.handleException)
#self.QNAM4Playlist = QgsNetworkAccessManager.instance()
#theReply2.error.connect(self.handleErrors)
# GUI
self.parent.progressBar.setRange(0, 0)
self.parent.lblProcess.setText("Reading output playlist...")
def start(self):
""" Start fetching """
self.fetchPlaylist(self.playlistUrl) # First call
def stop(self):
""" Stop fetching """
self.timer.stop()
self.QNAM4Playlist.finished.disconnect(self.handlePlaylist)
self.QNAM4Chunks.finished.disconnect(self.handleChunk)
self.removeTempGeometry(self.__geometryType)
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("Stop streaming!")
def validateCompletedStream(self):
""" Is the stream complete (Did the end tag appeared?) """
#return (self.__loadedChunks >= self.chunks and self.chunks != 0)
return self.__playlistFinished
def allChunksDelivered(self):
""" Are all chunks already loaded into the map? """
return ((self.__loadedChunks == self.__deliveredChunks
and self.__playlistFinished) or self.__exceptionFound)
def fetchPlaylist(self, playlistLink):
url = QUrl(playlistLink)
self.QNAM4Playlist.get(QNetworkRequest(url)) # SLOT: handlePlaylist
def handlePlaylist(self, reply):
""" Parse the chunk URLs and update the loadedChunks counter """
# Check if there is redirection
reDir = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute).toUrl()
if not reDir.isEmpty():
self.fetchPlaylist(reDir.toString())
return
# Parse URLs only if there is new data in the reply
if reply.bytesAvailable() > self.__bytesInlastReply:
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print(" Parsing the playlist...")
startFrom = reply.bytesAvailable(
) - self.__bytesInlastReply # Delta in bytes
self.__bytesInlastReply = reply.bytesAvailable()
newURLs = self.parseURLs(reply, startFrom)
else:
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print(" No new data in the playlist...")
newURLs = {}
# Store new URLs
if len(newURLs) > 0:
self.__loadedChunks += len(newURLs)
if self.chunks:
self.parent.progressBar.setRange(0, self.chunks)
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print(
str(self.__loadedChunks) + " chunks loaded" +
((" out of " + str(self.chunks)) if self.chunks else ""))
# If not complete, make additional calls
if not self.validateCompletedStream():
if not self.timer.isActive():
self.timer.start()
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("Timer started...")
else:
self.timer.stop()
self.QNAM4Playlist.finished.disconnect(self.handlePlaylist)
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("Playlist finished!")
if self.allChunksDelivered():
self.finishLoading()
if self.__exceptionFound:
self.fetchException()
if len(newURLs) > 0:
self.playlistHandled.emit(newURLs) # SLOT: fetchChunks
def parseURLs(self, reply, startFrom):
""" Get a dict of new IDs:URLs from the current playlist (newURLs) """
newURLs = {} # {0:URL0, 1:URL1, ...}
count = 0
#Get the delta and start reading it
allData = reply.readAll()
allData = allData.right(startFrom) # Get rid of old data
response = QTextStream(allData, QIODevice.ReadOnly)
data = response.readLine()
# Parse
while (data):
data = str(data.split("\n")[0])
if data:
if "#" in data: # It's a playlist comment
if self.__endTag in data:
self.__playlistFinished = True
elif self.__exceptionTag in data:
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("Exception found!")
self.__exceptionFound = True
self.__exceptionUrl = data.split(":", 1)[1].strip()
else:
newURLs[count + self.__loadedChunks] = data
count += 1
data = response.readLine()
return newURLs
def fetchChunks(self, newURLs):
""" Fetch each url """
for chunkId in newURLs:
self.urlReady.emit(self.encoding, chunkId,
newURLs[chunkId]) # SLOT: fetchResult
def fetchResult(self, encoding, chunkId, fileLink):
""" Send the GET request """
url = QUrl(fileLink)
theReply2 = self.QNAM4Chunks.get(QNetworkRequest(url))
theReply2.setProperty("chunkId", pystring(chunkId))
theReply2.setProperty("encoding", pystring(encoding))
def handleErrors(self, error): # TODO connect it
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("ERROR!!!", error)
def fetchException(self):
""" Send the GET request for the exception """
url = QUrl(self.__exceptionUrl)
theReply3 = self.QNAM4Exception.get(QNetworkRequest(url))
def handleException(self, reply):
""" Display the exception """
# Check if there is redirection
reDir = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute).toUrl()
if not reDir.isEmpty():
self.__exceptionUrl = reDir.toString()
self.fetchException()
return
resultXML = reply.readAll().data()
self.parent.setStatusLabel('error')
self.parent.progressBar.setMinimum(0)
self.parent.progressBar.setMaximum(100)
self.parent.errorHandler(resultXML)
def handleChunk(self, reply):
""" Store the file received """
#reply.deleteLater() # Recommended way to delete the reply
chunkId = reply.property("chunkId").toInt()[0]
encoding = reply.property("encoding").toString()
# Check if there is redirection
reDir = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute).toUrl()
if not reDir.isEmpty():
self.urlReady.emit(encoding, chunkId, reDir.toString())
return
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("GET chunk", chunkId)
# Update progressBar
if self.chunks:
self.parent.progressBar.setValue(self.__deliveredChunks + 1)
self.parent.lblProcess.setText("Downloading chunks... (" +
str(self.__deliveredChunks + 1) +
"/" + str(self.chunks) + ")")
# Get a unique temporary file name
tmpFile = tempfile.NamedTemporaryFile(prefix="base64",
suffix=getFileExtension(
self.mimeType),
dir=self.__chunksDir,
delete=False)
# TODO: Check if the file name already exists!!!
# Write the data to the temporary file
outFile = QFile(tmpFile.name)
outFile.open(QIODevice.WriteOnly)
outFile.write(reply.readAll())
outFile.close()
# Decode?
if encoding == "base64":
resultFile = decodeBase64(tmpFile.name, self.mimeType,
self.__chunksDir)
else:
resultFile = tmpFile.name
# Finally, load the data
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("READY to be loaded (", resultFile, ", chunkId:", chunkId,
")")
self.dataReady.emit(resultFile, chunkId) # SLOT: loadData
def loadData(self, resultFile, chunkId):
""" Load data to the map """
if isMimeTypeVector(self.mimeType, True) != None:
# Memory layer:
geometryTypes = [
"Point", "LineString", "Polygon", "Unknown", "NoGeometry"
]
vlayer = QgsVectorLayer(resultFile, "chunk", "ogr")
if self.__bFirstChunk:
self.__bFirstChunk = False
self.__geometryType = geometryTypes[vlayer.geometryType()]
self.__bGeomMulti = vlayer.wkbType() in [4, 5, 6, 11, 12, 13]
self.__memoryLayer = QgsVectorLayer(self.__geometryType,
"Streamed data", "memory")
self.__memoryLayer.dataProvider().addAttributes(
list(vlayer.pendingFields().values()))
self.__memoryLayer.updateFieldMap()
provider = vlayer.dataProvider()
allAttrs = provider.attributeIndexes()
vlayer.select(allAttrs)
# Visualize temporal geometries during the downloading process
# Don't add temporal geometries if last chunk
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("Loaded chunkId:", chunkId)
res = self.__memoryLayer.dataProvider().addFeatures(
[feat for feat in vlayer])
self.__deliveredChunks += 1
if not self.allChunksDelivered():
inFeat = QgsFeature()
inGeom = QgsGeometry()
self.createTempGeometry(chunkId, self.__geometryType)
while provider.nextFeature(inFeat):
inGeom = inFeat.geometry()
featList = self.extractAsSingle(
self.__geometryType,
inGeom) if self.__bGeomMulti else [inGeom]
for geom in featList:
self.addTempGeometry(chunkId, self.__geometryType,
geom)
else:
self.finishLoading()
# Raster data
elif isMimeTypeRaster(self.mimeType, True) != None:
# We can directly attach the new layer
if self.__bFirstChunk:
self.__bFirstChunk = False
self.__groupIndex = self.__legend.addGroup("Streamed-raster")
rLayer = QgsRasterLayer(resultFile, "raster_" + str(chunkId))
bLoaded = QgsProject.instance().addMapLayer(rLayer)
self.stretchRaster(rLayer)
self.__legend.moveLayer(rLayer, self.__groupIndex + 1)
self.__deliveredChunks += 1
if self.allChunksDelivered():
self.finishLoading()
def finishLoading(self):
""" Finish the loading process, load the definite assembled layer """
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("DONE!")
if not self.__bFirstChunk:
if isMimeTypeVector(self.mimeType, True) != None:
self.removeTempGeometry(self.__geometryType)
QgsProject.instance().addMapLayer(self.__memoryLayer)
elif isMimeTypeRaster(self.mimeType, True) != None:
self.parent.lblProcess.setText(
"All tiles are loaded. Merging them...")
# Generate gdal virtual raster
# Code adapted from GdalTools (C) 2009 by L. Masini and G. Sucameli (Faunalia)
self.process = QProcess(self)
self.process.finished.connect(self.loadVirtualRaster)
#self.setProcessEnvironment(self.process) Required in Windows?
cmd = "gdalbuildvrt"
arguments = pystringlist()
if platform.system() == "Windows" and cmd[-3:] == ".py":
command = cmd[:-3] + ".bat"
else:
command = cmd
tmpFile = tempfile.NamedTemporaryFile(prefix="virtual",
suffix=".vrt")
self.__virtualFile = tmpFile.name
arguments.append(self.__virtualFile)
rasters = self.getRasterFiles(self.__chunksDir,
getFileExtension(self.mimeType))
for raster in rasters:
arguments.append(raster)
self.process.start(command, arguments, QIODevice.ReadOnly)
if not self.__exceptionFound:
self.parent.setStatusLabel('finished')
self.parent.progressBar.setRange(0, 100)
self.parent.progressBar.setValue(100)
def createTempGeometry(self, chunkId, geometryType):
""" Create rubber bands for rapid visualization of geometries """
if geometryType == "Polygon":
self.__tmpGeometry[chunkId] = QgsRubberBand(
self.iface.mapCanvas(), True)
self.__tmpGeometry[chunkId].setColor(QColor(0, 255, 0, 255))
self.__tmpGeometry[chunkId].setWidth(2)
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print("rubberBand created")
elif geometryType == "LineString":
self.__tmpGeometry[chunkId] = QgsRubberBand(
self.iface.mapCanvas(), False)
self.__tmpGeometry[chunkId].setColor(QColor(255, 121, 48, 255))
self.__tmpGeometry[chunkId].setWidth(3)
elif geometryType == "Point":
# In the case of points, they will be added as vertex objects later
self.__tmpGeometry[chunkId] = []
def addTempGeometry(self, chunkId, geometryType, geometry):
""" Add geometries as rubber bands or vertex objects """
if geometryType == "Polygon" or geometryType == "LineString":
self.__tmpGeometry[chunkId].addGeometry(geometry, None)
elif geometryType == "Point":
vertex = QgsVertexMarker(self.iface.mapCanvas())
vertex.setCenter(geometry.asPoint())
vertex.setColor(QColor(0, 255, 0))
vertex.setIconSize(6)
vertex.setIconType(
QgsVertexMarker.ICON_BOX) # or ICON_CROSS, ICON_X
vertex.setPenWidth(3)
self.__tmpGeometry[chunkId].append(vertex)
def removeTempGeometry(self, geometryType):
""" Remove rubber bands or vertex objects from the map """
if geometryType == "Polygon" or geometryType == "LineString":
for chunkId in list(self.__tmpGeometry.keys()):
self.iface.mapCanvas().scene().removeItem(
self.__tmpGeometry[chunkId])
del self.__tmpGeometry[chunkId]
elif geometryType == "Point":
for chunkId in list(self.__tmpGeometry.keys()):
if len(self.__tmpGeometry[chunkId]) > 0:
for vertex in self.__tmpGeometry[chunkId]:
self.iface.mapCanvas().scene().removeItem(vertex)
del vertex
def extractAsSingle(self, geometryType, geom):
""" Extract multi geometries as single ones.
Required because of a QGIS bug regarding multipolygons and rubber bands
"""
# Code adapted from QGIS fTools plugin, (C) 2008-2011 Carson Farmer
multi_geom = QgsGeometry()
temp_geom = []
if geometryType == "Point":
multi_geom = geom.asMultiPoint()
for i in multi_geom:
temp_geom.append(QgsGeometry().fromPoint(i))
elif geometryType == "LineString":
multi_geom = geom.asMultiPolyline()
for i in multi_geom:
temp_geom.append(QgsGeometry().fromPolyline(i))
elif geometryType == "Polygon":
multi_geom = geom.asMultiPolygon()
for i in multi_geom:
temp_geom.append(QgsGeometry().fromPolygon(i))
return temp_geom
def loadVirtualRaster(self, exitCode, status):
""" Load a virtual raster to QGIS """
if exitCode == 0:
self.__legend.setGroupVisible(self.__groupIndex, False)
rLayer = QgsRasterLayer(self.__virtualFile, "virtual")
bLoaded = QgsProject.instance().addMapLayer(rLayer)
self.stretchRaster(rLayer)
self.process.kill()
def stretchRaster(self, raster):
raster.setMinimumMaximumUsingLastExtent()
raster.setContrastEnhancementAlgorithm(1)
raster.triggerRepaint()
def setProcessEnvironment(self, process):
""" From GdalTools. Set environment variables for running gdalbuildvrt """
envvar_list = {
"PATH": self.getGdalBinPath(),
"PYTHONPATH": self.getGdalPymodPath()
}
if self.DEBUG: # fix_print_with_import
# fix_print_with_import
print(envvar_list)
sep = os.pathsep
for name, val in envvar_list.items():
if val == None or val == "":
continue
envval = os.getenv(name)
if envval == None or envval == "":
envval = str(val)
elif not pystring(envval).split(sep).contains(
val, Qt.CaseInsensitive):
envval += "%s%s" % (sep, str(val))
else:
envval = None
if envval != None:
os.putenv(name, envval)
if False: # not needed because os.putenv() has already updated the environment for new child processes
env = QProcess.systemEnvironment()
if env.contains(QRegExp("^%s=(.*)" % name,
Qt.CaseInsensitive)):
env.replaceInStrings(
QRegExp("^%s=(.*)" % name, Qt.CaseInsensitive),
"%s=\\1%s%s" % (name, sep, gdalPath))
else:
env << "%s=%s" % (name, val)
process.setEnvironment(env)
def getRasterFiles(self, dir, extension):
rasters = pystringlist()
for name in glob.glob(dir + '/*' + extension):
rasters.append(name)
return rasters
def getGdalBinPath(self):
""" Retrieves GDAL binaries location """
settings = QSettings()
return settings.value("/GdalTools/gdalPath", pystring("")).toString()
def getGdalPymodPath(self):
""" Retrieves GDAL python modules location """
settings = QSettings()
return settings.value("/GdalTools/gdalPymodPath",
pystring("")).toString()
class ClassDownloadMasc:
"""
Class allowing to download needed files
"""
def __init__(self, path_work=None, url_base=None, parent=None):
self.masplug_path = None
if url_base is None:
self.url_base = ''
else:
self.url_base = url_base
self.parent = parent
if parent is None:
self.dbg = True
else:
self.dbg = self.parent.DEBUG
if path_work is None:
self.masplug_path = ''
else:
self.masplug_path = path_work
self.file_install = None
self.url = None
self.manager = QgsNetworkAccessManager()
self.manager = self.manager.instance()
def download_dir(self, dir):
"""
download dir represitory
:param dir : (dict)the keys are represitory and the element is list_file
:return:
"""
for rep in dir.keys():
self.print_('Downloading executable file in "{}" directory\n'
'Download ...'.format(rep))
url = posixpath.join(self.url_base, rep)
os.makedirs(os.path.join(self.masplug_path, rep), exist_ok=True)
for filen in dir[rep]:
url2 = posixpath.join(url, filen)
paht_file = os.path.join(self.masplug_path, rep, filen)
self.download_file(url2, paht_file)
self.print_('Downloading Done')
def download_file(self, url, path_file):
"""
download function
:param url: url path of file
:param path_file: path to save file
:return:
"""
self.file_install = path_file
self.url = url
loop = QEventLoop()
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.exit(1))
timer.start(100000) # 10 second time-out
# req = QNetworkRequest(QUrl('https://www.python.org/'))
req = QNetworkRequest(QUrl(url))
result = self.manager.get(req)
result.finished.connect(lambda: self.fin_req(loop, result))
self.print_('fetching request...', self.dbg)
if loop.exec_() == 0:
timer.stop()
self.print_(
'{} is received: {}'.format(os.path.basename(path_file),
result.readAll().count()),
self.dbg)
else:
self.print_('request timed-out')
def print_(self, txt, dbg=True):
if self.parent is None and dbg:
print(txt)
else:
if dbg:
self.parent.add_info(txt)
def fin_req(self, loop, result):
"""
action when received the request
:param loop (obj)
:param result (obj) reply in request
:return:
"""
if result.error() != QNetworkReply.NoError:
self.print_("Error of request : {}".format(self.url))
loop.exit(1)
return
loop.exit()
# save file
self.save_file(result.readAll())
def save_file(self, data):
"""
Save file
:param data: data in file
:return:
"""
fch = open(self.file_install, 'wb')
with fch:
fch.write(data)
class CloudqubeClient:
"""Handles calls to Terraqube Cloud."""
def __init__(self, server):
self._nam = QgsNetworkAccessManager()
self._server = server
self._token = None
self._replies = []
# Private methods
def get_url(self, url):
if url.startswith('http'):
return url
else:
return "{0}/api/1.0.0/{1}".format(self._server, url)
def finished(self, reply):
self._replies.remove(reply)
def set_token(self, token):
self._token = token
def str_to_byte_array(self, input_string):
return bytes(input_string, encoding='utf-8')
def byte_array_to_string(self, byte_array):
return byte_array.data().decode('utf-8')
def prepare_request(self, url, content_type):
url = self.get_url(url)
req = QNetworkRequest(QUrl(url))
req.setHeader(QNetworkRequest.ContentTypeHeader, content_type)
req.setRawHeader(b'Accept', b'application/json')
if self._token:
req.setRawHeader(b'Authorization', self.str_to_byte_array(
'Bearer {0}'.format(self._token)))
return req
def get(self, url, callback, error, content_type='application/json'):
req = self.prepare_request(url, content_type)
self._replies.append(CloudqubeJsonReply(
self._nam.get(req), None, callback, self.finished, error))
def post_json(self, url, data, callback, error,
content_type='application/json'):
req = self.prepare_request(url, content_type)
byte_array = self.str_to_byte_array(json.dumps(data)) if data else None
self._replies.append(CloudqubeJsonReply(self._nam.post(
req, byte_array), data, callback, self.finished, error))
def put_json(self, url, data, callback, error,
content_type='application/json'):
req = self.prepare_request(url, content_type)
byte_array = self.str_to_byte_array(json.dumps(data)) if data else None
self._replies.append(CloudqubeJsonReply(self._nam.put(
req, byte_array), data, callback, self.finished, error))
def post_bytes(self, url, data, callback, error,
content_type='application/octet-stream'):
req = self.prepare_request(url, content_type)
self._replies.append(CloudqubeJsonReply(self._nam.post(
req, data), data, callback, self.finished, error))
def delete_nam(self, url, callback, error, content_type='application/json'):
req = self.prepare_request(url, content_type)
self._replies.append(CloudqubeJsonReply(
self._nam.deleteResource(req), None, callback, self.finished, error))
def add_text_parts(self, multi_part, fields):
"""Adds all text fields to the multipart object."""
for key in fields:
text_part = QHttpPart()
text_part.setHeader(QNetworkRequest.ContentDispositionHeader,
QVariant('form-data; name="{0}"'.format(key)))
text_part.setBody(self.str_to_byte_array(fields[key]))
multi_part.append(text_part)
def add_image_part(self, multi_part, filename):
"""Adds the image field to the multipart object."""
image_part = QHttpPart()
image_part.setHeader(QNetworkRequest.ContentTypeHeader,
QVariant('application/octet-stream'))
image_part.setHeader(
QNetworkRequest.ContentDispositionHeader,
QVariant('form-data; name="file"'))
f = QFile(filename)
f.open(QIODevice.ReadOnly)
image_part.setBodyDevice(f)
f.setParent(multi_part)
multi_part.append(image_part)
def complete_hiperqube_upload(self, hiperqube_id, uploaded_parts, callback, error):
QgsMessageLog.logMessage('complete_hiperqube_upload: started')
payload = uploaded_parts
self.put_json(
"hiperqubes/{0}/upload".format(hiperqube_id),
payload,
callback,
error)
def create_hiperqube_upload(self, hiperqube_id, size, callback, error):
"""Creates a new upload for the given hiperqube."""
QgsMessageLog.logMessage('create_hiperqube_upload: started')
payload = {
'size': size
}
self.post_json(
"hiperqubes/{0}/upload".format(hiperqube_id),
payload,
callback,
error)
# Public methods
# Authentication
def login_user(self, username, password, callback, error):
"""Login user to Terraqube Cloud using username and password."""
login_callback = LoginCallback(self, callback)
response = self.post_json(
'user/login',
{'username': username, 'password': password},
login_callback.notify,
error)
# Projects
def create_project(self, name, callback, error):
"""Creates a new project with the specified name."""
payload = {
'name': name
}
self.post_json("projects", payload, callback, error)
def get_projects(self, callback, error):
"""Get list of projects for current user."""
self.get('projects', callback, error)
def delete_project(self, project_id, callback, error):
"""Deletes a project."""
self.delete_nam("projects/{0}".format(project_id), callback, error)
# Hiperqubes
def get_hiperqubes(self, project_id, callback, error):
"""Get list of hiperqubes for current user."""
self.get(
"projects/{0}/hiperqubes".format(project_id), callback, error)
def get_hiperqube_details(self, hiperqube_id, callback, error):
"""Get hiperqube details."""
self.get("hiperqubes/{0}".format(hiperqube_id), callback, error)
def create_hiperqube(self, project_id, name, captured_date, callback,
error):
"""Create a new hiperqube."""
captured_date = captured_date.replace(microsecond=0).replace(
tzinfo=pytz.reference.LocalTimezone())
captured_date_str = captured_date.isoformat()
payload = {
'name': name,
'capturedDate': captured_date_str
}
self.post_json(
"projects/{0}/hiperqubes".format(project_id),
payload,
callback,
error)
def delete_hiperqube(self, hiperqube_id, callback, error):
"""Deletes a hiperqube."""
self.delete_nam("hiperqubes/{0}".format(hiperqube_id), callback, error)
def upload_hiperqube_hdr(self, hiperqube_id, filename, callback, error):
"""Upload an HDR file to an existing hiperqube."""
f = QFile(filename)
f.open(QIODevice.ReadOnly)
self.post_bytes("hiperqubes/{0}/hdr".format(hiperqube_id),
f,
callback,
error,
content_type='application/octet-stream')
def upload_hiperqube_bil(self, url, fields, filename, progress, callback,
error):
"""Upload a BIL file to an existing hiperqube."""
req = QNetworkRequest(QUrl(url))
multi_part = QHttpMultiPart(QHttpMultiPart.FormDataType)
self.add_text_parts(multi_part, fields)
f = self.add_image_part(multi_part, filename)
rep = self._nam.post(req, multi_part)
multi_part.setParent(rep)
reply = CloudqubeProgressReply(
req, progress, callback, self.finished, error)
self._replies.append(reply)
def upload_hiperqube_bil_multipart(self, hiperqube_id, filename, progress, callback,
error):
"""Upload a BIL file to an existing hiperqube in multiple parts."""
def hiperqube_upload_created(parts):
QgsMessageLog.logMessage('hiperqube_upload_created: sorting part numbers')
parts = sorted(parts, key=lambda part: part['partNumber'])
QgsMessageLog.logMessage('hiperqube_upload_created: sorted!')
uploaded_parts = []
f = QFile(filename)
QgsMessageLog.logMessage('hiperqube_upload_created: opening file')
f.open(QIODevice.ReadOnly)
QgsMessageLog.logMessage('hiperqube_upload_created: file open')
def upload_part(index):
QgsMessageLog.logMessage('hiperqube_upload_created: uploading part {0}'.format(index))
def part_uploaded():
nonlocal uploaded_size
nonlocal index
e_tag = self.byte_array_to_string(rep.rawHeader(b'ETag'))
QgsMessageLog.logMessage('part_uplaoded: eTag {0}'.format(e_tag))
uploaded_size = uploaded_size + size
uploaded_parts.append({ 'eTag': e_tag, 'partNumber': part_number })
index = index + 1
if index < len(parts):
upload_part(index)
else:
QgsMessageLog.logMessage('part_uplaoded: closing file'.format(e_tag))
f.close()
self.complete_hiperqube_upload(hiperqube_id, uploaded_parts, callback, error)
nonlocal uploaded_size
part = parts[index]
url = part['url']
size = part['size']
part_number = part['partNumber']
req = QNetworkRequest(QUrl(url))
data = f.read(size)
QgsMessageLog.logMessage('upload_part: making put request')
rep = self._nam.put(req, data)
QgsMessageLog.logMessage('upload_part: got reply')
reply = CloudqubeMultipartProgressReply(
uploaded_size, total_size, rep, progress, part_uploaded, self.finished, error)
QgsMessageLog.logMessage('upload_part: appending reply')
self._replies.append(reply)
upload_part(0)
uploaded_size = 0
total_size = os.path.getsize(filename)
self.create_hiperqube_upload(hiperqube_id, total_size, hiperqube_upload_created, error)
# Signatures
def create_signature(self, hiperqube_id, name, pixels, callback, error):
"""Creates a new signature with the given name and pixels."""
points = []
for pixel in pixels:
points.append({
'line': pixel[0],
'col': pixel[1]
})
payload = {
'name': name,
'points': points
}
self.post_json(
"hiperqubes/{0}/signatures".format(hiperqube_id),
payload,
callback,
error)
def get_signatures(self, hiperqube_id, callback, error):
"""Gets all signatures from a hiperqube."""
self.get(
"hiperqubes/{0}/signatures".format(hiperqube_id), callback, error)
def get_signature(self, signature_id, callback, error):
"""Gets a signatures by id."""
self.get(
"signatures/{0}".format(signature_id), callback, error)
def delete_signature(self, signature_id, callback, error):
"""Deletes a signature."""
self.delete_nam(
"signatures/{0}".format(signature_id),
callback,
error)
def post_signature_chart(self, signature_ids, callback, error):
"""Creates a chart from an array of signature_ids."""
self.post_json(
'signatures/chart',
signature_ids,
callback,
error)
# File download
def download_file(self, uri, callback, error):
"""Downloads the file in the uri in a temporary file and returns its
name."""
req = QNetworkRequest(QUrl(uri))
reply = CloudqubeFileReply(self._nam.get(
req), callback, self.finished, error)
self._replies.append(reply)
return reply.filename()
# Abort
def abort_upload(self):
"""Aborts any pending upload."""
for reply in self._replies:
reply.abort()
self._replies = []