def getLayer(self,
user,
repo,
layer,
refspec,
extent=None,
screenWidth=None,
screenHeight=None,
limit=None,
simplifyGeom=True,
filepath=None):
user = cleanseUserName(user)
repo = cleanseRepoName(repo)
layer = cleanseLayerName(layer)
refspec = cleanseRefSpec(refspec)
query = QueryForLayerProto(self.connector)
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if limit != 1:
startProgressBar("Transferring data from GeoGig", 0,
currentWindow().messageBar())
setProgressValue(0)
result = query.query(user,
repo,
layer,
refspec,
extent,
screenWidth,
screenHeight,
limit,
simplifyGeom=simplifyGeom,
filepath=filepath)
if limit != 1:
closeProgressBar()
QApplication.restoreOverrideCursor()
return result
def linkClicked(self):
timepositions = self.coverage.timePositions()
dlg = DownloadDialog(timepositions, self.tree)
dlg.show()
dlg.exec_()
if dlg.timepositions:
folder = os.path.join(dlg.folder, self.coverage.name())
if not os.path.exists(folder):
try:
os.makedirs(folder)
except:
iface.messageBar().pushMessage("",
"Wrong output directory or error creating it",
level=QgsMessageBar.WARNING)
return
bandsFile = os.path.join(folder, "bands.json")
with open(bandsFile, "w") as f:
json.dump(self.coverage.bands, f)
startProgressBar("Downloading datacube subset", len(timepositions))
for i, time in enumerate(timepositions):
setProgressValue(i)
layer = self.coverage.layerForTimePosition(time)
execute(lambda: layer.saveTo(folder, dlg.roi))
closeProgressBar()
if dlg.openInDatacubePanel:
execute(lambda: self.widget.addEndpoint(dlg.folder))
def addLayer(self, user, repo, layer, branch, message):
user = cleanseUserName(user)
repo = cleanseRepoName(repo)
layer = cleanseLayerName(layer)
branch = cleanseBranchName(branch)
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
startProgressBar("Sending to Geogig Server", 2,
currentWindow().messageBar())
# tx for this
transactionid = self.openTransaction(user, repo)
# create branch
self.checkoutBranch(user, repo, branch, transactionid)
# create layer (with feature type)
headers = {
"Accept": "application/json",
"geogig-transaction-id": transactionid
}
layerdef = self.getLayerDef(layer)
self.connector.post("layers/{}/{}".format(user, repo),
json=layerdef,
headers=headers)
# send features
nfeatures = layer.featureCount()
featureIterator = layer.getFeatures()
BATCHSIZE = 1000
featuresSent = 0
def _batch(iterable, size):
sourceiter = iter(iterable)
while True:
batchiter = islice(sourceiter, size)
try:
yield chain([next(batchiter)], batchiter)
except StopIteration:
return
for batch in _batch(featureIterator, BATCHSIZE):
batch = list(batch)
progressText ="Sending features {:,}-{:,} of {:,}"\
.format(featuresSent,featuresSent+len(batch),nfeatures)
setProgressText(progressText)
self.addFeaturesToWorking(batch, user, repo, layer.name(),
transactionid)
featuresSent += len(batch)
#close transaction
self.commitTransaction(user, repo, transactionid, message)
closeProgressBar()
QApplication.restoreOverrideCursor()
def __init__(self, connector, taskId, msg=None):
QObject.__init__(self)
taskId = cleanseTaskId(taskId)
self.taskId = taskId
self.connector = connector
self.url = "{}tasks/{}".format(self.connector.url, str(self.taskId))
self.canceled = False
self.response = self.connector.getHttp("tasks/{}".format(
str(self.taskId)))
progress = self.response["progress"]
if progress:
self.maxvalue = max(0, progress["max_progress"])
self.text = msg or progress["task_description"]
else:
self.maxvalue = 0
self.text = "Waiting for task in server..."
startProgressBar(self.text, self.maxvalue,
currentWindow().messageBar())
setProgressValue(0)
def saveTiles(self, _extent):
folder = tempFolderInTempFolder()
layer = self.layer()
if _extent.intersects(layer.extent()):
extent = _extent.intersect(layer.extent())
xSize = extent.width() / layer.rasterUnitsPerPixelX()
ySize = extent.height() / layer.rasterUnitsPerPixelY()
xTiles = math.ceil(xSize / self.TILESIZE)
yTiles = math.ceil(ySize / self.TILESIZE)
i = 0
startProgressBar(
"Retrieving and preparing data [layer %s]" % self.name(),
xTiles * yTiles)
for x in xrange(xTiles):
for y in xrange(yTiles):
minX = extent.xMinimum(
) + x * layer.rasterUnitsPerPixelX() * self.TILESIZE
maxX = min(
extent.xMaximum(),
extent.xMinimum() +
(x + 1) * layer.rasterUnitsPerPixelX() * self.TILESIZE)
minY = extent.yMinimum(
) + y * layer.rasterUnitsPerPixelY() * self.TILESIZE
maxY = min(
extent.yMaximum(),
extent.yMinimum() +
(y + 1) * layer.rasterUnitsPerPixelY() * self.TILESIZE)
pt1 = QgsPoint(minX, minY)
pt2 = QgsPoint(maxX, maxY)
tileExtent = QgsRectangle(pt1, pt2)
filename = os.path.join(folder, "%i_%i.tif" % (x, y))
self._save(filename, tileExtent)
i += 1
setProgressValue(i)
closeProgressBar()
return folder
def _createMosaic(self):
mosaicFunction = mosaicFunctions[self.comboMosaicType.currentIndex()]
def getValue(textbox, paramName):
try:
v = float(textbox.text())
return v
except:
iface.messageBar().pushMessage(
"",
"Wrong value for parameter %s: %s" %
(paramName, textbox.text()),
level=QgsMessageBar.WARNING)
raise
try:
widgets = [
self.textXMin, self.textXMax, self.textYMin, self.textYMax
]
names = ["X min", "X max", "Y min", "Y max"]
xmin, xmax, ymin, ymax = [
getValue(w, n) for w, n in zip(widgets, names)
]
except:
return
extent = QgsRectangle(QgsPoint(xmin, ymin), QgsPoint(xmax, ymax))
txt = self.comboCoverage.currentText()
if not txt:
iface.messageBar().pushMessage("",
"No coverage selected",
level=QgsMessageBar.WARNING)
return
name, coverageName = txt.split(" : ")
loadedLayers = self._loadedLayersForCoverage(name, coverageName)
minDays = self.sliderStartDate.value()
maxDays = self.sliderEndDate.value()
validLayers = []
for layer in loadedLayers:
time = daysFromDate(parser.parse(layer.time()))
if (time >= minDays and time <= maxDays):
validLayers.append(layer)
bandNames = layers._coverages[name][coverageName].bands
if validLayers:
newBands = []
tilesFolders = []
dstFolder = tempFolderInTempFolder()
'''We download the layers so we can access them locally'''
lay = validLayers[0]
xSize = extent.width() / lay.layer().rasterUnitsPerPixelX()
ySize = extent.height() / lay.layer().rasterUnitsPerPixelY()
xTiles = math.ceil(xSize / lay.TILESIZE)
yTiles = math.ceil(ySize / lay.TILESIZE)
logger.info(
"Downloading datacube layers to local files. Extent:%sx%s. Tiles count: %sx%s"
% (extent.width(), extent.height(), xTiles, yTiles))
for i, lay in enumerate(validLayers):
start = timelib.time()
tilesFolders.append(lay.saveTiles(extent))
end = timelib.time()
logger.info("Layer %s downloaded in %s seconds." %
(str(i), str(end - start)))
try:
qaBand = bandNames.index("pixel_qa")
except:
qaBand = None
tileFiles = os.listdir(tilesFolders[0])
if not tileFiles:
iface.messageBar().pushMessage(
"",
"No available data within the selected extent.",
level=QgsMessageBar.WARNING)
return
startProgressBar("Processing mosaic data", len(tileFiles))
'''Now we process all tiles separately'''
for i, filename in enumerate(tileFiles):
tilestart = timelib.time()
setProgressValue(i)
start = timelib.time()
newBands = {}
files = [
os.path.join(folder, filename) for folder in tilesFolders
]
if qaBand is not None:
qaData = [getArray(f, qaBand + 1) for f in files]
else:
qaData = None
end = timelib.time()
logger.info("QA band prepared in %s seconds" %
(str(end - start)))
if mosaicFunction.bandByBand:
'''
We operate band by band, since a given band in the the final result
layer depends only on the values of that band in the input layers,
not the value of other bands'''
start = timelib.time()
for band, bandName in enumerate(bandNames):
if band == qaBand:
newBands[bandName] = mosaicFunction.computeQAMask(
qaData)
else:
bandData = [getArray(f, band + 1) for f in files]
newBands[bandName] = mosaicFunction.compute(
bandData, qaData)
bandData = None
end = timelib.time()
logger.info("Tile %s read and processed in %s seconds." %
(str(i), str(end - start)))
else:
'''
We operate with all bands at once, and the output layer will
have only each band computed from the set of them in the input
layers'''
bandData = []
bandNamesArray = []
start = timelib.time()
for i, band in enumerate(bandNames):
if i != qaBand:
bandData.append(
[getArray(f, i + 1) for f in files])
bandNamesArray.append(band)
end = timelib.time()
logger.info(
"Tile %s data read and prepared in %s seconds." %
(str(i), str(end - start)))
start = timelib.time()
newBandsArray = mosaicFunction.compute(bandData, qaData)
end = timelib.time()
logger.info("Tile %s data processed in %s seconds." %
(str(i), str(end - start)))
newBands = {
k: v
for k, v in zip(bandNamesArray, newBandsArray)
}
if qaBand is not None:
start = timelib.time()
newBands[bandNames[
qaBand]] = mosaicFunction.computeQAMask(qaData)
end = timelib.time()
logger.info("QA band prepared in %s seconds" %
(str(end - start)))
bandData = None
start = timelib.time()
'''We write the set of bands as a new layer. That will be an output tile'''
templateFilename = os.path.join(tilesFolders[0], filename)
ds = gdal.Open(templateFilename, GA_ReadOnly)
bandCount = ds.RasterCount
datatype = ds.GetRasterBand(1).DataType
width = ds.RasterXSize
height = ds.RasterYSize
geotransform = ds.GetGeoTransform()
projection = ds.GetProjection()
del ds
driver = gdal.GetDriverByName("GTiff")
dstFilename = os.path.join(dstFolder, filename)
dstDs = driver.Create(dstFilename, width, height, bandCount,
datatype)
for b, band in enumerate(bandNames):
gdalBand = dstDs.GetRasterBand(b + 1)
gdalBand.SetNoDataValue(NO_DATA)
gdalBand.WriteArray(newBands[band])
gdalBand.FlushCache()
del newBands
dstDs.SetGeoTransform(geotransform)
dstDs.SetProjection(projection)
del dstDs
end = timelib.time()
logger.info("Tile %s written to local file in %s seconds." %
(str(i), str(end - start)))
tileend = timelib.time()
logger.info("Total time to process tile: %s seconds." %
(str(tileend - tilestart)))
'''With all the tiles, we create a virtual raster'''
toMerge = ";".join([os.path.join(dstFolder, f) for f in tileFiles])
outputFile = os.path.join(dstFolder, "mosaic.vrt")
processing.runalg("gdalogr:buildvirtualraster", {
"INPUT": toMerge,
"SEPARATE": False,
"OUTPUT": outputFile
})
layer = QgsRasterLayer(outputFile,
"Mosaic [%s]" % mosaicFunction.name, "gdal")
layers._mosaicLayers[validLayers[0].datasetName()][
validLayers[0].coverageName()].append(outputFile)
addLayerIntoGroup(layer, validLayers[0].datasetName(),
validLayers[0].coverageName(), bandNames)
closeProgressBar()
iface.messageBar().pushMessage(
"",
"Mosaic has been correctly created and added to project.",
level=QgsMessageBar.INFO)
else:
iface.messageBar().pushMessage(
"",
"No layers available from the selected coverage.",
level=QgsMessageBar.WARNING)
def _plot(self):
self.figure.clear()
self.canvas.draw()
self.buttonSave.setEnabled(False)
if self.pt is None and self.rectangle is None:
return
if self.parameter is None or self.coverage is None or self.dataset is None:
return
canvasLayers = []
try:
allCoverageLayers = layers._layers[self.dataset][self.coverage]
except KeyError:
return
for layerdef in allCoverageLayers:
source = layerdef.source()
time = parser.parse(layerdef.time())
try:
layer = layerFromSource(source)
canvasLayers.append((layerdef, time))
except WrongLayerSourceException:
pass
if not canvasLayers:
return
minDate = None
maxDate = None
minY = None
maxY = None
if self.filter:
if self.filter[0] is not None:
minDate = self.filter[0]
if self.filter[1] is not None:
maxDate = self.filter[1]
minY = self.filter[2] or None
maxY = self.filter[3] or None
try:
bands = allCoverageLayers[0].bands()
if self.rectangle is None:
self.data = {}
startProgressBar("Retrieving plot data", len(canvasLayers))
for (i, (layerdef, time)) in enumerate(canvasLayers):
if ((minDate is not None and time < minDate)
or (maxDate is not None and time > maxDate)):
continue
start = timelib.time()
layer = layerdef.layer()
v = self.parameter.value(layer, self.pt, bands)
end = timelib.time()
logger.info(
"Plot data for layer %i retrieved in %s seconds" %
(i, str(end - start)))
setProgressValue(i + 1)
if v is not None:
self.data[time] = [(v, (self.pt.x(), self.pt.y()))]
closeProgressBar()
if not self.data:
return
y = [v[0][0] for v in self.data.values()]
ymin = min(y)
ymax = max(y)
else:
self.data = {}
startProgressBar("Retrieving plot data", len(canvasLayers))
for (i, (layerdef, time)) in enumerate(canvasLayers):
if ((minDate is not None and time < minDate)
or (maxDate is not None and time > maxDate)):
continue
start = timelib.time()
layer = layerdef.layer()
if not self.rectangle.intersects(layer.extent()):
continue
rectangle = self.rectangle.intersect(layer.extent())
xsteps = int(rectangle.width() /
layer.rasterUnitsPerPixelX())
ysteps = int(rectangle.height() /
layer.rasterUnitsPerPixelY())
filename = layerdef.layerFile(rectangle)
roi = layers.getBandArrays(filename)
end = timelib.time()
logger.info(
"ROI data for layer %i retrieved in %s seconds" %
(i, str(end - start)))
start = timelib.time()
setProgressValue(i + 1)
self.data[time] = []
for col in range(xsteps):
x = rectangle.xMinimum(
) + col * layer.rasterUnitsPerPixelX()
for row in range(ysteps):
y = rectangle.yMinimum(
) + row * layer.rasterUnitsPerPixelY()
pixel = QgsPoint(col, row)
value = self.parameter.value(roi, pixel, bands)
if value:
self.data[time].append((value, (x, y)))
end = timelib.time()
logger.info(
"Plot data computed from ROI data in %s seconds" %
(str(end - start)))
closeProgressBar()
if not self.data:
return
y = [[v[0] for v in lis] for lis in self.data.values()]
ymin = min([min(v) for v in y])
ymax = max([max(v) for v in y])
xmin = min(self.data.keys())
xmax = max(self.data.keys())
if self.filter is None:
self.plotDataChanged.emit(xmin, xmax, ymin, ymax)
self.dataToPlot = copy.deepcopy(self.data)
if self.filter:
for key, values in self.data.iteritems():
for v in values[::-1]:
if ((minY is not None and v[0] < minY)
or (maxY is not None and v[0] > maxY)):
try:
self.dataToPlot[key].remove(v)
except:
pass
datesToRemove = []
for key, values in self.dataToPlot.iteritems():
if not values:
datesToRemove.append(key)
for d in datesToRemove:
del self.dataToPlot[d]
axes = self.figure.add_subplot(1, 1, 1)
x = matplotlib.dates.date2num(self.dataToPlot.keys())
if self.rectangle is None:
y = [v[0][0] for v in self.dataToPlot.values() if v]
axes.scatter(self.dataToPlot.keys(), y)
else:
sortedKeys = sorted(self.dataToPlot.keys())
y = [[v[0] for v in self.dataToPlot[k]] for k in sortedKeys]
axes.boxplot(y)
axes.set_xticklabels(
[str(d).split(" ")[0] for d in sortedKeys], rotation=70)
self.figure.autofmt_xdate()
except Exception, e:
traceback.print_exc()
closeProgressBar()
return
def start(self):
startProgressBar("Downloading from GeoGig server", 0,
currentWindow().messageBar())
self.worker.start() # start load