def processAlgorithm(self, progress):
fileName = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(fileName)
crs = QgsCoordinateReferenceSystem(self.getParameterValue(self.CRS))
provider = layer.dataProvider()
ds = provider.dataSourceUri()
p = re.compile('\|.*')
dsPath = p.sub('', ds)
if dsPath.lower().endswith('.shp'):
dsPath = dsPath[:-4]
wkt = crs.toWkt()
with open(dsPath + '.prj', 'w') as f:
f.write(wkt)
qpjFile = dsPath + '.qpj'
if os.path.exists(qpjFile):
with open(qpjFile, 'w') as f:
f.write(wkt)
layer.setCrs(crs)
iface.mapCanvas().refresh()
self.setOutputValue(self.OUTPUT, fileName)
def processAlgorithm(self, feedback):
fileName = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(fileName)
crs = QgsCoordinateReferenceSystem(self.getParameterValue(self.CRS))
provider = layer.dataProvider()
ds = provider.dataSourceUri()
p = re.compile("\|.*")
dsPath = p.sub("", ds)
if dsPath.lower().endswith(".shp"):
dsPath = dsPath[:-4]
wkt = crs.toWkt()
with open(dsPath + ".prj", "w") as f:
f.write(wkt)
qpjFile = dsPath + ".qpj"
if os.path.exists(qpjFile):
with open(qpjFile, "w") as f:
f.write(wkt)
layer.setCrs(crs)
iface.mapCanvas().refresh()
self.setOutputValue(self.OUTPUT, fileName)
def processAlgorithm(self, parameters, context, feedback):
fileName = self.getParameterValue(self.INPUT)
layer = QgsProcessingUtils.mapLayerFromString(fileName, context)
crs = QgsCoordinateReferenceSystem(self.getParameterValue(self.CRS))
provider = layer.dataProvider()
ds = provider.dataSourceUri()
p = re.compile('\|.*')
dsPath = p.sub('', ds)
if dsPath.lower().endswith('.shp'):
dsPath = dsPath[:-4]
wkt = crs.toWkt()
with open(dsPath + '.prj', 'w') as f:
f.write(wkt)
qpjFile = dsPath + '.qpj'
if os.path.exists(qpjFile):
with open(qpjFile, 'w') as f:
f.write(wkt)
layer.setCrs(crs)
iface.mapCanvas().refresh()
self.setOutputValue(self.OUTPUT, fileName)
def set_point(self, variable: str, point: QgsPointXY,
crs: QgsCoordinateReferenceSystem) -> list:
"""
Produces R code that creates a variable from point input.
:param variable: string. Name of the variable.
:param point: QgsPointXY. Point to extract x and y coordinates from.
:param crs: QgsCoordinateReferenceSystem. Coordinate reference system for the point.
:return: string. R code that constructs the point.
"""
commands = []
if self.use_sf:
commands.append('point_crs <- st_crs(\'{}\')'.format(crs.toWkt()))
commands.append(
'{0} <- st_sfc(st_point(c({1},{2})), crs = point_crs)'.format(
variable, point.x(), point.y()))
else:
commands.append('xy_df <- cbind(c({}), c({}))'.format(
point.x(), point.y()))
commands.append('point_crs <- CRS(\'{}\')'.format(crs.toProj4()))
commands.append(
'{} <- SpatialPoints(xy_df, proj4string = point_crs)'.format(
variable))
return commands
def convert_and_import(xml_file):
QgsProject.instance().clear()
with tempfile.NamedTemporaryFile(delete=True) as f:
out_f = f.name
config_file = os.path.join(os.path.dirname(__file__), "gmlasconf.xml")
gdal.SetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF")
ds = gdal.OpenEx(
"GMLAS:{}".format(xml_file),
open_options=[
"EXPOSE_METADATA_LAYERS=YES",
"CONFIG_FILE={}".format(config_file),
],
)
srs = osr.SpatialReference()
qgs_srs = QgsCoordinateReferenceSystem("EPSG:4326")
srs.ImportFromWkt(qgs_srs.toWkt())
params = {
"destNameOrDestDS": out_f,
"srcDS": ds,
"format": "SQLite",
"accessMode": "overwrite",
"datasetCreationOptions": ["SPATIALITE=YES"],
"options": ["-forceNullable", "-skipfailures"]
# , 'srcSRS': srs
# , 'dstSRS': srs
,
"geometryType": "CONVERT_TO_LINEAR",
"reproject": False,
}
# call gdal to convert
gdal.VectorTranslate(**params)
# fix geometry types
ds = None
# populate the qgis project
import_in_qgis(out_f, "SQLite")
layers = []
for lid in sorted(QgsProject.instance().mapLayers().keys()):
vl = QgsProject.instance().mapLayer(lid)
layers.append((vl.name(), vl.wkbType()))
rels = []
relations = QgsProject.instance().relationManager().relations()
for relid in sorted(relations.keys()):
rel = relations[relid]
p = rel.fieldPairs()
rels.append(
(
rel.id()[0:3],
rel.referencingLayer().name(),
list(p.keys())[0],
rel.referencedLayer().name(),
list(p.values())[0],
)
)
return sorted(layers), sorted(rels)
def testWidget(self):
"""
Test widget logic
"""
w = QgsCrsDefinitionWidget()
self.assertFalse(w.crs().isValid())
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
spy = QSignalSpy(w.crsChanged)
c = QgsCoordinateReferenceSystem('EPSG:3111')
w.setCrs(c)
self.assertEqual(w.crs(), c)
self.assertEqual(len(spy), 1)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
self.assertEqual(w.definitionString(), c.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
# native proj string definition
w.setCrs(c, QgsCoordinateReferenceSystem.FormatProj)
self.assertEqual(w.crs(), c)
self.assertEqual(len(spy), 2)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatProj)
self.assertEqual(w.definitionString(), c.toProj())
# native WKT string definition
w.setCrs(c, QgsCoordinateReferenceSystem.FormatWkt)
self.assertEqual(w.crs(), c)
self.assertEqual(len(spy), 3)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
self.assertEqual(w.definitionString(), c.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
# change format
w.setFormat(QgsCoordinateReferenceSystem.FormatProj)
self.assertEqual(len(spy), 4)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatProj)
self.assertEqual(w.definitionString(), c.toProj())
w.setFormat(QgsCoordinateReferenceSystem.FormatWkt)
# trip through proj string is lossy -- don't compare to previous wkt!
self.assertEqual(len(spy), 5)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
def test_definition_string(self):
"""
Test definition string logic
"""
w = QgsCrsDefinitionWidget()
w.setFormat(QgsCoordinateReferenceSystem.FormatWkt)
c = QgsCoordinateReferenceSystem('EPSG:3111')
spy = QSignalSpy(w.crsChanged)
w.setDefinitionString(c.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
self.assertEqual(w.crs(), c)
self.assertEqual(len(spy), 1)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
self.assertEqual(w.definitionString(), c.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
c2 = QgsCoordinateReferenceSystem('EPSG:3113')
w.setDefinitionString(c2.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
self.assertEqual(w.crs(), c2)
self.assertEqual(len(spy), 2)
self.assertEqual(w.format(), QgsCoordinateReferenceSystem.FormatWkt)
self.assertEqual(w.definitionString(), c2.toWkt(QgsCoordinateReferenceSystem.WKT_PREFERRED))
def convert_feature_service_workspace(
name, workspace_name: WorkspaceName, base: str,
crs: QgsCoordinateReferenceSystem, subset: str,
context: Context) -> DataSourceProperties:
"""
Convert FeatureServiceWorkspaceFactory
"""
# todo -- auto create auth service?
crs_text = crs.authid()
if not crs_text:
crs_text = 'WKT:{}'.format(crs.toWkt())
uri = "crs='{}' url='{}/{}'".format(crs_text, workspace_name.name,
name.name)
if context.ignore_online_sources:
uri = ''
provider = 'arcgisfeatureserver'
wkb_type = DatasetNameConverter.geometry_type_to_wkb(name.shape_type)
return DataSourceProperties(uri=uri,
wkb_type=wkb_type,
provider=provider)
def convert_and_import(xml_file):
QgsProject.instance().clear()
with tempfile.NamedTemporaryFile(delete=True) as f:
out_f = f.name
config_file = os.path.join(os.path.dirname(__file__), "gmlasconf.xml")
gdal.SetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF")
ds = gdal.OpenEx("GMLAS:{}".format(xml_file), open_options=['EXPOSE_METADATA_LAYERS=YES', 'CONFIG_FILE={}'.format(config_file)])
srs = osr.SpatialReference()
qgs_srs = QgsCoordinateReferenceSystem("EPSG:4326")
srs.ImportFromWkt(qgs_srs.toWkt())
params = {
'destNameOrDestDS': out_f
, 'srcDS': ds
, 'format': "SQLite"
, 'accessMode': "overwrite"
, 'datasetCreationOptions': ['SPATIALITE=YES']
, 'options' : ['-forceNullable', '-skipfailures']
#, 'srcSRS': srs
#, 'dstSRS': srs
, 'geometryType': 'CONVERT_TO_LINEAR'
, 'reproject': False
}
# call gdal to convert
gdal.VectorTranslate(**params)
# fix geometry types
ds = None
# populate the qgis project
import_in_qgis(out_f, "SQLite")
layers = []
for lid in sorted(QgsProject.instance().mapLayers().keys()):
vl = QgsProject.instance().mapLayer(lid)
layers.append((vl.name(), vl.wkbType()))
rels = []
relations = QgsProject.instance().relationManager().relations()
for relid in sorted(relations.keys()):
rel = relations[relid]
p = rel.fieldPairs()
rels.append((rel.id()[0:3], rel.referencingLayer().name(), list(p.keys())[0], rel.referencedLayer().name(), list(p.values())[0]))
return sorted(layers), sorted(rels)
class GSIElevTileProvider:
def __init__(self, dest_wkt):
self.dest_wkt = dest_wkt
# crs transformer, which aims to calculate bbox in EPSG:3857
self.crs3857 = QgsCoordinateReferenceSystem(3857)
self.dest_crs = QgsCoordinateReferenceSystem()
if not self.dest_crs.createFromWkt(dest_wkt):
logMessage("Failed to create CRS from WKT: {0}".format(dest_wkt))
self.transform = QgsCoordinateTransform(self.dest_crs, self.crs3857,
QgsProject.instance())
# approximate bbox of this data
self.boundingbox = QgsRectangle(13667807, 2320477, 17230031, 5713298)
self.downloader = Downloader()
self.downloader.userAgent = "QGIS/{0} Qgis2threejs GSIElevTileProvider".format(
Qgis.QGIS_VERSION_INT
) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value(
"/qgis/defaultTileExpiry", 24, type=int)
self.driver = gdal.GetDriverByName("MEM")
self.last_dataset = None
def name(self):
return "GSI Elevation Tile"
def read(self, width, height, extent):
# calculate bounding box in EPSG:3857
geometry = extent.geometry()
geometry.transform(self.transform)
merc_rect = geometry.boundingBox()
# if the bounding box doesn't intersect with the bounding box of this data, return a list filled with nodata value
if not self.boundingbox.intersects(merc_rect):
return [NODATA_VALUE] * width * height
# get tiles
over_smpl = 1
segments_x = 1 if width == 1 else width - 1
res = extent.width() / segments_x / over_smpl
ds = self.getDataset(merc_rect.xMinimum(), merc_rect.yMinimum(),
merc_rect.xMaximum(), merc_rect.yMaximum(), res)
geotransform = extent.geotransform(width, height)
return self._read(ds, width, height, geotransform)
def readValue(self, x, y):
"""Get value at the position using 1px * 1px memory raster. The value is calculated using a tile of max zoom level"""
# coordinate transformation into EPSG:3857
pt = self.transform.transform(QgsPoint(x, y))
# if the point is not within the bounding box of this data, return nodata value
if not self.boundingbox.contains(pt):
return NODATA_VALUE
res = 0.1
hres = res / 2
ds = self.getDataset(pt.x() - hres,
pt.y() - hres,
pt.x() + hres,
pt.y() + hres, res)
geotransform = [x - hres, res, 0, y + hres, 0, -res]
return self._read(ds, 1, 1, geotransform)[0]
def _read(self, ds, width, height, geotransform):
# create a memory dataset
warped_ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32)
warped_ds.SetProjection(self.dest_wkt)
warped_ds.SetGeoTransform(geotransform)
# reproject image
gdal.ReprojectImage(ds, warped_ds, None, None, gdal.GRA_Bilinear)
# load values into an array
band = warped_ds.GetRasterBand(1)
fs = "f" * width * height
return struct.unpack(
fs, band.ReadRaster(0, 0, width, height,
buf_type=gdal.GDT_Float32))
def getDataset(self, xmin, ymin, xmax, ymax, mapUnitsPerPixel):
# calculate zoom level
mpp1 = TSIZE1 / TILE_SIZE
zoom = int(math.ceil(math.log(mpp1 / mapUnitsPerPixel, 2) + 1))
zoom = max(0, min(zoom, ZMAX))
# calculate tile range (yOrigin is top)
size = TSIZE1 / 2**(zoom - 1)
matrixSize = 2**zoom
ulx = max(0, int((xmin + TSIZE1) / size))
uly = max(0, int((TSIZE1 - ymax) / size))
lrx = min(int((xmax + TSIZE1) / size), matrixSize - 1)
lry = min(int((TSIZE1 - ymin) / size), matrixSize - 1)
cols = lrx - ulx + 1
rows = lry - uly + 1
# download count limit
if cols * rows > 128:
logMessage("Number of tiles to fetch is too large!")
width = height = 1
return self.driver.Create("", width, height, 1, gdal.GDT_Float32,
[])
if self.last_dataset and self.last_dataset[0] == [
zoom, ulx, uly, lrx, lry
]: # if same as last tile set, return cached dataset
return self.last_dataset[1]
urltmpl = "http://cyberjapandata.gsi.go.jp/xyz/dem/{z}/{x}/{y}.txt"
#urltmpl = "http://localhost/xyz/dem/{z}/{x}/{y}.txt"
tiles = self.fetchFiles(urltmpl, zoom, ulx, uly, lrx, lry)
# create a memory dataset
width = cols * TILE_SIZE
height = rows * TILE_SIZE
res = size / TILE_SIZE
geotransform = [
ulx * size - TSIZE1, res, 0, TSIZE1 - uly * size, 0, -res
]
#mem_driver = gdal.GetDriverByName("GTiff")
#ds = mem_driver.Create("D:/fetched_tile.tif", width, height, 1, gdal.GDT_Float32, [])
ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
ds.SetProjection(str(self.crs3857.toWkt()))
ds.SetGeoTransform(geotransform)
band = ds.GetRasterBand(1)
for i, tile in enumerate(tiles):
if tile:
col = i % cols
row = i // cols
band.WriteRaster(col * TILE_SIZE, row * TILE_SIZE, TILE_SIZE,
TILE_SIZE, tile)
ds.FlushCache()
self.last_dataset = [[zoom, ulx, uly, lrx, lry], ds] # cache dataset
return ds
def fetchFiles(self, urltmpl, zoom, xmin, ymin, xmax, ymax):
downloadTimeout = 60
urls = []
for y in range(ymin, ymax + 1):
for x in range(xmin, xmax + 1):
urls.append(
urltmpl.replace("{x}",
str(x)).replace("{y}", str(y)).replace(
"{z}", str(zoom)))
files = self.downloader.fetchFiles(urls, downloadTimeout)
for url in urls:
data = files[url]
if data:
yield numpy.fromstring(data.replace(
b"e", NODATA_VALUE_BYTES).replace(b"\n", b","),
dtype=numpy.float32,
sep=",").tostring() # to byte array
else:
array = numpy.empty(TILE_SIZE * TILE_SIZE, dtype=numpy.float32)
array.fill(NODATA_VALUE)
yield array.tostring()
class TECfile(QListWidgetItem):
def __init__(self, parent, widgetType, filePath, iface):
super(TECfile, self).__init__(parent, widgetType)
self.filePath = filePath
self.fileName = os.path.basename(filePath)
self.headerLinesCount = 0
self.iface = iface
self.TECTile = ''
self.settings = QSettings('ManySplendid', 'SRH2D_TEC_Viewer')
self.variables = list()
self.composition = dict()
self.variableType = list()
self.readTEC(filePath)
self.outDir = ''
self.xAttr = ''
self.yAttr = ''
refId = QgsCoordinateReferenceSystem.PostgisCrsId
if self.settings.value('crs'):
self.crs = QgsCoordinateReferenceSystem(self.settings.value('crs'),
refId)
else:
self.crs = QgsCoordinateReferenceSystem(3826, refId)
self.iface = ''
self.setText(os.path.basename(filePath))
def readTEC(self, filePath):
f = open(filePath)
dat = f.readlines()
self.getVariables(dat)
self.dat = dat
def loadTEC(self):
self.readVariables(self.dat)
for i in range(0, len(self.variables[3]['Water_Elev_m'])):
if float(self.variables[3]['Water_Elev_m'][i]) == -999.0:
self.variables[3]['Water_Elev_m'][i] = self.variables[2][
'Bed_Elev_meter'][i]
Xkey = self.variables[0].keys()[0]
X = self.toFloat(self.variables[0][Xkey])
self.Xmax = max(X)
self.Xmin = min(X)
Ykey = self.variables[1].keys()[0]
Y = self.toFloat(self.variables[1][Ykey])
self.Ymax = max(Y)
self.Ymin = min(Y)
# ######## Make Othogonal Grid Points #########
points2 = geometry.MultiPoint(list(zip(X, Y)))
concave_hull2, edge_points2 = self.alpha_shape(points2, alpha=0.01)
self.hull = concave_hull2
reso = float(self.settings.value('resolution'))
self.reso = reso
X_coordinate = np.arange(self.Xmin, self.Xmax + 0.001, reso)
Y_coordinate = np.arange(self.Ymax, self.Ymin, -reso)
self.Nx = len(X_coordinate)
self.Ny = len(Y_coordinate)
X_ref, Y_ref = np.meshgrid(X_coordinate, Y_coordinate)
self.X_ref = X_ref
self.Y_ref = Y_ref
widx = np.zeros([self.Ny, self.Nx])
# Identify points inside mesh area
for i in range(0, len(X_ref)):
for j in range(0, len(X_ref[0])):
widx[i, j] = self.within_polygon(
geometry.Point(X_ref[i, j], Y_ref[i, j]), concave_hull2, 0)
self.widx = widx
# self.makeMeshLayer()
# self.makeSieve()
# self.genNodeLayer()
def alpha_shape(self, points, alpha):
"""
Compute the alpha shape (concave hull) of a set of points.
@param points: Iterable container of points.
@param alpha: alpha value to influence the gooeyness of the border.
Smaller numbers don't fall inward as much as larger
numbers. Too large, and you lose everything!
"""
if len(points) < 4:
# When you have a triangle, there is no sense in computing an alpha
# shape.
return geometry.MultiPoint(list(points)).convex_hull
def add_edge(edges, edge_points, coords, i, j):
"""Add a line between the i-th and j-th points, if not in the list
already"""
if (i, j) in edges or (j, i) in edges:
# already added
return
edges.add((i, j))
edge_points.append(coords[[i, j]])
coords = np.array([point.coords[0] for point in points])
tri = Delaunay(coords)
edges = set()
edge_points = []
# loop over triangles:
# ia, ib, ic = indices of corner points of the triangle
for ia, ib, ic in tri.vertices:
pa = coords[ia]
pb = coords[ib]
pc = coords[ic]
# Lengths of sides of triangle
a = sqrt((pa[0] - pb[0])**2 + (pa[1] - pb[1])**2)
b = sqrt((pb[0] - pc[0])**2 + (pb[1] - pc[1])**2)
c = sqrt((pc[0] - pa[0])**2 + (pc[1] - pa[1])**2)
# Semiperimeter of triangle
s = (a + b + c) / 2.0
# Area of triangle by Heron's formula
area = sqrt(s * (s - a) * (s - b) * (s - c))
try:
circum_r = a * b * c / (4.0 * area)
except (ZeroDivisionError):
circum_r = 0
# Here's the radius filter.
# print circum_r
if circum_r < 1.0 / alpha:
add_edge(edges, edge_points, coords, ia, ib)
add_edge(edges, edge_points, coords, ib, ic)
add_edge(edges, edge_points, coords, ic, ia)
m = geometry.MultiLineString(edge_points)
triangles = list(polygonize(m))
return cascaded_union(triangles), edge_points
def within_polygon(self, grid_point, concave_hull, buffer):
intw = 0
if grid_point.within(concave_hull.buffer(buffer)):
intw = 1
return intw
def export(self):
self.outDir = os.path.join(self.outDir,
self.fileName.replace('.dat', ''))
if os.path.isdir(self.outDir):
subprocess.call(['cmd', '/c', 'RD', '/S', '/Q', self.outDir])
subprocess.call(['cmd', '/c', 'mkdir', self.outDir])
subprocess.call([
'cmd', '/c', 'mkdir',
os.path.join(self.outDir, 'supplements')
])
else:
subprocess.call(['cmd', '/c', 'mkdir', self.outDir])
subprocess.call([
'cmd', '/c', 'mkdir',
os.path.join(self.outDir, 'supplements')
])
self.loadTEC()
group = QgsProject.instance().layerTreeRoot().addGroup(self.fileName)
attrCounter = 0
for attr in self.attributes:
if attr[1] == 1:
attrCounter += 1
_attrProgress = int(75 / attrCounter / 3)
progress = 75
self.contentLayers = list()
for attr in self.attributes:
idx = self.attributes.index(attr) + 2
if attr[1] == 1:
self.makeContentLayers(attr[0], idx)
progress += _attrProgress
if len(attr[0]) > 10:
rasterName = attr[0][0:10]
else:
rasterName = attr[0]
rasterPath = os.path.join(self.outDir, rasterName + '.tif')
baseName = QFileInfo(rasterPath).baseName()
rasterLayer = QgsRasterLayer(rasterPath, baseName)
rasMapLayer = QgsMapLayerRegistry.instance().addMapLayer(
rasterLayer, False)
progress += _attrProgress
self.contentLayers.append([attr[0], rasMapLayer.id()])
group.addLayer(rasterLayer)
progress += _attrProgress
def attributeList(self):
attributes = self.attributes
_attrs = list()
for i in range(2, len(attributes)):
_attrs.append([attributes[i], 0])
self.attributes = _attrs
def getVariables(self, dat):
def readDat(dat, title, _variable, ZONE, DT):
mode = 0
lineCount = 0
for line in dat:
if line.startswith(' TITLE'):
mode = 0
elif line.startswith(' VARIABLES'):
mode = 1
elif line.startswith(' ZONE'):
mode = 2
elif line.startswith(' DT'):
mode = 3
elif line.endswith(')\n'):
mode = 4
if mode == 0:
title.append(line)
elif mode == 1:
_variables.append(line)
elif mode == 2:
ZONE.append(line)
elif mode == 3 or mode == 4:
DT.append(line)
if line.endswith(')\n'):
mode = 5
else:
return (title, _variable, ZONE, DT, lineCount)
lineCount += 1
title = list()
_variables = list()
ZONE = list()
DT = list()
title, _variables, ZONE, DT, lineCount = readDat(
dat, title, _variables, ZONE, DT)
self.headerLinesCount = lineCount
titleString = ''
for line in title:
titleString += line
titleString.replace('\n', '')
titleString.replace('"', '')
titleString = re.split('=', titleString)
self.TECTitle = titleString[1].replace('\n', '').strip()
variableString = ''
for line in _variables:
variableString += line
variableString.replace('\n', '')
self.variables = re.findall(r'\S+', variableString)
self.attributes = re.findall(r'\S+', variableString)
self.variables.pop(0)
self.attributes.pop(0)
self.variables[0] = self.variables[0].replace('=', '')
self.attributes[0] = self.attributes[0].replace('=', '')
self.attributeList()
zoneString = ''
for line in ZONE:
zoneString += line
zoneSplit = re.split(',', zoneString)
zoneSplit[0] = zoneSplit[0].replace(' ZONE ', '')
for unit in zoneSplit:
self.composition.update(
{unit.split('=')[0]: unit.split('=')[1].strip()})
DTstring = ''
for line in DT:
DTstring += line
DTstring = DTstring.replace(' DT=(', '')
DTstring = DTstring.replace(' \n', '')
DTstring = DTstring.replace(')\n', '')
vtype = re.split(r'\s', DTstring.strip())
self.variableType = vtype
def readVariables(self, dat):
lineCounter = int(ceil(float(self.composition['N']) / 5))
variables = self.variables
totalVaraibles = len(variables)
readCounter = 0
variableCounter = 0
data = list()
mesh = list()
for i in range(self.headerLinesCount, len(dat)):
row = re.findall(r'\S+', dat[i])
for number in row:
data.append(number)
readCounter += 1
if ((readCounter == lineCounter)
and (variableCounter < totalVaraibles)):
variables[variableCounter] = {
str(variables[variableCounter]): data
}
variableCounter += 1
data = list()
readCounter = 0
elif variableCounter >= totalVaraibles:
polygon = list()
for node in dat[i].split():
polygon.append(int(node))
mesh.append(polygon)
self.mesh = mesh
def toFloat(self, array):
for i in range(0, len(array)):
array[i] = float(array[i])
return array
def makeMeshLayer(self):
mesh = self.mesh
# ######## Make Polygon Mesh Layer ##########
c_folder = self.outDir
crs = self.crs
path = os.path.join(c_folder, 'mesh.shp')
self.layerPath = path
fields = QgsFields()
fields.append(QgsField("id", QVariant.Int))
fields.append(QgsField("val", QVariant.Int))
writer = QgsVectorFileWriter(path, 'utf-8', fields, QGis.WKBPolygon,
crs, "ESRI Shapefile")
feat_id = 1
for polygon in mesh:
feature = QgsFeature()
geoString = 'POLYGON (('
if polygon[-1] == polygon[-2]:
polygon[-1] = polygon[0]
else:
polygon.append(polygon[0])
for node in polygon:
geoString += (str(X[node - 1]) + " " + str(Y[node - 1]) + ",")
geoString = geoString[:-1] + "))"
feature.setGeometry(QgsGeometry.fromWkt(geoString))
feature.setAttributes([feat_id, 1])
writer.addFeature(feature)
feat_id += 1
layer = QgsVectorLayer(path, QFileInfo(path).baseName(), 'ogr')
self.meshLayer = layer
def makeSieve(self):
layer = self.meshLayer
sievePath = os.path.join(self.outDir, 'sieve.tif')
xmin = float(self.Xmin)
xmax = float(self.Xmax)
ymin = float(self.Ymin)
ymax = float(self.Ymax)
processing.runalg(
'gdalogr:rasterize', {
"INPUT": layer,
"FIELD": "val",
"DIMENSIONS": 1,
"WIDTH": float(self.settings.value('resolution')),
"HEIGHT": float(self.settings.value('resolution')),
"RAST_EXT": "%f,%f,%f,%f" % (xmin, xmax, ymin, ymax),
"TFW": 1,
"RTYPE": 4,
"NO_DATA": -1,
"COMPRESS": 0,
"JPEGCOMPRESSION": 1,
"ZLEVEL": 1,
"PREDICTOR": 1,
"TILED": False,
"BIGTIFF": 0,
"EXTRA": '',
"OUTPUT": sievePath
})
crs = self.crs
dataset = gdal.Open(sievePath, GA_Update)
# band = dataset.GetRasterBand(1)
srs = osr.SpatialReference()
srs.ImportFromEPSG(3826)
dataset.SetProjection(srs.ExportToWkt())
# dataset = None
sieveLayer = QgsRasterLayer(sievePath, QFileInfo(sievePath).baseName())
self.sieveLayer = sieveLayer
self.sieveLayer.setCrs(crs)
def genNodeLayer(self):
crs = self.crs
baseName = os.path.splitext(self.fileName)[0]
path = os.path.join(self.outDir, baseName + '.shp')
fields = QgsFields()
for i in range(2, len(self.variables)):
fieldName = self.variables[i].keys()[0]
if self.variableType[i] == 'DOUBLE':
fields.append(QgsField(fieldName, QVariant.Double, len=20))
elif self.variableType[i] == 'INT':
fields.append(QgsField(fieldName, QVariant.Int, len=20))
writer = QgsVectorFileWriter(path, 'utf-8', fields, QGis.WKBPoint, crs,
"ESRI Shapefile")
Xkey = self.variables[0].keys()[0]
X = self.variables[0][Xkey]
Ykey = self.variables[1].keys()[0]
Y = self.variables[1][Ykey]
for i in range(0, len(X)):
geoString = 'POINT (' + str(X[i]) + ' ' + str(Y[i]) + ')'
attr = list()
#
for j in range(2, len(self.variables)):
fieldName = self.variables[j].keys()[0]
attr.append(self.variables[j][fieldName][i])
#
point = QgsFeature()
point.setGeometry(QgsGeometry.fromWkt(geoString))
point.setAttributes(attr)
writer.addFeature(point)
del writer
nodeLayer = QgsVectorLayer(path, QFileInfo(path).baseName(), 'ogr')
self.nodeLayer = nodeLayer
def makeContentLayers(self, fieldKey, idx):
c_folder = self.outDir
# original grid data
Xkey = self.variables[0].keys()[0]
X = self.toFloat(self.variables[0][Xkey])
Ykey = self.variables[1].keys()[0]
Y = self.toFloat(self.variables[1][Ykey])
ZKey = self.variables[idx].keys()[0]
Z = self.toFloat(self.variables[idx][ZKey])
if len(fieldKey) > 10:
fieldName = fieldKey[0:10]
else:
fieldName = fieldKey
rasterName = fieldName
X_ref = self.X_ref
Y_ref = self.Y_ref
widx = self.widx
driver = gdal.GetDriverByName('GTiff')
dst_filename = os.path.join(c_folder, rasterName + '.tif')
dataset = driver.Create(dst_filename, self.Nx, self.Ny, 1,
gdal.GDT_Float32)
dataset.SetProjection(str(self.crs.toWkt()))
geotransform = (self.Xmin, self.reso, 0, self.Ymax, 0, -self.reso)
dataset.SetGeoTransform(geotransform)
ValueArray = np.zeros([self.Ny, self.Nx])
# Interpolate on grid data using new orthogonal grids coordinate
points = list()
for i in range(0, len(X_ref)):
points = points + zip(X_ref[i, :], Y_ref[i, :])
Values = griddata(zip(X, Y), Z, points)
for i in range(0, len(X_ref)):
ValueArray[i, :] = Values[i * self.Nx:(i + 1) * self.Nx]
for i in range(0, len(widx)):
for j in range(0, len(widx[i])):
if int(widx[i, j]) == 0:
ValueArray[i, j] = -9999
# ###### Create Empty geotiff ######
band = dataset.GetRasterBand(1)
band.SetNoDataValue(-9999) # Set no data value to -9999 in raster file
band.WriteArray(ValueArray)
dataset.FlushCache()
class GSIElevTileProvider:
def __init__(self, dest_wkt):
self.dest_wkt = dest_wkt
# crs transformer, which aims to calculate bbox in EPSG:3857
self.crs3857 = QgsCoordinateReferenceSystem(3857)
self.dest_crs = QgsCoordinateReferenceSystem()
if not self.dest_crs.createFromWkt(dest_wkt):
logMessage("Failed to create CRS from WKT: {0}".format(dest_wkt))
self.transform = QgsCoordinateTransform(self.dest_crs, self.crs3857)
# approximate bbox of this data
self.boundingbox = QgsRectangle(13667807, 2320477, 17230031, 5713298)
self.downloader = Downloader()
self.downloader.userAgent = "QGIS/{0} Qgis2threejs GSIElevTileProvider".format(QGis.QGIS_VERSION) # not written since QGIS 2.2
self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value("/qgis/defaultTileExpiry", 24, type=int)
self.driver = gdal.GetDriverByName("MEM")
self.last_dataset = None
def name(self):
return "GSI Elevation Tile"
def read(self, width, height, extent):
# calculate bounding box in EPSG:3857
geometry = extent.geometry()
geometry.transform(self.transform)
merc_rect = geometry.boundingBox()
# if the bounding box doesn't intersect with the bounding box of this data, return a list filled with nodata value
if not self.boundingbox.intersects(merc_rect):
return [NODATA_VALUE] * width * height
# get tiles
over_smpl = 1
segments_x = 1 if width == 1 else width - 1
res = extent.width() / segments_x / over_smpl
ds = self.getDataset(merc_rect.xMinimum(), merc_rect.yMinimum(), merc_rect.xMaximum(), merc_rect.yMaximum(), res)
geotransform = extent.geotransform(width, height)
return self._read(ds, width, height, geotransform)
def readValue(self, x, y):
"""Get value at the position using 1px * 1px memory raster. The value is calculated using a tile of max zoom level"""
# coordinate transformation into EPSG:3857
pt = self.transform.transform(QgsPoint(x, y))
# if the point is not within the bounding box of this data, return nodata value
if not self.boundingbox.contains(pt):
return NODATA_VALUE
res = 0.1
hres = res / 2
ds = self.getDataset(pt.x() - hres, pt.y() - hres, pt.x() + hres, pt.y() + hres, res)
geotransform = [x - hres, res, 0, y + hres, 0, -res]
return self._read(ds, 1, 1, geotransform)[0]
def _read(self, ds, width, height, geotransform):
# create a memory dataset
warped_ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32)
warped_ds.SetProjection(self.dest_wkt)
warped_ds.SetGeoTransform(geotransform)
# reproject image
gdal.ReprojectImage(ds, warped_ds, None, None, gdal.GRA_Bilinear)
# load values into an array
band = warped_ds.GetRasterBand(1)
fs = "f" * width * height
return struct.unpack(fs, band.ReadRaster(0, 0, width, height, buf_type=gdal.GDT_Float32))
def getDataset(self, xmin, ymin, xmax, ymax, mapUnitsPerPixel):
# calculate zoom level
mpp1 = TSIZE1 / TILE_SIZE
zoom = int(math.ceil(math.log(mpp1 / mapUnitsPerPixel, 2) + 1))
zoom = max(0, min(zoom, ZMAX))
# calculate tile range (yOrigin is top)
size = TSIZE1 / 2 ** (zoom - 1)
matrixSize = 2 ** zoom
ulx = max(0, int((xmin + TSIZE1) / size))
uly = max(0, int((TSIZE1 - ymax) / size))
lrx = min(int((xmax + TSIZE1) / size), matrixSize - 1)
lry = min(int((TSIZE1 - ymin) / size), matrixSize - 1)
cols = lrx - ulx + 1
rows = lry - uly + 1
# download count limit
if cols * rows > 128:
logMessage("Number of tiles to fetch is too large!")
width = height = 1
return self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
if self.last_dataset and self.last_dataset[0] == [zoom, ulx, uly, lrx, lry]: # if same as last tile set, return cached dataset
return self.last_dataset[1]
urltmpl = "http://cyberjapandata.gsi.go.jp/xyz/dem/{z}/{x}/{y}.txt"
#urltmpl = "http://localhost/xyz/dem/{z}/{x}/{y}.txt"
tiles = self.fetchFiles(urltmpl, zoom, ulx, uly, lrx, lry)
# create a memory dataset
width = cols * TILE_SIZE
height = rows * TILE_SIZE
res = size / TILE_SIZE
geotransform = [ulx * size - TSIZE1, res, 0, TSIZE1 - uly * size, 0, -res]
#mem_driver = gdal.GetDriverByName("GTiff")
#ds = mem_driver.Create("D:/fetched_tile.tif", width, height, 1, gdal.GDT_Float32, [])
ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
ds.SetProjection(str(self.crs3857.toWkt()))
ds.SetGeoTransform(geotransform)
band = ds.GetRasterBand(1)
for i, tile in enumerate(tiles):
if tile:
col = i % cols
row = i / cols
band.WriteRaster(col * TILE_SIZE, row * TILE_SIZE, TILE_SIZE, TILE_SIZE, tile)
ds.FlushCache()
self.last_dataset = [[zoom, ulx, uly, lrx, lry], ds] # cache dataset
return ds
def fetchFiles(self, urltmpl, zoom, xmin, ymin, xmax, ymax):
downloadTimeout = 60
urls = []
for y in range(ymin, ymax + 1):
for x in range(xmin, xmax + 1):
urls.append(urltmpl.replace("{x}", str(x)).replace("{y}", str(y)).replace("{z}", str(zoom)))
files = self.downloader.fetchFiles(urls, downloadTimeout)
for url in urls:
data = files[url]
if data:
yield numpy.fromstring(data.replace("e", str(NODATA_VALUE)).replace("\n", ","), dtype=numpy.float32, sep=",").tostring() # to byte array
else:
array = numpy.empty(TILE_SIZE * TILE_SIZE, dtype=numpy.float32)
array.fill(NODATA_VALUE)
yield array.tostring()
