def testIntersection(self):
rect1 = QgsRectangle(0.0, 0.0, 5.0, 5.0)
rect2 = QgsRectangle(2.0, 2.0, 7.0, 7.0)
myMessage = "Expected: %s\nGot: %s\n" % (True, rect1.intersects(rect2))
assert rect1.intersects(rect2), myMessage
rect3 = rect1.intersect(rect2)
self.assertFalse(rect3.isEmpty(), "Empty rectangle returned")
myMessage = "Expected: %s\nGot: %s\n" % (3.0, rect3.width())
assert rect3.width() == 3.0, myMessage
myMessage = "Expected: %s\nGot: %s\n" % (3.0, rect3.height())
assert rect3.height() == 3.0, myMessage
def testIntersection(self):
rect1 = QgsRectangle(0.0, 0.0, 5.0, 5.0)
rect2 = QgsRectangle(2.0, 2.0, 7.0, 7.0)
myMessage = ('Expected: %s\nGot: %s\n' %
(True, rect1.intersects(rect2)))
assert rect1.intersects(rect2), myMessage
rect3 = rect1.intersect(rect2)
self.assertFalse(rect3.isEmpty(), "Empty rectangle returned")
myMessage = ('Expected: %s\nGot: %s\n' % (3.0, rect3.width()))
assert rect3.width() == 3.0, myMessage
myMessage = ('Expected: %s\nGot: %s\n' % (3.0, rect3.height()))
assert rect3.height() == 3.0, myMessage
def testIntersection(self):
rect1 = QgsRectangle(0.0, 0.0, 5.0, 5.0)
rect2 = QgsRectangle(2.0, 2.0, 7.0, 7.0)
self.assertTrue(rect1.intersects(rect2))
rect3 = rect1.intersect(rect2)
self.assertFalse(rect3.isEmpty(), "Empty rectangle returned")
self.assertEqual(rect3.width(), 3.0)
self.assertEqual(rect3.height(), 3.0)
def subdivideRecursively(self, rect, maxHeight):
if maxHeight <= self.height:
return
self.subNodes = []
for y in range(2):
for x in range(2):
xmin = self.rect.xMinimum() + 0.5 * x * self.rect.width()
ymin = self.rect.yMinimum() + 0.5 * (1 - y) * self.rect.height()
xmax = xmin + 0.5 * self.rect.width()
ymax = ymin + 0.5 * self.rect.height()
quadrect = QgsRectangle(xmin, ymin, xmax, ymax)
node = QuadNode(self, quadrect, 2 * y + x, self.height + 1)
self.subNodes.append(node)
if quadrect.intersects(rect):
node.subdivideRecursively(rect, maxHeight)
def subdivideRecursively(self, rect, maxHeight):
if maxHeight <= self.height:
return
self.subNodes = []
for y in range(2):
for x in range(2):
xmin = self.rect.xMinimum() + 0.5 * x * self.rect.width()
ymin = self.rect.yMinimum() + 0.5 * (1 - y) * self.rect.height()
xmax = xmin + 0.5 * self.rect.width()
ymax = ymin + 0.5 * self.rect.height()
quadrect = QgsRectangle(xmin, ymin, xmax, ymax)
node = self.__class__(self, quadrect, 2 * y + x, self.height + 1)
self.subNodes.append(node)
if quadrect.intersects(rect):
node.subdivideRecursively(rect, maxHeight)
class GSIElevTileProvider:
def __init__(self, dest_wkt):
self.dest_wkt = dest_wkt
# crs transformer, which aims to calculate bbox in EPSG:3857
self.crs3857 = QgsCoordinateReferenceSystem(3857)
self.dest_crs = QgsCoordinateReferenceSystem()
if not self.dest_crs.createFromWkt(dest_wkt):
logMessage("Failed to create CRS from WKT: {0}".format(dest_wkt))
self.transform = QgsCoordinateTransform(self.dest_crs, self.crs3857,
QgsProject.instance())
# approximate bbox of this data
self.boundingbox = QgsRectangle(13667807, 2320477, 17230031, 5713298)
self.downloader = Downloader()
self.downloader.userAgent = "QGIS/{0} Qgis2threejs GSIElevTileProvider".format(
Qgis.QGIS_VERSION_INT
) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value(
"/qgis/defaultTileExpiry", 24, type=int)
self.driver = gdal.GetDriverByName("MEM")
self.last_dataset = None
def name(self):
return "GSI Elevation Tile"
def read(self, width, height, extent):
# calculate bounding box in EPSG:3857
geometry = extent.geometry()
geometry.transform(self.transform)
merc_rect = geometry.boundingBox()
# if the bounding box doesn't intersect with the bounding box of this data, return a list filled with nodata value
if not self.boundingbox.intersects(merc_rect):
return [NODATA_VALUE] * width * height
# get tiles
over_smpl = 1
segments_x = 1 if width == 1 else width - 1
res = extent.width() / segments_x / over_smpl
ds = self.getDataset(merc_rect.xMinimum(), merc_rect.yMinimum(),
merc_rect.xMaximum(), merc_rect.yMaximum(), res)
geotransform = extent.geotransform(width, height)
return self._read(ds, width, height, geotransform)
def readValue(self, x, y):
"""Get value at the position using 1px * 1px memory raster. The value is calculated using a tile of max zoom level"""
# coordinate transformation into EPSG:3857
pt = self.transform.transform(QgsPoint(x, y))
# if the point is not within the bounding box of this data, return nodata value
if not self.boundingbox.contains(pt):
return NODATA_VALUE
res = 0.1
hres = res / 2
ds = self.getDataset(pt.x() - hres,
pt.y() - hres,
pt.x() + hres,
pt.y() + hres, res)
geotransform = [x - hres, res, 0, y + hres, 0, -res]
return self._read(ds, 1, 1, geotransform)[0]
def _read(self, ds, width, height, geotransform):
# create a memory dataset
warped_ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32)
warped_ds.SetProjection(self.dest_wkt)
warped_ds.SetGeoTransform(geotransform)
# reproject image
gdal.ReprojectImage(ds, warped_ds, None, None, gdal.GRA_Bilinear)
# load values into an array
band = warped_ds.GetRasterBand(1)
fs = "f" * width * height
return struct.unpack(
fs, band.ReadRaster(0, 0, width, height,
buf_type=gdal.GDT_Float32))
def getDataset(self, xmin, ymin, xmax, ymax, mapUnitsPerPixel):
# calculate zoom level
mpp1 = TSIZE1 / TILE_SIZE
zoom = int(math.ceil(math.log(mpp1 / mapUnitsPerPixel, 2) + 1))
zoom = max(0, min(zoom, ZMAX))
# calculate tile range (yOrigin is top)
size = TSIZE1 / 2**(zoom - 1)
matrixSize = 2**zoom
ulx = max(0, int((xmin + TSIZE1) / size))
uly = max(0, int((TSIZE1 - ymax) / size))
lrx = min(int((xmax + TSIZE1) / size), matrixSize - 1)
lry = min(int((TSIZE1 - ymin) / size), matrixSize - 1)
cols = lrx - ulx + 1
rows = lry - uly + 1
# download count limit
if cols * rows > 128:
logMessage("Number of tiles to fetch is too large!")
width = height = 1
return self.driver.Create("", width, height, 1, gdal.GDT_Float32,
[])
if self.last_dataset and self.last_dataset[0] == [
zoom, ulx, uly, lrx, lry
]: # if same as last tile set, return cached dataset
return self.last_dataset[1]
urltmpl = "http://cyberjapandata.gsi.go.jp/xyz/dem/{z}/{x}/{y}.txt"
#urltmpl = "http://localhost/xyz/dem/{z}/{x}/{y}.txt"
tiles = self.fetchFiles(urltmpl, zoom, ulx, uly, lrx, lry)
# create a memory dataset
width = cols * TILE_SIZE
height = rows * TILE_SIZE
res = size / TILE_SIZE
geotransform = [
ulx * size - TSIZE1, res, 0, TSIZE1 - uly * size, 0, -res
]
#mem_driver = gdal.GetDriverByName("GTiff")
#ds = mem_driver.Create("D:/fetched_tile.tif", width, height, 1, gdal.GDT_Float32, [])
ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
ds.SetProjection(str(self.crs3857.toWkt()))
ds.SetGeoTransform(geotransform)
band = ds.GetRasterBand(1)
for i, tile in enumerate(tiles):
if tile:
col = i % cols
row = i // cols
band.WriteRaster(col * TILE_SIZE, row * TILE_SIZE, TILE_SIZE,
TILE_SIZE, tile)
ds.FlushCache()
self.last_dataset = [[zoom, ulx, uly, lrx, lry], ds] # cache dataset
return ds
def fetchFiles(self, urltmpl, zoom, xmin, ymin, xmax, ymax):
downloadTimeout = 60
urls = []
for y in range(ymin, ymax + 1):
for x in range(xmin, xmax + 1):
urls.append(
urltmpl.replace("{x}",
str(x)).replace("{y}", str(y)).replace(
"{z}", str(zoom)))
files = self.downloader.fetchFiles(urls, downloadTimeout)
for url in urls:
data = files[url]
if data:
yield numpy.fromstring(data.replace(
b"e", NODATA_VALUE_BYTES).replace(b"\n", b","),
dtype=numpy.float32,
sep=",").tostring() # to byte array
else:
array = numpy.empty(TILE_SIZE * TILE_SIZE, dtype=numpy.float32)
array.fill(NODATA_VALUE)
yield array.tostring()
def make_coco_dataset(self):
layers = QgsProject.instance().mapLayers().values()
vectorlayers = [layer for layer in layers if layer.type() == VectorLayer]
rasterLayers = [layer for layer in layers if layer.type() == RasterLayer]
vectorlayer = vectorlayers[self.dlg.vectorlayerselector.currentIndex()]
rasterlayer = rasterLayers[self.dlg.rasterlayerselector.currentIndex()]
img_size = int(self.dlg.imagesizeselector.currentText())
scale_realworld = int(self.dlg.zoomlevel.currentText())
buffer_size = int(self.dlg.buffersize.text()) / 100
target_dir = self.dlg.dirselectline.text()
dataset_description = str(self.dlg.datasetdescription.text())
contributor = str(self.dlg.creatorname.text())
url = str(self.dlg.url.text())
version = str(self.dlg.datasetversion.text())
license = str(self.dlg.licenseselector.currentText())
# to implement
scale_to_fit = self.dlg.scaletofit.isChecked()
use_fieldcategory = bool(self.dlg.usecategoryfields.isChecked())
categoryfield = self.dlg.categoryfields.currentText()
QgsMessageLog.logMessage(str(scale_to_fit), "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage(str(use_fieldcategory), "cocotrainer", level=Qgis.Info)
# prepare directories
os.makedirs(os.path.join(target_dir, "train"), exist_ok=True)
os.makedirs(os.path.join(target_dir, "val"), exist_ok=True)
QgsMessageLog.logMessage("====================================", "cocotrainer", level=Qgis.Info)
# TODO:
# - use field categories
features_iterator = vectorlayer.getFeatures(QgsFeatureRequest().setFilterExpression('$area > 1'))
features = [feature for feature in features_iterator]
QgsMessageLog.logMessage("vector layer: " + vectorlayer.name(), "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage("Filtered polygons smaller than 1m2 from data", "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage("features: " + str(len(list(features))), "cocotrainer", level=Qgis.Info)
options = QgsMapSettings()
options.setLayers([rasterlayer])
options.setOutputSize(QSize(int(img_size), int(img_size)))
QgsMessageLog.logMessage(str(scale_realworld), "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage(str(img_size), "cocotrainer", level=Qgis.Info)
date = datetime.now()
cat_dict = {}
if use_fieldcategory:
categories = []
# uniqueprovider = rasterlayer.dataProvider()
fields = vectorlayer.fields()
id = fields.indexFromName(categoryfield)
uniquevalues = vectorlayer.uniqueValues(id)
for i, val in enumerate(uniquevalues):
categories.append({"supercategory": "object", "id": i, "name": str(val)})
cat_dict[str(val)] = i
else:
categories = [{"supercategory": "object", "id": 0, "name": "CATEGORYNAME"}]
cat_dict["CATEGORYNAME"] = 0
coco_annotation = {
"info": {
"description": dataset_description,
"url": url,
"version": version,
"year": int(date.strftime("%Y")),
"contributor": contributor,
"date_created": date.strftime("%d/%m/%y")
},
"licenses": [self.license_dict[license]],
"images": [],
"annotations": [],
"categories": categories,
# < -- Not in Captionsannotations
"segment_info": [] # < -- Only in Panoptic annotations
}
bboxes, polygons = {}, {}
for i, feature in enumerate(features):
polygons[i] = feature.geometry()
bboxes[i] = feature.geometry().boundingBox()
if use_fieldcategory:
cats = {}
for i, feature in enumerate(features):
cats[i] = str(feature[categoryfield])
image_id = 0
annotation_id = 0
for i, feature in enumerate(features):
self.dlg.progressBar.setValue((i / len(polygons) * 100))
QgsMessageLog.logMessage("something is happening", "cocotrainer", level=Qgis.Info)
geoms = feature.geometry()
bbox = geoms.boundingBox()
if scale_to_fit:
xmax = bbox.xMaximum()
ymax = bbox.yMaximum()
ymin = bbox.yMinimum()
xmin = bbox.xMinimum()
diffx = xmax - xmin
diffy = ymax - ymin
xmax = xmax + (buffer_size * diffx)
xmin = xmin - (buffer_size * diffx)
ymax = ymax + (buffer_size * diffy)
ymin = ymin - (buffer_size * diffy)
else:
midpoint = bbox.center()
QgsMessageLog.logMessage("trying to log centerpoint", "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage(str(midpoint.x()), "cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage(str(midpoint.y()), "cocotrainer", level=Qgis.Info)
xmin = midpoint.x() - scale_realworld / 2
xmax = midpoint.x() + scale_realworld / 2
ymin = midpoint.y() - scale_realworld / 2
ymax = midpoint.y() + scale_realworld / 2
extent = QgsRectangle(xmin, ymin, xmax, ymax)
fileid = "{}_{}_{}".format(xmin, ymin, scale_realworld)
image = {
"license": 0,
"file_name": fileid + ".png",
"coco_url": None,
"height": img_size,
"width": img_size,
"date_captured": date.strftime("%d/%m/%y"),
"flickr_url": None,
"id": image_id
}
coco_annotation["images"].append(image)
for j, geo in bboxes.items():
if extent.contains(geo) or extent.intersects(geo):
points = polygons[j].asMultiPolygon()
xs = np.array([p.x() - xmin for p in points[0][0]])
ys = np.array([(ymax - ymin) - (p.y() - ymin) for p in points[0][0]])
xs *= (img_size / scale_realworld)
xs = np.round(xs)
xs = xs.astype(np.int32)
ys *= (img_size / scale_realworld)
ys = np.round(ys)
ys = ys.astype(np.int32)
ys[ys < 0] = 0
xs[xs < 0] = 0
polygon = [[int(p[0]), int(p[1])] for p in zip(xs, ys)]
flat_list = [item for sublist in polygon for item in sublist]
QgsMessageLog.logMessage(str(flat_list), "cocotrainer", level=Qgis.Info)
pixelposbbox = [int(np.min(xs)), (int(np.min(ys))), geo.width() * (img_size / scale_realworld),
geo.height() * (img_size / scale_realworld)]
pixelposbbox = [number if number > 0 else 0 for number in pixelposbbox]
QgsMessageLog.logMessage(str([geo.xMinimum(), geo.yMaximum(), geo.width(), geo.height()]),
"cocotrainer", level=Qgis.Info)
QgsMessageLog.logMessage(str(pixelposbbox), "cocotrainer", level=Qgis.Info)
annotation = {
"segmentation": [flat_list], # format is [x1,y1,x2,y2]
"area": polygons[j].area(),
"iscrowd": 0,
"image_id": image_id,
"bbox": pixelposbbox, # format is [top left x position, top left y position, width, height]
"category_id": cat_dict[cats[j]],
"id": annotation_id
}
annotation_id += 1
coco_annotation["annotations"].append(annotation)
options.setExtent(extent)
render = QgsMapRendererParallelJob(options)
def finished():
QgsMessageLog.logMessage("saving image to disk", "cocotrainer", level=Qgis.Info)
fname = os.path.join(target_dir, 'train', "{}.png".format(fileid))
img = render.renderedImage()
if not img.save(fname, "png"):
print("Error saving image")
render.finished.connect(finished)
render.start()
render.waitForFinished()
image_id += 1
QgsMessageLog.logMessage(str(coco_annotation), "cocotrainer", level=Qgis.Info)
with open(os.path.join(target_dir, "annotation.json"), "w") as outfile:
json.dump(coco_annotation, outfile)
self.dlg.progressBar.setValue(100)
self.dlg.finished_label.setText("parsed {} features".format(len(list(features))))
class GSIElevTileProvider:
def __init__(self, dest_wkt):
self.dest_wkt = dest_wkt
# crs transformer, which aims to calculate bbox in EPSG:3857
self.crs3857 = QgsCoordinateReferenceSystem(3857)
self.dest_crs = QgsCoordinateReferenceSystem()
if not self.dest_crs.createFromWkt(dest_wkt):
logMessage("Failed to create CRS from WKT: {0}".format(dest_wkt))
self.transform = QgsCoordinateTransform(self.dest_crs, self.crs3857)
# approximate bbox of this data
self.boundingbox = QgsRectangle(13667807, 2320477, 17230031, 5713298)
self.downloader = Downloader()
self.downloader.userAgent = "QGIS/{0} Qgis2threejs GSIElevTileProvider".format(QGis.QGIS_VERSION) # not written since QGIS 2.2
self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value("/qgis/defaultTileExpiry", 24, type=int)
self.driver = gdal.GetDriverByName("MEM")
self.last_dataset = None
def name(self):
return "GSI Elevation Tile"
def read(self, width, height, extent):
# calculate bounding box in EPSG:3857
geometry = extent.geometry()
geometry.transform(self.transform)
merc_rect = geometry.boundingBox()
# if the bounding box doesn't intersect with the bounding box of this data, return a list filled with nodata value
if not self.boundingbox.intersects(merc_rect):
return [NODATA_VALUE] * width * height
# get tiles
over_smpl = 1
segments_x = 1 if width == 1 else width - 1
res = extent.width() / segments_x / over_smpl
ds = self.getDataset(merc_rect.xMinimum(), merc_rect.yMinimum(), merc_rect.xMaximum(), merc_rect.yMaximum(), res)
geotransform = extent.geotransform(width, height)
return self._read(ds, width, height, geotransform)
def readValue(self, x, y):
"""Get value at the position using 1px * 1px memory raster. The value is calculated using a tile of max zoom level"""
# coordinate transformation into EPSG:3857
pt = self.transform.transform(QgsPoint(x, y))
# if the point is not within the bounding box of this data, return nodata value
if not self.boundingbox.contains(pt):
return NODATA_VALUE
res = 0.1
hres = res / 2
ds = self.getDataset(pt.x() - hres, pt.y() - hres, pt.x() + hres, pt.y() + hres, res)
geotransform = [x - hres, res, 0, y + hres, 0, -res]
return self._read(ds, 1, 1, geotransform)[0]
def _read(self, ds, width, height, geotransform):
# create a memory dataset
warped_ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32)
warped_ds.SetProjection(self.dest_wkt)
warped_ds.SetGeoTransform(geotransform)
# reproject image
gdal.ReprojectImage(ds, warped_ds, None, None, gdal.GRA_Bilinear)
# load values into an array
band = warped_ds.GetRasterBand(1)
fs = "f" * width * height
return struct.unpack(fs, band.ReadRaster(0, 0, width, height, buf_type=gdal.GDT_Float32))
def getDataset(self, xmin, ymin, xmax, ymax, mapUnitsPerPixel):
# calculate zoom level
mpp1 = TSIZE1 / TILE_SIZE
zoom = int(math.ceil(math.log(mpp1 / mapUnitsPerPixel, 2) + 1))
zoom = max(0, min(zoom, ZMAX))
# calculate tile range (yOrigin is top)
size = TSIZE1 / 2 ** (zoom - 1)
matrixSize = 2 ** zoom
ulx = max(0, int((xmin + TSIZE1) / size))
uly = max(0, int((TSIZE1 - ymax) / size))
lrx = min(int((xmax + TSIZE1) / size), matrixSize - 1)
lry = min(int((TSIZE1 - ymin) / size), matrixSize - 1)
cols = lrx - ulx + 1
rows = lry - uly + 1
# download count limit
if cols * rows > 128:
logMessage("Number of tiles to fetch is too large!")
width = height = 1
return self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
if self.last_dataset and self.last_dataset[0] == [zoom, ulx, uly, lrx, lry]: # if same as last tile set, return cached dataset
return self.last_dataset[1]
urltmpl = "http://cyberjapandata.gsi.go.jp/xyz/dem/{z}/{x}/{y}.txt"
#urltmpl = "http://localhost/xyz/dem/{z}/{x}/{y}.txt"
tiles = self.fetchFiles(urltmpl, zoom, ulx, uly, lrx, lry)
# create a memory dataset
width = cols * TILE_SIZE
height = rows * TILE_SIZE
res = size / TILE_SIZE
geotransform = [ulx * size - TSIZE1, res, 0, TSIZE1 - uly * size, 0, -res]
#mem_driver = gdal.GetDriverByName("GTiff")
#ds = mem_driver.Create("D:/fetched_tile.tif", width, height, 1, gdal.GDT_Float32, [])
ds = self.driver.Create("", width, height, 1, gdal.GDT_Float32, [])
ds.SetProjection(str(self.crs3857.toWkt()))
ds.SetGeoTransform(geotransform)
band = ds.GetRasterBand(1)
for i, tile in enumerate(tiles):
if tile:
col = i % cols
row = i / cols
band.WriteRaster(col * TILE_SIZE, row * TILE_SIZE, TILE_SIZE, TILE_SIZE, tile)
ds.FlushCache()
self.last_dataset = [[zoom, ulx, uly, lrx, lry], ds] # cache dataset
return ds
def fetchFiles(self, urltmpl, zoom, xmin, ymin, xmax, ymax):
downloadTimeout = 60
urls = []
for y in range(ymin, ymax + 1):
for x in range(xmin, xmax + 1):
urls.append(urltmpl.replace("{x}", str(x)).replace("{y}", str(y)).replace("{z}", str(zoom)))
files = self.downloader.fetchFiles(urls, downloadTimeout)
for url in urls:
data = files[url]
if data:
yield numpy.fromstring(data.replace("e", str(NODATA_VALUE)).replace("\n", ","), dtype=numpy.float32, sep=",").tostring() # to byte array
else:
array = numpy.empty(TILE_SIZE * TILE_SIZE, dtype=numpy.float32)
array.fill(NODATA_VALUE)
yield array.tostring()