def vector_layer_id(ngw_resource_group):
with transaction.manager:
obj = VectorLayer(
parent_id=ngw_resource_group,
display_name='vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=uuid4().hex,
).persist()
geojson = {
'type': 'FeatureCollection',
'features': get_features_for_orderby_test()
}
dsource = ogr.Open(json.dumps(geojson))
layer = dsource.GetLayer(0)
obj.setup_from_ogr(layer)
obj.load_from_ogr(layer)
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=obj.id).one())
def test_size_limit(size_limit, width, height, band_count, datatype, ok, ngw_env, ngw_resource_group):
res = RasterLayer(
parent_id=ngw_resource_group, display_name='test-raster-limit',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
).persist()
driver = gdal.GetDriverByName('GTiff')
proj = res.srs.to_osr()
proj_wkt = proj.ExportToWkt()
with ngw_env.raster_layer.options.override(dict(size_limit=size_limit)):
with NamedTemporaryFile('w') as f:
ds = driver.Create(f.name, width, height, band_count, datatype)
ds.SetProjection(proj_wkt)
ds.FlushCache()
ds = None
f.flush()
if ok:
res.load_file(f.name, ngw_env)
else:
with pytest.raises(ValidationError):
res.load_file(f.name, ngw_env)
DBSession.expunge(res)
def type_layer(ngw_resource_group):
with transaction.manager:
vl_type = VectorLayer(
parent_id=ngw_resource_group,
display_name='type',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=uuid4().hex,
).persist()
import nextgisweb.vector_layer.test
path = Path(
nextgisweb.vector_layer.test.__file__).parent / 'data/type.geojson'
ogrds = ogr.Open(str(path))
ogrlayer = ogrds.GetLayer(0)
vl_type.setup_from_ogr(ogrlayer)
vl_type.load_from_ogr(ogrlayer)
DBSession.flush()
DBSession.expunge(vl_type)
yield vl_type.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=vl_type.id).one())
def test_storage_estimate_all(ngw_env, ngw_resource_group, ngw_webtest_app,
ngw_auth_administrator):
with transaction.manager:
res = vector_layer('test-vector-layer', ngw_resource_group)
DBSession.flush()
DBSession.expunge(res)
feature = dict(geom='POINT (0 0)')
ngw_webtest_app.post_json('/api/resource/%d/feature/' % res.id, feature)
content = 'some-content'
resp = ngw_webtest_app.put('/api/component/file_upload/', content)
file_upload = resp.json
ngw_webtest_app.post_json(
'/api/resource/%d/feature/%d/attachment/' % (res.id, 1),
dict(file_upload=file_upload))
ngw_webtest_app.post('/api/component/pyramid/estimate_storage', status=200)
sleep(0.05) # Give a chance to start a thread and acquire the lock
with transaction.manager:
# Wait estimation end
DBSession.execute(SQL_LOCK)
cur = ngw_env.core.query_storage(
dict(resource_id=lambda col: col == res.id))
assert FeatureAttachmentData.identity in cur
assert cur[FeatureAttachmentData.identity]['data_volume'] == len(content)
with transaction.manager:
DBSession.delete(res)
def vector_layer_id(ngw_resource_group):
with transaction.manager:
obj = VectorLayer(
parent_id=ngw_resource_group, display_name='vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=uuid4().hex,
).persist()
geojson = {
"type": "FeatureCollection",
"name": "polygon_extent",
"crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::3857"}},
"features": [
{
"type": "Feature",
"properties": {"name": "west"},
"geometry": {
"type": "Polygon",
"coordinates": [[
[5542180, 8799167],
[6191082, 7551279],
[4668659, 7126998],
[5542180, 8799167],
]],
},
},
{
"type": "Feature",
"properties": {"name": "east"},
"geometry": {
"type": "Polygon",
"coordinates": [[
[15100999, 10396463],
[16498633, 10546209],
[16673337, 9223449],
[15175872, 8948913],
[15100999, 10396463],
]],
},
},
],
}
dsource = ogr.Open(json.dumps(geojson))
layer = dsource.GetLayer(0)
obj.setup_from_ogr(layer)
obj.load_from_ogr(layer)
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=obj.id).one())
def test_create(filename, options, checks, ngw_resource_group, ngw_txn):
obj = VectorLayer(parent_id=ngw_resource_group,
display_name='vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex)).persist()
src = str(path / filename)
ds = ogr.Open(src)
layer = ds.GetLayer(0)
geom_cast_params = dict(geometry_type=options.get('geometry_type'),
is_multi=options.get('is_multi'),
has_z=options.get('has_z'))
def setup_and_load():
setup_kwargs = dict()
load_kwargs = dict()
if 'skip_other_geometry_types' in options:
setup_kwargs['skip_other_geometry_types'] = options[
'skip_other_geometry_types']
load_kwargs['skip_other_geometry_types'] = options[
'skip_other_geometry_types']
if 'fix_errors' in options:
load_kwargs['fix_errors'] = options['fix_errors']
if 'skip_errors' in options:
load_kwargs['skip_errors'] = options['skip_errors']
obj.setup_from_ogr(layer,
geom_cast_params=geom_cast_params,
**setup_kwargs)
obj.load_from_ogr(layer, **load_kwargs)
if 'exception' in checks:
with pytest.raises(checks['exception']):
setup_and_load()
DBSession.expunge(obj)
else:
setup_and_load()
DBSession.flush()
if 'geometry_type' in checks:
exp_geometry_type = checks['geometry_type']
assert obj.geometry_type == exp_geometry_type, \
"Expected geometry type was {} but actually got {}".format(
exp_geometry_type, obj.geometry_type)
if 'feature_count' in checks:
exp_feature_count = checks['feature_count']
query = obj.feature_query()
feature_count = query().total_count
assert feature_count == exp_feature_count, \
"Expected feature count was {} but got {}".format(
exp_feature_count, feature_count)
def vlayer_id(ngw_resource_group):
with transaction.manager:
res_vl = VectorLayer(
parent_id=ngw_resource_group,
display_name='test_cyrillic',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=uuid4().hex,
).persist()
geojson = {
'type':
'FeatureCollection',
'crs': {
'type': 'name',
'properties': {
'name': 'urn:ogc:def:crs:EPSG::3857'
}
},
'features': [{
'type': 'Feature',
'properties': {
'field1': 1,
'поле2': 'значение1',
'!field3': '!@#$%^&*()значение1'
},
'geometry': {
'type': 'Point',
'coordinates': [0, 0]
}
}, {
'type': 'Feature',
'properties': {
'field1': 2,
'поле2': 'значение2',
'!field3': '!@#$%^&*()значение2'
},
'geometry': {
'type': 'Point',
'coordinates': [10, 10]
}
}]
}
dsource = ogr.Open(json.dumps(geojson))
layer = dsource.GetLayer(0)
res_vl.setup_from_ogr(layer)
res_vl.load_from_ogr(layer)
DBSession.flush()
DBSession.expunge(res_vl)
yield res_vl.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=res_vl.id).one())
def vector_layer_id(ngw_resource_group):
with transaction.manager:
obj = VectorLayer(
parent_id=ngw_resource_group,
display_name='vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=uuid4().hex,
).persist()
geojson = {
'type':
'FeatureCollection',
'crs': {
'type': 'name',
'properties': {
'name': 'urn:ogc:def:crs:EPSG::3857'
}
},
'features': [{
'type': 'Feature',
'properties': {
'price': -1,
'name': 'feature1'
},
'geometry': {
'type': 'Point',
'coordinates': [0, 0]
}
}, {
'type': 'Feature',
'properties': {
'price': -2,
'name': 'feature2'
},
'geometry': {
'type': 'Point',
'coordinates': [0, 0]
}
}]
}
dsource = ogr.Open(json.dumps(geojson))
layer = dsource.GetLayer(0)
obj.setup_from_ogr(layer)
obj.load_from_ogr(layer)
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=obj.id).one())
def webmap(ngw_env, ngw_resource_group):
with transaction.manager:
obj = WebMap(parent_id=ngw_resource_group,
display_name=__name__,
owner_user=User.by_keyname('administrator'),
root_item=WebMapItem(item_type='root')).persist()
DBSession.flush()
DBSession.expunge(obj)
yield obj
with transaction.manager:
DBSession.delete(WebMap.filter_by(id=obj.id).one())
def srs_msk23_id():
with transaction.manager:
obj = SRS(
display_name="MSK23",
wkt=
'PROJCS["МСК 23 зона 1",GEOGCS["GCS_Pulkovo_1942",DATUM["Pulkovo_1942",SPHEROID["Krassowsky_1940",6378245.0,298.3]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",1300000.0],PARAMETER["False_Northing",-4511057.628],PARAMETER["Central_Meridian",37.98333333333],PARAMETER["Scale_Factor",1.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]', # NOQA: E501
).persist()
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(SRS.filter_by(id=obj.id).one())
def service_id(ngw_resource_group):
with transaction.manager:
res_vl = VectorLayer(
parent_id=ngw_resource_group, display_name='test_vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex),
).persist()
geojson = {
'type': 'FeatureCollection',
'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:EPSG::3857'}},
'features': [{
'type': 'Feature',
'properties': {'name': 'feature1'},
'geometry': {'type': 'Point', 'coordinates': [0, 0]}
}, {
'type': 'Feature',
'properties': {'price': -1},
'geometry': {'type': 'Point', 'coordinates': [10, 10]}
}]
}
dsource = ogr.Open(json.dumps(geojson))
layer = dsource.GetLayer(0)
res_vl.setup_from_ogr(layer, lambda x: x)
res_vl.load_from_ogr(layer, lambda x: x)
DBSession.flush()
res_wfs = WFSService(
parent_id=ngw_resource_group, display_name='test_wfsserver_service',
owner_user=User.by_keyname('administrator'),
).persist()
res_wfs.layers.append(WFSLayer(
resource=res_vl, keyname='test',
display_name='test', maxfeatures=1000,
))
DBSession.flush()
DBSession.expunge(res_vl)
DBSession.expunge(res_wfs)
yield res_wfs.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=res_vl.id).one())
DBSession.delete(WFSService.filter_by(id=res_wfs.id).one())
def rstyle_id(ngw_env, rlayer_id):
with transaction.manager:
obj = RasterStyle(
parent_id=rlayer_id,
display_name='test-wms-rstyle',
owner_user=User.by_keyname('administrator')).persist()
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(RasterStyle.filter_by(id=obj.id).one())
def resource_id(ngw_resource_group):
with transaction.manager:
res = ResourceGroup(
parent_id=ngw_resource_group,
display_name='test-resource-group',
owner_user=User.by_keyname('administrator')).persist()
DBSession.flush()
DBSession.expunge(res)
yield res.id
with transaction.manager:
DBSession.delete(res)
def resource(ngw_resource_group):
with transaction.manager:
obj = ResourceGroup(
parent_id=ngw_resource_group, display_name='Test Юникод Symbols ~%',
keyname='Test-Keyname',
owner_user=User.by_keyname('administrator'),
).persist()
DBSession.flush()
DBSession.expunge(obj)
yield obj
with transaction.manager:
DBSession.delete(ResourceGroup.filter_by(id=obj.id).one())
def test_size_limit_reproj(source, size_expect, ngw_env, ngw_resource_group):
res = RasterLayer(
parent_id=ngw_resource_group, display_name='test-raster-limit-reproj',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
).persist()
filename = os.path.join(os.path.split(__file__)[0], 'data', source)
with ngw_env.raster_layer.options.override(dict(size_limit=size_expect - 100)):
with pytest.raises(ValidationError):
res.load_file(filename, ngw_env)
with ngw_env.raster_layer.options.override(dict(size_limit=size_expect)):
res.load_file(filename, ngw_env)
DBSession.expunge(res)
def test_resource_storage(ngw_env, ngw_resource_group, ngw_webtest_app,
ngw_auth_administrator):
reserve_storage = ngw_env.core.reserve_storage
with transaction.manager:
res1 = vector_layer('test-resource-1', ngw_resource_group)
res2 = vector_layer('test-resource-2', ngw_resource_group)
reserve_storage('test_comp',
TestKOD1,
resource=res1,
value_data_volume=100)
reserve_storage('test_comp',
TestKOD2,
resource=res1,
value_data_volume=10)
reserve_storage('test_comp',
TestKOD1,
resource=res2,
value_data_volume=200)
DBSession.flush()
DBSession.expunge(res1)
DBSession.expunge(res2)
resp = ngw_webtest_app.get('/api/resource/%d/volume' % res1.id, status=200)
assert resp.json['volume'] == 110
resp = ngw_webtest_app.get('/api/resource/%d/volume' % res2.id, status=200)
assert resp.json['volume'] == 200
cur = ngw_env.core.query_storage()
assert cur['']['data_volume'] == 310
assert cur[TestKOD1.identity]['data_volume'] == 300
with transaction.manager:
DBSession.delete(res1)
cur = ngw_env.core.query_storage()
assert cur['']['data_volume'] == 200
with transaction.manager:
DBSession.delete(res2)
cur = ngw_env.core.query_storage()
assert cur['']['data_volume'] == 0
def wfs_service_path(ngw_resource_group, ngw_httptest_app):
with transaction.manager:
vl_type = VectorLayer(
parent_id=ngw_resource_group,
display_name='type',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex),
).persist()
dsource = type_geojson_dataset('type.geojson')
layer = dsource.GetLayer(0)
vl_type.setup_from_ogr(layer, lambda x: x)
vl_type.load_from_ogr(layer, lambda x: x)
DBSession.flush()
# NOTE: GDAL doesn't support time fields in GML / WFS. It completely breaks
# XSD schema parsing. Delete the time field to pass tests.
DBSession.delete(vl_type.field_by_keyname('time'))
wfs_service = WFSService(
parent_id=ngw_resource_group,
display_name='test_wfsserver_service',
owner_user=User.by_keyname('administrator'),
).persist()
wfs_service.layers.append(
WFS_Service_Layer(resource=vl_type,
keyname='type',
display_name='type',
maxfeatures=1000))
DBSession.flush()
DBSession.expunge(vl_type)
DBSession.expunge(wfs_service)
path = '{}/api/resource/{}/wfs'.format(ngw_httptest_app.base_url,
wfs_service.id)
yield path
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=vl_type.id).one())
DBSession.delete(WFSService.filter_by(id=wfs_service.id).one())
def service(ngw_resource_group):
with transaction.manager:
res_vl = VectorLayer(
parent_id=ngw_resource_group,
display_name='test_vector_layer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex),
).persist()
dsource = type_geojson_dataset()
layer = dsource.GetLayer(0)
res_vl.setup_from_ogr(layer, lambda x: x)
res_vl.load_from_ogr(layer, lambda x: x)
DBSession.flush()
res_wfs = WFSService(
parent_id=ngw_resource_group,
display_name='test_wfsserver_service',
owner_user=User.by_keyname('administrator'),
).persist()
res_wfs.layers.append(
WFSLayer(
resource=res_vl,
keyname='test',
display_name='test',
maxfeatures=1000,
))
DBSession.flush()
DBSession.expunge(res_vl)
DBSession.expunge(res_wfs)
yield res_wfs.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=res_vl.id).one())
DBSession.delete(WFSService.filter_by(id=res_wfs.id).one())
def service_id(ngw_resource_group, rstyle_id):
with transaction.manager:
obj = Service(parent_id=ngw_resource_group,
display_name='test-wms-service',
owner_user=User.by_keyname('administrator')).persist()
DBSession.flush()
obj.layers.append(
Layer(resource_id=rstyle_id,
keyname='test_rounds',
display_name='test-rounds'))
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(Service.filter_by(id=obj.id).one())
def rlayer_id(ngw_env, ngw_resource_group):
with transaction.manager:
obj = RasterLayer(parent_id=ngw_resource_group,
display_name='test-wms-rlayer',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one()).persist()
import nextgisweb.raster_layer.test
path = Path(
nextgisweb.raster_layer.test.__file__).parent / 'data/rounds.tif'
obj.load_file(str(path), ngw_env)
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(RasterLayer.filter_by(id=obj.id).one())
def raster_layer_id(ngw_env, ngw_resource_group):
with transaction.manager:
obj = RasterLayer(
parent_id=ngw_resource_group,
display_name="raster_layer.test:export",
owner_user=User.by_keyname("administrator"),
srs=SRS.filter_by(id=3857).one(),
).persist()
obj.load_file(
os.path.join(
os.path.split(__file__)[0], "data",
"sochi-aster-colorized.tif"), ngw_env)
DBSession.flush()
DBSession.expunge(obj)
yield obj.id
with transaction.manager:
DBSession.delete(RasterLayer.filter_by(id=obj.id).one())
def make_layer_style(ngw_resource_group, num):
layer = RasterLayer(
parent_id=ngw_resource_group,
display_name=f'test-render-layer-{num}',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
xsize=100,
ysize=100,
dtype='Byte',
band_count=3,
).persist()
style = RasterStyle(
parent=layer,
display_name=f'test-render-style-{num}',
owner_user=User.by_keyname('administrator'),
).persist()
DBSession.flush()
DBSession.expunge(style)
return layer.id, style.id
def test_cog(ngw_webtest_app, ngw_auth_administrator, ngw_resource_group,
ngw_env):
source = "sochi-aster-dem.tif"
source_srs_id = 4326
src = os.path.join(os.path.split(__file__)[0], "data", source)
with transaction.manager:
res = RasterLayer(
parent_id=ngw_resource_group,
display_name="test_cog:{}".format(source),
owner_user=User.by_keyname("administrator"),
srs=SRS.filter_by(id=source_srs_id).one(),
).persist()
res.load_file(src, ngw_env, cog=False)
DBSession.flush()
DBSession.expunge(res)
ds = gdal.Open(ngw_env.raster_layer.workdir_filename(res.fileobj))
cs = ds.GetRasterBand(1).Checksum()
ngw_webtest_app.put_json(f"/api/resource/{res.id}",
dict(raster_layer=dict(cog=True)))
res = RasterLayer.filter_by(id=res.id).one()
fn = ngw_env.raster_layer.workdir_filename(res.fileobj)
warnings, errors, _ = validate(fn, full_check=True)
assert len(errors) == 0
ngw_webtest_app.put_json(f"/api/resource/{res.id}",
dict(raster_layer=dict(cog=False)))
res = RasterLayer.filter_by(id=res.id).one()
fn = ngw_env.raster_layer.workdir_filename(res.fileobj)
ds = gdal.Open(fn)
warnings, errors, _ = validate(fn, full_check=True)
assert len(errors) == 1
assert cs == ds.GetRasterBand(1).Checksum()
def service_id(vlayer_id, ngw_resource_group):
with transaction.manager:
res_wfs = WFSService(
parent_id=ngw_resource_group,
display_name='test_cyrillic_service',
owner_user=User.by_keyname('administrator'),
).persist()
res_wfs.layers.append(
WFSLayer(
resource_id=vlayer_id,
keyname='test',
display_name='test',
maxfeatures=1000,
))
DBSession.flush()
DBSession.expunge(res_wfs)
yield res_wfs.id
with transaction.manager:
DBSession.delete(WFSService.filter_by(id=res_wfs.id).one())
def service(ngw_resource_group):
with transaction.manager:
vl_type = VectorLayer(
parent_id=ngw_resource_group,
display_name='type',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex),
).persist()
dsource = type_geojson_dataset('type.geojson')
layer = dsource.GetLayer(0)
vl_type.setup_from_ogr(layer, lambda x: x)
vl_type.load_from_ogr(layer, lambda x: x)
DBSession.flush()
# NOTE: GDAL doesn't support time fields in GML / WFS. It completely breaks
# XSD schema parsing. Delete the time field to pass tests.
DBSession.delete(vl_type.field_by_keyname('time'))
vl_pointz = VectorLayer(
parent_id=ngw_resource_group,
display_name='pointz',
owner_user=User.by_keyname('administrator'),
srs=SRS.filter_by(id=3857).one(),
tbl_uuid=six.text_type(uuid4().hex),
).persist()
dsource = type_geojson_dataset('pointz.geojson')
layer = dsource.GetLayer(0)
vl_pointz.setup_from_ogr(layer, lambda x: x)
vl_pointz.load_from_ogr(layer, lambda x: x)
DBSession.flush()
res_wfs = WFSService(
parent_id=ngw_resource_group,
display_name='test_wfsserver_service',
owner_user=User.by_keyname('administrator'),
).persist()
res_wfs.layers.extend((
WFSLayer(resource=vl_type,
keyname='type',
display_name='type',
maxfeatures=1000),
WFSLayer(resource=vl_pointz,
keyname='pointz',
display_name='pointz',
maxfeatures=1000),
))
DBSession.flush()
DBSession.expunge(vl_type)
DBSession.expunge(vl_pointz)
DBSession.expunge(res_wfs)
yield res_wfs.id
with transaction.manager:
DBSession.delete(VectorLayer.filter_by(id=vl_type.id).one())
DBSession.delete(VectorLayer.filter_by(id=vl_pointz.id).one())
DBSession.delete(WFSService.filter_by(id=res_wfs.id).one())