def generate_layer_thumbnail(workspace, layername):
headers = {
settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE: settings.LAYMAN_GS_USER,
}
layer_info = get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['wms', 'native_bounding_box', 'native_crs', ]})
wms_url = layer_info['_wms']['url']
native_bbox = layer_info['native_bounding_box']
native_crs = layer_info['native_crs']
raw_bbox = native_bbox if not bbox_util.is_empty(native_bbox) else crs_def.CRSDefinitions[native_crs].default_bbox
bbox = bbox_util.ensure_bbox_with_area(raw_bbox, crs_def.CRSDefinitions[native_crs].no_area_bbox_padding)
tn_bbox = gs_util.get_square_bbox(bbox)
# Reason: https://github.com/geopython/OWSLib/issues/709
# tn_img = wms.getmap(
# layers=[layername],
# srs='EPSG:3857',
# bbox=tn_bbox,
# size=(300, 300),
# format='image/png',
# transparent=True,
# )
ensure_layer_thumbnail_dir(workspace, layername)
tn_path = get_layer_thumbnail_path(workspace, layername)
# out = open(tn_path, 'wb')
# out.write(tn_img.read())
# out.close()
from layman.layer.geoserver.wms import VERSION
response = gs_util.get_layer_thumbnail(wms_url, layername, tn_bbox, native_crs, headers=headers, wms_version=VERSION)
if "png" not in response.headers['content-type'].lower():
raise LaymanError("Thumbnail rendering failed", data=response.content)
response.raise_for_status()
with open(tn_path, "wb") as out_file:
out_file.write(response.content)
def get_template_path_and_values(username,
mapname,
http_method=None,
actor_name=None):
assert http_method in [
common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH
]
uuid_file_path = get_publication_uuid_file(MAP_TYPE, username, mapname)
publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path))
revision_date = datetime.now()
map_json = get_map_json(username, mapname)
operates_on = map_json_to_operates_on(map_json, editor=actor_name)
publ_info = get_publication_info(
username,
MAP_TYPE,
mapname,
context={
'keys': ['title', 'bounding_box', 'description'],
})
bbox_3857 = publ_info.get('bounding_box')
if bbox_util.is_empty(bbox_3857):
bbox_3857 = settings.LAYMAN_DEFAULT_OUTPUT_BBOX
extent = bbox_util.transform(tuple(bbox_3857),
epsg_from=3857,
epsg_to=4326)
title = publ_info['title']
abstract = publ_info.get('description')
md_language = next(
iter(
common_language.get_languages_iso639_2(' '.join([
title or '',
abstract or '',
]))), None)
prop_values = _get_property_values(
username=username,
mapname=mapname,
uuid=get_map_uuid(username, mapname),
title=title,
abstract=abstract or None,
publication_date=publ_datetime.strftime('%Y-%m-%d'),
revision_date=revision_date.strftime('%Y-%m-%d'),
md_date_stamp=date.today().strftime('%Y-%m-%d'),
identifier=url_for('rest_workspace_map.get',
workspace=username,
mapname=mapname),
identifier_label=mapname,
extent=extent,
epsg_codes=map_json_to_epsg_codes(map_json),
md_organisation_name=None,
organisation_name=None,
operates_on=operates_on,
md_language=md_language,
)
if http_method == common.REQUEST_METHOD_POST:
prop_values.pop('revision_date', None)
template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'record-template.xml')
return template_path, prop_values
def test_fill_project_template(workspace, publ_type, publication):
ensure_publication(workspace, publ_type, publication)
qgs_path = f'{settings.LAYMAN_QGIS_DATA_DIR}/{publication}.qgs'
wms_url = f'{settings.LAYMAN_QGIS_URL}?MAP={qgs_path}'
wms_version = '1.3.0'
layer_info = process_client.get_workspace_publication(publ_type, workspace, publication)
layer_uuid = layer_info['uuid']
with pytest.raises(requests.exceptions.HTTPError) as excinfo:
WebMapService(wms_url, version=wms_version)
assert excinfo.value.response.status_code == 500
with app.app_context():
layer_bbox = layer_db.get_bbox(workspace, publication)
layer_crs = layer_db.get_crs(workspace, publication)
layer_bbox = layer_bbox if not bbox_util.is_empty(layer_bbox) else crs_def.CRSDefinitions[layer_crs].default_bbox
with app.app_context():
qml_path = qgis_util.get_original_style_path(workspace, publication)
parser = ET.XMLParser(remove_blank_text=True)
qml_xml = ET.parse(qml_path, parser=parser)
exp_min_scale = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('min_scale')
if exp_min_scale is not None:
assert qml_xml.getroot().attrib['minScale'] == exp_min_scale
with app.app_context():
db_types = layer_db.get_geometry_types(workspace, publication)
db_cols = [
col for col in layer_db.get_all_column_infos(workspace, publication)
if col.name not in ['wkb_geometry', 'ogc_fid']
]
qml_geometry = qgis_util.get_qml_geometry_from_qml(qml_xml)
source_type = qgis_util.get_source_type(db_types, qml_geometry)
with app.app_context():
layer_qml_str = qgis_util.fill_layer_template(workspace, publication, layer_uuid, layer_bbox, layer_crs, qml_xml, source_type, db_cols)
layer_qml = ET.fromstring(layer_qml_str.encode('utf-8'), parser=parser)
if exp_min_scale is not None:
assert layer_qml.attrib['minScale'] == exp_min_scale
with app.app_context():
qgs_str = qgis_util.fill_project_template(workspace, publication, layer_uuid, layer_qml_str, layer_crs, settings.LAYMAN_OUTPUT_SRS_LIST,
layer_bbox, source_type)
with open(qgs_path, "w") as qgs_file:
print(qgs_str, file=qgs_file)
wmsi = WebMapService(wms_url, version=wms_version)
assert publication in wmsi.contents
wms_layer = wmsi.contents[publication]
exp_output_srs = set(settings.LAYMAN_OUTPUT_SRS_LIST)
assert exp_output_srs.issubset(set(wms_layer.crsOptions))
wms_layer_bbox = next((tuple(bbox_crs[:4]) for bbox_crs in wms_layer.crs_list if bbox_crs[4] == layer_crs))
assert_util.assert_same_bboxes(wms_layer_bbox, layer_bbox, 0.1)
os.remove(qgs_path)
with pytest.raises(requests.exceptions.HTTPError) as excinfo:
WebMapService(wms_url, version=wms_version)
assert excinfo.value.response.status_code == 500
def get_layer_bbox(workspace, layer):
db_bbox = layman_util.get_publication_info(workspace,
LAYER_TYPE,
layer,
context={
'keys': ['bounding_box']
})['bounding_box']
# GeoServer is not working good with degradeted bbox
return bbox_util.ensure_bbox_with_area(
db_bbox, settings.NO_AREA_BBOX_PADDING) if not bbox_util.is_empty(
db_bbox) else settings.LAYMAN_DEFAULT_OUTPUT_BBOX
def get_template_path_and_values(workspace, layername, http_method=None):
assert http_method in [common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH]
publ_info = get_publication_info(workspace, LAYER_TYPE, layername, context={
'keys': ['title', 'bounding_box', 'description'],
})
title = publ_info['title']
abstract = publ_info.get('description')
bbox_3857 = publ_info.get('bounding_box')
if bbox_util.is_empty(bbox_3857):
bbox_3857 = settings.LAYMAN_DEFAULT_OUTPUT_BBOX
extent = bbox_util.transform(tuple(bbox_3857), epsg_from=3857, epsg_to=4326)
uuid_file_path = get_publication_uuid_file(LAYER_TYPE, workspace, layername)
publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path))
revision_date = datetime.now()
md_language = next(iter(common_language.get_languages_iso639_2(' '.join([
title or '',
abstract or ''
]))), None)
try:
languages = db.get_text_languages(workspace, layername)
except LaymanError:
languages = []
try:
scale_denominator = db.guess_scale_denominator(workspace, layername)
except LaymanError:
scale_denominator = None
prop_values = _get_property_values(
workspace=workspace,
layername=layername,
uuid=get_layer_uuid(workspace, layername),
title=title,
abstract=abstract or None,
publication_date=publ_datetime.strftime('%Y-%m-%d'),
revision_date=revision_date.strftime('%Y-%m-%d'),
md_date_stamp=date.today().strftime('%Y-%m-%d'),
identifier=url_for('rest_workspace_layer.get', workspace=workspace, layername=layername),
identifier_label=layername,
extent=extent,
wms_url=wms.get_wms_url(workspace, external_url=True),
wfs_url=wfs.get_wfs_url(workspace, external_url=True),
md_organisation_name=None,
organisation_name=None,
md_language=md_language,
languages=languages,
scale_denominator=scale_denominator,
epsg_codes=settings.LAYMAN_OUTPUT_SRS_LIST,
)
if http_method == common.REQUEST_METHOD_POST:
prop_values.pop('revision_date', None)
template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'record-template.xml')
return template_path, prop_values
def get_layer_bbox(workspace, layer):
layer_info = layman_util.get_publication_info(
workspace,
LAYER_TYPE,
layer,
context={'keys': [
'native_bounding_box',
'native_crs',
]})
db_bbox = layer_info['native_bounding_box']
crs = layer_info['native_crs']
# GeoServer is not working good with degradeted bbox
result = bbox_util.ensure_bbox_with_area(db_bbox, crs_def.CRSDefinitions[crs].no_area_bbox_padding) \
if not bbox_util.is_empty(db_bbox) else crs_def.CRSDefinitions[crs].default_bbox
return result
def set_bbox(workspace, publication_type, publication, bbox, crs, ):
max_bbox = crs_def.CRSDefinitions[crs].max_bbox if crs else None
cropped_bbox = (
min(max(bbox[0], max_bbox[0]), max_bbox[2]),
min(max(bbox[1], max_bbox[1]), max_bbox[3]),
max(min(bbox[2], max_bbox[2]), max_bbox[0]),
max(min(bbox[3], max_bbox[3]), max_bbox[1]),
) if not bbox_util.is_empty(bbox) and max_bbox else bbox
srid = db_util.get_srid(crs)
query = f'''update {DB_SCHEMA}.publications set
bbox = ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s ,%s)),
srid = %s
where type = %s
and name = %s
and id_workspace = (select w.id from {DB_SCHEMA}.workspaces w where w.name = %s);'''
params = cropped_bbox + (srid, publication_type, publication, workspace,)
db_util.run_statement(query, params)
def save_qgs_file(workspace, layer):
info = layer_util.get_layer_info(workspace, layer)
uuid = info['uuid']
qgis.ensure_layer_dir(workspace, layer)
layer_bbox = db.get_bbox(workspace, layer)
layer_bbox = layer_bbox if not bbox_util.is_empty(
layer_bbox) else settings.LAYMAN_DEFAULT_OUTPUT_BBOX
qml = util.get_original_style_xml(workspace, layer)
qml_geometry = util.get_qml_geometry_from_qml(qml)
db_types = db.get_geometry_types(workspace, layer)
db_cols = [
col for col in db.get_all_column_infos(workspace, layer)
if col.name not in ['wkb_geometry', 'ogc_fid']
]
source_type = util.get_source_type(db_types, qml_geometry)
layer_qml = util.fill_layer_template(workspace, layer, uuid, layer_bbox,
qml, source_type, db_cols)
qgs_str = util.fill_project_template(workspace, layer, uuid, layer_qml,
settings.LAYMAN_OUTPUT_SRS_LIST,
layer_bbox, source_type)
with open(get_layer_file_path(workspace, layer), "w") as qgs_file:
print(qgs_str, file=qgs_file)
def test_wms_layer(workspace, publ_type, publication):
ensure_publication(workspace, publ_type, publication)
style = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA]['style_type']
style_file_type = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('style_file_type')
expected_style_file = f'/layman_data_test/workspaces/{workspace}/layers/{publication}/input_style/{publication}'
expected_qgis_file = f'/qgis/data/test/workspaces/{workspace}/layers/{publication}/{publication}.qgis'
wms_stores_url = urljoin(GS_REST_WORKSPACES, f'{workspace}_wms/wmsstores/')
wms_layers_url = urljoin(GS_REST_WORKSPACES, f'{workspace}_wms/wmslayers/')
with app.app_context():
info = layman_util.get_publication_info(workspace, publ_type, publication, context={'keys': ['wms']})
if style_file_type:
assert (os.path.exists(expected_style_file + '.qml')) == (style_file_type == 'qml')
assert (os.path.exists(expected_style_file + '.sld')) == (style_file_type == 'sld')
assert (os.path.exists(expected_qgis_file)) == (style == 'qml')
response = requests.get(wms_stores_url,
auth=settings.LAYMAN_GS_AUTH,
timeout=settings.DEFAULT_CONNECTION_TIMEOUT,
)
assert response.status_code == 200, response.json()
if style == 'qml':
wms_stores = [store['name'] for store in response.json()['wmsStores']['wmsStore']]
assert f'{DEFAULT_WMS_QGIS_STORE_PREFIX}_{publication}' in wms_stores, response.json()
elif style == 'sld':
url = urljoin(GS_REST, f'workspaces/{workspace}_wms/styles/{publication}')
response = requests.get(url,
auth=GS_AUTH,
headers=headers_sld,
timeout=settings.DEFAULT_CONNECTION_TIMEOUT,
)
response.raise_for_status()
response = requests.get(wms_layers_url,
auth=settings.LAYMAN_GS_AUTH,
timeout=settings.DEFAULT_CONNECTION_TIMEOUT,
)
assert response.status_code == 200, response.json()
if style == 'qml':
wms_layers = [layer['name'] for layer in response.json()['wmsLayers']['wmsLayer']]
assert publication in wms_layers, response.json()
get_map = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('get_map')
if get_map:
url_detail, expected_file, pixel_tolerance = get_map
url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{workspace}_wms/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image/png&TRANSPARENT=true&STYLES=&LAYERS={workspace}:{publication}&" + url_detail
obtained_file = f'tmp/artifacts/test_sld_style_applied_in_wms_{publication}.png'
assert_util.assert_same_images(url, obtained_file, expected_file, pixel_tolerance)
gs_workspace = info['_wms']['workspace']
authn_headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
for service_endpoint in {'ows', 'wms'}:
wms_url = geoserver_client.get_wms_url(gs_workspace, service_endpoint)
layer_info = process_client.get_workspace_layer(workspace, publication, headers=authn_headers)
crs = layer_info['native_crs']
raw_bbox = layer_info['bounding_box'] if not bbox_util.is_empty(layer_info['bounding_box']) \
else crs_def.CRSDefinitions[crs].default_bbox
bbox = bbox_util.ensure_bbox_with_area(raw_bbox, crs_def.CRSDefinitions[crs].no_area_bbox_padding)
tn_bbox = gs_util.get_square_bbox(bbox)
response = gs_util.get_layer_thumbnail(wms_url, publication, tn_bbox, crs_def.EPSG_3857, headers=authn_headers, wms_version=VERSION)
response.raise_for_status()
assert 'image' in response.headers['content-type'], f'response.headers={response.headers}, response.content={response.content}'
all_auth_info = util.get_users_and_headers_for_publication(workspace, publ_type, publication)
headers_list_in = all_auth_info['read'][util.KEY_AUTH][util.KEY_HEADERS]
headers_list_out = all_auth_info['read'][util.KEY_NOT_AUTH][util.KEY_HEADERS]
for in_headers in headers_list_in:
wms = geoserver_client.get_wms_capabilities(gs_workspace, headers=in_headers)
assert publication in set(wms.contents)
for out_headers in headers_list_out:
wms = geoserver_client.get_wms_capabilities(gs_workspace, headers=out_headers)
assert publication not in set(wms.contents)
def test_fill_project_template():
workspace = 'test_fill_project_template_workspace'
layer = 'test_fill_project_template_layer'
qgs_path = f'{settings.LAYMAN_QGIS_DATA_DIR}/{layer}.qgs'
wms_url = f'{settings.LAYMAN_QGIS_URL}?MAP={qgs_path}'
wms_version = '1.3.0'
layer_info = process_client.publish_workspace_layer(
workspace,
layer,
file_paths=[
'/code/tmp/naturalearth/10m/cultural/ne_10m_admin_0_countries.geojson'
],
)
layer_uuid = layer_info['uuid']
with pytest.raises(requests.exceptions.HTTPError) as excinfo:
WebMapService(wms_url, version=wms_version)
assert excinfo.value.response.status_code == 500
with app.app_context():
layer_bbox = db.get_bbox(workspace, layer)
layer_bbox = layer_bbox if not bbox_util.is_empty(
layer_bbox) else settings.LAYMAN_DEFAULT_OUTPUT_BBOX
qml_path = '/code/sample/style/ne_10m_admin_0_countries.qml'
parser = ET.XMLParser(remove_blank_text=True)
qml_xml = ET.parse(qml_path, parser=parser)
exp_min_scale = '200000000'
template_xml = ET.parse(util.get_layer_template_path(), parser=parser)
assert qml_xml.getroot().attrib['minScale'] == exp_min_scale
assert template_xml.getroot().attrib['minScale'] != exp_min_scale
with app.app_context():
db_types = db.get_geometry_types(workspace, layer)
db_cols = [
col for col in db.get_all_column_infos(workspace, layer)
if col.name not in ['wkb_geometry', 'ogc_fid']
]
qml_geometry = util.get_qml_geometry_from_qml(qml_xml)
source_type = util.get_source_type(db_types, qml_geometry)
layer_qml_str = util.fill_layer_template(workspace, layer, layer_uuid,
layer_bbox, qml_xml, source_type,
db_cols)
layer_qml = ET.fromstring(layer_qml_str.encode('utf-8'), parser=parser)
assert layer_qml.attrib['minScale'] == exp_min_scale
qgs_str = util.fill_project_template(workspace, layer, layer_uuid,
layer_qml_str,
settings.LAYMAN_OUTPUT_SRS_LIST,
layer_bbox, source_type)
with open(qgs_path, "w") as qgs_file:
print(qgs_str, file=qgs_file)
wmsi = WebMapService(wms_url, version=wms_version)
assert layer in wmsi.contents
wms_layer = wmsi.contents[layer]
for expected_output_srs in settings.LAYMAN_OUTPUT_SRS_LIST:
assert f"EPSG:{expected_output_srs}" in wms_layer.crsOptions
wms_layer_bbox = next((tuple(bbox_crs[:4])
for bbox_crs in wms_layer.crs_list
if bbox_crs[4] == f"EPSG:3857"))
precision = 0.1
for idx, expected_coordinate in enumerate(layer_bbox):
assert abs(expected_coordinate - wms_layer_bbox[idx]) <= precision
os.remove(qgs_path)
with pytest.raises(requests.exceptions.HTTPError) as excinfo:
WebMapService(wms_url, version=wms_version)
assert excinfo.value.response.status_code == 500
process_client.delete_workspace_layer(workspace, layer)
def refresh_wms(
self,
workspace,
layername,
store_in_geoserver,
description=None,
title=None,
access_rights=None,
):
info = layman_util.get_publication_info(
workspace,
LAYER_TYPE,
layername,
context={'keys': [
'file',
'native_bounding_box',
'native_crs',
]})
file_type = info['file']['file_type']
crs = info['native_crs']
assert description is not None
assert title is not None
geoserver_workspace = wms.get_geoserver_workspace(workspace)
geoserver.ensure_workspace(workspace)
if self.is_aborted():
raise AbortedException
coverage_store_name = wms.get_geotiff_store_name(layername)
if file_type == settings.FILE_TYPE_VECTOR:
if store_in_geoserver:
gs_util.delete_wms_layer(geoserver_workspace, layername,
settings.LAYMAN_GS_AUTH)
gs_util.delete_wms_store(geoserver_workspace,
settings.LAYMAN_GS_AUTH,
wms.get_qgis_store_name(layername))
geoserver.publish_layer_from_db(
workspace,
layername,
description,
title,
crs=crs,
geoserver_workspace=geoserver_workspace,
)
else:
gs_util.delete_feature_type(geoserver_workspace, layername,
settings.LAYMAN_GS_AUTH)
geoserver.publish_layer_from_qgis(
workspace,
layername,
description,
title,
geoserver_workspace=geoserver_workspace,
)
elif file_type == settings.FILE_TYPE_RASTER:
file_path = info['_file']['normalized_file']['gs_path']
real_bbox = info['native_bounding_box']
bbox = bbox_util.ensure_bbox_with_area(real_bbox, crs_def.CRSDefinitions[crs].no_area_bbox_padding)\
if not bbox_util.is_empty(real_bbox) else crs_def.CRSDefinitions[crs].default_bbox
lat_lon_bbox = bbox_util.transform(bbox, crs, crs_def.EPSG_4326)
gs_util.create_coverage_store(geoserver_workspace,
settings.LAYMAN_GS_AUTH,
coverage_store_name, file_path)
gs_util.publish_coverage(geoserver_workspace,
settings.LAYMAN_GS_AUTH,
coverage_store_name,
layername,
title,
description,
bbox,
crs,
lat_lon_bbox=lat_lon_bbox)
else:
raise NotImplementedError(f"Unknown file type: {file_type}")
geoserver.set_security_rules(workspace, layername, access_rights,
settings.LAYMAN_GS_AUTH, geoserver_workspace)
wms.clear_cache(workspace)
if self.is_aborted():
wms.delete_layer(workspace, layername)
raise AbortedException
def test_empty_table_bbox(empty_table):
username, layername = empty_table
with layman.app_context():
bbox = db.get_bbox(username, layername)
assert bbox_util.is_empty(bbox), bbox
def get_template_path_and_values(workspace,
mapname,
http_method=None,
actor_name=None):
assert http_method in [
common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH
]
uuid_file_path = get_publication_uuid_file(MAP_TYPE, workspace, mapname)
publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path))
revision_date = datetime.now()
map_json = get_map_json(workspace, mapname)
operates_on = map_json_to_operates_on(map_json, editor=actor_name)
publ_info = get_publication_info(
workspace,
MAP_TYPE,
mapname,
context={
'keys':
['title', 'native_bounding_box', 'description', 'native_crs'],
})
native_bbox = publ_info.get('native_bounding_box')
crs = publ_info.get('native_crs')
if bbox_util.is_empty(native_bbox):
native_bbox = crs_def.CRSDefinitions[crs].default_bbox
extent = bbox_util.transform(native_bbox,
crs_from=publ_info.get('native_crs'),
crs_to=crs_def.EPSG_4326)
title = publ_info['title']
abstract = publ_info.get('description')
md_language = next(
iter(
common_language.get_languages_iso639_2(' '.join([
title or '',
abstract or '',
]))), None)
prop_values = _get_property_values(
workspace=workspace,
mapname=mapname,
uuid=get_map_uuid(workspace, mapname),
title=title,
abstract=abstract or None,
publication_date=publ_datetime.strftime('%Y-%m-%d'),
revision_date=revision_date.strftime('%Y-%m-%d'),
md_date_stamp=date.today().strftime('%Y-%m-%d'),
identifier=url_for('rest_workspace_map.get',
workspace=workspace,
mapname=mapname),
identifier_label=mapname,
extent=extent,
crs_list=[crs],
md_organisation_name=None,
organisation_name=None,
operates_on=operates_on,
md_language=md_language,
)
if http_method == common.REQUEST_METHOD_POST:
prop_values.pop('revision_date', None)
template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'record-template.xml')
return template_path, prop_values