def get_template_path_and_values(username,
mapname,
http_method=None,
actor_name=None):
assert http_method in [
common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH
]
uuid_file_path = get_publication_uuid_file(MAP_TYPE, username, mapname)
publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path))
revision_date = datetime.now()
map_json = get_map_json(username, mapname)
operates_on = map_json_to_operates_on(map_json, editor=actor_name)
publ_info = get_publication_info(
username,
MAP_TYPE,
mapname,
context={
'keys': ['title', 'bounding_box', 'description'],
})
bbox_3857 = publ_info.get('bounding_box')
if bbox_util.is_empty(bbox_3857):
bbox_3857 = settings.LAYMAN_DEFAULT_OUTPUT_BBOX
extent = bbox_util.transform(tuple(bbox_3857),
epsg_from=3857,
epsg_to=4326)
title = publ_info['title']
abstract = publ_info.get('description')
md_language = next(
iter(
common_language.get_languages_iso639_2(' '.join([
title or '',
abstract or '',
]))), None)
prop_values = _get_property_values(
username=username,
mapname=mapname,
uuid=get_map_uuid(username, mapname),
title=title,
abstract=abstract or None,
publication_date=publ_datetime.strftime('%Y-%m-%d'),
revision_date=revision_date.strftime('%Y-%m-%d'),
md_date_stamp=date.today().strftime('%Y-%m-%d'),
identifier=url_for('rest_workspace_map.get',
workspace=username,
mapname=mapname),
identifier_label=mapname,
extent=extent,
epsg_codes=map_json_to_epsg_codes(map_json),
md_organisation_name=None,
organisation_name=None,
operates_on=operates_on,
md_language=md_language,
)
if http_method == common.REQUEST_METHOD_POST:
prop_values.pop('revision_date', None)
template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'record-template.xml')
return template_path, prop_values
def test_migrate_maps_on_wms_workspace(ensure_map):
layer_workspace = 'test_migrate_maps_on_wms_workspace_layer_workspace'
layer = 'test_migrate_maps_on_wms_workspace_layer'
workspace = 'test_migrate_maps_on_wms_workspace_workspace'
map = 'test_migrate_maps_on_wms_workspace_map'
expected_file = 'sample/style/test_sld_style_applied_in_map_thumbnail_map.png'
ensure_map(workspace, map, layer_workspace, layer)
with app.app_context():
map_json = input_file.get_map_json(workspace, map)
assert map_json['layers'][0]['url'] == 'http://localhost:8000/geoserver/test_migrate_maps_on_wms_workspace_layer_workspace/ows',\
map_json
thumbnail_path = thumbnail.get_map_thumbnail_path(workspace, map)
diffs_before = util.compare_images(expected_file, thumbnail_path)
shutil.copyfile(
thumbnail_path,
'/code/tmp/artifacts/upgrade_v1_10_map_thumbnail_before.png')
assert 28000 < diffs_before < 35000
with app.app_context():
upgrade_v1_10.migrate_maps_on_wms_workspace()
with app.app_context():
map_json = input_file.get_map_json(workspace, map)
assert map_json['layers'][0][
'url'] == 'http://localhost:8000/geoserver/test_migrate_maps_on_wms_workspace_layer_workspace_wms/ows', map_json
thumbnail.generate_map_thumbnail(workspace, map, '')
diffs_after = util.compare_images(expected_file, thumbnail_path)
shutil.copyfile(
thumbnail_path,
'/code/tmp/artifacts/upgrade_v1_10_map_thumbnail_after.png')
assert diffs_after < 1000
process_client.delete_workspace_layer(layer_workspace, layer)
process_client.delete_workspace_map(workspace, map)
def find_maps_containing_layer(layer_workspace, layer_name):
from layman.layer import LAYER_TYPE
from layman.layer.geoserver import util as layer_gs_util
from layman.map.filesystem import input_file as map_input_file
from layman.map.util import find_maps_by_grep
from layman.util import get_publication_info
gs_url = layer_gs_util.get_gs_proxy_base_url()
gs_url = gs_url if gs_url.endswith('/') else f"{gs_url}/"
gs_domain = urlparse(gs_url).hostname
layer_info = get_publication_info(layer_workspace, LAYER_TYPE, layer_name, context={'keys': ['wms']})
layer_wms_workspace = layer_info.get('_wms', {}).get('workspace')
# first rough filters
url_pattern = fr'^\s*.?url.?:\s*.*{gs_domain}.*/geoserver(/({layer_workspace}|{layer_wms_workspace}))?/(ows|wms|wfs).*,\s*$'
url_maps = find_maps_by_grep(url_pattern)
layer_pattern = fr'^\s*.?(layers|LAYERS).?:\s*.*{layer_name}.*\s*$'
layer_maps = find_maps_by_grep(layer_pattern)
maps = url_maps.intersection(layer_maps)
# verify layer for map
gs_ows_url_pattern = fr'^{re.escape(gs_url)}(({layer_workspace}|{layer_wms_workspace})/)?(?:ows|wms|wfs).*$'
result_maps = set()
for workspace, map in maps:
map_json_raw = map_input_file.get_map_json(workspace, map)
map_json = map_input_file.unquote_urls(map_json_raw)
for map_layer in map_json['layers']:
layer_url = map_layer.get('url')
if not layer_url:
continue
match = re.match(gs_ows_url_pattern, layer_url)
if not match:
continue
map_layers = CaseInsensitiveDict(**map_layer.get('params', dict())).get('layers')
layers = unquote(map_layers).split(',')
if layer_name in layers or f'{layer_workspace}:{layer_name}' in layers or f'{layer_wms_workspace}:{layer_name}' in layers:
result_maps.add((workspace, map))
return result_maps
def get_template_path_and_values(workspace,
mapname,
http_method=None,
actor_name=None):
assert http_method in [
common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH
]
uuid_file_path = get_publication_uuid_file(MAP_TYPE, workspace, mapname)
publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path))
revision_date = datetime.now()
map_json = get_map_json(workspace, mapname)
operates_on = map_json_to_operates_on(map_json, editor=actor_name)
publ_info = get_publication_info(
workspace,
MAP_TYPE,
mapname,
context={
'keys':
['title', 'native_bounding_box', 'description', 'native_crs'],
})
native_bbox = publ_info.get('native_bounding_box')
crs = publ_info.get('native_crs')
if bbox_util.is_empty(native_bbox):
native_bbox = crs_def.CRSDefinitions[crs].default_bbox
extent = bbox_util.transform(native_bbox,
crs_from=publ_info.get('native_crs'),
crs_to=crs_def.EPSG_4326)
title = publ_info['title']
abstract = publ_info.get('description')
md_language = next(
iter(
common_language.get_languages_iso639_2(' '.join([
title or '',
abstract or '',
]))), None)
prop_values = _get_property_values(
workspace=workspace,
mapname=mapname,
uuid=get_map_uuid(workspace, mapname),
title=title,
abstract=abstract or None,
publication_date=publ_datetime.strftime('%Y-%m-%d'),
revision_date=revision_date.strftime('%Y-%m-%d'),
md_date_stamp=date.today().strftime('%Y-%m-%d'),
identifier=url_for('rest_workspace_map.get',
workspace=workspace,
mapname=mapname),
identifier_label=mapname,
extent=extent,
crs_list=[crs],
md_organisation_name=None,
organisation_name=None,
operates_on=operates_on,
md_language=md_language,
)
if http_method == common.REQUEST_METHOD_POST:
prop_values.pop('revision_date', None)
template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'record-template.xml')
return template_path, prop_values
def adjust_data_for_bbox_search():
logger.info(f' Starting - Set bbox for all publications')
query = f'''
select w.name,
p.type,
p.name
from {DB_SCHEMA}.publications p inner join
{DB_SCHEMA}.workspaces w on w.id = p.id_workspace
where p.type = %s
'''
params = (LAYER_TYPE, )
publications = db_util.run_query(query, params)
for (workspace, _, layer) in publications:
logger.info(f' Migrate layer {workspace}.{layer}')
table_query = f'''
SELECT count(*)
FROM pg_tables
WHERE schemaname = '{workspace}'
AND tablename = '{layer}'
AND tableowner = '{settings.LAYMAN_PG_USER}'
;'''
cnt = db_util.run_query(table_query)[0][0]
if cnt == 0:
logger.warning(
f' Layer DB table not available, not migrating.')
continue
bbox_query = f'''
select ST_Extent(l.wkb_geometry) bbox
from {workspace}.{layer} l
;'''
bbox = db_util.run_query(bbox_query)[0]
set_layer_bbox_query = f'''
update {DB_SCHEMA}.publications set
bbox = %s
where type = %s
and name = %s
and id_workspace = (select w.id from {DB_SCHEMA}.workspaces w where w.name = %s);'''
db_util.run_statement(set_layer_bbox_query, (
bbox,
LAYER_TYPE,
layer,
workspace,
))
params = (MAP_TYPE, )
publications = db_util.run_query(query, params)
for (workspace, _, map) in publications:
from layman.map.filesystem import input_file
logger.info(f' Migrate map {workspace}.{map}')
map_json = input_file.get_map_json(workspace, map)
bbox_4326 = float(map_json['extent'][0]), float(map_json['extent'][1]),\
float(map_json['extent'][2]), float(map_json['extent'][3])
query_transform = f'''
with tmp as (select ST_Transform(ST_SetSRID(ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s, %s)), %s), %s) bbox)
select st_xmin(bbox),
st_ymin(bbox),
st_xmax(bbox),
st_ymax(bbox)
from tmp
;'''
params = bbox_4326 + (
4326,
3857,
)
try:
bbox_3857 = db_util.run_query(query_transform,
params,
encapsulate_exception=False)[0]
except psycopg2.errors.InternalError_: # pylint: disable=no-member
logger.warning(
f' Bounding box not transformed, so set to None. Bounding box in 4326: "{bbox_4326}"'
)
bbox_3857 = (None, None, None, None)
set_map_bbox_query = f'''
update {DB_SCHEMA}.publications set
bbox = ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s ,%s))
where type = %s
and name = %s
and id_workspace = (select w.id from {DB_SCHEMA}.workspaces w where w.name = %s);'''
params = bbox_3857 + (
MAP_TYPE,
map,
workspace,
)
db_util.run_statement(set_map_bbox_query, params)
logger.info(f' DONE - Set bbox for all publications')