def test_patch_layer_style(client):
with app.app_context():
workspace = 'testuser1'
layername = 'ne_110m_admin_0_countries'
rest_path = url_for('rest_workspace_layer.patch', workspace=workspace, layername=layername)
sld_path = 'sample/style/generic-blue_sld.xml'
assert os.path.isfile(sld_path)
response = client.patch(rest_path, data={
'style': (open(sld_path, 'rb'), os.path.basename(sld_path)),
'title': 'countries in blue'
})
assert response.status_code == 200
# last_task = util._get_layer_task(workspace, layername)
# Time to generate testing thumbnail is probably shorter than getting & parsing WMS/WFS capabilities documents
# so it's finished before PATCH request is completed
#
# assert last_task is not None and not util._is_task_ready(last_task)
# resp_json = rv.get_json()
# keys_to_check = ['thumbnail']
# for key_to_check in keys_to_check:
# assert 'status' in resp_json[key_to_check]
flask_client.wait_till_layer_ready(workspace, layername)
# last_task['last'].get()
resp_json = response.get_json()
assert resp_json['title'] == "countries in blue"
wms_url = geoserver_wms.get_wms_url(workspace)
wms = wms_proxy(wms_url)
assert layername in wms.contents
assert wms[layername].title == 'countries in blue'
assert wms[layername].styles[
workspace + '_wms:' + layername]['title'] == 'Generic Blue'
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
expected_md_values = {
'abstract': "and new description",
'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944],
'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername),
'identifier': {
'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'label': 'ne_110m_admin_0_countries'
},
'language': ['eng'],
'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'organisation_name': None,
'publication_date': TODAY_DATE,
'reference_system': EXP_REFERENCE_SYSTEMS,
'revision_date': TODAY_DATE,
'spatial_resolution': {
'scale_denominator': 100000000,
},
'title': 'countries in blue',
}
check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_post_layers_shp(client):
workspace = 'testuser1'
layername = 'ne_110m_admin_0_countries_shp'
rest_path = url_for('rest_workspace_layers.post', workspace=workspace)
file_paths = [
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.cpg',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.dbf',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.prj',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.README.html',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shp',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shx',
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.VERSION.txt',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths]
response = client.post(rest_path, data={
'file': files,
'name': layername,
})
assert response.status_code == 200
finally:
for file_path in files:
file_path[0].close()
chain_info = util.get_layer_chain(workspace, layername)
assert chain_info is not None and not celery_util.is_chain_ready(chain_info)
flask_client.wait_till_layer_ready(workspace, layername)
# last_task['last'].get()
wms_url = geoserver_wms.get_wms_url(workspace)
wms = wms_proxy(wms_url)
assert 'ne_110m_admin_0_countries_shp' in wms.contents
publication_counter.increase()
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
def patch_layer(client):
username = TEST_USER
layername = TEST_LAYER
with app.app_context():
rest_path = url_for('rest_workspace_layer.patch', workspace=username, layername=layername)
file_paths = [
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths]
response = client.patch(rest_path, data={
'file': files,
'title': 'patched layer',
})
assert response.status_code == 200
finally:
for file_path in files:
file_path[0].close()
flask_client.wait_till_layer_ready(username, layername)
def test_patch_layer_data(client):
with app.app_context():
workspace = 'testuser2'
layername = 'countries'
rest_path = url_for('rest_workspace_layer.patch', workspace=workspace, layername=layername)
file_paths = [
'tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in
file_paths]
response = client.patch(rest_path, data={
'file': files,
'title': 'populated places'
})
assert response.status_code == 200
finally:
for file_path in files:
file_path[0].close()
chain_info = util.get_layer_chain(workspace, layername)
assert chain_info is not None and not celery_util.is_chain_ready(chain_info)
resp_json = response.get_json()
keys_to_check = ['db_table', 'wms', 'wfs', 'thumbnail', 'metadata']
for key_to_check in keys_to_check:
assert 'status' in resp_json[key_to_check]
flask_client.wait_till_layer_ready(workspace, layername)
# last_task['last'].get()
with app.app_context():
rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername)
response = client.get(rest_path)
assert 200 <= response.status_code < 300
resp_json = response.get_json()
assert resp_json['title'] == "populated places"
feature_type = get_feature_type(workspace, 'postgresql', layername)
attributes = feature_type['attributes']['attribute']
assert next((
a for a in attributes if a['name'] == 'sovereignt'
), None) is None
assert next((
a for a in attributes if a['name'] == 'adm0cap'
), None) is not None
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
with app.app_context():
expected_md_values = {
'abstract': "popis st\u00e1t\u016f",
'extent': [-175.22056435043098, -41.29999116752133, 179.21664802661394, 64.15002486626597],
'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername),
'identifier': {
'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
"label": "countries"
},
'language': ["eng", 'chi', 'rus'],
'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'organisation_name': None,
'publication_date': TODAY_DATE,
'reference_system': EXP_REFERENCE_SYSTEMS,
'revision_date': TODAY_DATE,
'spatial_resolution': None, # it's point data now and we can't guess scale from point data
'title': 'populated places',
}
check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_uppercase_attr(client):
with app.app_context():
workspace = 'testuser2'
rest_path = url_for('rest_workspace_layers.post', workspace=workspace)
file_paths = [
'sample/data/upper_attr.geojson',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
sld_path = 'sample/data/upper_attr.sld'
assert os.path.isfile(sld_path)
layername = 'upper_attr'
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths]
response = client.post(rest_path, data={
'file': files,
'name': layername,
'style': (open(sld_path, 'rb'), os.path.basename(sld_path)),
})
assert response.status_code == 200
resp_json = response.get_json()
# print(resp_json)
finally:
for file_path in files:
file_path[0].close()
chain_info = util.get_layer_chain(workspace, layername)
assert chain_info is not None and not celery_util.is_chain_ready(chain_info)
flask_client.wait_till_layer_ready(workspace, layername)
# last_task['last'].get()
assert celery_util.is_chain_ready(chain_info)
with app.app_context():
rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername)
response = client.get(rest_path)
assert 200 <= response.status_code < 300
resp_json = response.get_json()
# print(resp_json)
for source in [
'wms',
'wfs',
'thumbnail',
'file',
'db_table',
'metadata',
]:
assert 'status' not in resp_json[source], f"{source}: {resp_json[source]}"
style_url = geoserver_sld.get_workspace_style_url(workspace, layername)
response = requests.get(style_url + '.sld',
auth=settings.LAYMAN_GS_AUTH
)
response.raise_for_status()
sld_file = io.BytesIO(response.content)
tree = ET.parse(sld_file)
root = tree.getroot()
assert root.attrib['version'] == '1.1.0'
feature_type = get_feature_type(workspace, 'postgresql', layername)
attributes = feature_type['attributes']['attribute']
attr_names = ["id", "dpr_smer_k", "fid_zbg", "silnice", "silnice_bs", "typsil_p", "cislouseku", "jmeno",
"typsil_k", "peazkom1", "peazkom2", "peazkom3", "peazkom4", "vym_tahy_k", "vym_tahy_p",
"r_indsil7", "kruh_obj_k", "etah1", "etah2", "etah3", "etah4", "kruh_obj_p", "dpr_smer_p"]
for attr_name in attr_names:
assert next((
a for a in attributes if a['name'] == attr_name
), None) is not None
th_path = get_layer_thumbnail_path(workspace, layername)
assert os.path.getsize(th_path) > 5000
with app.app_context():
rest_path = url_for('rest_workspace_layer.delete_layer', workspace=workspace, layername=layername)
response = client.delete(rest_path)
assert 200 <= response.status_code < 300
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
def test_post_layers_complex(client):
with app.app_context():
workspace = 'testuser2'
rest_path = url_for('rest_workspace_layers.post', workspace=workspace)
file_paths = [
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
sld_path = 'sample/style/generic-blue_sld.xml'
assert os.path.isfile(sld_path)
layername = ''
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths]
response = client.post(rest_path, data={
'file': files,
'name': 'countries',
'title': 'staty',
'description': 'popis států',
'style': (open(sld_path, 'rb'), os.path.basename(sld_path)),
})
assert response.status_code == 200
resp_json = response.get_json()
# print(resp_json)
layername = resp_json[0]['name']
finally:
for file_path in files:
file_path[0].close()
chain_info = util.get_layer_chain(workspace, layername)
assert chain_info is not None and not celery_util.is_chain_ready(chain_info)
flask_client.wait_till_layer_ready(workspace, layername)
# last_task['last'].get()
assert celery_util.is_chain_ready(chain_info)
wms_url = geoserver_wms.get_wms_url(workspace)
wms = wms_proxy(wms_url)
assert 'countries' in wms.contents
assert wms['countries'].title == 'staty'
assert wms['countries'].abstract == 'popis států'
assert wms['countries'].styles[workspace + '_wms:countries']['title'] == 'Generic Blue'
assert layername != ''
rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername)
response = client.get(rest_path)
assert 200 <= response.status_code < 300
resp_json = response.get_json()
# print(resp_json)
assert resp_json['title'] == 'staty'
assert resp_json['description'] == 'popis států'
for source in [
'wms',
'wfs',
'thumbnail',
'file',
'db_table',
'metadata',
]:
assert 'status' not in resp_json[source]
style_url = geoserver_sld.get_workspace_style_url(workspace, layername)
response = requests.get(style_url + '.sld',
auth=settings.LAYMAN_GS_AUTH
)
response.raise_for_status()
sld_file = io.BytesIO(response.content)
tree = ET.parse(sld_file)
root = tree.getroot()
assert root.attrib['version'] == '1.0.0'
feature_type = get_feature_type(workspace, 'postgresql', layername)
attributes = feature_type['attributes']['attribute']
assert next((
a for a in attributes if a['name'] == 'sovereignt'
), None) is not None
publication_counter.increase()
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
with app.app_context():
expected_md_values = {
'abstract': "popis st\u00e1t\u016f",
'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944],
'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername),
'identifier': {
"identifier": url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
"label": "countries"
},
'language': ["eng"],
'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'organisation_name': None,
'publication_date': TODAY_DATE,
'reference_system': EXP_REFERENCE_SYSTEMS,
'revision_date': None,
'spatial_resolution': {
'scale_denominator': 100000000,
},
'title': "staty",
}
check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_post_layers_simple(client):
with app.app_context():
workspace = 'testuser1'
rest_path = url_for('rest_workspace_layers.post', workspace=workspace)
file_paths = [
'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson',
]
for file_path in file_paths:
assert os.path.isfile(file_path)
files = []
try:
files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths]
response = client.post(rest_path, data={
'file': files,
})
assert response.status_code == 200
finally:
for file_path in files:
file_path[0].close()
layername = 'ne_110m_admin_0_countries'
chain_info = util.get_layer_chain(workspace, layername)
assert chain_info is not None and not celery_util.is_chain_ready(chain_info)
layer_info = util.get_layer_info(workspace, layername)
keys_to_check = ['db_table', 'wms', 'wfs', 'thumbnail', 'metadata']
for key_to_check in keys_to_check:
assert 'status' in layer_info[key_to_check]
# For some reason this hangs forever on get() if run (either with src/layman/authz/read_everyone_write_owner_auth2_test.py::test_authn_map_access_rights or src/layman/authn/oauth2_test.py::test_patch_current_user_without_username) and with src/layman/common/metadata/util.csw_insert
# last_task['last'].get()
# e.g. python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/authn/oauth2_test.py::test_patch_current_user_without_username src/layman/layer/rest_workspace_test.py::test_post_layers_simple
# this can badly affect also .get(propagate=False) in layman.celery.abort_task_chain
# but hopefully this is only related to magic flask&celery test suite
flask_client.wait_till_layer_ready(workspace, layername)
layer_info = util.get_layer_info(workspace, layername)
for key_to_check in keys_to_check:
assert isinstance(layer_info[key_to_check], str) \
or 'status' not in layer_info[key_to_check]
wms_url = geoserver_wms.get_wms_url(workspace)
wms = wms_proxy(wms_url)
assert layername in wms.contents
from layman.layer import get_layer_type_def
from layman.common.filesystem import uuid as common_uuid
uuid_filename = common_uuid.get_publication_uuid_file(
get_layer_type_def()['type'], workspace, layername)
assert os.path.isfile(uuid_filename)
uuid_str = None
with open(uuid_filename, "r") as file:
uuid_str = file.read().strip()
assert uuid.is_valid_uuid(uuid_str)
assert settings.LAYMAN_REDIS.sismember(uuid.UUID_SET_KEY, uuid_str)
assert settings.LAYMAN_REDIS.exists(uuid.get_uuid_metadata_key(uuid_str))
assert settings.LAYMAN_REDIS.hexists(
uuid.get_workspace_type_names_key(workspace, '.'.join(__name__.split('.')[:-1])),
layername
)
layer_info = client.get(url_for('rest_workspace_layer.get', workspace=workspace, layername=layername)).get_json()
assert set(layer_info['metadata'].keys()) == {'identifier', 'csw_url', 'record_url', 'comparison_url'}
assert layer_info['metadata']['identifier'] == f"m-{uuid_str}"
assert layer_info['metadata']['csw_url'] == settings.CSW_PROXY_URL
md_record_url = f"http://micka:80/record/basic/m-{uuid_str}"
assert layer_info['metadata']['record_url'].replace("http://localhost:3080", "http://micka:80") == md_record_url
assert layer_info['metadata']['comparison_url'] == url_for_external('rest_workspace_layer_metadata_comparison.get',
workspace=workspace, layername=layername)
assert 'id' not in layer_info.keys()
assert 'type' not in layer_info.keys()
response = requests.get(md_record_url, auth=settings.CSW_BASIC_AUTHN)
response.raise_for_status()
assert layername in response.text
publication_counter.increase()
uuid.check_redis_consistency(expected_publ_num_by_type={
f'{LAYER_TYPE}': publication_counter.get()
})
with app.app_context():
expected_md_values = {
'abstract': None,
'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944],
'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername),
'identifier': {
'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'label': 'ne_110m_admin_0_countries'
},
'language': ['eng'],
'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername),
'organisation_name': None,
'publication_date': TODAY_DATE,
'reference_system': EXP_REFERENCE_SYSTEMS,
'revision_date': None,
'spatial_resolution': {
'scale_denominator': 100000000,
},
'title': 'ne_110m_admin_0_countries',
}
check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)