def test_patch_layer_style(client): with app.app_context(): workspace = 'testuser1' layername = 'ne_110m_admin_0_countries' rest_path = url_for('rest_workspace_layer.patch', workspace=workspace, layername=layername) sld_path = 'sample/style/generic-blue_sld.xml' assert os.path.isfile(sld_path) response = client.patch(rest_path, data={ 'style': (open(sld_path, 'rb'), os.path.basename(sld_path)), 'title': 'countries in blue' }) assert response.status_code == 200 # last_task = util._get_layer_task(workspace, layername) # Time to generate testing thumbnail is probably shorter than getting & parsing WMS/WFS capabilities documents # so it's finished before PATCH request is completed # # assert last_task is not None and not util._is_task_ready(last_task) # resp_json = rv.get_json() # keys_to_check = ['thumbnail'] # for key_to_check in keys_to_check: # assert 'status' in resp_json[key_to_check] flask_client.wait_till_layer_ready(workspace, layername) # last_task['last'].get() resp_json = response.get_json() assert resp_json['title'] == "countries in blue" wms_url = geoserver_wms.get_wms_url(workspace) wms = wms_proxy(wms_url) assert layername in wms.contents assert wms[layername].title == 'countries in blue' assert wms[layername].styles[ workspace + '_wms:' + layername]['title'] == 'Generic Blue' uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() }) expected_md_values = { 'abstract': "and new description", 'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944], 'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername), 'identifier': { 'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'label': 'ne_110m_admin_0_countries' }, 'language': ['eng'], 'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'organisation_name': None, 'publication_date': TODAY_DATE, 'reference_system': EXP_REFERENCE_SYSTEMS, 'revision_date': TODAY_DATE, 'spatial_resolution': { 'scale_denominator': 100000000, }, 'title': 'countries in blue', } check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_post_layers_shp(client): workspace = 'testuser1' layername = 'ne_110m_admin_0_countries_shp' rest_path = url_for('rest_workspace_layers.post', workspace=workspace) file_paths = [ 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.cpg', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.dbf', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.prj', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.README.html', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shp', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shx', 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.VERSION.txt', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.post(rest_path, data={ 'file': files, 'name': layername, }) assert response.status_code == 200 finally: for file_path in files: file_path[0].close() chain_info = util.get_layer_chain(workspace, layername) assert chain_info is not None and not celery_util.is_chain_ready(chain_info) flask_client.wait_till_layer_ready(workspace, layername) # last_task['last'].get() wms_url = geoserver_wms.get_wms_url(workspace) wms = wms_proxy(wms_url) assert 'ne_110m_admin_0_countries_shp' in wms.contents publication_counter.increase() uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() })
def patch_layer(client): username = TEST_USER layername = TEST_LAYER with app.app_context(): rest_path = url_for('rest_workspace_layer.patch', workspace=username, layername=layername) file_paths = [ 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.patch(rest_path, data={ 'file': files, 'title': 'patched layer', }) assert response.status_code == 200 finally: for file_path in files: file_path[0].close() flask_client.wait_till_layer_ready(username, layername)
def test_patch_layer_data(client): with app.app_context(): workspace = 'testuser2' layername = 'countries' rest_path = url_for('rest_workspace_layer.patch', workspace=workspace, layername=layername) file_paths = [ 'tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.patch(rest_path, data={ 'file': files, 'title': 'populated places' }) assert response.status_code == 200 finally: for file_path in files: file_path[0].close() chain_info = util.get_layer_chain(workspace, layername) assert chain_info is not None and not celery_util.is_chain_ready(chain_info) resp_json = response.get_json() keys_to_check = ['db_table', 'wms', 'wfs', 'thumbnail', 'metadata'] for key_to_check in keys_to_check: assert 'status' in resp_json[key_to_check] flask_client.wait_till_layer_ready(workspace, layername) # last_task['last'].get() with app.app_context(): rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername) response = client.get(rest_path) assert 200 <= response.status_code < 300 resp_json = response.get_json() assert resp_json['title'] == "populated places" feature_type = get_feature_type(workspace, 'postgresql', layername) attributes = feature_type['attributes']['attribute'] assert next(( a for a in attributes if a['name'] == 'sovereignt' ), None) is None assert next(( a for a in attributes if a['name'] == 'adm0cap' ), None) is not None uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() }) with app.app_context(): expected_md_values = { 'abstract': "popis st\u00e1t\u016f", 'extent': [-175.22056435043098, -41.29999116752133, 179.21664802661394, 64.15002486626597], 'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername), 'identifier': { 'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), "label": "countries" }, 'language': ["eng", 'chi', 'rus'], 'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'organisation_name': None, 'publication_date': TODAY_DATE, 'reference_system': EXP_REFERENCE_SYSTEMS, 'revision_date': TODAY_DATE, 'spatial_resolution': None, # it's point data now and we can't guess scale from point data 'title': 'populated places', } check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_uppercase_attr(client): with app.app_context(): workspace = 'testuser2' rest_path = url_for('rest_workspace_layers.post', workspace=workspace) file_paths = [ 'sample/data/upper_attr.geojson', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] sld_path = 'sample/data/upper_attr.sld' assert os.path.isfile(sld_path) layername = 'upper_attr' try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.post(rest_path, data={ 'file': files, 'name': layername, 'style': (open(sld_path, 'rb'), os.path.basename(sld_path)), }) assert response.status_code == 200 resp_json = response.get_json() # print(resp_json) finally: for file_path in files: file_path[0].close() chain_info = util.get_layer_chain(workspace, layername) assert chain_info is not None and not celery_util.is_chain_ready(chain_info) flask_client.wait_till_layer_ready(workspace, layername) # last_task['last'].get() assert celery_util.is_chain_ready(chain_info) with app.app_context(): rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername) response = client.get(rest_path) assert 200 <= response.status_code < 300 resp_json = response.get_json() # print(resp_json) for source in [ 'wms', 'wfs', 'thumbnail', 'file', 'db_table', 'metadata', ]: assert 'status' not in resp_json[source], f"{source}: {resp_json[source]}" style_url = geoserver_sld.get_workspace_style_url(workspace, layername) response = requests.get(style_url + '.sld', auth=settings.LAYMAN_GS_AUTH ) response.raise_for_status() sld_file = io.BytesIO(response.content) tree = ET.parse(sld_file) root = tree.getroot() assert root.attrib['version'] == '1.1.0' feature_type = get_feature_type(workspace, 'postgresql', layername) attributes = feature_type['attributes']['attribute'] attr_names = ["id", "dpr_smer_k", "fid_zbg", "silnice", "silnice_bs", "typsil_p", "cislouseku", "jmeno", "typsil_k", "peazkom1", "peazkom2", "peazkom3", "peazkom4", "vym_tahy_k", "vym_tahy_p", "r_indsil7", "kruh_obj_k", "etah1", "etah2", "etah3", "etah4", "kruh_obj_p", "dpr_smer_p"] for attr_name in attr_names: assert next(( a for a in attributes if a['name'] == attr_name ), None) is not None th_path = get_layer_thumbnail_path(workspace, layername) assert os.path.getsize(th_path) > 5000 with app.app_context(): rest_path = url_for('rest_workspace_layer.delete_layer', workspace=workspace, layername=layername) response = client.delete(rest_path) assert 200 <= response.status_code < 300 uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() })
def test_post_layers_complex(client): with app.app_context(): workspace = 'testuser2' rest_path = url_for('rest_workspace_layers.post', workspace=workspace) file_paths = [ 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] sld_path = 'sample/style/generic-blue_sld.xml' assert os.path.isfile(sld_path) layername = '' try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.post(rest_path, data={ 'file': files, 'name': 'countries', 'title': 'staty', 'description': 'popis států', 'style': (open(sld_path, 'rb'), os.path.basename(sld_path)), }) assert response.status_code == 200 resp_json = response.get_json() # print(resp_json) layername = resp_json[0]['name'] finally: for file_path in files: file_path[0].close() chain_info = util.get_layer_chain(workspace, layername) assert chain_info is not None and not celery_util.is_chain_ready(chain_info) flask_client.wait_till_layer_ready(workspace, layername) # last_task['last'].get() assert celery_util.is_chain_ready(chain_info) wms_url = geoserver_wms.get_wms_url(workspace) wms = wms_proxy(wms_url) assert 'countries' in wms.contents assert wms['countries'].title == 'staty' assert wms['countries'].abstract == 'popis států' assert wms['countries'].styles[workspace + '_wms:countries']['title'] == 'Generic Blue' assert layername != '' rest_path = url_for('rest_workspace_layer.get', workspace=workspace, layername=layername) response = client.get(rest_path) assert 200 <= response.status_code < 300 resp_json = response.get_json() # print(resp_json) assert resp_json['title'] == 'staty' assert resp_json['description'] == 'popis států' for source in [ 'wms', 'wfs', 'thumbnail', 'file', 'db_table', 'metadata', ]: assert 'status' not in resp_json[source] style_url = geoserver_sld.get_workspace_style_url(workspace, layername) response = requests.get(style_url + '.sld', auth=settings.LAYMAN_GS_AUTH ) response.raise_for_status() sld_file = io.BytesIO(response.content) tree = ET.parse(sld_file) root = tree.getroot() assert root.attrib['version'] == '1.0.0' feature_type = get_feature_type(workspace, 'postgresql', layername) attributes = feature_type['attributes']['attribute'] assert next(( a for a in attributes if a['name'] == 'sovereignt' ), None) is not None publication_counter.increase() uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() }) with app.app_context(): expected_md_values = { 'abstract': "popis st\u00e1t\u016f", 'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944], 'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername), 'identifier': { "identifier": url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), "label": "countries" }, 'language': ["eng"], 'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'organisation_name': None, 'publication_date': TODAY_DATE, 'reference_system': EXP_REFERENCE_SYSTEMS, 'revision_date': None, 'spatial_resolution': { 'scale_denominator': 100000000, }, 'title': "staty", } check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)
def test_post_layers_simple(client): with app.app_context(): workspace = 'testuser1' rest_path = url_for('rest_workspace_layers.post', workspace=workspace) file_paths = [ 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson', ] for file_path in file_paths: assert os.path.isfile(file_path) files = [] try: files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] response = client.post(rest_path, data={ 'file': files, }) assert response.status_code == 200 finally: for file_path in files: file_path[0].close() layername = 'ne_110m_admin_0_countries' chain_info = util.get_layer_chain(workspace, layername) assert chain_info is not None and not celery_util.is_chain_ready(chain_info) layer_info = util.get_layer_info(workspace, layername) keys_to_check = ['db_table', 'wms', 'wfs', 'thumbnail', 'metadata'] for key_to_check in keys_to_check: assert 'status' in layer_info[key_to_check] # For some reason this hangs forever on get() if run (either with src/layman/authz/read_everyone_write_owner_auth2_test.py::test_authn_map_access_rights or src/layman/authn/oauth2_test.py::test_patch_current_user_without_username) and with src/layman/common/metadata/util.csw_insert # last_task['last'].get() # e.g. python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/authn/oauth2_test.py::test_patch_current_user_without_username src/layman/layer/rest_workspace_test.py::test_post_layers_simple # this can badly affect also .get(propagate=False) in layman.celery.abort_task_chain # but hopefully this is only related to magic flask&celery test suite flask_client.wait_till_layer_ready(workspace, layername) layer_info = util.get_layer_info(workspace, layername) for key_to_check in keys_to_check: assert isinstance(layer_info[key_to_check], str) \ or 'status' not in layer_info[key_to_check] wms_url = geoserver_wms.get_wms_url(workspace) wms = wms_proxy(wms_url) assert layername in wms.contents from layman.layer import get_layer_type_def from layman.common.filesystem import uuid as common_uuid uuid_filename = common_uuid.get_publication_uuid_file( get_layer_type_def()['type'], workspace, layername) assert os.path.isfile(uuid_filename) uuid_str = None with open(uuid_filename, "r") as file: uuid_str = file.read().strip() assert uuid.is_valid_uuid(uuid_str) assert settings.LAYMAN_REDIS.sismember(uuid.UUID_SET_KEY, uuid_str) assert settings.LAYMAN_REDIS.exists(uuid.get_uuid_metadata_key(uuid_str)) assert settings.LAYMAN_REDIS.hexists( uuid.get_workspace_type_names_key(workspace, '.'.join(__name__.split('.')[:-1])), layername ) layer_info = client.get(url_for('rest_workspace_layer.get', workspace=workspace, layername=layername)).get_json() assert set(layer_info['metadata'].keys()) == {'identifier', 'csw_url', 'record_url', 'comparison_url'} assert layer_info['metadata']['identifier'] == f"m-{uuid_str}" assert layer_info['metadata']['csw_url'] == settings.CSW_PROXY_URL md_record_url = f"http://micka:80/record/basic/m-{uuid_str}" assert layer_info['metadata']['record_url'].replace("http://localhost:3080", "http://micka:80") == md_record_url assert layer_info['metadata']['comparison_url'] == url_for_external('rest_workspace_layer_metadata_comparison.get', workspace=workspace, layername=layername) assert 'id' not in layer_info.keys() assert 'type' not in layer_info.keys() response = requests.get(md_record_url, auth=settings.CSW_BASIC_AUTHN) response.raise_for_status() assert layername in response.text publication_counter.increase() uuid.check_redis_consistency(expected_publ_num_by_type={ f'{LAYER_TYPE}': publication_counter.get() }) with app.app_context(): expected_md_values = { 'abstract': None, 'extent': [-180.0, -85.60903859383285, 180.0, 83.64513109859944], 'graphic_url': url_for_external('rest_workspace_layer_thumbnail.get', workspace=workspace, layername=layername), 'identifier': { 'identifier': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'label': 'ne_110m_admin_0_countries' }, 'language': ['eng'], 'layer_endpoint': url_for_external('rest_workspace_layer.get', workspace=workspace, layername=layername), 'organisation_name': None, 'publication_date': TODAY_DATE, 'reference_system': EXP_REFERENCE_SYSTEMS, 'revision_date': None, 'spatial_resolution': { 'scale_denominator': 100000000, }, 'title': 'ne_110m_admin_0_countries', } check_metadata(client, workspace, layername, METADATA_PROPERTIES_EQUAL, expected_md_values)