def check_workspace_names():
logger.info(
f' Starting - checking workspace names - for {settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX} suffix, '
f'for `{settings.REST_WORKSPACES_PREFIX}` name')
workspaces = prime_db_schema.get_workspaces()
for workspace in workspaces:
if workspace.endswith(settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX):
raise LaymanError(
f"A workspace has name with reserved suffix '{settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX}'. "
f"In that case, please downgrade to the previous minor release version of Layman and contact Layman "
f"contributors. One way how to do that is to create an issue in Layman repository: "
f"https://github.com/LayerManager/layman/issues/",
data={
'workspace': workspace,
})
if settings.REST_WORKSPACES_PREFIX in workspaces:
raise LaymanError(
f"A workspace has reserved name '{settings.REST_WORKSPACES_PREFIX}'. "
f"In that case, please downgrade to the previous minor release version of Layman and contact Layman "
f"contributors. One way how to do that is to create an issue in Layman repository: "
f"https://github.com/LayerManager/layman/issues/",
data={'workspace': settings.REST_WORKSPACES_PREFIX})
logger.info(
f' DONE - checking workspace names - for {settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX} suffix, '
f'for `{settings.REST_WORKSPACES_PREFIX}` name')
def refresh_input_chunk(self, workspace, layername, check_crs=True):
if self.is_aborted():
raise AbortedException
last_change = time.time()
num_files_saved = 0
num_chunks_saved = 0
chunk_info = input_chunk.layer_file_chunk_info(workspace, layername)
logger.debug(f'chunk_info {str(chunk_info)}')
while not chunk_info[0]:
if time.time() - last_change > settings.UPLOAD_MAX_INACTIVITY_TIME:
logger.info(
f'UPLOAD_MAX_INACTIVITY_TIME reached {workspace}.{layername}')
input_file.delete_layer(workspace, layername)
raise LaymanError(22)
time.sleep(0.5)
if self.is_aborted():
logger.info(f'Aborting for layer {workspace}.{layername}')
input_file.delete_layer(workspace, layername)
logger.info(f'Aborted for layer {workspace}.{layername}')
raise AbortedException
chunk_info = input_chunk.layer_file_chunk_info(workspace, layername)
logger.debug(f'chunk_info {str(chunk_info)}')
if num_files_saved != chunk_info[1] \
or num_chunks_saved != chunk_info[2]:
last_change = time.time()
num_files_saved = chunk_info[1]
num_chunks_saved = chunk_info[2]
logger.info(f'Layer chunks uploaded {workspace}.{layername}')
input_files = input_file.get_layer_input_files(workspace, layername)
input_file.check_filenames(workspace,
layername,
input_files,
check_crs,
ignore_existing_files=True)
main_filepath = layman_util.get_publication_info(workspace,
LAYER_TYPE,
layername,
context={
'keys': ['file']
})['_file']['gdal_path']
input_file.check_main_file(main_filepath, check_crs=check_crs)
file_type = input_file.get_file_type(
input_files.raw_or_archived_main_file_path)
style_type_for_check = layman_util.get_publication_info(
workspace, LAYER_TYPE, layername,
context={'keys': ['style_type']})['style_type']
if file_type == settings.FILE_TYPE_RASTER and style_type_for_check == 'qml':
raise LaymanError(48,
f'Raster layers are not allowed to have QML style.')
def check_username(username):
if username == settings.LAYMAN_GS_USER:
raise LaymanError(41, {'username': username})
if username in gs_util.RESERVED_WORKSPACE_NAMES:
raise LaymanError(35, {'reserved_by': __name__, 'workspace': username})
if username.endswith(settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX):
raise LaymanError(45, {'workspace_name': username})
rolename = gs_util.username_to_rolename(username)
if rolename in gs_util.RESERVED_ROLE_NAMES:
raise LaymanError(35, {'reserved_by': __name__, 'role': rolename})
def get_text_data(username, layername, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
col_names = get_text_column_names(username, layername, conn_cur=conn_cur)
if len(col_names) == 0:
return [], 0
num_features = get_number_of_features(username, layername, conn_cur=conn_cur)
if num_features == 0:
return [], 0
limit = max(100, num_features // 10)
try:
cur.execute(f"""
select {', '.join(col_names)}
from {username}.{layername}
order by ogc_fid
limit {limit}
""")
except BaseException as exc:
logger.error(f'get_text_data ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
col_texts = defaultdict(list)
for row in rows:
for idx, col_name in enumerate(col_names):
val = row[idx]
if val is not None and len(val) > 0:
col_texts[col_name].append(val)
col_texts = [
' '.join(texts)
for _, texts in col_texts.items()
]
# print(f"result col_texts={col_texts}")
return col_texts, limit
def refresh_input_chunk(self, username, layername, check_crs=True):
if self.is_aborted():
raise AbortedException
last_change = time.time()
num_files_saved = 0
num_chunks_saved = 0
chunk_info = input_chunk.layer_file_chunk_info(username, layername)
logger.debug(f'chunk_info {str(chunk_info)}')
while not chunk_info[0]:
if time.time() - last_change > settings.UPLOAD_MAX_INACTIVITY_TIME:
logger.info(
f'UPLOAD_MAX_INACTIVITY_TIME reached {username}.{layername}')
input_file.delete_layer(username, layername)
raise LaymanError(22)
time.sleep(0.5)
if self.is_aborted():
logger.info(f'Aborting for layer {username}.{layername}')
input_file.delete_layer(username, layername)
logger.info(f'Aborted for layer {username}.{layername}')
raise AbortedException
chunk_info = input_chunk.layer_file_chunk_info(username, layername)
logger.debug(f'chunk_info {str(chunk_info)}')
if num_files_saved != chunk_info[1] \
or num_chunks_saved != chunk_info[2]:
last_change = time.time()
num_files_saved = chunk_info[1]
num_chunks_saved = chunk_info[2]
logger.info(f'Layer chunks uploaded {username}.{layername}')
if check_crs:
main_filepath = input_file.get_layer_main_file_path(
username, layername)
input_file.check_layer_crs(main_filepath)
def check_main_file(main_filepath):
# check feature layers in source file
in_driver = get_ogr_driver(main_filepath)
in_data_source = in_driver.Open(main_filepath, 0)
n_layers = in_data_source.GetLayerCount()
if n_layers != 1:
raise LaymanError(5, {'found': n_layers, 'expected': 1})
def get_missing_attributes(attribute_tuples, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
# Find all triples which do not already exist
query = f"""select attribs.*
from (""" + "\n union all\n".join([f"select '{username}' username, '{layername}' layername, '{attrname}' attrname" for username, layername, attrname in attribute_tuples]) + """) attribs left join
information_schema.columns c on c.table_schema = attribs.username
and c.table_name = attribs.layername
and c.column_name = attribs.attrname
where c.column_name is null"""
try:
if attribute_tuples:
cur.execute(query)
except BaseException as exc:
logger.error(f'get_missing_attributes ERROR')
raise LaymanError(7) from exc
missing_attributes = set()
rows = cur.fetchall()
for row in rows:
missing_attributes.add((row[0],
row[1],
row[2]))
return missing_attributes
def check_vector_main_file(main_filepath, *, check_crs=True):
in_data_source = ogr.Open(main_filepath, 0)
n_layers = in_data_source.GetLayerCount()
if n_layers != 1:
raise LaymanError(5, {'found': n_layers, 'expected': 1})
if check_crs:
check_vector_layer_crs(main_filepath)
def refresh_table(self, username, layername, crs_id=None, ensure_user=False):
if ensure_user:
db.ensure_workspace(username)
if self.is_aborted():
raise AbortedException
main_filepath = get_layer_main_file_path(username, layername)
process = db.import_layer_vector_file_async(username, layername,
main_filepath, crs_id)
while process.poll() is None and not self.is_aborted():
pass
if self.is_aborted():
logger.info(f'terminating {username} {layername}')
process.terminate()
logger.info(f'terminating {username} {layername}')
delete_layer(username, layername)
raise AbortedException
return_code = process.poll()
if return_code != 0:
pg_error = str(process.stdout.read())
logger.error(f"STDOUT: {pg_error}")
if "ERROR: zero-length delimited identifier at or near" in pg_error:
err_code = 28
else:
err_code = 11
raise LaymanError(err_code, private_data=pg_error)
def check_spatial_ref_crs(spatial_ref):
crs_id = spatial_ref_crs_to_crs_id(spatial_ref)
if crs_id not in settings.INPUT_SRS_LIST:
raise LaymanError(4, {
'found': crs_id,
'supported_values': settings.INPUT_SRS_LIST
})
def check_raster_layer_crs(main_filepath):
crs = get_raster_crs(main_filepath)
if not crs:
raise LaymanError(4, {
'found': None,
'supported_values': settings.INPUT_SRS_LIST
})
check_spatial_ref_crs(crs)
def check_schema_name(db_schema):
usernames = global_get_workspaces(use_cache=False,
skip_modules=(
'layman.map.prime_db_schema',
'layman.layer.prime_db_schema',
))
if db_schema in usernames:
raise LaymanError(42, {'workspace': db_schema})
def check_new_layername(workspace, layername, conn_cur=None):
if conn_cur is None:
conn_cur = db_util.get_connection_cursor()
_, cur = conn_cur
# DB table name conflicts
try:
cur.execute(f"""SELECT n.nspname AS schemaname, c.relname, c.relkind
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname IN ('{workspace}', '{settings.PG_POSTGIS_SCHEMA}') AND c.relname='{layername}'""")
except BaseException as exc:
logger.error(f'check_new_layername ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
if len(rows) > 0:
raise LaymanError(9, {'db_object_name': layername})
def create_string_attributes(attribute_tuples, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
query = "\n".join([f"""ALTER TABLE {username}.{layername} ADD COLUMN {attrname} VARCHAR(1024);""" for username, layername, attrname in attribute_tuples]) + "\n COMMIT;"
try:
cur.execute(query)
except BaseException as exc:
logger.error(f'create_string_attributes ERROR')
raise LaymanError(7) from exc
def import_layer_vector_file(username, layername, main_filepath, crs_id):
process = import_layer_vector_file_async(username, layername, main_filepath,
crs_id)
while process.poll() is None:
pass
return_code = process.poll()
if return_code != 0:
pg_error = str(process.stdout.read())
raise LaymanError(11, private_data=pg_error)
def check_workspace_name(workspace, pattern_only=False):
if not re.match(WORKSPACE_NAME_PATTERN, workspace):
raise LaymanError(2, {
'parameter': 'workspace',
'expected': WORKSPACE_NAME_PATTERN
})
if pattern_only:
return
check_reserved_workspace_names(workspace)
providers = get_internal_providers()
call_modules_fn(providers, 'check_workspace_name', [workspace])
def delete_workspace(workspace, conn_cur=None):
if conn_cur is None:
conn_cur = db_util.get_connection_cursor()
conn, cur = conn_cur
try:
cur.execute(f"""DROP SCHEMA IF EXISTS "{workspace}" RESTRICT""")
conn.commit()
except BaseException as exc:
logger.error(f'delete_workspace ERROR')
raise LaymanError(7) from exc
def check_username(username, pattern_only=False):
if not re.match(USERNAME_PATTERN, username):
raise LaymanError(2, {
'parameter': 'user',
'expected': USERNAME_PATTERN
})
if pattern_only:
return
check_reserved_workspace_names(username)
providers = get_internal_providers()
call_modules_fn(providers, 'check_username', [username])
def ensure_workspace(workspace, conn_cur=None):
if conn_cur is None:
conn_cur = db_util.get_connection_cursor()
conn, cur = conn_cur
try:
cur.execute(
f"""CREATE SCHEMA IF NOT EXISTS "{workspace}" AUTHORIZATION {settings.LAYMAN_PG_USER}""")
conn.commit()
except BaseException as exc:
logger.error(f'ensure_workspace ERROR')
raise LaymanError(7) from exc
def delete_layer(workspace, layername, conn_cur=None):
if conn_cur is None:
conn_cur = db_util.get_connection_cursor()
conn, cur = conn_cur
query = f"""
DROP TABLE IF EXISTS "{workspace}"."{layername}" CASCADE
"""
try:
cur.execute(query)
conn.commit()
except BaseException as exc:
raise LaymanError(7)from exc
def check_filenames(username,
layername,
filenames,
check_crs,
ignore_existing_files=False):
main_filename = get_main_file_name(filenames)
if main_filename is None:
raise LaymanError(
2, {
'parameter':
'file',
'expected':
'At least one file with any of extensions: ' +
', '.join(settings.MAIN_FILE_EXTENSIONS)
})
basename, ext = map(lambda s: s.lower(), os.path.splitext(main_filename))
if ext == '.shp':
lower_filenames = list(map(lambda fn: fn.lower(), filenames))
shp_exts = ['.dbf', '.shx']
if check_crs:
shp_exts.append('.prj')
missing_exts = list(
filter(lambda e: basename + e not in lower_filenames, shp_exts))
if len(missing_exts) > 0:
detail = {'missing_extensions': missing_exts}
if '.prj' in missing_exts:
detail['suggestion'] = 'Missing .prj file can be fixed also ' \
'by setting "crs" parameter.'
raise LaymanError(18, detail)
input_file_dir = get_layer_input_file_dir(username, layername)
filename_mapping, _ = get_file_name_mappings(filenames, main_filename,
layername, input_file_dir)
if not ignore_existing_files:
conflict_paths = [
filename_mapping[k] for k, v in filename_mapping.items() if
v is not None and os.path.exists(os.path.join(input_file_dir, v))
]
if len(conflict_paths) > 0:
raise LaymanError(3, conflict_paths)
def get_number_of_features(username, layername, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
try:
cur.execute(f"""
select count(*)
from {username}.{layername}
""")
except BaseException as exc:
logger.error(f'get_number_of_features ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
return rows[0][0]
def authenticate():
user = None
username = request.headers.get(settings.LAYMAN_AUTHN_HTTP_HEADER_NAME,
None)
if username is None:
return user
user = users.get_user_infos(username).get(username)
if not user:
raise LaymanError(44,
f'Username {username} not recognized.',
sub_code=1)
user = {'username': username}
g.user = user
return user
def check_layer_crs(main_filepath):
in_driver = get_ogr_driver(main_filepath)
in_data_source = in_driver.Open(main_filepath, 0)
feature_layer = in_data_source.GetLayerByIndex(0)
crs = feature_layer.GetSpatialRef()
crs_auth_name = crs.GetAuthorityName(None)
crs_code = crs.GetAuthorityCode(None)
crs_id = crs_auth_name + ":" + crs_code
if crs_id not in settings.INPUT_SRS_LIST:
raise LaymanError(4, {
'found': crs_id,
'supported_values': settings.INPUT_SRS_LIST
})
def get_workspaces(conn_cur=None):
if conn_cur is None:
conn_cur = db_util.get_connection_cursor()
_, cur = conn_cur
try:
cur.execute(f"""select schema_name
from information_schema.schemata
where schema_name NOT IN ('{"', '".join(settings.PG_NON_USER_SCHEMAS)}\
') AND schema_owner = '{settings.LAYMAN_PG_USER}'""")
except BaseException as exc:
logger.error(f'get_workspaces ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
return [r[0] for r in rows]
def authenticate():
actor = None
actor_name = request.headers.get(settings.LAYMAN_AUTHN_HTTP_HEADER_NAME,
None)
if actor_name is None:
return actor
actor = users.get_user_infos(actor_name).get(actor_name)
if not actor:
raise LaymanError(44,
f'Username {actor_name} not recognized.',
sub_code=1)
actor = {'username': actor_name}
# pylint: disable=assigning-non-slot
g.user = actor
return actor
def get_all_column_infos(username, layername, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
try:
cur.execute(f"""
SELECT QUOTE_IDENT(column_name) AS column_name, data_type
FROM information_schema.columns
WHERE table_schema = '{username}'
AND table_name = '{layername}'
""")
except BaseException as exc:
logger.error(f'get_all_column_names ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
return [ColumnInfo(name=r[0], data_type=r[1]) for r in rows]
def get_geometry_types(username, layername, conn_cur=None):
conn, cur = conn_cur or db_util.get_connection_cursor()
try:
sql = f"""
select distinct ST_GeometryType(wkb_geometry) as geometry_type_name
from {username}.{layername}
"""
cur.execute(sql)
except BaseException as exc:
logger.error(f'get_geometry_types ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
conn.commit()
result = [row[0] for row in rows]
return result
def get_text_column_names(username, layername, conn_cur=None):
_, cur = conn_cur or db_util.get_connection_cursor()
try:
cur.execute(f"""
SELECT QUOTE_IDENT(column_name) AS column_name
FROM information_schema.columns
WHERE table_schema = '{username}'
AND table_name = '{layername}'
AND data_type IN ('character varying', 'varchar', 'character', 'char', 'text')
""")
except BaseException as exc:
logger.error(f'get_text_column_names ERROR')
raise LaymanError(7) from exc
rows = cur.fetchall()
return [r[0] for r in rows]
def raise_layman_error(response, status_codes_to_skip=None):
status_codes_to_skip = status_codes_to_skip or set()
status_codes_to_skip.add(200)
if 400 <= response.status_code < 500 and response.status_code not in status_codes_to_skip:
details = json.loads(response.text)
raise LaymanError(details['code'],
details.get('detail'),
http_code=response.status_code,
sub_code=details.get('sub_code'))
if response.status_code not in status_codes_to_skip:
logger.error(
f'raise_layman_error: response.status_code={response.status_code}, response.text={response.text}'
)
response.raise_for_status()
assert response.status_code in status_codes_to_skip, f"response.status_code={response.status_code}\nresponse.text={response.text}"
assert 'Deprecation' not in response.headers, f'This is deprecated URL! Use new one. headers={response.headers}'