def check_workspace_names(): logger.info( f' Starting - checking workspace names - for {settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX} suffix, ' f'for `{settings.REST_WORKSPACES_PREFIX}` name') workspaces = prime_db_schema.get_workspaces() for workspace in workspaces: if workspace.endswith(settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX): raise LaymanError( f"A workspace has name with reserved suffix '{settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX}'. " f"In that case, please downgrade to the previous minor release version of Layman and contact Layman " f"contributors. One way how to do that is to create an issue in Layman repository: " f"https://github.com/LayerManager/layman/issues/", data={ 'workspace': workspace, }) if settings.REST_WORKSPACES_PREFIX in workspaces: raise LaymanError( f"A workspace has reserved name '{settings.REST_WORKSPACES_PREFIX}'. " f"In that case, please downgrade to the previous minor release version of Layman and contact Layman " f"contributors. One way how to do that is to create an issue in Layman repository: " f"https://github.com/LayerManager/layman/issues/", data={'workspace': settings.REST_WORKSPACES_PREFIX}) logger.info( f' DONE - checking workspace names - for {settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX} suffix, ' f'for `{settings.REST_WORKSPACES_PREFIX}` name')
def refresh_input_chunk(self, workspace, layername, check_crs=True): if self.is_aborted(): raise AbortedException last_change = time.time() num_files_saved = 0 num_chunks_saved = 0 chunk_info = input_chunk.layer_file_chunk_info(workspace, layername) logger.debug(f'chunk_info {str(chunk_info)}') while not chunk_info[0]: if time.time() - last_change > settings.UPLOAD_MAX_INACTIVITY_TIME: logger.info( f'UPLOAD_MAX_INACTIVITY_TIME reached {workspace}.{layername}') input_file.delete_layer(workspace, layername) raise LaymanError(22) time.sleep(0.5) if self.is_aborted(): logger.info(f'Aborting for layer {workspace}.{layername}') input_file.delete_layer(workspace, layername) logger.info(f'Aborted for layer {workspace}.{layername}') raise AbortedException chunk_info = input_chunk.layer_file_chunk_info(workspace, layername) logger.debug(f'chunk_info {str(chunk_info)}') if num_files_saved != chunk_info[1] \ or num_chunks_saved != chunk_info[2]: last_change = time.time() num_files_saved = chunk_info[1] num_chunks_saved = chunk_info[2] logger.info(f'Layer chunks uploaded {workspace}.{layername}') input_files = input_file.get_layer_input_files(workspace, layername) input_file.check_filenames(workspace, layername, input_files, check_crs, ignore_existing_files=True) main_filepath = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={ 'keys': ['file'] })['_file']['gdal_path'] input_file.check_main_file(main_filepath, check_crs=check_crs) file_type = input_file.get_file_type( input_files.raw_or_archived_main_file_path) style_type_for_check = layman_util.get_publication_info( workspace, LAYER_TYPE, layername, context={'keys': ['style_type']})['style_type'] if file_type == settings.FILE_TYPE_RASTER and style_type_for_check == 'qml': raise LaymanError(48, f'Raster layers are not allowed to have QML style.')
def check_username(username): if username == settings.LAYMAN_GS_USER: raise LaymanError(41, {'username': username}) if username in gs_util.RESERVED_WORKSPACE_NAMES: raise LaymanError(35, {'reserved_by': __name__, 'workspace': username}) if username.endswith(settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX): raise LaymanError(45, {'workspace_name': username}) rolename = gs_util.username_to_rolename(username) if rolename in gs_util.RESERVED_ROLE_NAMES: raise LaymanError(35, {'reserved_by': __name__, 'role': rolename})
def get_text_data(username, layername, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() col_names = get_text_column_names(username, layername, conn_cur=conn_cur) if len(col_names) == 0: return [], 0 num_features = get_number_of_features(username, layername, conn_cur=conn_cur) if num_features == 0: return [], 0 limit = max(100, num_features // 10) try: cur.execute(f""" select {', '.join(col_names)} from {username}.{layername} order by ogc_fid limit {limit} """) except BaseException as exc: logger.error(f'get_text_data ERROR') raise LaymanError(7) from exc rows = cur.fetchall() col_texts = defaultdict(list) for row in rows: for idx, col_name in enumerate(col_names): val = row[idx] if val is not None and len(val) > 0: col_texts[col_name].append(val) col_texts = [ ' '.join(texts) for _, texts in col_texts.items() ] # print(f"result col_texts={col_texts}") return col_texts, limit
def refresh_input_chunk(self, username, layername, check_crs=True): if self.is_aborted(): raise AbortedException last_change = time.time() num_files_saved = 0 num_chunks_saved = 0 chunk_info = input_chunk.layer_file_chunk_info(username, layername) logger.debug(f'chunk_info {str(chunk_info)}') while not chunk_info[0]: if time.time() - last_change > settings.UPLOAD_MAX_INACTIVITY_TIME: logger.info( f'UPLOAD_MAX_INACTIVITY_TIME reached {username}.{layername}') input_file.delete_layer(username, layername) raise LaymanError(22) time.sleep(0.5) if self.is_aborted(): logger.info(f'Aborting for layer {username}.{layername}') input_file.delete_layer(username, layername) logger.info(f'Aborted for layer {username}.{layername}') raise AbortedException chunk_info = input_chunk.layer_file_chunk_info(username, layername) logger.debug(f'chunk_info {str(chunk_info)}') if num_files_saved != chunk_info[1] \ or num_chunks_saved != chunk_info[2]: last_change = time.time() num_files_saved = chunk_info[1] num_chunks_saved = chunk_info[2] logger.info(f'Layer chunks uploaded {username}.{layername}') if check_crs: main_filepath = input_file.get_layer_main_file_path( username, layername) input_file.check_layer_crs(main_filepath)
def check_main_file(main_filepath): # check feature layers in source file in_driver = get_ogr_driver(main_filepath) in_data_source = in_driver.Open(main_filepath, 0) n_layers = in_data_source.GetLayerCount() if n_layers != 1: raise LaymanError(5, {'found': n_layers, 'expected': 1})
def get_missing_attributes(attribute_tuples, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() # Find all triples which do not already exist query = f"""select attribs.* from (""" + "\n union all\n".join([f"select '{username}' username, '{layername}' layername, '{attrname}' attrname" for username, layername, attrname in attribute_tuples]) + """) attribs left join information_schema.columns c on c.table_schema = attribs.username and c.table_name = attribs.layername and c.column_name = attribs.attrname where c.column_name is null""" try: if attribute_tuples: cur.execute(query) except BaseException as exc: logger.error(f'get_missing_attributes ERROR') raise LaymanError(7) from exc missing_attributes = set() rows = cur.fetchall() for row in rows: missing_attributes.add((row[0], row[1], row[2])) return missing_attributes
def check_vector_main_file(main_filepath, *, check_crs=True): in_data_source = ogr.Open(main_filepath, 0) n_layers = in_data_source.GetLayerCount() if n_layers != 1: raise LaymanError(5, {'found': n_layers, 'expected': 1}) if check_crs: check_vector_layer_crs(main_filepath)
def refresh_table(self, username, layername, crs_id=None, ensure_user=False): if ensure_user: db.ensure_workspace(username) if self.is_aborted(): raise AbortedException main_filepath = get_layer_main_file_path(username, layername) process = db.import_layer_vector_file_async(username, layername, main_filepath, crs_id) while process.poll() is None and not self.is_aborted(): pass if self.is_aborted(): logger.info(f'terminating {username} {layername}') process.terminate() logger.info(f'terminating {username} {layername}') delete_layer(username, layername) raise AbortedException return_code = process.poll() if return_code != 0: pg_error = str(process.stdout.read()) logger.error(f"STDOUT: {pg_error}") if "ERROR: zero-length delimited identifier at or near" in pg_error: err_code = 28 else: err_code = 11 raise LaymanError(err_code, private_data=pg_error)
def check_spatial_ref_crs(spatial_ref): crs_id = spatial_ref_crs_to_crs_id(spatial_ref) if crs_id not in settings.INPUT_SRS_LIST: raise LaymanError(4, { 'found': crs_id, 'supported_values': settings.INPUT_SRS_LIST })
def check_raster_layer_crs(main_filepath): crs = get_raster_crs(main_filepath) if not crs: raise LaymanError(4, { 'found': None, 'supported_values': settings.INPUT_SRS_LIST }) check_spatial_ref_crs(crs)
def check_schema_name(db_schema): usernames = global_get_workspaces(use_cache=False, skip_modules=( 'layman.map.prime_db_schema', 'layman.layer.prime_db_schema', )) if db_schema in usernames: raise LaymanError(42, {'workspace': db_schema})
def check_new_layername(workspace, layername, conn_cur=None): if conn_cur is None: conn_cur = db_util.get_connection_cursor() _, cur = conn_cur # DB table name conflicts try: cur.execute(f"""SELECT n.nspname AS schemaname, c.relname, c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname IN ('{workspace}', '{settings.PG_POSTGIS_SCHEMA}') AND c.relname='{layername}'""") except BaseException as exc: logger.error(f'check_new_layername ERROR') raise LaymanError(7) from exc rows = cur.fetchall() if len(rows) > 0: raise LaymanError(9, {'db_object_name': layername})
def create_string_attributes(attribute_tuples, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() query = "\n".join([f"""ALTER TABLE {username}.{layername} ADD COLUMN {attrname} VARCHAR(1024);""" for username, layername, attrname in attribute_tuples]) + "\n COMMIT;" try: cur.execute(query) except BaseException as exc: logger.error(f'create_string_attributes ERROR') raise LaymanError(7) from exc
def import_layer_vector_file(username, layername, main_filepath, crs_id): process = import_layer_vector_file_async(username, layername, main_filepath, crs_id) while process.poll() is None: pass return_code = process.poll() if return_code != 0: pg_error = str(process.stdout.read()) raise LaymanError(11, private_data=pg_error)
def check_workspace_name(workspace, pattern_only=False): if not re.match(WORKSPACE_NAME_PATTERN, workspace): raise LaymanError(2, { 'parameter': 'workspace', 'expected': WORKSPACE_NAME_PATTERN }) if pattern_only: return check_reserved_workspace_names(workspace) providers = get_internal_providers() call_modules_fn(providers, 'check_workspace_name', [workspace])
def delete_workspace(workspace, conn_cur=None): if conn_cur is None: conn_cur = db_util.get_connection_cursor() conn, cur = conn_cur try: cur.execute(f"""DROP SCHEMA IF EXISTS "{workspace}" RESTRICT""") conn.commit() except BaseException as exc: logger.error(f'delete_workspace ERROR') raise LaymanError(7) from exc
def check_username(username, pattern_only=False): if not re.match(USERNAME_PATTERN, username): raise LaymanError(2, { 'parameter': 'user', 'expected': USERNAME_PATTERN }) if pattern_only: return check_reserved_workspace_names(username) providers = get_internal_providers() call_modules_fn(providers, 'check_username', [username])
def ensure_workspace(workspace, conn_cur=None): if conn_cur is None: conn_cur = db_util.get_connection_cursor() conn, cur = conn_cur try: cur.execute( f"""CREATE SCHEMA IF NOT EXISTS "{workspace}" AUTHORIZATION {settings.LAYMAN_PG_USER}""") conn.commit() except BaseException as exc: logger.error(f'ensure_workspace ERROR') raise LaymanError(7) from exc
def delete_layer(workspace, layername, conn_cur=None): if conn_cur is None: conn_cur = db_util.get_connection_cursor() conn, cur = conn_cur query = f""" DROP TABLE IF EXISTS "{workspace}"."{layername}" CASCADE """ try: cur.execute(query) conn.commit() except BaseException as exc: raise LaymanError(7)from exc
def check_filenames(username, layername, filenames, check_crs, ignore_existing_files=False): main_filename = get_main_file_name(filenames) if main_filename is None: raise LaymanError( 2, { 'parameter': 'file', 'expected': 'At least one file with any of extensions: ' + ', '.join(settings.MAIN_FILE_EXTENSIONS) }) basename, ext = map(lambda s: s.lower(), os.path.splitext(main_filename)) if ext == '.shp': lower_filenames = list(map(lambda fn: fn.lower(), filenames)) shp_exts = ['.dbf', '.shx'] if check_crs: shp_exts.append('.prj') missing_exts = list( filter(lambda e: basename + e not in lower_filenames, shp_exts)) if len(missing_exts) > 0: detail = {'missing_extensions': missing_exts} if '.prj' in missing_exts: detail['suggestion'] = 'Missing .prj file can be fixed also ' \ 'by setting "crs" parameter.' raise LaymanError(18, detail) input_file_dir = get_layer_input_file_dir(username, layername) filename_mapping, _ = get_file_name_mappings(filenames, main_filename, layername, input_file_dir) if not ignore_existing_files: conflict_paths = [ filename_mapping[k] for k, v in filename_mapping.items() if v is not None and os.path.exists(os.path.join(input_file_dir, v)) ] if len(conflict_paths) > 0: raise LaymanError(3, conflict_paths)
def get_number_of_features(username, layername, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() try: cur.execute(f""" select count(*) from {username}.{layername} """) except BaseException as exc: logger.error(f'get_number_of_features ERROR') raise LaymanError(7) from exc rows = cur.fetchall() return rows[0][0]
def authenticate(): user = None username = request.headers.get(settings.LAYMAN_AUTHN_HTTP_HEADER_NAME, None) if username is None: return user user = users.get_user_infos(username).get(username) if not user: raise LaymanError(44, f'Username {username} not recognized.', sub_code=1) user = {'username': username} g.user = user return user
def check_layer_crs(main_filepath): in_driver = get_ogr_driver(main_filepath) in_data_source = in_driver.Open(main_filepath, 0) feature_layer = in_data_source.GetLayerByIndex(0) crs = feature_layer.GetSpatialRef() crs_auth_name = crs.GetAuthorityName(None) crs_code = crs.GetAuthorityCode(None) crs_id = crs_auth_name + ":" + crs_code if crs_id not in settings.INPUT_SRS_LIST: raise LaymanError(4, { 'found': crs_id, 'supported_values': settings.INPUT_SRS_LIST })
def get_workspaces(conn_cur=None): if conn_cur is None: conn_cur = db_util.get_connection_cursor() _, cur = conn_cur try: cur.execute(f"""select schema_name from information_schema.schemata where schema_name NOT IN ('{"', '".join(settings.PG_NON_USER_SCHEMAS)}\ ') AND schema_owner = '{settings.LAYMAN_PG_USER}'""") except BaseException as exc: logger.error(f'get_workspaces ERROR') raise LaymanError(7) from exc rows = cur.fetchall() return [r[0] for r in rows]
def authenticate(): actor = None actor_name = request.headers.get(settings.LAYMAN_AUTHN_HTTP_HEADER_NAME, None) if actor_name is None: return actor actor = users.get_user_infos(actor_name).get(actor_name) if not actor: raise LaymanError(44, f'Username {actor_name} not recognized.', sub_code=1) actor = {'username': actor_name} # pylint: disable=assigning-non-slot g.user = actor return actor
def get_all_column_infos(username, layername, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() try: cur.execute(f""" SELECT QUOTE_IDENT(column_name) AS column_name, data_type FROM information_schema.columns WHERE table_schema = '{username}' AND table_name = '{layername}' """) except BaseException as exc: logger.error(f'get_all_column_names ERROR') raise LaymanError(7) from exc rows = cur.fetchall() return [ColumnInfo(name=r[0], data_type=r[1]) for r in rows]
def get_geometry_types(username, layername, conn_cur=None): conn, cur = conn_cur or db_util.get_connection_cursor() try: sql = f""" select distinct ST_GeometryType(wkb_geometry) as geometry_type_name from {username}.{layername} """ cur.execute(sql) except BaseException as exc: logger.error(f'get_geometry_types ERROR') raise LaymanError(7) from exc rows = cur.fetchall() conn.commit() result = [row[0] for row in rows] return result
def get_text_column_names(username, layername, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() try: cur.execute(f""" SELECT QUOTE_IDENT(column_name) AS column_name FROM information_schema.columns WHERE table_schema = '{username}' AND table_name = '{layername}' AND data_type IN ('character varying', 'varchar', 'character', 'char', 'text') """) except BaseException as exc: logger.error(f'get_text_column_names ERROR') raise LaymanError(7) from exc rows = cur.fetchall() return [r[0] for r in rows]
def raise_layman_error(response, status_codes_to_skip=None): status_codes_to_skip = status_codes_to_skip or set() status_codes_to_skip.add(200) if 400 <= response.status_code < 500 and response.status_code not in status_codes_to_skip: details = json.loads(response.text) raise LaymanError(details['code'], details.get('detail'), http_code=response.status_code, sub_code=details.get('sub_code')) if response.status_code not in status_codes_to_skip: logger.error( f'raise_layman_error: response.status_code={response.status_code}, response.text={response.text}' ) response.raise_for_status() assert response.status_code in status_codes_to_skip, f"response.status_code={response.status_code}\nresponse.text={response.text}" assert 'Deprecation' not in response.headers, f'This is deprecated URL! Use new one. headers={response.headers}'