def __init__(self, config_models, logger):
"""Constructor
:param Logger logger: Logger
"""
super().__init__(
'search',
'https://github.com/qwc-services/qwc-fulltext-search-service/raw/master/schemas/qwc-search-service.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
def __init__(self, config_models, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param Logger logger: Logger
"""
super().__init__(
'mapViewer',
'https://github.com/qwc-services/qwc-map-viewer/raw/master/schemas/qwc-map-viewer.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
def __init__(self, config_models, logger):
"""Constructor
:param Logger logger: Logger
"""
super().__init__(
'data',
'https://github.com/qwc-services/qwc-data-service/raw/master/schemas/qwc-data-service.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __init__(self, config_models, db_engine, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param DatabaseEngine db_engine: Database engine with DB connections
:param Logger logger: Logger
"""
super().__init__(
'ogc',
'https://github.com/qwc-services/qwc-ogc-service/raw/master/schemas/qwc-ogc-service.json',
logger)
self.config_models = config_models
self.db_engine = db_engine
self.permissions_query = PermissionsQuery(config_models, logger)
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __init__(self, config_models, generator_config, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param Logger logger: Logger
"""
super().__init__(
'dataproduct',
'https://github.com/qwc-services/sogis-dataproduct-service/raw/master/schemas/sogis-dataproduct-service.json',
logger)
self.config_models = config_models
self.generator_config = generator_config
self.permissions_query = PermissionsQuery(config_models, logger)
self.db_engine = DatabaseEngine()
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __init__(self, config_models, generator_config, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param obj generator_config: ConfigGenerator config
:param Logger logger: Logger
"""
super().__init__(
'legend',
'https://github.com/qwc-services/qwc-legend-service/raw/master/schemas/qwc-legend-service.json',
logger
)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
# get path for writing custom legend images from ConfigGenerator config
self.legend_images_output_path = generator_config.get(
'legend_images_path', '/tmp'
)
class OGCServiceConfig(ServiceConfig):
"""OGCServiceConfig class
Generate OGC service config and permissions.
"""
def __init__(self, config_models, db_engine, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param DatabaseEngine db_engine: Database engine with DB connections
:param Logger logger: Logger
"""
super().__init__(
'ogc',
'https://github.com/qwc-services/qwc-ogc-service/raw/master/schemas/qwc-ogc-service.json',
logger)
self.config_models = config_models
self.db_engine = db_engine
self.permissions_query = PermissionsQuery(config_models, logger)
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __del__(self):
"""Destructor"""
if self.session:
# close shared session
self.session.close()
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
# get base config
config = super().config(service_config)
# additional service config
cfg_config = service_config.get('config', {})
if 'default_qgis_server_url' not in cfg_config:
# use default QGIS server URL from ConfigGenerator config
# if not set in service config
qgis_server_url = service_config.get('defaults', {}).get(
'qgis_server_url',
'http://localhost:8001/ows/').rstrip('/') + '/'
cfg_config['default_qgis_server_url'] = qgis_server_url
config['config'] = cfg_config
resources = OrderedDict()
config['resources'] = resources
# collect resources from ConfigDB
session = self.config_models.session()
# NOTE: keep precached resources in memory while querying resources
cached_resources = self.precache_resources(session)
resources['wms_services'] = self.wms_services(service_config, session)
resources['wfs_services'] = self.wfs_services(service_config, session)
session.close()
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
if self.session is None:
# create shared session
self.session = self.config_models.session()
# NOTE: keep precached resources in memory while querying
# permissions
self.cached_resources = self.precache_resources(self.session)
permissions['wms_services'] = self.wms_permissions(role, self.session)
permissions['wfs_services'] = self.wfs_permissions(role, self.session)
return permissions
# service config
def wms_services(self, service_config, session):
"""Collect WMS Service resources from ConfigDB.
:param obj service_config: Additional service config
:param Session session: DB session
"""
wms_services = []
# additional service config
cfg_config = service_config.get('config', {})
cfg_resources = service_config.get('resources', {})
cfg_wms_services = cfg_resources.get('wms_services', [])
default_qgis_server_url = cfg_config.get(
'default_qgis_server_url',
'http://localhost:8001/ows/').rstrip('/') + '/'
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter(WmsWfs.ows_type == 'WMS')
for wms in query.all():
# find additional config for WMS service
cfg_wms = {}
for cfg in cfg_wms_services:
if cfg.get('name') == wms.name:
cfg_wms = cfg
break
# NOTE: use ordered keys
wms_service = OrderedDict()
wms_service['name'] = wms.name
# set any online resources
wms_service['online_resources'] = cfg_wms.get(
'online_resources', {})
# collect WMS layers
wms_service['root_layer'] = self.collect_wms_layers(
wms.root_layer, False)
# use separate QGIS project for printing
wms_service['print_url'] = cfg_wms.get(
'print_url',
urljoin(default_qgis_server_url, "%s_print" % wms.name))
wms_service['print_templates'] = self.print_templates(session)
wms_service['internal_print_layers'] = self.print_layers(session)
wms_services.append(wms_service)
return wms_services
def collect_wms_layers(self, layer, facade):
"""Recursively collect WMS layer info for layer subtree from ConfigDB
and return nested WMS layers.
:param obj layer: Group or Data layer object
:param bool facade: Whether this is a facade sub layer
"""
# NOTE: use ordered keys
wms_layer = OrderedDict()
wms_layer['name'] = layer.name
if layer.type == 'group':
wms_layer['type'] = 'layergroup'
else:
wms_layer['type'] = 'layer'
if layer.title:
wms_layer['title'] = layer.title
if layer.type == 'group':
# group layer
# set facade if layer is a facade group or is a facade sub layer
in_facade = layer.facade or facade
sublayers = []
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sub layer
sublayers.append(self.collect_wms_layers(sublayer, in_facade))
wms_layer['layers'] = sublayers
if layer.facade:
wms_layer['hide_sublayers'] = True
else:
# data layer
queryable = False
data_source = layer.data_set_view.data_set.data_source
if data_source.connection_type == 'database':
# vector data layer
# collect attribute names
attributes = [
attr.name for attr in layer.data_set_view.attributes
]
wms_layer['attributes'] = attributes
# add geometry column
attributes.append('geometry')
# layer is queryable if there are any attributes
queryable = len(attributes) > 0
else:
# raster data layers are always queryable
queryable = True
wms_layer['queryable'] = queryable
if facade and layer.layer_transparency != 0:
# add opacity to facade sublayers
wms_layer['opacity'] = 100 - layer.layer_transparency
return wms_layer
def print_templates(self, session):
"""Return print templates from ConfigDB.
:param Session session: DB session
"""
TemplateQGIS = self.config_models.model('template_qgis')
query = session.query(TemplateQGIS).order_by(TemplateQGIS.name)
return [template.name for template in query.all()]
def print_layers(self, session):
"""Return internal print layers for background layers from ConfigDB.
:param Session session: DB session
"""
BackgroundLayer = self.config_models.model('background_layer')
query = session.query(BackgroundLayer).order_by(BackgroundLayer.name)
return [layer.name for layer in query.all()]
def wfs_services(self, service_config, session):
"""Collect WFS Service resources from ConfigDB.
:param obj service_config: Additional service config
:param Session session: DB session
"""
wfs_services = []
# additional service config
cfg_config = service_config.get('config', {})
cfg_resources = service_config.get('resources', {})
cfg_wfs_services = cfg_resources.get('wfs_services', [])
default_qgis_server_url = cfg_config.get(
'default_qgis_server_url',
'http://localhost:8001/ows/').rstrip('/') + '/'
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter(WmsWfs.ows_type == 'WFS')
for wfs in query.all():
# find additional config for WFS service
cfg_wfs = {}
for cfg in cfg_wfs_services:
if cfg.get('name') == wfs.name:
cfg_wfs = cfg
break
# NOTE: use ordered keys
wfs_service = OrderedDict()
wfs_service['name'] = wfs.name
# use separate QGIS project
wfs_service['wfs_url'] = cfg_wfs.get(
'wfs_url', urljoin(default_qgis_server_url,
"%s_wfs" % wfs.name))
# set any online resource
wfs_service['online_resource'] = cfg_wfs.get('online_resource')
# collect WFS layers
wfs_service['layers'] = self.collect_wfs_layers(wfs.root_layer)
wfs_services.append(wfs_service)
return wfs_services
def collect_wfs_layers(self, layer):
"""Recursively collect WFS layer info for layer subtree from ConfigDB
and return flat WFS layer list.
:param obj layer: Group or Data layer object
"""
wfs_layers = []
if layer.type == 'group':
# group layer
sublayers = []
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sub layer
wfs_layers += self.collect_wfs_layers(sublayer)
else:
# data layer
data_source = layer.data_set_view.data_set.data_source
if data_source.connection_type == 'database':
# vector data layer
# NOTE: use ordered keys
wfs_layer = OrderedDict()
wfs_layer['name'] = layer.name
attributes = []
# add primary key
# NOTE: QGIS Server 3.10 returns incomplete FIDs if
# primary key property is excluded
pkey = self.postgis_primary_key(layer.data_set_view)
if pkey:
attributes.append(pkey)
else:
self.logger.warning(
"Could not find primary key for layer '%s'" %
layer.name)
# collect attribute names
attributes += [
attr.name for attr in layer.data_set_view.attributes
]
# add geometry column
attributes.append('geometry')
wfs_layer['attributes'] = attributes
wfs_layers.append(wfs_layer)
return wfs_layers
def postgis_primary_key(self, data_set_view):
"""Return primary key for a PostGIS DataSetView.
:param obj data_set_view: DataSetView object
"""
data_set = data_set_view.data_set
if data_set.primary_key:
# primary key if view
return data_set.primary_key
# database connection URL
conn_str = data_set.data_source.connection
# parse schema and table name
data_set_name = data_set.data_set_name
parts = data_set_name.split('.')
if len(parts) > 1:
schema = parts[0]
table_name = parts[1]
else:
schema = 'public'
table_name = data_set_name
primary_key = None
try:
# connect to GeoDB
conn = None
engine = self.db_engine.db_engine(conn_str)
conn = engine.connect()
# build query SQL
sql = sql_text("""
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid
AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = '{schema}.{table}'::regclass
AND i.indisprimary;
""".format(schema=schema, table=table_name))
# execute query
result = conn.execute(sql)
for row in result:
primary_key = row['attname']
except Exception as e:
self.logger.error("Could not get primary key of '%s':\n%s" %
(data_set_name, e))
finally:
if conn:
# close database connection
conn.close()
return primary_key
# permissions
def wms_permissions(self, role, session):
"""Collect WMS Service permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = []
# get IDs for permitted resources required for WMS
table_names = [
'wms_wfs', 'ows_layer', 'data_set_view', 'data_set', 'data_source',
'data_set_view_attributes', 'background_layer', 'template'
]
# get IDs for role permissions
permitted_ids = self.permissions_query.resource_ids(
table_names, role, session)
# get IDs for public permissions
public_ids = self.permissions_query.resource_ids(
table_names, self.permissions_query.public_role(), session)
if role != self.permissions_query.public_role():
# use only additional role permissions
# as role permissions without public permissions
permitted_ids = permitted_ids - public_ids
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter(WmsWfs.ows_type == 'WMS')
for wms in query.all():
if wms.gdi_oid not in permitted_ids | public_ids:
# WMS not permitted
continue
# NOTE: use ordered keys
wms_permissions = OrderedDict()
wms_permissions['name'] = wms.name
# collect WMS layers
layers = self.collect_wms_layer_permissions(
wms.root_layer, permitted_ids, public_ids)
# add internal print layers
layers += self.permitted_print_layers(permitted_ids, session)
wms_permissions['layers'] = layers
# print templates
print_templates = self.permitted_print_templates(
permitted_ids, session)
if print_templates:
wms_permissions['print_templates'] = print_templates
if layers or print_templates:
permissions.append(wms_permissions)
return permissions
def collect_wms_layer_permissions(self, layer, permitted_ids, public_ids):
"""Recursively collect WMS layer permissions for a role for layer
subtree from ConfigDB and return flat list of permitted WMS layers.
:param obj layer: Group or Data layer object
:param set<int> permitted_ids: Set of permitted resource IDs for role
:param set<int> public_ids: Set of permitted resource IDs for
public role
"""
wms_layers = []
# NOTE: use ordered keys
wms_layer = OrderedDict()
wms_layer['name'] = layer.name
if layer.type == 'group':
# group layer
sublayers = []
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sub layer
sublayers += self.collect_wms_layer_permissions(
sublayer, permitted_ids, public_ids)
if sublayers:
# add group layer if any sub layers are permitted
wms_layers.append(wms_layer)
# add sub layers
wms_layers += sublayers
else:
# data layer
# check permissions for data layer and required resources
data_set_view = layer.data_set_view
data_set = data_set_view.data_set
data_source = data_set.data_source
if (layer.gdi_oid in permitted_ids
and data_set_view.gdi_oid in permitted_ids
and data_set.gdi_oid in permitted_ids and
# NOTE: data_source permissions may be only public
data_source.gdi_oid in permitted_ids | public_ids):
if data_source.connection_type == 'database':
# vector data layer
# collect attribute names
attributes = [
attr.name for attr in layer.data_set_view.attributes
if attr.gdi_oid in permitted_ids
]
wms_layer['attributes'] = attributes
# add geometry column
attributes.append('geometry')
# info template (NOTE: used in FeatureInfo service)
if layer.templateinfo:
wms_layer['info_template'] = \
layer.templateinfo.gdi_oid in (
permitted_ids | public_ids
)
# add layer
wms_layers.append(wms_layer)
elif (layer.templateinfo and layer.gdi_oid in public_ids
and data_set_view.gdi_oid in public_ids
and data_set.gdi_oid in public_ids
and data_source.gdi_oid in public_ids):
# public layer with potential restricted info template
# or restricted feature report
if (layer.templateinfo
and layer.templateinfo.gdi_oid in permitted_ids):
# public layer with restricted info template
wms_layer['info_template'] = True
# add layer
wms_layers.append(wms_layer)
return wms_layers
def permitted_print_layers(self, permitted_ids, session):
"""Return permitted internal print layers for background layers from
ConfigDB.
:param set<int> permitted_ids: Set of permitted resource IDs for role
:param Session session: DB session
"""
BackgroundLayer = self.config_models.model('background_layer')
query = session.query(BackgroundLayer).order_by(BackgroundLayer.name)
internal_print_layers = []
for layer in query.all():
if layer.gdi_oid in permitted_ids:
# NOTE: use ordered keys
wms_layer = OrderedDict()
wms_layer['name'] = layer.name
internal_print_layers.append(wms_layer)
return internal_print_layers
def permitted_print_templates(self, permitted_ids, session):
"""Return permitted print templates from ConfigDB.
:param Session session: DB session
"""
TemplateQGIS = self.config_models.model('template_qgis')
query = session.query(TemplateQGIS).order_by(TemplateQGIS.name)
return [
template.name for template in query.all()
if template.gdi_oid in permitted_ids
]
def wfs_permissions(self, role, session):
"""Collect WFS Service permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = []
# get IDs for permitted resources required for WMS
table_names = [
'wms_wfs', 'ows_layer', 'data_set_view', 'data_set', 'data_source',
'data_set_view_attributes'
]
# get IDs for role permissions
permitted_ids = self.permissions_query.resource_ids(
table_names, role, session)
# get IDs for public permissions
public_ids = self.permissions_query.resource_ids(
table_names, self.permissions_query.public_role(), session)
if role != self.permissions_query.public_role():
# use only additional role permissions
# as role permissions without public permissions
permitted_ids = permitted_ids - public_ids
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter(WmsWfs.ows_type == 'WFS')
for wfs in query.all():
if wfs.gdi_oid not in permitted_ids | public_ids:
# WFS not permitted
continue
# NOTE: use ordered keys
wfs_permissions = OrderedDict()
wfs_permissions['name'] = wfs.name
# collect WMS layers
layers = self.collect_wfs_layer_permissions(
wfs.root_layer, permitted_ids, public_ids)
if layers:
wfs_permissions['layers'] = layers
permissions.append(wfs_permissions)
return permissions
def collect_wfs_layer_permissions(self, layer, permitted_ids, public_ids):
"""Recursively collect WFS layer info for layer subtree from ConfigDB
and return flat WFS layer list.
:param obj layer: Group or Data layer object
:param set<int> permitted_ids: Set of permitted resource IDs for role
:param set<int> public_ids: Set of permitted resource IDs for
public role
"""
wfs_layers = []
if layer.type == 'group':
# group layer
sublayers = []
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sub layer
wfs_layers += self.collect_wfs_layer_permissions(
sublayer, permitted_ids, public_ids)
else:
# data layer
# check permissions for data layer and required resources
data_set_view = layer.data_set_view
data_set = data_set_view.data_set
data_source = data_set.data_source
if (layer.gdi_oid in permitted_ids
and data_set_view.gdi_oid in permitted_ids
and data_set.gdi_oid in permitted_ids and
# NOTE: data_source permissions may be only public
data_source.gdi_oid in permitted_ids | public_ids):
if data_source.connection_type == 'database':
# vector data layer
# NOTE: use ordered keys
wfs_layer = OrderedDict()
wfs_layer['name'] = layer.name
attributes = []
# add primary key
# NOTE: QGIS Server 3.10 returns incomplete FIDs if
# primary key property is excluded
pkey = self.postgis_primary_key(layer.data_set_view)
if pkey:
attributes.append(pkey)
else:
self.logger.warning(
"Could not find primary key for layer '%s'" %
layer.name)
# collect attribute names
attributes = [
attr.name for attr in layer.data_set_view.attributes
if attr.gdi_oid in permitted_ids
]
# add geometry column
attributes.append('geometry')
wfs_layer['attributes'] = attributes
wfs_layers.append(wfs_layer)
return wfs_layers
# helpers
def precache_resources(self, session):
"""Precache some resources using eager loaded relations to reduce the
number of actual separate DB requests in later queries.
NOTE: The lookup tables do not have to be actually used, but remain
in memory, so SQLAlchemy will use the cached records.
E.g. accessing ows_layer_data.data_set_view afterwards won't generate
a separate DB query
:param Session session: DB session
"""
OWSLayerGroup = self.config_models.model('ows_layer_group')
OWSLayerData = self.config_models.model('ows_layer_data')
GroupLayer = self.config_models.model('group_layer')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
# precache OWSLayerData and eager load relations
ows_layer_data_lookup = {}
query = session.query(OWSLayerData)
query = query.options(
joinedload(OWSLayerData.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for layer in query.all():
ows_layer_data_lookup[layer.gdi_oid] = layer
# precache DataSetView and eager load attributes
query = session.query(DataSetView)
query = query.options(joinedload(DataSetView.attributes))
data_set_view_lookup = {}
for data_set_view in query.all():
data_set_view_lookup[data_set_view.gdi_oid] = data_set_view
# precache OWSLayerGroup and eager load sub layers
query = session.query(OWSLayerGroup)
query = query.options(
joinedload(OWSLayerGroup.sub_layers).joinedload(
GroupLayer.sub_layer))
ows_layer_group_lookup = {}
for group in query.all():
ows_layer_group_lookup[group.gdi_oid] = group
# NOTE: return precached resources so they stay in memory
return {
'ows_layer_data_lookup': ows_layer_data_lookup,
'data_set_view_lookup': data_set_view_lookup,
'ows_layer_group_lookup': ows_layer_group_lookup
}
class MapViewerConfig(ServiceConfig):
"""MapViewerServiceConfig class
Generate Map Viewer service config and permissions.
"""
# value for data_set_view.searchable if always searchable
ALWAYS_SEARCHABLE = 2
# lookup for edit field types:
# PostgreSQL data_type -> QWC2 edit field type
EDIT_FIELD_TYPES = {
'bigint': 'number',
'boolean': 'boolean',
'character varying': 'text',
'date': 'date',
'double precision': 'number',
'integer': 'number',
'numeric': 'number',
'real': 'number',
'smallint': 'number',
'text': 'text',
'timestamp with time zone': 'date',
'timestamp without time zone': 'date',
'uuid': 'text'
}
# lookup for edit geometry types:
# PostGIS geometry type -> QWC2 edit geometry type
EDIT_GEOM_TYPES = {
'POINT': 'Point',
'MULTIPOINT': 'Point',
'LINESTRING': 'LineString',
'MULTILINESTRING': 'LineString',
'POLYGON': 'Polygon',
'MULTIPOLYGON': 'Polygon'
}
def __init__(self, config_models, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param Logger logger: Logger
"""
super().__init__(
'mapViewer',
'https://github.com/qwc-services/qwc-map-viewer/raw/master/schemas/qwc-map-viewer.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
# get base config
config = super().config(service_config)
config['service'] = 'map-viewer'
resources = OrderedDict()
config['resources'] = resources
# collect resources from ConfigDB
session = self.config_models.session()
# NOTE: keep precached resources in memory while querying resources
cached_resources = self.precache_resources(session)
# collect resources from QWC2 config and ConfigDB
resources['qwc2_config'] = self.qwc2_config(service_config)
resources['qwc2_themes'] = self.qwc2_themes(service_config, session)
session.close()
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
session = self.config_models.session()
# NOTE: WMS service permissions collected by OGC service config
permissions['wms_services'] = []
permissions['background_layers'] = self.permitted_background_layers(
role, session)
# NOTE: edit permissions collected by Data service config
permissions['data_datasets'] = []
session.close()
return permissions
# service config
def qwc2_config(self, service_config):
"""Collect QWC2 application configuration from config.json.
:param obj service_config: Additional service config
"""
# NOTE: use ordered keys
qwc2_config = OrderedDict()
# additional service config
cfg_resources = service_config.get('resources', {})
cfg_qwc2_config = cfg_resources.get('qwc2_config', {})
# read QWC2 config.json
config = OrderedDict()
try:
config_file = cfg_qwc2_config.get('qwc2_config_file',
'config.json')
with open(config_file) as f:
# parse config JSON with original order of keys
config = json.load(f, object_pairs_hook=OrderedDict)
except Exception as e:
self.logger.error("Could not load QWC2 config.json:\n%s" % e)
config['ERROR'] = str(e)
# remove service URLs
service_urls = [
'permalinkServiceUrl', 'elevationServiceUrl', 'editServiceUrl',
'dataproductServiceUrl', 'searchServiceUrl',
'searchDataServiceUrl', 'authServiceUrl', 'mapInfoService',
'featureReportService', 'landRegisterService', 'cccConfigService',
'plotInfoService'
]
for service_url in service_urls:
config.pop(service_url, None)
# apply custom settings
if 'wmsDpi' in cfg_qwc2_config:
config['wmsDpi'] = cfg_qwc2_config['wmsDpi']
if 'minResultsExanded' in cfg_qwc2_config:
config['minResultsExanded'] = cfg_qwc2_config['minResultsExanded']
qwc2_config['config'] = config
return qwc2_config
def qwc2_themes(self, service_config, session):
"""Collect QWC2 themes configuration from ConfigDB.
:param obj service_config: Additional service config
:param Session session: DB session
"""
# NOTE: use ordered keys
qwc2_themes = OrderedDict()
# additional service config
cfg_resources = service_config.get('resources', {})
cfg_qwc2_themes = cfg_resources.get('qwc2_themes', {})
# collect resources from ConfigDB
themes = OrderedDict()
themes['title'] = 'root'
theme_external_layers = {}
themes['items'] = self.themes_items(theme_external_layers,
service_config, session)
themes['subdirs'] = []
themes['defaultTheme'] = cfg_qwc2_themes.get('default_theme')
themes['backgroundLayers'] = self.background_layers(session)
themes['defaultScales'] = cfg_qwc2_themes.get('default_scales', [
4000000, 2000000, 1000000, 400000, 200000, 80000, 40000, 20000,
10000, 8000, 6000, 4000, 2000, 1000, 500, 250, 100
])
themes['defaultWMSVersion'] = cfg_qwc2_themes.get(
'default_wms_version', '1.3.0')
themes['defaultPrintResolutions'] = cfg_qwc2_themes.get(
'default_print_resolutions', [300])
themes['defaultPrintGrid'] = cfg_qwc2_themes.get(
'default_print_grid', [{
's': 10000,
'x': 1000,
'y': 1000
}, {
's': 1000,
'x': 100,
'y': 100
}, {
's': 100,
'x': 10,
'y': 10
}])
themes["externalLayers"] = list(theme_external_layers.values())
qwc2_themes['themes'] = themes
return qwc2_themes
def themes_items(self, theme_external_layers, service_config, session):
"""Collect theme items from ConfigDB.
:param list theme_external_layers: external layers added to themes
:param obj service_config: Additional service config
:param Session session: DB session
"""
items = []
# additional service config
cfg_config = service_config.get('config', {})
cfg_resources = service_config.get('resources', {})
cfg_qwc2_themes = cfg_resources.get('qwc2_themes', {})
ogc_service_url = cfg_config.get('ogc_service_url', '/ows/')
# print layouts and labels
default_print_layout = cfg_qwc2_themes.get('default_print_layout')
print_layouts, print_label_config = self.print_layouts(
default_print_layout, session)
# search providers
search_providers = ['coordinates', self.solr_search_provider(session)]
# default layer bbox
layer_bbox = OrderedDict()
layer_bbox['crs'] = cfg_qwc2_themes.get('default_crs', 'EPSG:2056')
layer_bbox['bounds'] = cfg_qwc2_themes.get(
'default_layer_bounds', [2590000, 1210000, 2650000, 1270000])
themes_background_layers = self.background_layers(session)
# collect layers referenced by group
self.background_layer_group_refs = []
for l in themes_background_layers:
for it in l.get('items', []):
self.background_layer_group_refs.append(it.get('ref'))
# collect maps
Map = self.config_models.model('map')
query = session.query(Map).distinct(Map.name)
for map_obj in query.all():
wms_wfs = map_obj.wms_wfs
# extend default item
item = self.default_item(cfg_qwc2_themes)
item['id'] = map_obj.name
item['name'] = map_obj.name
item['title'] = map_obj.title
item['wms_name'] = wms_wfs.name
item['url'] = urljoin(ogc_service_url, wms_wfs.name)
# parse map extent
initial_bounds = [
float(c) for c in map_obj.initial_extent.split(',')
]
item['initialBbox']['bounds'] = initial_bounds
# collect layers
external_layers = []
layers, drawing_order = self.map_layers(map_obj, layer_bbox,
external_layers,
theme_external_layers)
item['sublayers'] = layers
item['drawingOrder'] = drawing_order
item['externalLayers'] = external_layers
background_layers = self.item_background_layers(session)
if map_obj.background_layer:
# set default background layer
bg_name = map_obj.background_layer.qwc2_bg_layer_name
for background_layer in background_layers:
if background_layer['name'] == bg_name:
background_layer['visibility'] = True
break
# use printLayer from themes_background_layers
for background_layer in background_layers:
themes_bgl = next((l for l in themes_background_layers
if l['name'] == background_layer['name']),
{})
if themes_bgl.get('printLayer'):
background_layer['printLayer'] = themes_bgl['printLayer']
item['backgroundLayers'] = background_layers
item['print'] = print_layouts
item['printLabelConfig'] = print_label_config
item['searchProviders'] = search_providers
item['editConfig'] = self.edit_config(session)
if map_obj.thumbnail_image:
item['thumbnail'] = ("img/custommapthumbs/%s" %
map_obj.thumbnail_image)
# NOTE: temp map order for sorting
item['map_order'] = map_obj.map_order
items.append(item)
# order by map order or title
items.sort(key=lambda i: (i['map_order'], i['title']))
# remove map_order
for item in items:
del item['map_order']
return items
def default_item(self, cfg_qwc2_themes):
"""Return theme item with default values.
:param obj cfg_qwc2_themes: Additional QWC2 themes config
"""
# NOTE: use ordered keys
item = OrderedDict()
item['id'] = None
item['name'] = None
item['title'] = None
item['wms_name'] = None
item['url'] = None
attribution = OrderedDict()
attribution['Title'] = cfg_qwc2_themes.get(
'default_theme_attribution_title', '')
attribution['OnlineResource'] = cfg_qwc2_themes.get(
'default_theme_attribution_online_resource', '')
item['attribution'] = attribution
item['keywords'] = ''
item['abstract'] = ''
item['mapCrs'] = 'EPSG:2056'
bbox = OrderedDict()
bbox['crs'] = cfg_qwc2_themes.get('default_crs', 'EPSG:2056')
bbox['bounds'] = cfg_qwc2_themes.get(
'default_theme_item_bounds', [2590000, 1210000, 2650000, 1270000])
item['bbox'] = bbox
initial_bbox = OrderedDict()
initial_bbox['crs'] = cfg_qwc2_themes.get('default_crs', 'EPSG:2056')
initial_bbox['bounds'] = cfg_qwc2_themes.get(
'default_theme_item_bounds', [2590000, 1210000, 2650000, 1270000])
item['initialBbox'] = initial_bbox
item['sublayers'] = []
item['expanded'] = True
item['drawingOrder'] = []
item['backgroundLayers'] = []
item['print'] = []
item['printLabelConfig'] = {}
item['searchProviders'] = []
item['editConfig'] = None
item['additionalMouseCrs'] = ['EPSG:21781', 'EPSG:2056']
item['tiled'] = False
item['availableFormats'] = cfg_qwc2_themes.get(
'default_image_formats', ['image/jpeg', 'image/png'])
item['skipEmptyFeatureAttributes'] = True
item['infoFormats'] = [
'text/plain', 'text/html', 'text/xml', 'application/vnd.ogc.gml',
'application/vnd.ogc.gml/3.1.1'
]
item['thumbnail'] = 'img/mapthumbs/default.jpg'
return item
def print_layouts(self, default_print_layout, session):
"""Return QWC2 print layouts and labels from ConfigDB.
:param str default_print_layout: Name of default print layout
:param Session session: DB session
"""
print_layouts = []
print_label_config = {}
TemplateQGIS = self.config_models.model('template_qgis')
query = session.query(TemplateQGIS).order_by(TemplateQGIS.name)
for template in query.all():
# NOTE: use ordered keys
print_layout = OrderedDict()
print_layout['name'] = template.name
map_cfg = OrderedDict()
map_cfg['name'] = 'map0'
map_cfg['width'] = template.map_width
map_cfg['height'] = template.map_height
print_layout['map'] = map_cfg
if template.print_labels:
# add print labels
labels = []
print_labels = template.print_labels.split(',')
for label in print_labels:
# add label
labels.append(label)
# printLabelConfig
label_cfg = OrderedDict()
label_cfg['rows'] = 1
label_cfg['maxLength'] = 128
print_label_config[label] = label_cfg
print_layout['labels'] = labels
print_layout['default'] = (template.name == default_print_layout)
print_layouts.append(print_layout)
return print_layouts, print_label_config
def solr_search_provider(self, session):
"""Return Solr search provider config with default search identifiers
from ConfigDB.
:param Session session: DB session
"""
searches = ['foreground']
facets = []
# collect always searchable datasets
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
query = session.query(DataSetEdit)
query = query.options(joinedload(DataSetEdit.data_set_view))
for dataset_edit in query.all():
data_set_view = dataset_edit.data_set_view
if data_set_view.searchable == self.ALWAYS_SEARCHABLE:
facets.append(data_set_view.facet)
# sort and add facets
facets.sort()
searches += facets
# NOTE: use ordered keys
cfg = OrderedDict()
cfg['provider'] = 'solr'
cfg['default'] = searches
return cfg
def map_layers(self, map_obj, layer_bbox, external_layers,
theme_external_layers):
"""Return theme item layers and drawing order for a map from ConfigDB.
:param obj map_obj: Map object
:param obj layer_bbox: Default layer extent
:param obj external_layers: Collected external layers
"""
layers = []
drawing_order = []
for map_layer in map_obj.map_layers:
ows_layer = map_layer.owslayer
opacity = round(
(100.0 - map_layer.layer_transparency) / 100.0 * 255)
res = self.collect_layers(ows_layer, opacity,
map_layer.layer_active, layer_bbox,
external_layers, theme_external_layers)
layers += res['layers']
drawing_order += res['drawing_order']
drawing_order.reverse()
return layers, drawing_order
def collect_layers(self, layer, opacity, visibility, layer_bbox,
external_layers, theme_external_layers):
"""Recursively collect layers for layer subtree from ConfigDB
and return nested theme item sublayers and drawing order.
:param obj layer: Group or Data layer object
:param int opacity: Layer Opacity between [0..100]
:param bool visibility: Whether layer is active
:param obj layer_bbox: Default layer extent
:param obj external_layers: Collected external layers
"""
layers = []
drawing_order = []
data_set_view = None
searchterms = []
# NOTE: use ordered keys
item_layer = OrderedDict()
item_layer['name'] = layer.name
if layer.title:
item_layer['title'] = layer.title
if layer.type == 'group' and not layer.facade:
# group layer
sublayers = []
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sublayer
res = self.collect_layers(sublayer, opacity, visibility,
layer_bbox, external_layers,
theme_external_layers)
sublayers += res['layers']
drawing_order += res['drawing_order']
item_layer['sublayers'] = sublayers
item_layer['expanded'] = True
else:
# data layer or facade group layer
queryable = False
display_field = None
if layer.ows_metadata:
# get abstract from layer metadata
try:
# load JSON from ows_metadata
ows_metadata = json.loads(layer.ows_metadata)
if 'abstract' in ows_metadata:
item_layer['abstract'] = ows_metadata['abstract']
except Exception as e:
self.logger.warning(
"Invalid JSON in ows_metadata of layer '%s':\n%s" %
(layer.name, e))
item_layer['visibility'] = visibility
if layer.type == 'data':
# data layer
data_set_view = layer.data_set_view
data_source = data_set_view.data_set.data_source
if data_source.connection_type == 'database':
# get any display field
for attr in data_set_view.attributes:
if attr.displayfield:
display_field = attr.alias or attr.name
break
item_layer['queryable'] = self.layer_queryable(layer)
if display_field:
item_layer['displayField'] = display_field
item_layer['opacity'] = opacity
item_layer['bbox'] = layer_bbox
drawing_order.append(layer.name)
if data_set_view:
data_set = data_set_view.data_set
data_source = data_set.data_source
if data_source.connection_type == "wms" or data_source.connection_type == "wmts":
external_layer_name = data_source.connection_type + ":" + data_source.connection + "#" + data_set.data_set_name
external_layers.append({
"internalLayer": item_layer['name'],
"name": external_layer_name
})
if not external_layer_name in theme_external_layers:
theme_external_layers[
external_layer_name] = self.build_external_layer(
external_layer_name, data_source.connection_type,
data_source.connection, data_set.data_set_name)
if theme_external_layers[external_layer_name]:
item_layer['abstract'] = theme_external_layers[
external_layer_name]['abstract']
if data_set_view.facet:
item_layer['searchterms'] = [data_set_view.facet]
searchterms.append(data_set_view.facet)
elif len(searchterms) > 0:
item_layer['searchterms'] = searchterms
layers.append(item_layer)
return {'layers': layers, 'drawing_order': drawing_order}
def build_external_layer(self, name, conn_type, url, layername):
if conn_type == "wms":
data = get_wms_layer_data(self.logger, url, layername)
return {
"name": name,
"type": conn_type,
"url": url,
"params": {
"LAYERS": layername
},
"infoFormats": ["text/plain"],
"abstract": data["abstract"]
}
elif conn_type == "wmts":
data = get_wmts_layer_data(self.logger, url, layername)
return {
"name": name,
"type": conn_type,
"url": data["res_url"],
"tileMatrixPrefix": "",
"tileMatrixSet": data["tileMatrixSet"],
"originX": data["origin"][0],
"originY": data["origin"][1],
"projection:": data["crs"],
"resolutions": data["resolutions"],
"tileSize": data["tile_size"],
"abstract": data["abstract"]
}
else:
self.logger.warning(
"Skipping external layer %s of unknown type %s" %
(name, conn_type))
return None
def layer_queryable(self, layer):
"""Recursively check whether a layer is queryable.
:param obj layer: Group or Data layer object
"""
queryable = False
if layer.type == 'group':
# group layer
for group_layer in layer.sub_layers:
sublayer = group_layer.sub_layer
# recursively collect sublayer
# group layer is queryable if any sublayer is queryable
queryable |= self.layer_queryable(sublayer)
else:
# data layer
data_set_view = layer.data_set_view
data_source = data_set_view.data_set.data_source
if data_source.connection_type == 'database':
if data_set_view.attributes:
# make layer queryable if there are any attributes
queryable = True
else:
# raster data layers are always queryable
queryable = True
return queryable
def item_background_layers(self, session):
"""Return background layers for item from ConfigDB.
:param Session session: DB session
"""
background_layers = []
BackgroundLayer = self.config_models.model('background_layer')
query = session.query(BackgroundLayer).order_by(BackgroundLayer.name)
for layer in query.all():
# NOTE: use ordered keys
background_layer = OrderedDict()
background_layer['name'] = layer.qwc2_bg_layer_name
background_layer['printLayer'] = layer.name
# Ignore background layers referenced by groups
if background_layer[
'name'] not in self.background_layer_group_refs:
background_layers.append(background_layer)
return background_layers
def background_layers(self, session):
"""Return available background layers from ConfigDB.
:param Session session: DB session
"""
background_layers = []
BackgroundLayer = self.config_models.model('background_layer')
query = session.query(BackgroundLayer).order_by(BackgroundLayer.name)
for layer in query.all():
try:
background_layer = json.loads(layer.qwc2_bg_layer_config,
object_pairs_hook=OrderedDict)
if layer.thumbnail_image:
# set custom thumbnail
background_layer['thumbnail'] = ("img/custommapthumbs/%s" %
layer.thumbnail_image)
background_layers.append(background_layer)
except Exception as e:
self.logger.warning(
"Could not load background layer '%s':\n%s" %
(layer.name, e))
return background_layers
def edit_config(self, session):
"""Return edit config for all available edit layers.
:param Session session: DB session
"""
# NOTE: use ordered keys
edit_config = OrderedDict()
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
ResourcePermission = self.config_models.model('resource_permission')
GDIResource = self.config_models.model('gdi_resource')
# get IDs for edit datasets with any write permissions
table_name = DataSetEdit.__table__.name
query = session.query(ResourcePermission)\
.join(ResourcePermission.resource) \
.filter(GDIResource.table_name == table_name) \
.filter(ResourcePermission.write) \
.distinct(GDIResource.gdi_oid)
# pluck resource IDs
writable_ids = [p.gdi_oid_resource for p in query.all()]
# get writable dataset edit configs and relations
query = session.query(DataSetEdit) \
.filter(DataSetEdit.gdi_oid.in_(writable_ids)) \
.order_by(DataSetEdit.name)
# eager load nested relations (except Attribute)
query = query.options(
joinedload(DataSetEdit.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for dataset_edit in query.all():
data_set_view = dataset_edit.data_set_view
data_set = data_set_view.data_set
# layer title
try:
title = data_set_view.ows_layers[0].title
except Exception as e:
title = dataset_edit.name
# parse schema and table name
data_set_name = data_set.data_set_name
parts = data_set_name.split('.')
if len(parts) > 1:
schema = parts[0]
table_name = parts[1]
else:
schema = 'public'
table_name = data_set_name
fields = []
geometry_type = None
# database connection URL
conn_str = data_set.data_source.connection
# get attributes
for attribute in data_set_view.attributes:
# get data type and constraints
attr_meta = self.permissions_query.attribute_metadata(
conn_str, schema, table_name, attribute.name)
field = OrderedDict()
field['id'] = attribute.name
field['name'] = attribute.alias or attribute.name
field['type'] = self.EDIT_FIELD_TYPES.get(
attr_meta['data_type'], 'text')
if attr_meta['constraints']:
# add any constraints
field['constraints'] = attr_meta['constraints']
fields.append(field)
# get geometry type
# primary key if view
primary_key = data_set.primary_key
# geometry column if multiple
geometry_column = data_set_view.geometry_column
# PostGIS metadata
pgmeta = self.permissions_query.postgis_metadata(
conn_str, schema, table_name, geometry_column)
if pgmeta.get('geometry_column'):
geometry_type = pgmeta.get('geometry_type')
if geometry_type not in self.EDIT_GEOM_TYPES:
# unsupported geometry type
self.logger.warning(
"Unsupported geometry type %s for editing %s.%s" %
(geometry_type, schema, table_name))
geometry_type = None
else:
geometry_type = self.EDIT_GEOM_TYPES.get(geometry_type)
if fields and geometry_type is not None:
# add only datasets with attributes and geometry
# NOTE: use ordered keys
dataset = OrderedDict()
dataset['editDataset'] = dataset_edit.name
dataset['layerName'] = title
dataset['fields'] = fields
dataset['geomType'] = geometry_type
edit_config[dataset_edit.name] = dataset
if edit_config:
return edit_config
else:
return None
# permissions
def permitted_background_layers(self, role, session):
"""Return permitted internal print layers for background layers from
ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
background_layers = []
# get IDs for permitted resources required for FeatureInfo
table_names = ['background_layer']
# get IDs for role permissions
permitted_ids = self.permissions_query.resource_ids(
table_names, role, session)
# get IDs for public permissions
public_ids = self.permissions_query.resource_ids(
table_names, self.permissions_query.public_role(), session)
if role != self.permissions_query.public_role():
# use only additional role permissions
# as role permissions without public permissions
permitted_ids = permitted_ids - public_ids
BackgroundLayer = self.config_models.model('background_layer')
query = session.query(BackgroundLayer).order_by(BackgroundLayer.name)
background_layers = [
layer.qwc2_bg_layer_name for layer in query.all()
if layer.gdi_oid in permitted_ids
]
return background_layers
# helpers
def precache_resources(self, session):
"""Precache some resources using eager loaded relations to reduce the
number of actual separate DB requests in later queries.
NOTE: The lookup tables do not have to be actually used, but remain
in memory, so SQLAlchemy will use the cached records.
E.g. accessing ows_layer_data.data_set_view afterwards won't generate
a separate DB query
:param Session session: DB session
"""
OWSLayerGroup = self.config_models.model('ows_layer_group')
OWSLayerData = self.config_models.model('ows_layer_data')
GroupLayer = self.config_models.model('group_layer')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
# precache OWSLayerData and eager load relations
ows_layer_data_lookup = {}
query = session.query(OWSLayerData)
query = query.options(
joinedload(OWSLayerData.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for layer in query.all():
ows_layer_data_lookup[layer.gdi_oid] = layer
# precache DataSetView and eager load attributes
query = session.query(DataSetView)
query = query.options(joinedload(DataSetView.attributes))
data_set_view_lookup = {}
for data_set_view in query.all():
data_set_view_lookup[data_set_view.gdi_oid] = data_set_view
# precache OWSLayerGroup and eager load sub layers
query = session.query(OWSLayerGroup)
query = query.options(
joinedload(OWSLayerGroup.sub_layers).joinedload(
GroupLayer.sub_layer))
ows_layer_group_lookup = {}
for group in query.all():
ows_layer_group_lookup[group.gdi_oid] = group
# NOTE: return precached resources so they stay in memory
return {
'ows_layer_data_lookup': ows_layer_data_lookup,
'data_set_view_lookup': data_set_view_lookup,
'ows_layer_group_lookup': ows_layer_group_lookup
}
class SearchServiceConfig(ServiceConfig):
"""SearchServiceConfig class
Generate Search service config.
"""
def __init__(self, config_models, logger):
"""Constructor
:param Logger logger: Logger
"""
super().__init__(
'search',
'https://github.com/qwc-services/qwc-fulltext-search-service/raw/master/schemas/qwc-search-service.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
config = super().config(service_config)
session = self.config_models.session()
resources = OrderedDict()
resources['facets'] = self._facets(session)
config['resources'] = resources
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
session = self.config_models.session()
permissions['solr_facets'] = self._facet_permissions(role, session)
# collect feature reports
session.close()
return permissions
def _facets(self, session):
"""Return search service resources.
:param str username: User name
:param str group: Group name
:param Session session: DB session
"""
searches = [{
'name': 'foreground',
'filter_word': 'Karte',
'default': True
}, {
'name': 'background',
'filter_word': 'Hintergrundkarte',
'default': False
}]
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
dataset_edit_ids = self.permissions_query.all_resource_ids(
[DataSetEdit.__table__.name], session)
query = session.query(DataSetEdit)
query = query.options(
joinedload(DataSetEdit.data_set_view)
# .joinedload(DataSetView.data_set)
)
# Da die 1:n Beziehung von DataSetView zu DataSet
# zwar im DB-Schema vorgesehen, aber nur teilweise
# umgesetzt ist (z.B. AGDI) wird vorläufig der
# indizierte Product-Identifier in DataSetView.facet
# gespeichert.
facets = {}
query = query.filter(DataSetEdit.gdi_oid.in_(dataset_edit_ids))
for dataset_edit in query.all():
data_set_view = dataset_edit.data_set_view
if data_set_view.searchable != 0:
feature_search = {
'name': data_set_view.facet,
# 'dataproduct_id': data_set_view.name,
'filter_word': data_set_view.filter_word,
'default': (data_set_view.searchable == 2)
}
if data_set_view.facet not in facets:
searches.append(feature_search)
facets[data_set_view.facet] = [feature_search]
# Only add feature_search entry, if filter_word differs
unique = True
for entry in facets[data_set_view.facet]:
if entry['filter_word'] == data_set_view.filter_word:
unique = False
if unique:
searches.append(feature_search)
facets[data_set_view.facet].append(feature_search)
return searches
def _facet_permissions(self, role, session):
"""Collect dataset_edit permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = []
if role == self.permissions_query.public_role():
# add default public permissions
permissions = ['foreground', 'background']
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
resource_ids = self.permissions_query.resource_ids(
[DataSetEdit.__table__.name], role, session)
query = session.query(DataSetEdit).\
filter(DataSetEdit.gdi_oid.in_(resource_ids))
query = query.options(
joinedload(DataSetEdit.data_set_view)
# .joinedload(DataSetView.data_set)
)
for dataset_edit in query.all():
data_set_view = dataset_edit.data_set_view
if data_set_view.searchable != 0:
permissions.append(data_set_view.facet)
return permissions
class DataServiceConfig(ServiceConfig):
"""DataServiceConfig class
Generate Data service config.
"""
def __init__(self, config_models, logger):
"""Constructor
:param Logger logger: Logger
"""
super().__init__(
'data',
'https://github.com/qwc-services/qwc-data-service/raw/master/schemas/qwc-data-service.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __del__(self):
"""Destructor"""
if self.session:
# close shared session
self.session.close()
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
config = super().config(service_config)
session = self.config_models.session()
# NOTE: keep precached resources in memory while querying resources
cached_resources = self.precache_resources(session)
resources = OrderedDict()
resources['datasets'] = self._datasets(session)
config['resources'] = resources
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
if self.session is None:
# create shared session
self.session = self.config_models.session()
# NOTE: keep precached resources in memory while querying
# permissions
self.cached_resources = self.precache_resources(self.session)
permissions['data_datasets'] = self._dataset_permissions(
role, self.session)
return permissions
def _datasets(self, session):
"""Return data service resources.
:param Session session: DB session
"""
datasets = []
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
DataSource = self.config_models.model('data_source')
Attribute = self.config_models.model('data_set_view_attributes')
# get dataset edit configs and relations
query = session.query(DataSetEdit).order_by(DataSetEdit.name)
# eager load nested relations (except Attribute)
query = query.options(
joinedload(DataSetEdit.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for dataset_edit in query.all():
data_set_view = dataset_edit.data_set_view
data_set = data_set_view.data_set
data_source = data_set.data_source
# database connection URLs
conn_str = data_set.data_source.connection
# use separate service for write access
# by appending suffix to service name, e.g.
# postgresql:///?service=sogis_services
# ->
# postgresql:///?service=sogis_services_write
conn_write_str = "%s_write" % conn_str
# parse schema and table name
data_set_name = data_set.data_set_name
parts = data_set_name.split('.')
if len(parts) > 1:
schema = parts[0]
table_name = parts[1]
else:
schema = 'public'
table_name = data_set_name
# primary key if view
primary_key = data_set.primary_key
# geometry column if multiple
geometry_column = data_set_view.geometry_column
pgmeta = self.permissions_query.postgis_metadata(
conn_str, schema, table_name, geometry_column)
# -> {'primary_key': 'id', 'geometry_column': 'geom',
# 'geometry_type': 'POLYGON', 'srid': 2056}
if not pgmeta:
# could not get PostGIS metadata
continue
# get attributes
attributes = []
for attribute in data_set_view.attributes:
# get data type and constraints
attr_meta = self.permissions_query.attribute_metadata(
conn_str, schema, table_name, attribute.name)
# NOTE: use ordered keys
field = OrderedDict()
field['name'] = attribute.name
field['data_type'] = attr_meta['data_type']
if attr_meta['constraints']:
# add any constraints
field['constraints'] = attr_meta['constraints']
attributes.append(field)
# NOTE: use ordered keys
dataset = OrderedDict()
dataset['name'] = dataset_edit.name
dataset['db_url'] = conn_str
dataset['db_write_url'] = conn_write_str
dataset['schema'] = schema
dataset['table_name'] = table_name
dataset['primary_key'] = primary_key or pgmeta.get('primary_key')
dataset['fields'] = attributes
if pgmeta.get('geometry_column'):
# NOTE: use ordered keys
geometry = OrderedDict()
geometry['geometry_column'] = pgmeta['geometry_column']
geometry['geometry_type'] = pgmeta['geometry_type']
geometry['srid'] = pgmeta['srid']
geometry['allow_null'] = True
dataset['geometry'] = geometry
datasets.append(dataset)
return datasets
def _dataset_permissions(self, role, session):
"""Collect edit dataset permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = []
# get IDs for permitted resources required for editing
table_names = [
'data_set_edit', 'data_set_view', 'data_set', 'data_source',
'data_set_view_attributes'
]
# get IDs for role permissions
permitted_ids = self.permissions_query.resource_ids(
table_names, role, session)
# get IDs for public permissions
public_ids = self.permissions_query.resource_ids(
table_names, self.permissions_query.public_role(), session)
# combined role and public permissions
role_and_public_ids = permitted_ids | public_ids
if role != self.permissions_query.public_role():
# use only additional role permissions
# as role permissions without public permissions
permitted_ids = permitted_ids - public_ids
# collect write permissions with highest priority for all edit datasets
writeable_datasets = set()
edit_permissions = self.permissions_query.resource_permissions(
'data_set_edit', None, role, session)
for edit_permission in edit_permissions:
if (edit_permission.write and edit_permission.resource.name
not in writeable_datasets):
writeable_datasets.add(edit_permission.resource.name)
DataSetEdit = self.config_models.model('data_set_edit')
query = session.query(DataSetEdit).order_by(DataSetEdit.name)
for dataset_edit in query.all():
# check permissions for edit dataset and required resources
data_set_view = dataset_edit.data_set_view
data_set = data_set_view.data_set
data_source = data_set.data_source
if not (dataset_edit.gdi_oid in role_and_public_ids
and data_set_view.gdi_oid in role_and_public_ids
and data_set.gdi_oid in role_and_public_ids
and data_source.gdi_oid in role_and_public_ids):
# edit dataset not permitted
continue
# NOTE: use ordered keys
dataset_permissions = OrderedDict()
dataset_permissions['name'] = dataset_edit.name
# collect attribute names
attributes = []
attributes = [
attr.name for attr in data_set_view.attributes
if attr.gdi_oid in permitted_ids
]
dataset_permissions['attributes'] = attributes
# get CRUD permissions for edit permission
writable = dataset_edit.name in writeable_datasets
dataset_permissions['writable'] = writable
dataset_permissions['creatable'] = writable
dataset_permissions['readable'] = True
dataset_permissions['updatable'] = writable
dataset_permissions['deletable'] = writable
if attributes or writable:
# only add additional permissions
permissions.append(dataset_permissions)
return permissions
# helpers
def precache_resources(self, session):
"""Precache some resources using eager loaded relations to reduce the
number of actual separate DB requests in later queries.
NOTE: The lookup tables do not have to be actually used, but remain
in memory, so SQLAlchemy will use the cached records.
E.g. accessing ows_layer_data.data_set_view afterwards won't generate
a separate DB query
:param Session session: DB session
"""
DataSetEdit = self.config_models.model('data_set_edit')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
# precache DataSetEdit and eager load relations
data_set_edit_lookup = {}
query = session.query(DataSetEdit)
query = query.options(
joinedload(DataSetEdit.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for data_set_edit in query.all():
data_set_edit_lookup[data_set_edit.gdi_oid] = data_set_edit
# precache DataSetView and eager load attributes
query = session.query(DataSetView)
query = query.options(joinedload(DataSetView.attributes))
data_set_view_lookup = {}
for data_set_view in query.all():
data_set_view_lookup[data_set_view.gdi_oid] = data_set_view
# NOTE: return precached resources so they stay in memory
return {
'data_set_edit_lookup': data_set_edit_lookup,
'data_set_view_lookup': data_set_view_lookup
}
class DataproductServiceConfig(ServiceConfig):
"""DataproductServiceConfig class
Generate Dataproduct service config and permissions.
"""
def __init__(self, config_models, generator_config, logger):
"""Constructor
:param ConfigModels config_models: Helper for ORM models
:param Logger logger: Logger
"""
super().__init__(
'dataproduct',
'https://github.com/qwc-services/sogis-dataproduct-service/raw/master/schemas/sogis-dataproduct-service.json',
logger)
self.config_models = config_models
self.generator_config = generator_config
self.permissions_query = PermissionsQuery(config_models, logger)
self.db_engine = DatabaseEngine()
# shared session and cached resources for collecting permissions
self.session = None
self.cached_resources = None
def __del__(self):
"""Destructor"""
if self.session:
# close shared session
self.session.close()
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
# get base config
config = super().config(service_config)
# additional service config
self.service_config = service_config.get('config', {})
session = self.config_models.session()
config['config'] = {}
resources = OrderedDict()
config['resources'] = resources
# collect resources from ConfigDB
# NOTE: keep precached resources in memory while querying resources
cached_resources = self.precache_resources(session)
resources['dataproducts'] = self._dataproducts(session)
session.close()
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
if self.session is None:
# create shared session
self.session = self.config_models.session()
# NOTE: keep precached resources in memory while querying
# permissions
self.cached_resources = self.precache_resources(self.session)
permissions['dataproducts'] = self._dataproduct_permissions(
role, self.session)
return permissions
# permissions
def _dataproduct_permissions(self, role, session):
"""Collect dataproduct permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = set()
OWSLayer = self.config_models.model('ows_layer')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
DataSource = self.config_models.model('data_source')
# collect permitted resource IDs
table_names = [
OWSLayer.__table__.name, DataSetView.__table__.name,
DataSet.__table__.name, DataSource.__table__.name
]
resource_ids = self.permissions_query.resource_ids(
table_names, role, session)
# collect permissions for nested Group or Data layer objects
# NOTE: do not filter here by permitted resource IDs,
# as Group layers do not have explicit permissions
query = session.query(OWSLayer).order_by(OWSLayer.name)
for ows_layer in query.all():
self._collect_layer_permissions(ows_layer, resource_ids,
permissions)
# collect permissions for basic DataSets
query = session.query(DataSetView).order_by(DataSetView.name) \
.filter(DataSetView.gdi_oid.in_(resource_ids))
# filter by DataSetViews without OWSLayers
query = query.options(joinedload(DataSetView.ows_layers)) \
.filter(~DataSetView.ows_layers.any())
for data_set_view in query.all():
permissions.add(data_set_view.name)
return sorted(list(permissions), key=str.lower)
def _collect_layer_permissions(self, ows_layer, permitted_ids,
permissions):
"""Recursively collect layers for layer subtree from ConfigDB.
:param obj ows_layer: Group or Data layer object
:param set<int> permitted_ids: Set of permitted resource IDs
:param permissions: Partial set of permitted DataProduct names
"""
if ows_layer.type == 'group':
# collect sub layers
sublayers = []
for group_layer in ows_layer.sub_layers:
sub_layer = group_layer.sub_layer
# update sub layer permissions
self._collect_layer_permissions(sub_layer, permitted_ids,
permissions)
if sub_layer.name in permissions:
sublayers.append(sub_layer.name)
if sublayers:
# add group if any sub layers permitted
permissions.add(ows_layer.name)
else:
# data layer
# NOTE: only checking data layer permissions,
# not for required resources
if ows_layer.gdi_oid in permitted_ids:
permissions.add(ows_layer.name)
# service config
def _dataproducts(self, session):
"""Collect dataproduct resources from ConfigDB.
:param Session session: DB session
"""
dataproducts = []
# get WFS root layer
wfs_root_layer_id = None
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter_by(ows_type='WFS')
wms_wfs = query.first()
if wms_wfs is not None:
wfs_root_layer_id = wms_wfs.gdi_oid_root_layer
# collect Group or Data layer objects
OWSLayer = self.config_models.model('ows_layer')
query = session.query(OWSLayer).order_by(OWSLayer.name)
# ignore WFS root layer
query = query.filter(OWSLayer.gdi_oid != wfs_root_layer_id)
for ows_layer in query.all():
metadata, searchterms = self._dataproduct_metadata(
ows_layer, session)
if len(metadata) > 0:
dataproducts.append(metadata)
# collect DataSetViews for basic DataSets
DataSetView = self.config_models.model('data_set_view')
query = session.query(DataSetView).order_by(DataSetView.name)
# filter by DataSetViews without OWSLayers
query = query.options(joinedload(DataSetView.ows_layers)) \
.filter(~DataSetView.ows_layers.any())
for data_set_view in query.all():
metadata = self._basic_dataset_metadata(data_set_view, session)
dataproducts.append(metadata)
return dataproducts
def _dataproduct_metadata(self, ows_layer, session):
"""Recursively collect metadata of a dataproduct.
:param obj ows_layer: Group or Data layer object
:param Session session: DB session
"""
metadata = OrderedDict()
# type
sublayers = None
data_set_view = None
searchterms = []
if ows_layer.type == 'group':
if ows_layer.facade:
dataproduct_type = 'facadelayer'
else:
dataproduct_type = 'layergroup'
# collect sub layers
sublayers = []
for group_layer in ows_layer.sub_layers:
sub_layer = group_layer.sub_layer
submetadata, subsearchterms = self._dataproduct_metadata(
sub_layer, session)
if submetadata:
sublayers.append(sub_layer.name)
if dataproduct_type == 'facadelayer':
searchterms += subsearchterms
if not sublayers:
self.logger.warning(
"Skipping ProductSet %s with empty sublayers" %
ows_layer.name)
return (metadata, searchterms)
else:
dataproduct_type = 'datasetview'
# find matching DataSetView
DataSetView = self.config_models.model('data_set_view')
query = session.query(DataSetView).filter_by(name=ows_layer.name)
data_set_view = query.first()
try:
contacts = self._dataproduct_contacts(ows_layer, session)
datasource = self._dataproduct_datasource(ows_layer, session)
wms_datasource = self._dataproduct_wms(ows_layer, session)
ows_metadata = self._ows_metadata(ows_layer)
description = datasource.get('abstract') or ows_metadata.get(
'abstract')
except ObjectDeletedError as e:
self.logger.error("%s: %s" % (ows_layer.name, e))
return (metadata, searchterms)
# qml
qml = None
if ows_layer.type == 'data':
qml = ows_layer.client_qgs_style or ows_layer.qgs_style
# embed any uploaded symbols in QML
qml = self._update_qml(ows_layer.name, qml)
metadata['identifier'] = ows_layer.name
metadata['display'] = ows_layer.title
metadata['type'] = dataproduct_type
metadata['synonyms'] = self._split_values(ows_layer.synonyms)
metadata['keywords'] = self._split_values(ows_layer.keywords)
metadata['description'] = description or ''
metadata['contacts'] = contacts
metadata['wms_datasource'] = wms_datasource
metadata['qml'] = qml
metadata['sublayers'] = sublayers
if data_set_view:
if data_set_view.facet:
metadata['searchterms'] = [data_set_view.facet]
searchterms.append(data_set_view.facet)
elif len(searchterms) > 0:
metadata['searchterms'] = searchterms
if wms_datasource: # ?
metadata.update(self._layer_display_infos(ows_layer, session))
metadata.update(datasource)
# Filter null entries
filtered_metadata = OrderedDict()
for k, v in metadata.items():
if v is not None:
filtered_metadata[k] = v
return (filtered_metadata, searchterms)
def _layer_display_infos(self, ows_layer, session):
"""Return theme item layer infos from ConfigDB.
:param obj ows_layer: Group or Data layer object
"""
visible = True
# get visibility from parent group_layer
parents = ows_layer.parents
if len(parents) > 0:
group_layer = parents[0]
visible = group_layer.layer_active
queryable = self._layer_queryable(ows_layer)
display_field = None
if ows_layer.type == 'data':
data_set_view = ows_layer.data_set_view
data_source = data_set_view.data_set.data_source
if data_source.connection_type == 'database':
# get any display field
for attr in data_set_view.attributes:
if attr.displayfield:
display_field = attr.name
break
opacity = round((100.0 - ows_layer.layer_transparency) / 100.0 * 255)
metadata = OrderedDict()
metadata['visibility'] = visible
metadata['queryable'] = queryable
metadata['displayField'] = display_field
metadata['opacity'] = opacity
return metadata
def _layer_queryable(self, ows_layer):
"""Recursively check whether layer or any sublayers are queryable.
:param obj ows_layer: Group or Data layer object
"""
queryable = False
if ows_layer.type == 'data':
data_set_view = ows_layer.data_set_view
data_source = data_set_view.data_set.data_source
if data_source.connection_type == 'database':
if data_set_view.attributes:
# make layer queryable if there are any attributes
queryable = True
else:
# raster data layers are always queryable
queryable = True
elif ows_layer.type == 'group' and ows_layer.facade:
# check if any facade sublayer is queryable
for group_layer in ows_layer.sub_layers:
sub_layer = group_layer.sub_layer
if self._layer_queryable(sub_layer):
# make facade layer queryable
queryable = True
break
return queryable
def _basic_dataset_metadata(self, data_set_view, session):
"""Collect metadata of a basic DataSet dataproduct.
:param obj data_set_view: DataSetView object
:param Session session: DB session
"""
metadata = OrderedDict()
contacts = self._basic_dataset_contacts(data_set_view, session)
metadata['identifier'] = data_set_view.name
metadata['display'] = data_set_view.data_set.data_set_name
metadata['type'] = 'datasetview'
metadata['description'] = data_set_view.description
metadata['contacts'] = contacts
metadata['datatype'] = 'table'
if data_set_view.facet:
metadata['searchterms'] = [data_set_view.facet]
return metadata
def _dataproduct_contacts(self, ows_layer, session):
"""Return contacts metadata for a dataproduct.
:param obj ows_layer: Group or Data layer object
:param Session session: DB session
"""
# collect contacts for layer and related GDI resources
gdi_oids = [ows_layer.gdi_oid]
if ows_layer.type == 'data':
# include data source
gdi_oids.append(
ows_layer.data_set_view.data_set.gdi_oid_data_source)
return self._contacts(gdi_oids, session)
def _basic_dataset_contacts(self, data_set_view, session):
"""Return contacts metadata for a basic DataSet dataproduct.
:param obj data_set_view: DataSetView object
:param Session session: DB session
"""
# collect contacts for basic DataSet and related GDI resources
gdi_oids = [
data_set_view.gdi_oid, data_set_view.data_set.gdi_oid_data_source
]
return self._contacts(gdi_oids, session)
def _contacts(self, gdi_oids, session):
"""Return contacts metadata for a list of resource IDs.
:param list[int] gdi_oids: List of GDI resource IDs
:param Session session: DB session
"""
contacts = []
ResourceContact = self.config_models.model('resource_contact')
Contact = self.config_models.model('contact')
query = session.query(ResourceContact) \
.filter(ResourceContact.gdi_oid_resource.in_(gdi_oids)) \
.order_by(ResourceContact.id_contact_role)
# eager load relations
query = query.options(
joinedload(ResourceContact.contact).joinedload(
Contact.organisation))
for res_contact in query.all():
person = res_contact.contact
person_data = OrderedDict()
person_data['id'] = person.id
person_data['name'] = person.name
person_data['function'] = person.function
person_data['email'] = person.email
person_data['phone'] = person.phone
person_data['street'] = person.street
person_data['house_no'] = person.house_no
person_data['zip'] = person.zip
person_data['city'] = person.city
person_data['country_code'] = person.country_code
organisation_data = None
organisation = person.organisation
if organisation is not None:
organisation_data = OrderedDict()
organisation_data['id'] = organisation.id
organisation_data['name'] = organisation.name
organisation_data['unit'] = organisation.unit
organisation_data['abbreviation'] = organisation.abbreviation
organisation_data['street'] = organisation.street
organisation_data['house_no'] = organisation.house_no
organisation_data['zip'] = organisation.zip
organisation_data['city'] = organisation.city
organisation_data['country_code'] = organisation.country_code
# Filter null entries
filtered_data = OrderedDict()
for k, v in organisation_data.items():
if v is not None:
filtered_data[k] = v
organisation_data = filtered_data
contact = OrderedDict()
contact['person'] = person_data
contact['organisation'] = organisation_data
contacts.append(contact)
return contacts
def _dataproduct_datasource(self, ows_layer, session):
"""Return datasource metadata for a dataproduct.
:param obj ows_layer: Group or Data layer object
:param Session session: DB session
"""
metadata = OrderedDict()
if ows_layer.type == 'group':
# group layer
metadata['bbox'] = self.service_config.get('default_extent')
metadata['crs'] = 'EPSG:2056'
return metadata
data_set = ows_layer.data_set_view.data_set
data_source = data_set.data_source
if data_source.connection_type == 'database':
# vector DataSet
# get table metadata
postgis_datasource = None
pg_metadata = self._dataset_info(data_source.gdi_oid,
data_set.data_set_name)
if 'error' not in pg_metadata:
data_set_name = "%s.%s" % (pg_metadata.get('schema'),
pg_metadata.get('table'))
primary_key = pg_metadata.get('primary_key')
if primary_key is None:
# get primary key if view
primary_key = data_set.primary_key
geom = {}
if len(pg_metadata.get('geometry_columns')) > 1:
used_col = ows_layer.data_set_view.geometry_column
for geom_col in pg_metadata.get('geometry_columns'):
# get used geometry column if multiple
if geom_col.get('geometry_column') == used_col:
geom = geom_col
break
elif len(pg_metadata.get('geometry_columns')) == 1:
# use sole geometry column
geom = pg_metadata.get('geometry_columns')[0]
postgis_datasource = OrderedDict()
postgis_datasource['dbconnection'] = data_source.connection
postgis_datasource['data_set_name'] = data_set_name
postgis_datasource['primary_key'] = primary_key
postgis_datasource['geometry_field'] = geom.get(
'geometry_column')
postgis_datasource['geometry_type'] = geom.get('geometry_type')
postgis_datasource['srid'] = geom.get('srid')
else:
# show error message
postgis_datasource = {'error': pg_metadata.get('error')}
metadata['bbox'] = self.service_config.get('default_extent')
metadata['crs'] = 'EPSG:2056'
metadata['datatype'] = 'vector'
metadata['postgis_datasource'] = postgis_datasource
elif data_source.connection_type == 'wms':
# External WMS
url = data_source.connection
layername = data_set.data_set_name
conn = "wms:%s#%s" % (url, layername)
data = get_wms_layer_data(self.logger, url, layername)
metadata = OrderedDict()
metadata['datatype'] = 'raster'
metadata['abstract'] = data.get("abstract", "")
metadata['external_layer'] = {
"name": conn,
"type": "wms",
"url": url,
"params": {
"LAYERS": layername
},
"infoFormats": ["text/plain"]
}
elif data_source.connection_type == 'wmts':
# External WMTS
url = data_source.connection
layername = data_set.data_set_name
conn = "wmts:%s#%s" % (url, layername)
data = get_wmts_layer_data(self.logger, url, layername)
metadata = OrderedDict()
metadata['datatype'] = 'raster'
metadata['abstract'] = data["abstract"]
metadata['external_layer'] = {
"name": conn,
"type": "wmts",
"url": data["res_url"],
"tileMatrixPrefix": "",
"tileMatrixSet": data["tileMatrixSet"],
"originX": data["origin"][0],
"originY": data["origin"][1],
"projection:": data["crs"],
"resolutions": data["resolutions"],
"tileSize": data["tile_size"]
}
else:
# raster DataSet
raster_datasource_pattern = self.service_config.get(
'raster_datasource_pattern', '')
raster_datasource_repl = self.service_config.get(
'raster_datasource_repl', '')
# modify connection dir
connection = re.sub(raster_datasource_pattern,
raster_datasource_repl, data_source.connection)
# TODO: get srid
srid = 2056
metadata = OrderedDict()
metadata['datatype'] = 'raster'
raster_datasource = OrderedDict()
raster_datasource[
'datasource'] = connection + data_set.data_set_name
raster_datasource['srid'] = srid
metadata['raster_datasource'] = raster_datasource
return metadata
def _dataset_info(self, data_source_id, table_name):
"""Return table metadata for a data_set.
:param int data_source_id: data_source ID
:param str table_name: Table name as "<schema>.<table>"
"""
# NOTE: form field returns 'None' as string if not set
if not table_name or table_name == 'None':
# empty table name
return None
# parse schema and table name
parts = table_name.split('.')
if len(parts) > 1:
schema = parts[0]
table_name = parts[1]
else:
schema = 'public'
return self._postgis_metadata(data_source_id, schema, table_name)
def _postgis_metadata(self, data_source_id, schema, table_name):
"""Return primary key, geometry columns, types and srids
from a PostGIS table.
:param int data_source_id: data_source ID
:param str schema: DB schema name
:param str table_name: DB table name
"""
metadata = {}
try:
engine = self._engine_for_data_source(data_source_id)
if engine is None:
return {'error': "FEHLER: DataSource nicht gefunden"}
# connect to data_source
conn = engine.connect()
# get primary key
# build query SQL
sql = sql_text("""
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid
AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = '{schema}.{table}'::regclass
AND i.indisprimary;
""".format(schema=schema, table=table_name))
# execute query
primary_key = None
result = conn.execute(sql)
for row in result:
primary_key = row['attname']
# get geometry column and srid
# build query SQL
sql = sql_text("""
SELECT f_geometry_column, srid, type
FROM geometry_columns
WHERE f_table_schema = '{schema}' AND f_table_name = '{table}';
""".format(schema=schema, table=table_name))
# execute query
geometry_columns = []
result = conn.execute(sql)
for row in result:
geometry_columns.append({
'geometry_column':
row['f_geometry_column'],
'geometry_type':
row['type'],
'srid':
row['srid']
})
# close database connection
conn.close()
metadata = {
'schema': schema,
'table': table_name,
'primary_key': primary_key,
'geometry_columns': geometry_columns
}
except OperationalError as e:
self.logger.error(e.orig)
return {'error': "OperationalError: %s" % e.orig}
except ProgrammingError as e:
self.logger.error(e.orig)
return {'error': "ProgrammingError: %s" % e.orig}
return metadata
def _engine_for_data_source(self, data_source_id):
"""Return SQLAlchemy engine for a data_source.
:param int data_source_id: data_source ID
"""
engine = None
# find data_source
DataSource = self.config_models.model('data_source')
session = self.config_models.session()
query = session.query(DataSource) \
.filter_by(gdi_oid=data_source_id)
data_source = query.first()
session.close()
if data_source is not None:
engine = self.db_engine.db_engine(data_source.connection)
return engine
def _dataproduct_wms(self, ows_layer, session):
"""Return any WMS datasource for a dataproduct.
:param obj ows_layer: Group or Data layer object
:param Session session: DB session
"""
wms_datasource = None
# get WMS root layer
root_layer = None
WmsWfs = self.config_models.model('wms_wfs')
query = session.query(WmsWfs).filter_by(ows_type='WMS')
# eager load relation
query = query.options(joinedload(WmsWfs.root_layer))
wms_wfs = query.first()
if wms_wfs is not None:
root_layer = wms_wfs.root_layer
if self._layer_in_ows(ows_layer, root_layer):
wms_datasource = OrderedDict()
wms_datasource['service_url'] = self.service_config.get(
'wms_service_url')
wms_datasource['name'] = ows_layer.name
return wms_datasource
def _layer_in_ows(self, ows_layer, root_layer):
"""Recursively check if layer is a WMS layer.
:param obj ows_layer: Group or Data layer object
:param obj root_layer: WMS root layer
"""
if root_layer is None:
# no WMS root layer
return False
in_wms = False
# get parent groups
parents = [p.group for p in ows_layer.parents]
for parent in parents:
if parent.gdi_oid == root_layer.gdi_oid:
# parent is WMS root layer
in_wms = True
else:
# check if parent group is a WMS layer
in_wms = in_wms or self._layer_in_ows(parent, root_layer)
if in_wms:
break
return in_wms
def _ows_metadata(self, layer):
"""Return ows_metadata for a layer.
:param obj layer: Group or Data layer object
"""
ows_metadata = {}
if layer.ows_metadata:
try:
# load JSON from ows_metadata
ows_metadata = json.loads(layer.ows_metadata)
except ValueError as e:
self.logger.warning(
"Invalid JSON in ows_metadata of layer %s: %s" %
(layer.name, e))
return ows_metadata
def _split_values(self, value):
"""Split comma separated values into list.
:param str value: Comma separated values
"""
if value:
return [s.strip() for s in value.split(',')]
else:
return []
def _update_qml(self, identifier, qml):
"""Update QML with embedded symbols.
param str identifer: Dataproduct ID
param str qml: QML XML string
"""
if not qml:
return qml
try:
# parse XML
root = ElementTree.fromstring(qml)
# embed symbols
self._embed_qml_symbols(root, 'SvgMarker', 'name')
self._embed_qml_symbols(root, 'SVGFill', 'svgFile')
self._embed_qml_symbols(root, 'RasterFill', 'imageFile')
# return updated QML
qml = ElementTree.tostring(root, encoding='utf-8', method='xml')
return qml.decode()
except Exception as e:
self.logger.warning(
"Could not embed QML symbols for dataproduct '%s':\n%s" %
(identifier, e))
return qml
def _embed_qml_symbols(self, root, layer_class, prop_key):
"""Embed symbol resources as base64 in QML.
:param xml.etree.ElementTree.Element root: XML root node
:param str layer_class: Symbol layer class
:param str prop_key: Symbol layer prop key for symbol path
"""
qgs_resources_dir = self.service_config.get('qgs_resources_dir', '')
for svgprop in root.findall(".//layer[@class='%s']/prop[@k='%s']" %
(layer_class, prop_key)):
symbol_path = svgprop.get('v')
path = os.path.abspath(os.path.join(qgs_resources_dir,
symbol_path))
# NOTE: assume symbols not included in ZIP are default symbols
if os.path.exists(path):
try:
# read symbol data and convert to base64
with open(path, 'rb') as f:
symbol_data = base64.b64encode(f.read())
# embed symbol in QML
svgprop.set('v', "base64:%s" % symbol_data.decode())
self.logger.info("Embed symbol in QML: %s" % symbol_path)
except Exception as e:
self.logger.warning("Could not embed QML symbol %s:\n%s" %
(symbol_path, e))
def precache_resources(self, session):
"""Precache some resources using eager loaded relations to reduce the
number of actual separate DB requests in later queries.
NOTE: The lookup tables do not have to be actually used, but remain
in memory, so SQLAlchemy will use the cached records.
E.g. accessing ows_layer_data.data_set_view afterwards won't generate
a separate DB query
:param Session session: DB session
"""
OWSLayerGroup = self.config_models.model('ows_layer_group')
OWSLayerData = self.config_models.model('ows_layer_data')
GroupLayer = self.config_models.model('group_layer')
DataSetView = self.config_models.model('data_set_view')
DataSet = self.config_models.model('data_set')
# precache OWSLayerData and eager load relations
ows_layer_data_lookup = {}
query = session.query(OWSLayerData)
query = query.options(
joinedload(OWSLayerData.data_set_view).joinedload(
DataSetView.data_set).joinedload(DataSet.data_source))
for layer in query.all():
ows_layer_data_lookup[layer.gdi_oid] = layer
# precache DataSetView and eager load attributes
query = session.query(DataSetView)
query = query.options(joinedload(DataSetView.attributes))
data_set_view_lookup = {}
for data_set_view in query.all():
data_set_view_lookup[data_set_view.gdi_oid] = data_set_view
# precache OWSLayerGroup and eager load sub layers
query = session.query(OWSLayerGroup)
query = query.options(
joinedload(OWSLayerGroup.sub_layers).joinedload(
GroupLayer.sub_layer))
ows_layer_group_lookup = {}
for group in query.all():
ows_layer_group_lookup[group.gdi_oid] = group
# NOTE: return precached resources so they stay in memory
return {
'ows_layer_data_lookup': ows_layer_data_lookup,
'data_set_view_lookup': data_set_view_lookup,
'ows_layer_group_lookup': ows_layer_group_lookup
}
class DocumentServiceConfig(ServiceConfig):
"""DocumentServiceConfig class
Generate Document service config.
"""
def __init__(self, config_models, logger):
"""Constructor
:param Logger logger: Logger
"""
super().__init__(
'document',
'https://github.com/qwc-services/qwc-document-service/raw/master/schemas/qwc-document-service.json',
logger)
self.config_models = config_models
self.permissions_query = PermissionsQuery(config_models, logger)
def config(self, service_config):
"""Return service config.
:param obj service_config: Additional service config
"""
config = super().config(service_config)
session = self.config_models.session()
resources = OrderedDict()
resources['document_templates'] = self._document_templates(session)
config['resources'] = resources
return config
def permissions(self, role):
"""Return service permissions for a role.
:param str role: Role name
"""
# NOTE: use ordered keys
permissions = OrderedDict()
# collect permissions from ConfigDB
session = self.config_models.session()
permissions['document_templates'] = \
self._document_template_permissions(role, session)
# collect feature reports
session.close()
return permissions
def _document_templates(self, session):
"""Return document service resources.
:param Session session: DB session
"""
templates = []
TemplateJasper = self.config_models.model('template_jasper')
query = session.query(TemplateJasper).order_by(TemplateJasper.name)
for template_obj in query.all():
# remove .jrxml extension from filename
report_filename = os.path.splitext(template_obj.report_filename)[0]
resource = {
'template': template_obj.name,
'report_filename': report_filename
}
templates.append(resource)
return templates
def _document_template_permissions(self, role, session):
"""Collect template permissions from ConfigDB.
:param str role: Role name
:param Session session: DB session
"""
permissions = []
TemplateJasper = self.config_models.model('template_jasper')
resource_ids = self.permissions_query.resource_ids(['template'], role,
session)
query = session.query(TemplateJasper).\
order_by(TemplateJasper.name).\
filter(TemplateJasper.gdi_oid.in_(resource_ids))
for template_obj in query.all():
permissions.append(template_obj.name)
return permissions