def get_sortkey(table):
"""Get a field to sort by
"""
# Just pick the first column in the table in alphabetical order.
# Ideally we would get the primary key from bcdc api, but it doesn't
# seem to be available
wfs = WebFeatureService(url=bcdata.OWS_URL, version="2.0.0")
return sorted(wfs.get_schema("pub:" + table)["properties"].keys())[0]
def test_schema_wfs_200():
wfs = WebFeatureService(
'https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288',
version='2.0.0')
schema = wfs.get_schema('footprint')
assert len(schema['properties']) == 4
assert schema['properties']['summary'] == 'string'
assert schema['geometry'] == '3D Polygon'
def index():
if not request.json:
abort(404)
body = request.json
print(body)
wfs = WebFeatureService(url=body['url'], version=body['version'])
response = wfs.get_schema(body['key'])
return jsonify(response)
def test_get_schema(self, mp_wfs_110, mp_remote_describefeaturetype):
"""Test the get_schema method for a standard schema.
Parameters
----------
mp_wfs_110 : pytest.fixture
Monkeypatch the call to the remote GetCapabilities request.
mp_remote_describefeaturetype : pytest.fixture
Monkeypatch the call to the remote DescribeFeatureType request.
"""
wfs110 = WebFeatureService(WFS_SERVICE_URL, version='1.1.0')
schema = wfs110.get_schema('dov-pub:Boringen')
def info(dataset, indent, meta_member):
"""Print basic metadata about a DataBC WFS layer as JSON.
Optionally print a single metadata item as a string.
"""
table = bcdata.validate_name(dataset)
wfs = WebFeatureService(url=bcdata.OWS_URL, version="2.0.0")
info = {}
info["name"] = table
info["count"] = bcdata.get_count(table)
info["schema"] = wfs.get_schema("pub:" + table)
if meta_member:
click.echo(info[meta_member])
else:
click.echo(json.dumps(info, indent=indent))
def test_schema_result(self, wfs_version):
"""Test whether the output from get_schema is a wellformed dictionary."""
wfs = WebFeatureService(WFS_SERVICE_URL, version=wfs_version)
schema = wfs.get_schema('dov-pub:Boringen')
assert isinstance(schema, dict)
assert 'properties' in schema or 'geometry' in schema
if 'geometry' in schema:
assert 'geometry_column' in schema
if 'properties' in schema:
assert isinstance(schema['properties'], dict)
assert 'required' in schema
assert isinstance(schema['required'], list)
def test_get_schema_typename_eq_attribute(
self, mp_wfs_110,
mp_remote_describefeaturetype_typename_eq_attribute):
"""Test the get_schema method for a schema where the typeName equals
one of the attributes.
Parameters
----------
mp_wfs_110 : pytest.fixture
Monkeypatch the call to the remote GetCapabilities request.
mp_remote_describefeaturetype : pytest.fixture
Monkeypatch the call to the remote DescribeFeatureType
request.
"""
wfs110 = WebFeatureService(WFS_SERVICE_URL, version='1.1.0')
schema = wfs110.get_schema('gw_varia:hhz')
def get_sortkey(table):
"""Check data for unique columns available for sorting paged requests
"""
wfs = WebFeatureService(url=bcdata.OWS_URL, version="2.0.0")
columns = list(wfs.get_schema("pub:" + table)["properties"].keys())
# use OBJECTID as default sort key, if present
if "OBJECTID" in columns:
return "OBJECTID"
# if OBJECTID is not present (several GSR tables), use SEQUENCE_ID
elif "SEQUENCE_ID" in columns:
return "SEQUENCE_ID"
# otherwise, it should be safe to presume first column is the primary key
# (WHSE_FOREST_VEGETATION.VEG_COMP_LYR_R1_POLY's FEATURE_ID appears to be
# the only public case, and very large veg downloads are likely better
# accessed via some other channel)
else:
return columns[0]
def test_schema_result(self, mp_wfs_110, mp_remote_describefeaturetype):
"""Test whether the output from get_schema is a wellformed dictionary.
Parameters
----------
mp_wfs_110 : pytest.fixture
Monkeypatch the call to the remote GetCapabilities request.
mp_remote_describefeaturetype : pytest.fixture
Monkeypatch the call to the remote DescribeFeatureType request.
"""
wfs110 = WebFeatureService(WFS_SERVICE_URL, version='1.1.0')
schema = wfs110.get_schema('dov-pub:Boringen')
assert isinstance(schema, dict)
assert 'properties' in schema or 'geometry' in schema
if 'geometry' in schema:
assert 'geometry_column' in schema
if 'properties' in schema:
assert isinstance(schema['properties'], dict)
assert 'required' in schema
assert isinstance(schema['required'], list)
def test_get_schema(self, wfs_version):
"""Test the get_schema method for a standard schema."""
wfs = WebFeatureService(WFS_SERVICE_URL, version=wfs_version)
schema = wfs.get_schema('dov-pub:Boringen')
class GeoServer(Server):
"""
Represents a server running GeoServer [1], an application that provides access to layers.
This class provides a concrete implementation of the more generic Server component (which is intentionally generic).
Currently the Server class does not dictate an interface for accessing resources but this class aims to present
GeoServer specific components (such as workspaces) as generic components (such as namespaces).
GeoServer instances typically represent individual instances (i.e. hosts are servers) rather than a wider and more
abstract platform offered by a service provider.
Information on layers and other resources are fetched using a combination of the GeoServer specific administrative
API [2] accessed through geoserver-restconfig [3] and OGC services accessed through OWSLib [4] (and currently
limited to WMS and WFS).
[1] https://geoserver.readthedocs.io/en/latest/
[2] https://geoserver.readthedocs.io/en/latest/rest/index.html
[3] https://pypi.org/project/geoserver-restconfig
[4] https://pypi.org/project/OWSLib/
"""
def __init__(
self,
server_id: str,
label: str,
hostname: str,
port: str,
api_path: str,
wms_path: str,
wfs_path: str,
username: str,
password: str,
):
"""
:param server_id: unique identifier, typically a ULID (Universally Unique Lexicographically Sortable Identifier)
:param label: a human readable, well-known, identifier for the server - typically based on the hostname
:param hostname: servers fully qualified hostname
:param port: port on which GeoServer is running (usually '80' or '8080')
:param api_path: URL path, relative to the root of the server, to the GeoServer API (usually '/geoserver/rest')
:param wms_path: URL path, relative to the root of the server, to the GeoServer WMS endpoint (usually
'/geoserver/ows?service=wms&version=1.3.0&request=GetCapabilities')
:param wfs_path: URL path, relative to the root of the server, to the GeoServer WFS endpoint (usually
'/geoserver/ows?service=wfs&version=2.0.0&request=GetCapabilities')
:param username: username for account to use for GeoServer API
:param password: password for account to use for GeoServer API
"""
endpoint = build_base_data_source_endpoint(data_source={
"hostname": hostname,
"port": port
})
self.client = Catalogue(service_url=f"{endpoint}{api_path}",
username=username,
password=password)
self.wms = WebMapService(url=f"{endpoint}{wms_path}",
version="1.3.0",
username=username,
password=password)
self.wfs = WebFeatureService(url=f"{endpoint}{wfs_path}",
version="2.0.0",
username=username,
password=password)
super().__init__(
server_id=server_id,
label=label,
hostname=hostname,
server_type=ServerType.GEOSERVER.value,
version=self._get_geoserver_version(),
)
def get_namespaces(self) -> List[str]:
"""
Gets all GeoServer workspace names as Namespace labels
:return: list of Namespace labels
"""
workspaces = []
for workspace in self.client.get_workspaces():
workspaces.append(workspace.name)
return workspaces
def get_namespace(self, namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific workspace as a Namespace
Note: GeoServer workspaces do not support the concept of a title, a static substitute value is therefore used
Note: GeoServer workspaces do support the concept of a namespace, but it is not yet implemented [#28]
:param namespace_reference: Namespace (workspace) label (name)
:return: dictionary of Namespace information that can be made into a Namespace object
"""
workspace = self.client.get_workspace(name=namespace_reference)
if workspace is None:
raise KeyError(
f"Namespace [{namespace_reference}] not found in server [{self.label}]"
)
return {"label": workspace.name, "title": "-", "namespace": "-"}
def get_repositories(self) -> List[Tuple[str, str]]:
"""
Gets all GeoServer store names as Repository labels
:return: list of Repository:Namespace label tuples
"""
stores = []
# Passing workspaces here is a workaround for a bug in the get stores method where workspaces aren't specified.
# The method says all workspaces should be checked but the logic to do this is in the wrong place so none are.
for store in self.client.get_stores(
workspaces=self.client.get_workspaces()):
stores.append((store.name, store.workspace.name))
return stores
def get_repository(self, repository_reference: str,
namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific store as a Repository
If a Namespace (workspace) label is specified the Repository must exist within that Namespace.
GeoServer store types are sometimes unsuitable or non-standard and so need to be mapped to a conventional value.
in the RepositoryType enum using the GeoServerRepositoryType enum.
Note: GeoServer stores do not support the concept of a title, a static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param repository_reference: Repository (store) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of repository information that can be made into a Repository object
"""
_store = self.client.get_store(name=repository_reference,
workspace=namespace_reference)
if _store is None:
raise KeyError(
f"Repository [{repository_reference}] not found in server [{self.label}]"
)
store = {
"label":
_store.name,
"title":
"-",
"repository_type":
RepositoryType[GeoServerRepositoryType(str(
_store.type).lower()).name].value,
"namespace_label":
_store.workspace.name,
}
if hasattr(_store, "description") and _store.description is not None:
store["title"] = _store.description
if (store["repository_type"] == RepositoryType.POSTGIS.value
or store["repository_type"] == RepositoryType.ORACLE.value):
store["hostname"] = _store.connection_parameters["host"]
store["database"] = _store.connection_parameters["database"]
store["schema"] = _store.connection_parameters["schema"]
return store
def get_styles(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer style names as Style labels
Python's None value will be used to represent the Namespace of global styles (i.e that don't have a Namespace
(workspace)).
:return: list of Style:Namespace label tuples
"""
styles = []
for _style in self.client.get_styles():
styles.append((_style.name, _style.workspace))
return styles
def get_style(self,
style_reference: str,
namespace_reference: str = None) -> Dict[str, str]:
"""
Gets a specific style as a Style
If a Namespace (workspace) label is specified the Style must exist within that Namespace.
Note: GeoServer styles do support the concept of a title, but it is not exposed through the admin API so a
static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param style_reference: Style (style) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of style information that can be made into a Style object
"""
_style = self.client.get_style(name=style_reference,
workspace=namespace_reference)
_type = str(_style.style_format).lower()
if _type == "sld10":
_type = "sld"
style = {
"label": _style.name,
"title": "-",
"style_type": _type,
}
if hasattr(_style, "workspace") and _style.workspace is not None:
style["namespace_label"] = _style.workspace
return style
def get_layers(self) -> List[str]:
"""
Gets all GeoServer layer names as Layer labels
:return: list of Layer labels
"""
layers = []
for _layer in self.client.get_layers():
layers.append(_layer.name)
return layers
def get_layer(
self, layer_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer as a Layer
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_reference: Layer (layer) label (name)
:return: dictionary of layer information that can be made into a Layer object
"""
_layer = self.client.get_layer(name=layer_reference)
layer = {
"label":
_layer.resource.name,
"title":
_layer.resource.title,
"layer_type":
str(_layer.type).lower(),
"geometry_type":
None,
"services": [],
"table_view":
None,
"namespace_label":
_layer.resource.workspace.name,
"repository_label":
_layer.resource.store.name,
"style_labels":
[(_layer.default_style.name, _layer.default_style.workspace)],
}
if layer_reference in list(
self.wms.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wms.contents):
layer["services"].append(LayerService.WMS.value)
if layer_reference in list(
self.wfs.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wfs.contents):
layer["services"].append(LayerService.WFS.value)
# WFS lookups don't seem to mind if the layer is namespaced or not
_properties = self.wfs.get_schema(layer_reference)
if "geometry" in _properties and isinstance(
_properties["geometry"], str):
try:
layer["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] for layer {layer_reference} not mapped to "
f"LayerGeometry enum.")
elif "properties" in _properties:
for geometry_column_name in GeoServerGeometryColumnNames:
if geometry_column_name.value in _properties[
"properties"].keys():
try:
layer["geometry_type"] = LayerGeometry[
GeoPropertyGeoServerLayerGeom(
str(_properties["properties"][
geometry_column_name.value])
).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['properties'][geometry_column_name.value]}] for layer "
f"{layer_reference} in column '{geometry_column_name.value}' not mapped to "
f"LayerGeometry enum.")
if (str(_layer.resource.store.type).lower()
== RepositoryType.POSTGIS.value
or str(_layer.resource.store.type).lower()
== RepositoryType.ORACLE.value):
layer["table_view"] = _layer.resource.native_name
return layer
def get_layer_groups(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer layer group names as LayerGroup labels
Python's None value will be used to represent the Namespace of global layer groups (i.e that don't have a
Namespace (workspace)).
:return: list of LayerGroup:Namespace label tuples
"""
layer_groups = []
for _layer_group in self.client.get_layergroups(
workspaces=self.client.get_workspaces()):
layer_groups.append((_layer_group.name, _layer_group.workspace))
return layer_groups
def get_layer_group(
self, layer_group_reference: str, namespace_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer group as a LayerGroup
If a Namespace (workspace) label is specified the LayerGroup must exist within that Namespace.
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_group_reference: LayerGroup (layer group) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of layer group information that can be made into a LayerGroup object
"""
_layer_group = self.client.get_layergroup(
name=layer_group_reference, workspace=namespace_reference)
layer_group = {
"label": _layer_group.name,
"title": _layer_group.title,
"services": [],
"namespace_label": _layer_group.workspace,
"layer_labels": [],
"style_labels": [],
}
for layer_label in _layer_group.layers:
layer_label = layer_label.split(":")
if len(layer_label) == 2:
layer_group["layer_labels"].append(
(layer_label[1], layer_label[0]))
elif len(layer_label) == 1:
layer_group["layer_labels"].append((layer_label[0], None))
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wms.contents):
layer_group["services"].append(LayerService.WMS.value)
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wfs.contents):
layer_group["services"].append(LayerService.WFS.value)
_properties = self.wfs.get_schema(
f"{namespace_reference}:{layer_group_reference}")
try:
layer_group["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] not mapped to LayerGeometry enum."
)
for style_label in _layer_group.styles:
if style_label is not None:
style_label = style_label.split(":")
if len(style_label) == 2 and (
style_label[1],
style_label[0]) not in layer_group["style_labels"]:
layer_group["style_labels"].append(
(style_label[1], style_label[0]))
if len(style_label) == 1 and (
style_label[0],
None) not in layer_group["style_labels"]:
layer_group["style_labels"].append((style_label[0], None))
return layer_group
def _get_geoserver_version(self) -> str:
"""
Gets the GeoServer version
:return: GeoServer version string
"""
return self.client.get_version()
class WfsConnect(DbConnect):
"""connecteur wwfs: simule un acces base a partir du schema"""
def __init__(self,
serveur,
base,
user,
passwd,
debug=0,
system=False,
params=None,
code=None):
super().__init__(serveur, base, user, passwd, debug, system, params,
code)
importer()
self.types_base.update(TYPES_A)
self.type_base = "wfs"
self.tablelist = []
self.connect()
self.geographique = True
self.accept_sql = "no"
self.curtable = ""
self.curnb = 0
def connect(self):
"""effectue un getcapabilities pour connaitre le schema"""
try:
print("connection wfs", self.serveur)
if "version=" in self.serveur:
serveur, vdef = self.serveur.split(" ", 1)
version = vdef.split("=")[1]
else:
serveur = self.serveur
version = "1.1.0"
self.connection = WebFeatureService(url=serveur, version=version)
self.connection.cursor = lambda: None
# simulation de curseur pour l'initialisation
except Error as err:
print("erreur wfs", err)
return False
self.tablelist = [
tuple(i.split(":", 1) if ":" in i else ["", i])
for i in self.connection.contents
]
# print("retour getcap", len(self.tablelist))
def commit(self):
pass
def get_tables(self):
""" retourne la liste des tables """
return list(self.tables.values())
@property
def rowcount(self):
return -1
def get_attr_of_classe(self, schemaclasse):
"""recupere la description d une classe"""
ident = schemaclasse.identclasse
print("analyse classe", ident)
groupe, nom = ident
wfsid = ":".join(ident)
schemadef = self.connection.get_schema(wfsid)
# print("recup attlist ", attdict)
del schemaclasse.attributs["__pending"]
if schemadef is None:
print("schema non present sur le serveur", ident)
return
attdict = schemadef["properties"]
if attdict is not None:
for nom_att, xmltype in attdict.items():
# print(nom_att, xmltype)
pyetltype = ALLTYPES.get(xmltype)
if pyetltype is None:
print(" type inconnu", xmltype)
pyetltype = "T"
schemaclasse.stocke_attribut(nom_att, pyetltype)
type_geom = schemadef.get("geometry")
if type_geom:
if type_geom in TYPES_G:
type_geom = TYPES_G[type_geom]
else:
print("geometrie inconnue", type_geom)
type_geom = "-1"
nom_geom = schemadef["geometry_column"]
dimension = 2
schemaclasse.stocke_geometrie(type_geom,
dimension=dimension,
srid="3948",
multiple=1,
nom=nom_geom)
def get_attributs(self):
"""description des attributs de la base sqlite
structure fournie :
nom_groupe;nom_classe;nom_attr;alias;type_attr;graphique;multiple;\
defaut;obligatoire;enum;dimension;num_attribut;index;unique;clef_primaire;\
clef_etrangere;cible_clef;taille;decimales"""
attlist = []
tables = self.tablelist
print("webservices: lecture tables", tables)
for groupe, nom in tables:
att = self.attdef(
groupe,
nom,
"__pending",
self.get_attr_of_classe,
"T",
"",
"",
"",
"",
"",
2,
0,
"",
"",
"",
"",
"",
"",
0,
0,
)
attlist.append(att)
ident = (groupe, nom)
nouv_table = [groupe, nom, "", "", "", -1, "", "", "", "", ""]
# print ('table', nouv_table)
self.tables[ident] = nouv_table
return attlist
def get_enums(self):
return ()
def get_type(self, nom_type):
if nom_type in TYPES_G:
return nom_type
return self.types_base.get(nom_type.upper(), "?")
def get_cursinfo(self, volume=0, nom="", regle=None):
"""recupere un curseur"""
# print(" postgres get cursinfo")
return (WfsCursinfo(self, volume=volume, nom=nom, regle=regle)
if self.connection else None)
def get_surf(self, nom):
return ""
def get_perim(self, nom):
return ""
def get_long(self, nom):
return ""
def get_geom(self, nom):
return ""
def set_geom(self, geom, srid):
return ""
def set_geomb(self, geom, srid, buffer):
return ""
def set_limit(self, maxi, _):
if maxi:
return "maxFeatures=" + str(maxi)
return ""
def cond_geom(self, nom_fonction, nom_geometrie, geom2):
cond = ""
fonction = ""
if nom_fonction == "dans_emprise":
bbox = getbbox(geom2)
return bbox
return ""
def req_alpha(self,
ident,
schema,
attribut,
valeur,
mods,
maxi=0,
ordre=None):
"""recupere les elements d'une requete alpha"""
niveau, classe = ident
requete = ""
data = ""
schema.resolve()
attlist = schema.get_liste_attributs()
self.get_attr_of_classe
params = {"typename": niveau + ":" + classe}
if attribut:
filter = F.PropertyIsLike(propertyname=attribut,
literal=valeur,
wildCard="*")
filterxml = etree.tostring(filter.toXML(), encoding="unicode")
params["filter"] = filterxml
print("envoi requete", params)
# reponse = self.connection.getfeature(**params)
reponse = self.connection.getfeature(typename=niveau + ":" + classe)
print("wfs apres reponse", type(reponse))
return reponse
class GeoCoder(WfsFilter):
"""
geocoder
Default settings
----------------
wfs_ver - 1.1.0 default!!!
wfs_timeout = None/int sec - max timeout to response wfs server
out_geom - (Default - None - json)|gml|wkt
"""
wfs_ver = '1.1.0'
wfs_timeout = None
out_geom = None
#----------------------------------------------------------------------
def __init__(self, url='http://localhost:3007', map_name='', debug=False):
WfsFilter.__init__(self)
if map_name:
wfs_url = u"{0}/{1}".format(url, map_name)
else:
wfs_url = url
self.debug = debug
self.map_name_use = map_name
self.wfs_args = {
"url": wfs_url,
"version": self.wfs_ver,
}
if isinstance(self.wfs_timeout, int):
self.wfs_args["timeout"] = self.wfs_timeout
try:
self.wfs = WebFeatureService(**self.wfs_args)
except Exception as err:
raise Exception(u"WFS is not support in '{0}'\n{1}".format(
wfs_url, err))
else:
self.capabilities = None
self.get_capabilities()
self.info = None
self.get_info()
self._set_def_resp_params()
def _set_def_resp_params(self):
self.epsg_code_cap = self.capabilities["epsg_code"]
self.epsg_code_use = None
self.layer_property_cap = self.capabilities["layer_property"]
self.layer_property_use = None
self.geom_property_cap = None
self.response = []
def echo2json(self, dict_):
print json.dumps(
dict_,
sort_keys=True,
indent=4,
separators=(',', ': '),
ensure_ascii=False,
)
def create_json_crs(self, crs_string):
crs_string = crs_string.split(":")
if len(crs_string) == 2:
if crs_string[0].lower() == "epsg":
return {
"type": "EPSG",
"properties": {
"code": crs_string[-1],
},
}
def gml2json(self, gml):
json_out = {
"type": "FeatureCollection",
"features": [],
}
geom_crs = None
tree = etree.fromstring(gml)
nsmap = tree.nsmap
tag_name = lambda t: t.tag.split("{%s}" % nsmap[t.prefix])[-1]
for feature in tree.getiterator("{%s}featureMember" % nsmap["gml"]):
json_feature = {
"type": "Feature",
"properties": {},
"geometry": None,
}
for layer in feature.iterfind('{%s}*' % nsmap["ms"]):
json_feature["properties"]["layer"] = tag_name(layer)
wfs_id = layer.get("{%s}id" % nsmap["gml"], None)
if wfs_id and self.map_name_use:
wfs_id = u"{0}.{1}".format(self.map_name_use, wfs_id)
json_feature["properties"]["id"] = wfs_id
for prop in layer.iterfind('{%s}*' % nsmap["ms"]):
get_prop = True
for geom in prop.iterfind("{%s}*" % nsmap["gml"]):
get_prop = False
geom_crs = geom.get("srsName", None)
ogr_geom = ogr.CreateGeometryFromGML(
etree.tostring(geom))
if isinstance(ogr_geom, ogr.Geometry):
json_feature["geometry"] = json.loads(
ogr_geom.ExportToJson())
if geom_crs:
json_feature["geometry"][
"crs"] = self.create_json_crs(geom_crs)
if self.out_geom:
ogr_geom = ogr.CreateGeometryFromJson(
str(
json.dumps(json_feature["geometry"],
ensure_ascii=False)))
if self.out_geom == "gml":
json_feature[
"geometry"] = ogr_geom.ExportToGML()
elif self.out_geom == "wkt":
json_feature[
"geometry"] = ogr_geom.ExportToWkt()
else:
raise Exception(
'out_geom="{} is not valid (None,gml,wkt)use"'
.format(self.out_geom))
if get_prop:
json_feature["properties"][tag_name(prop)] = prop.text
json_out["features"].append(json_feature)
if geom_crs:
json_out["crs"] = self.create_json_crs(geom_crs)
if self.debug:
self.echo2json(json_out)
return json_out
def get_help(self):
filter_tags = {my: self.filter_tags[my]() for my in self.filter_tags}
comparsion_opts = {
my: self.filter_opts[my]()
for my in self.filter_opts
if not isinstance(self.filter_opts[my](), dict)
}
spatial_opts = {
my: self.filter_opts[my]()
for my in self.filter_opts
if isinstance(self.filter_opts[my](), dict)
}
json_out = {
"filter": {
"tags": filter_tags,
"comparsion opts": comparsion_opts,
"spatial opts": spatial_opts,
"example": {
"tag": [
{
"tag": {
"property 1": {
"comparsion opt": "value",
},
"property 2": {
"comparsion opt 1": "value",
"comparsion opt 2": ["value 1", "value 2"],
"spatial opt": {
"spatial opt key 1": "value",
"spatial opt key 2": "value",
},
},
},
},
{
"any key": {
"spatial opt": {
"spatial opt key 1": "value",
"spatial opt key 2": "value",
},
},
},
],
},
},
}
if self.debug:
self.echo2json(json_out)
return json_out
def get_info(self):
if self.info is None:
json_out = {}
for layer_name in self.wfs.contents:
if self.wfs.contents[layer_name].metadataUrls:
wfs_opts = self.wfs.contents[layer_name].metadataUrls[0]
wfs_opts["gml"] = self.wfs.contents[
layer_name].outputFormats[0]
else:
wfs_opts = None
json_out[layer_name] = {
"wgs84_bbox":
list(self.wfs.contents[layer_name].boundingBoxWGS84),
"wfs_opts":
wfs_opts,
}
if self.debug:
self.echo2json(json_out)
self.info = json_out
return self.info
def get_capabilities(self):
if self.capabilities is None:
json_out = {
"max_features": None,
"filter": None,
"layers": {},
}
all_epsg_code = None
all_layer_property = None
all_geom_property = None
for layer_name in self.wfs.contents:
layer_schema = self.wfs.get_schema(layer_name)
if layer_schema:
geom_property = layer_schema['geometry_column']
layer_property = []
layer_property.append(geom_property)
layer_property += layer_schema['properties'].keys()
epsg_code = [
my.code
for my in self.wfs.contents[layer_name].crsOptions
]
json_out['layers'][layer_name] = {
"epsg_code": epsg_code,
"layer_property": layer_property,
"geom_property": geom_property,
"max_features": None,
"filter": None,
}
if all_layer_property is None:
all_layer_property = layer_property
else:
all_layer_property = list(
set(all_layer_property).intersection(
set(layer_property)))
if all_geom_property is None:
all_geom_property = geom_property
elif all_geom_property is not False:
if all_geom_property != geom_property:
all_geom_property = False
if all_epsg_code is None:
all_epsg_code = epsg_code
else:
all_epsg_code = list(
set(all_epsg_code).intersection(set(epsg_code)))
json_out.update({
"epsg_code": all_epsg_code,
"layer_property": all_layer_property,
"geom_property": all_geom_property,
})
if self.debug:
self.echo2json(json_out)
self.capabilities = json_out
return self.capabilities
def get_owslib_args(self, layer_name, filter_json=None, **kwargs):
feature_args = ["propertyname", "maxfeatures", "srsname"]
out_args = {
"typename": layer_name,
}
if isinstance(filter_json, dict):
out_args["filter"] = self.filter_engine(filter_json)
for arg in feature_args:
if kwargs.get(arg, None):
out_args[arg] = kwargs[arg]
return out_args
def merge_gjson(self, gjson):
if not isinstance(self.response, list) or not isinstance(gjson, dict):
return
elif not gjson['features']:
return
elif not self.response:
self.response = [gjson]
else:
gj = {}
gj_test_fea = copy.deepcopy(gjson['features'][0])
gj['props'] = set(gj_test_fea['properties'].keys())
if isinstance(gj_test_fea['geometry'], dict):
gj['crs'] = copy.deepcopy(gjson['crs'])
gj['geom'] = gj_test_fea['geometry']['type']
else:
gj['crs'] = None
gj['geom'] = None
merge = False
for lst_element in self.response:
if isinstance(lst_element, dict):
lst = {}
lst_test_fea = copy.deepcopy(lst_element['features'][0])
lst['props'] = set(lst_test_fea['properties'].keys())
if isinstance(lst_test_fea['geometry'], dict):
lst['crs'] = copy.deepcopy(lst_element['crs'])
lst['geom'] = lst_test_fea['geometry']['type']
else:
lst['crs'] = None
lst['geom'] = None
if gj == lst:
index = self.response.index(lst_element)
self.response[index]['features'].extend(
gjson['features'])
merge = True
break
if not merge:
self.response.append(gjson)
def get_response(self, request_):
def_com_opts = {
"layer_name": [u"{}", "layer"],
"filter_json": [{}, "filter"],
"propertyname": [[], "layer_property"],
"maxfeatures": [u"{}", "max_features"],
"srsname": [u"EPSG:{}", "epsg_code"],
}
self._set_def_resp_params() # start response
capabilities = copy.deepcopy(self.capabilities)
cap_layers = {my: {} for my in capabilities["layers"]}
req_layers = {
my: request_.get("layers", cap_layers)[my]
for my in request_.get("layers", cap_layers)
if cap_layers.has_key(my)
}
req_opts = copy.deepcopy(request_)
if req_opts.has_key("layers"):
del (req_opts["layers"])
all_opts = []
for layer in req_layers:
capabilities['layers'][layer]["layer"] = None
layer_opts = {"layer": layer}
layer_opts.update(req_opts)
if isinstance(req_layers[layer], dict):
layer_opts.update(req_layers[layer])
com_opts = copy.deepcopy(def_com_opts)
for opt in com_opts:
cap_param = capabilities['layers'][layer][def_com_opts[opt][1]]
param = layer_opts.get(com_opts[opt][1], None)
if cap_param:
# test param from capabilites
cap_param = [self.data2literal(my) for my in cap_param]
if isinstance(param, list):
param = [self.data2literal(my) for my in param]
param = list(set(param).intersection(set(cap_param)))
elif isinstance(param, (str, unicode, int, float)):
param = self.data2literal(param)
if param not in cap_param:
param = None
if param:
if isinstance(com_opts[opt][0], unicode):
if isinstance(param, (str, unicode, int, float)):
com_opts[opt] = com_opts[opt][0].format(param)
elif isinstance(com_opts[opt][0], (dict)):
if isinstance(param, dict):
com_opts[opt] = copy.deepcopy(param)
elif isinstance(com_opts[opt][0], list):
if isinstance(param, list):
com_opts[opt] = copy.deepcopy(param)
else:
com_opts[opt] = None
else:
com_opts[opt] = None
# data for use in filter
self.epsg_code_cap = capabilities["layers"][layer]["epsg_code"]
self.epsg_code_use = layer_opts.get(
"epsg_code", capabilities["layers"][layer]["epsg_code"][0])
self.layer_property_cap = capabilities["layers"][layer][
"layer_property"]
self.geom_property_cap = capabilities["layers"][layer][
"geom_property"]
self.layer_property_use = layer_opts.get(
"layer_property",
capabilities["layers"][layer]["layer_property"])
all_opts.append(self.get_owslib_args(**com_opts))
#return gjson & merge
multy_layer = MultyLayer(self.wfs_args, all_opts)
for gml_layer in multy_layer():
self.merge_gjson(self.gml2json(gml_layer))
del multy_layer
resp_out = copy.deepcopy(self.response)
self._set_def_resp_params() # end response
if len(resp_out) == 0:
return {}
elif len(resp_out) == 1:
return resp_out[0]
else:
return resp_out
def get_properties(self):
disable_prop = ["layer", "id", "osm_id"]
out = {}
layers = self.capabilities["layers"].keys()
for layer in layers:
out[layer] = {}
req = {
"layers": {
layer: None,
},
}
resp = self.get_response(req)
prop_list = resp["features"][0]["properties"].keys()
for prop in prop_list:
if prop not in disable_prop:
out[layer][prop] = list(
set([
my["properties"][prop] for my in resp["features"]
]))
return out
def __call__(self, *args, **kwargs):
return self.get_response(*args, **kwargs)
class WFS():
def __init__(self, url, version):
self.wfs = WebFeatureService(url=url, version=version)
def set_collection(self, collection):
collection_exist = self._check_collection(collection)
if collection_exist:
self.collection = collection
return collection_exist
def _check_collection(self, collection):
feature_types = list(self.wfs.contents)
collection_exist = True if collection in feature_types else False
return collection_exist
def get_schema(self):
if hasattr(self, 'collection'):
return self.wfs.get_schema(self.collection)
else:
return None
def make_request(self,
max_dataset=100,
sort_by=None,
start_index=0,
constraint=None):
output_format = self._get_outputformat()
result = self.wfs.getfeature(typename=self.collection,
filter=constraint,
maxfeatures=max_dataset,
sortby=sort_by,
startindex=start_index,
outputFormat=output_format)
result = result.read()
if 'json' in output_format:
return json.loads(result)
else:
return result
def get_request(self,
max_dataset=None,
sort_by=None,
start_index=0,
constraint=None):
output_format = self._get_outputformat()
result = self.wfs.getGETGetFeatureRequest(typename=self.collection,
filter=constraint,
maxfeatures=max_dataset,
sortby=sort_by,
startindex=start_index,
outputFormat=output_format)
return result
def _get_outputformat(self):
getfeature_param = self.wfs.getOperationByName('GetFeature').parameters
allowed_output_format = getfeature_param["outputFormat"]["values"]
for output_format in OUTPUT_FORMAT:
if output_format in allowed_output_format:
return output_format
return None
def set_filter_equal_to(self, propertyname, value):
constraint = PropertyIsLike(propertyname=propertyname, literal=value)
filterxml = etree.tostring(constraint.toXML()).decode("utf-8")
return filterxml
def define_request(
dataset,
query=None,
crs="epsg:4326",
bounds=None,
bounds_crs="EPSG:3005",
sortby=None,
pagesize=10000,
):
"""Define the getfeature request parameters required to download a dataset
References:
- http://www.opengeospatial.org/standards/wfs
- http://docs.geoserver.org/stable/en/user/services/wfs/vendor.html
- http://docs.geoserver.org/latest/en/user/tutorials/cql/cql_tutorial.html
"""
# validate the table name and find out how many features it holds
table = validate_name(dataset)
n = bcdata.get_count(table, query=query)
wfs = WebFeatureService(url=bcdata.OWS_URL, version="2.0.0")
geom_column = wfs.get_schema("pub:" + table)["geometry_column"]
# DataBC WFS getcapabilities says that it supports paging,
# and the spec says that responses should include 'next URI'
# (section 7.7.4.4.1)....
# But I do not see any next uri in the responses. Instead of following
# the paged urls, for datasets with >10k records, just generate urls
# based on number of features in the dataset.
chunks = math.ceil(n / pagesize)
# if making several requests, we need to sort by something
if chunks > 1 and not sortby:
sortby = get_sortkey(table)
# build the request parameters for each chunk
param_dicts = []
for i in range(chunks):
request = {
"service": "WFS",
"version": "2.0.0",
"request": "GetFeature",
"typeName": table,
"outputFormat": "json",
"SRSNAME": crs,
}
if sortby:
request["sortby"] = sortby
# build the CQL based on query and bounds
# (the bbox param shortcut is mutually exclusive with CQL_FILTER)
if query and not bounds:
request["CQL_FILTER"] = query
if bounds:
b0, b1, b2, b3 = [str(b) for b in bounds]
bnd_query = f"bbox({geom_column}, {b0}, {b1}, {b2}, {b3}, '{bounds_crs}')"
if not query:
request["CQL_FILTER"] = bnd_query
else:
request["CQL_FILTER"] = query + " AND " + bnd_query
if chunks > 1:
request["startIndex"] = i * pagesize
request["count"] = pagesize
param_dicts.append(request)
return param_dicts
from owslib.wfs import WebFeatureService
from owslib.util import Authentication
from pprint import pprint
import geopandas as gpd
wfs = WebFeatureService(
'https://geoservicos.cprm.gov.br/geoserver/geoquimica/wfs',
version='1.1.0',
auth=Authentication(verify=False))
pprint([operation.name for operation in wfs.operations])
pprint(list(wfs.contents))
pprint(wfs.get_schema('geoquimica:novo-fcampo'))
response = wfs.getfeature(typename='geoquimica:novo-fcampo',
bbox=(-41.26, -12.82, -40.88, -12.52),
srsname='urn:x-ogc:def:crs:EPSG:4326')
#with open('/tmp/geoquimica.gml', 'wb') as output:
# output.write(response.read())
df = gpd.read_file(response)
print(df.head())
