def patch_well_known_namespaces(etree_module): import warnings from owslib.namespaces import Namespaces ns = Namespaces() """Monkey patches the etree module to add some well-known namespaces.""" try: register_namespace = etree_module.register_namespace except AttributeError: try: etree_module._namespace_map def register_namespace(prefix, uri): etree_module._namespace_map[uri] = prefix except AttributeError: def register_namespace(prefix, uri): pass warnings.warn( "Only 'lxml.etree' >= 2.3 and 'xml.etree.ElementTree' >= 1.3 are fully supported!" ) for k, v in six.iteritems(ns.get_namespaces()): register_namespace(k, v)
def patch_well_known_namespaces(etree_module): from owslib.namespaces import Namespaces ns = Namespaces() """Monkey patches the etree module to add some well-known namespaces.""" etree_module._namespace_map.update(ns.get_namespaces())
def load_naps(self): ns = Namespaces() gmd = ns.get_namespace('gmd') session = connect_to_database() for napid in self.napids: print '{0}Full NAP Record for {1}{2}'.format( Fore.GREEN, Fore.CYAN, napid) self.csw.getrecordbyid(id=[napid], outputschema=gmd) ec_rec = find_record_by_uuid(session, napid, query_class=ECRecord) if ec_rec is None: ec_rec = ECRecord( uuid=self.csw.records[napid].identifier, title=self.csw.records[napid].identification.title, state='active', nap_record=self.csw.records[napid].xml, csw_scanned=datetime.now().isoformat()) else: ec_rec.title = self.csw.records[napid].identification.title, ec_rec.state = 'active', ec_rec.nap_record = self.csw.records[napid].xml, ec_rec.csw_scanned = datetime.now().isoformat() add_record(session, ec_rec) session.close_all()
def get_namespaces(): n = Namespaces() ns = n.get_namespaces( ["fes", "ogc", "om", "gml32", "sa", "sml", "swe20", "swes", "xlink"]) ns["ows"] = n.get_namespace("ows110") ns["sos"] = n.get_namespace("sos20") return ns
def get_namespaces(self): """ """ n = Namespaces() namespaces = n.get_namespaces(["sml", "gml", "xlink", "swe"]) namespaces["ism"] = "urn:us:gov:ic:ism:v2" return namespaces
def patch_well_known_namespaces(etree_module): from owslib.namespaces import Namespaces ns = Namespaces() """Monkey patches the etree module to add some well-known namespaces.""" for k, v in ns.get_namespaces().iteritems(): etree_module.register_namespace(k, v)
def get_namespaces(): """ Returns specified namespaces using owslib Namespaces function. """ n = Namespaces() ns = n.get_namespaces( ["gco", "gmd", "gml", "gml32", "gmx", "gts", "srv", "xlink", "dc"]) return ns
def patch_well_known_namespaces(etree_module): from owslib.namespaces import Namespaces ns = Namespaces() """Monkey patches the etree module to add some well-known namespaces.""" for k,v in ns.get_namespaces().iteritems(): etree_module.register_namespace(k, v)
def __init__(self, catalogue_url: str): self._catalogue_url = catalogue_url self._catalogue = None self._catalogue_service = None self._namespaces = Namespaces()
def __get_namespaces(): """Get default namespaces from OWSLib, extended with the 'gfc' namespace to be able to parse feature catalogues.""" n = Namespaces() ns = n.get_namespaces() ns[None] = n.get_namespace("gmd") ns['gfc'] = 'http://www.isotc211.org/2005/gfc' return ns
def get_namespaces(): n = Namespaces() ns = n.get_namespaces([ "fes", "gml32", "ogc", "om20", "sa", "sml", "swe20", "swes", "wml2", "xlink", "xsi" ]) ns["gda"] = 'http://www.opengis.net/sosgda/1.0' ns["ns"] = "http://www.opengis.net/gml/3.2" ns["ows"] = n.get_namespace("ows110") ns["sams"] = "http://www.opengis.net/samplingSpatial/2.0" ns["sf"] = "http://www.opengis.net/sampling/2.0" ns["sos"] = n.get_namespace("sos20") return ns
class IOOSSOSGCCheck(BaseSOSGCCheck, IOOSBaseCheck): # set up namespaces for XPath ns = Namespaces().get_namespaces(['sos', 'gml', 'xlink']) ns['ows'] = Namespaces().get_namespace('ows110') @check_has(BaseCheck.HIGH) def check_high(self, ds): return [] @check_has(BaseCheck.MEDIUM) def check_recommended(self, ds): return [ ('service_contact_email', XPath( "/sos:Capabilities/ows:ServiceProvider/ows:ServiceContact/ows:ContactInfo/ows:Address/ows:ElectronicMailAddress", namespaces=self.ns)), ('service_contact_name', XPath( "/sos:Capabilities/ows:ServiceProvider/ows:ServiceContact/ows:IndividualName", namespaces=self.ns)), ('service_provider_name', XPath("/sos:Capabilities/ows:ServiceProvider/ows:ProviderName", namespaces=self.ns)), ('service_title', XPath("/sos:Capabilities/ows:ServiceProvider/ows:ProviderName", namespaces=self.ns)), ('service_type_name', XPath( "/sos:Capabilities/ows:ServiceIdentification/ows:ServiceType", namespaces=self.ns)), ('service_type_version', XPath( "/sos:Capabilities/ows:ServiceIdentification/ows:ServiceTypeVersion", namespaces=self.ns)), # ds.identification[0].observed_properties has this as well, but # don't want to try to shoehorn a function here # ('variable_names', len(ds.identification[0].observed_properties) > 0) ('variable_names', XPath( "/sos:Capabilities/sos:Contents/sos:ObservationOfferingList/sos:ObservationOffering/sos:observedProperty", namespaces=self.ns)), ('data_format_template_version', XPath( "/sos:Capabilities/ows:OperationsMetadata/ows:ExtendedCapabilities/gml:metaDataProperty[@xlink:title='ioosTemplateVersion']/gml:version", namespaces=self.ns)) ] @check_has(BaseCheck.LOW) def check_suggested(self, ds): return ['altitude_units']
def patch_well_known_namespaces(etree_module): """Monkey patches the etree module to add some well-known namespaces.""" ns = Namespaces() try: register_namespace = etree_module.register_namespace except AttributeError: etree_module._namespace_map def register_namespace(prefix, uri): etree_module._namespace_map[uri] = prefix for k, v in list(ns.get_namespaces().items()): register_namespace(k, v)
def get_md_resource(file_path): """Read the file and parse into an XML tree. Parameters ---------- file_path : str Path of the file to read. Returns ------- etree.ElementTree XML tree of the resource on disk. """ namespaces = Namespaces().get_namespaces(keys=('gmd', 'gmi')) with io.open(file_path, mode='r', encoding='utf-8') as f: data = f.read().encode('utf-8') data = etree.fromstring(data) mdelem = data.find( './/' + util.nspath_eval('gmd:MD_Metadata', namespaces) ) or data.find('.//' + util.nspath_eval('gmi:MI_Metadata', namespaces)) if mdelem is None and data.tag in [ '{http://www.isotc211.org/2005/gmd}MD_Metadata', '{http://www.isotc211.org/2005/gmi}MI_Metadata' ]: mdelem = data return mdelem
def schema(self): if not self._schema: if self._schema_type == SchemaType.ISO19139: ns_name = 'gmd' else: ns_name = 'csw' self._schema = Namespaces().get_namespace(ns_name) return self._schema
def patch_well_known_namespaces(etree_module): import warnings from owslib.namespaces import Namespaces ns = Namespaces() """Monkey patches the etree module to add some well-known namespaces.""" try: register_namespace = etree_module.register_namespace except AttributeError: try: etree_module._namespace_map def register_namespace(prefix, uri): etree_module._namespace_map[uri] = prefix except AttributeError: def register_namespace(prefix, uri): pass warnings.warn("Only 'lxml.etree' >= 2.3 and 'xml.etree.ElementTree' >= 1.3 are fully supported!") for k, v in ns.get_namespaces().iteritems(): register_namespace(k, v)
def add_namespaces(root, ns_keys): if isinstance(ns_keys, six.string_types): ns_keys = [ns_keys] namespaces = Namespaces() ns_keys = [(x, namespaces.get_namespace(x)) for x in ns_keys] if etree.__name__ != 'lxml.etree': # We can just add more namespaces when not using lxml. # We can't re-add an existing namespaces. Get a list of current # namespaces in use existing_namespaces = set() for elem in root.getiterator(): if elem.tag[0] == "{": uri, tag = elem.tag[1:].split("}") existing_namespaces.add(namespaces.get_namespace_from_url(uri)) for key, link in ns_keys: if link is not None and key not in existing_namespaces: root.set("xmlns:%s" % key, link) return root else: # lxml does not support setting xmlns attributes # Update the elements nsmap with new namespaces new_map = root.nsmap for key, link in ns_keys: if link is not None: new_map[key] = link # Recreate the root element with updated nsmap new_root = etree.Element(root.tag, nsmap=new_map) # Carry over attributes for a, v in list(root.items()): new_root.set(a, v) # Carry over children for child in root: new_root.append(deepcopy(child)) return new_root
def get_namespaces_io100(): n = Namespaces() ns = n.get_namespaces(["ogc","swes","sml","xlink","xsi"]) ns["ows"] = n.get_namespace("ows110") ns["sos"] = n.get_namespace("sos20") ns["gml"] = n.get_namespace("gml32") ns["om"] = n.get_namespace("om20") ns['swe'] = 'http://www.opengis.net/swe/2.0' ns["sams"] = "http://www.opengis.net/samplingSpatial/2.0" ns["sf"] = "http://www.opengis.net/sampling/2.0" return ns
def create_metadata_record(**kwargs): """ Create a csw:Record XML document from harvested metadata """ if 'srs' in kwargs: srs = kwargs['srs'] else: srs = '4326' modified = '%sZ' % datetime.datetime.utcnow().isoformat().split('.')[0] nsmap = Namespaces().get_namespaces(['csw', 'dc', 'dct', 'ows']) e = etree.Element(nspath_eval('csw:Record', nsmap), nsmap=nsmap) etree.SubElement(e, nspath_eval('dc:identifier', nsmap)).text = kwargs['identifier'] etree.SubElement(e, nspath_eval('dc:title', nsmap)).text = kwargs['title'] if 'alternative' in kwargs: etree.SubElement(e, nspath_eval('dct:alternative', nsmap)).text = kwargs['alternative'] etree.SubElement(e, nspath_eval('dct:modified', nsmap)).text = modified etree.SubElement(e, nspath_eval('dct:abstract', nsmap)).text = kwargs['abstract'] etree.SubElement(e, nspath_eval('dc:type', nsmap)).text = kwargs['type'] etree.SubElement(e, nspath_eval('dc:format', nsmap)).text = kwargs['format'] etree.SubElement(e, nspath_eval('dc:source', nsmap)).text = kwargs['source'] if 'relation' in kwargs: etree.SubElement(e, nspath_eval('dc:relation', nsmap)).text = kwargs['relation'] if 'keywords' in kwargs: for keyword in kwargs['keywords']: etree.SubElement(e, nspath_eval('dc:subject', nsmap)).text = keyword for link in kwargs['links']: etree.SubElement(e, nspath_eval('dct:references', nsmap), scheme=link[0]).text = link[1] bbox2 = loads(kwargs['wkt_geometry']).bounds bbox = etree.SubElement(e, nspath_eval('ows:BoundingBox', nsmap), crs='http://www.opengis.net/def/crs/EPSG/0/%s' % srs, dimensions='2') etree.SubElement(bbox, nspath_eval('ows:LowerCorner', nsmap)).text = '%s %s' % (bbox2[1], bbox2[0]) etree.SubElement(bbox, nspath_eval('ows:UpperCorner', nsmap)).text = '%s %s' % (bbox2[3], bbox2[2]) return etree.tostring(e, pretty_print=True)
class IOOSSOSDSCheck(BaseSOSDSCheck, IOOSBaseCheck): # set up namespaces for XPath ns = Namespaces().get_namespaces(['sml', 'swe', 'gml', 'xlink']) @check_has(BaseCheck.HIGH) def check_high(self, ds): return [ ('platform_sponsor', XPath("/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='sponsor']/sml:Term/sml:value", namespaces=self.ns)), ('platform_type', XPath("/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='platformType']/sml:Term/sml:value", namespaces=self.ns)), ('station_publisher_name', XPath("/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/publisher']/sml:ResponsibleParty/sml:organizationName", namespaces=self.ns)), ('station_publisher_email', XPath("/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/publisher']/sml:ResponsibleParty/sml:contactInfo/address/sml:electronicMailAddress", namespaces=self.ns)), ('station_id', XPath("/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='stationID']/sml:Term/sml:value", namespaces=self.ns)), ('station_long_name', XPath("/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='longName']/sml:Term/sml:value", namespaces=self.ns)), ('station_short_name', XPath("/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='shortName']/sml:Term/sml:value", namespaces=self.ns)), ('station_wmo_id', XPath("/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier/sml:Term[@definition=\"http://mmisw.org/ont/ioos/definition/wmoID\"]/sml:value", namespaces=self.ns)), ('time_period', XPath("/sml:SensorML/sml:member/sml:System/sml:capabilities[@name='observationTimeRange']/swe:DataRecord/swe:field[@name='observationTimeRange']/swe:TimeRange/swe:value", namespaces=self.ns)), ('operator_email', XPath("/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/operator']/sml:ResponsibleParty/sml:contactInfo/address/sml:electronicMailAddress", namespaces=self.ns)), ('operator_name', XPath("/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/operator']/sml:ResponsibleParty/sml:organizationName", namespaces=self.ns)), ('station_description', XPath("/sml:SensorML/sml:member/sml:System/gml:description", namespaces=self.ns)), # replaced with lon/lat with point ('station_location_point', XPath("/sml:SensorML/sml:member/sml:System/sml:location/gml:Point/gml:pos", namespaces=self.ns)) ] @check_has(BaseCheck.MEDIUM) def check_recommended(self, ds): return [ ('sensor_descriptions', XPath("/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/gml:description", namespaces=self.ns)), ('sensor_ids', XPath("/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/@gml:id", namespaces=self.ns)), ('sensor_names', XPath("/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/@name", namespaces=self.ns)), ('data_format_template_version', XPath("/sml:SensorML/sml:capabilities/swe:SimpleDataRecord/swe:field[@name='ioosTemplateVersion']/swe:Text/swe:value", namespaces=self.ns)), ('variable_names', XPath("/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/sml:outputs/sml:OutputList/sml:output/swe:Quantity/@definition", namespaces=self.ns)), ('variable_units', XPath("/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/sml:outputs/sml:OutputList/sml:output/swe:Quantity/swe:uom/@code", namespaces=self.ns)), ('network_id', XPath("/sml:SensorML/sml:member/sml:System/sml:capabilities[@name='networkProcedures']/swe:SimpleDataRecord/gml:metaDataProperty/@xlink:href", namespaces=self.ns)), ('operator_sector', XPath("/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='operatorSector']/sml:Term/sml:value", namespaces=self.ns)), ] @check_has(BaseCheck.LOW) def check_suggested(self, ds): return [ ]
API for OGC Web Services Common (OWS) constructs and metadata. OWS Common: http://www.opengeospatial.org/standards/common Currently supports version 1.1.0 (06-121r3). """ import logging from owslib.etree import etree from owslib import crs, util from owslib.namespaces import Namespaces LOGGER = logging.getLogger(__name__) n = Namespaces() OWS_NAMESPACE_1_0_0 = n.get_namespace("ows") OWS_NAMESPACE_1_1_0 = n.get_namespace("ows110") OWS_NAMESPACE_2_0_0 = n.get_namespace("ows200") XSI_NAMESPACE = n.get_namespace("xsi") XLINK_NAMESPACE = n.get_namespace("xlink") DEFAULT_OWS_NAMESPACE = OWS_NAMESPACE_1_1_0 # Use this as default for OWSCommon objects class OwsCommon(object): """Initialize OWS Common object""" def __init__(self, version): self.version = version if version == '1.0.0':
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["ogc", "sml", "gml", "sos", "swe", "xlink"]) ns["ows"] = n.get_namespace("ows110") return ns
from owslib.crs import Crs from owslib.feature import WebFeatureService_ from owslib.feature.common import ( WFSCapabilitiesReader, AbstractContentMetadata, ) from owslib.namespaces import Namespaces # other imports from io import BytesIO from urllib.parse import urlencode import logging from owslib.util import log n = Namespaces() WFS_NAMESPACE = n.get_namespace("wfs20") OWS_NAMESPACE = n.get_namespace("ows110") OGC_NAMESPACE = n.get_namespace("ogc") GML_NAMESPACE = n.get_namespace("gml") FES_NAMESPACE = n.get_namespace("fes") class ServiceException(Exception): pass class WebFeatureService_2_0_0(WebFeatureService_): """Abstraction for OGC Web Feature Service (WFS). Implements IWebFeatureService.
def get_namespaces(): ns = Namespaces() return ns.get_namespaces(["om10", "swe101", "swe20", "gml311", "xlink"])
def get_namespaces(): n = Namespaces() return n.get_namespaces()
import warnings import six from owslib.etree import etree from owslib.util import (openURL, ServiceException, testXMLValue, extract_xml_list, xmltag_split, OrderedDict, nspath, bind_url) from owslib.util import nspath from owslib.fgdc import Metadata from owslib.iso import MD_Metadata from owslib.crs import Crs from owslib.namespaces import Namespaces from owslib.map.common import WMSCapabilitiesReader from owslib.util import log n = Namespaces() WMS_NAMESPACE = n.get_namespace("wms") OGC_NAMESPACE = n.get_namespace('ogc') class WebMapService_1_3_0(object): def __getitem__(self, name): ''' check contents dictionary to allow dict like access to service layers ''' if name in self.__getattribute__('contents'): return self.__getattribute__('contents')[name] else: raise KeyError("No content named %s" % name)
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["ogc"]) ns[None] = n.get_namespace("ogc") return ns
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["dif", "fes", "gml", "ogc", "xs", "xsi"]) ns[None] = n.get_namespace("ogc") return ns
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["gm03"]) ns[None] = n.get_namespace("gm03") return ns
def get_namespaces(): ns = Namespaces() return ns.get_namespaces(["swe20"])
""" API for OGC Web Services Common (OWS) constructs and metadata. OWS Common: http://www.opengeospatial.org/standards/common Currently supports version 1.1.0 (06-121r3). """ from __future__ import absolute_import, division, print_function from owslib.etree import etree from owslib import crs, util from owslib.namespaces import Namespaces n = Namespaces() OWS_NAMESPACE_1_0_0 = n.get_namespace("ows") OWS_NAMESPACE_1_1_0 = n.get_namespace("ows110") OWS_NAMESPACE_2_0 = n.get_namespace("ows200") XSI_NAMESPACE = n.get_namespace("xsi") XLINK_NAMESPACE = n.get_namespace("xlink") DEFAULT_OWS_NAMESPACE = OWS_NAMESPACE_1_1_0 # Use this as default for OWSCommon objects class OwsCommon(object): """Initialize OWS Common object""" def __init__(self, version): self.version = version
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["atom", "dc", "gml", "gml32", "xlink"]) ns.update(add_namespaces) ns[None] = n.get_namespace("atom") return ns
def get_namespaces(): n = Namespaces() namespaces = n.get_namespaces(["sml","gml","xlink"]) namespaces["ism"] = "urn:us:gov:ic:ism:v2" return namespaces
import cgi from cStringIO import StringIO from urllib import urlencode from urllib2 import urlopen from owslib.util import openURL, testXMLValue, extract_xml_list, ServiceException, xmltag_split from owslib.etree import etree from owslib.fgdc import Metadata from owslib.iso import MD_Metadata from owslib.crs import Crs from owslib.feature import WebFeatureService_ from owslib.namespaces import Namespaces from owslib.util import log from _socket import timeout n = Namespaces() WFS_NAMESPACE = n.get_namespace("wfs") OGC_NAMESPACE = n.get_namespace("ogc") #TODO: use nspath in util.py def nspath(path, ns=WFS_NAMESPACE): """ Prefix the given path with the given namespace identifier. Parameters ---------- path : string ElementTree API Compatible path expression ns : string
def get_namespaces(): n = Namespaces() ns = n.get_namespaces( ["gco", "gmd", "gml", "gml32", "gmx", "gts", "srv", "xlink"]) ns[None] = n.get_namespace("gmd") return ns
""" Set of functions, which are suitable for DescribeFeatureType parsing and generating layer schema description compatible with `fiona` """ import cgi, sys from owslib.util import openURL try: from urllib import urlencode except ImportError: from urllib.parse import urlencode from owslib.etree import etree from owslib.namespaces import Namespaces from owslib.util import which_etree, findall MYNS = Namespaces() XS_NAMESPACE = MYNS.get_namespace('xs') GML_NAMESPACES = (MYNS.get_namespace('gml'), MYNS.get_namespace('gml311'), MYNS.get_namespace('gml32')) def get_schema(url, typename, version='1.0.0', timeout=30, username=None, password=None): """Parses DescribeFeatureType response and creates schema compatible with :class:`fiona` :param str url: url of the service :param str version: version of the service :param str typename: name of the layer :param int timeout: request timeout """
def reindex_services(): region_map = { 'AOOS': '1706F520-2647-4A33-B7BF-592FAFDE4B45', 'CARICOOS': '117F1684-A5E3-400E-98D8-A270BDBA1603', 'CENCOOS': '4BA5624D-A61F-4C7E-BAEE-7F8BDDB8D9C4', 'GCOOS': '003747E7-4818-43CD-937D-44D5B8E2F4E9', 'GLOS': 'B664427E-6953-4517-A874-78DDBBD3893E', 'MARACOOS': 'C664F631-6E53-4108-B8DD-EFADF558E408', 'NANOOS': '254CCFC0-E408-4E13-BD62-87567E7586BB', 'NERACOOS': 'E41F4FCD-0297-415D-AC53-967B970C3A3E', 'PacIOOS': '68FF11D8-D66B-45EE-B33A-21919BB26421', 'SCCOOS': 'B70B3E3C-3851-4BA9-8E9B-C9F195DCEAC7', 'SECOORA': 'B3EA8869-B726-4E39-898A-299E53ABBC98' } #'NOS/CO-OPS': '72E748DF-23B1-4E80-A2C4-81E70783094A', #'USACE': '73019DFF-2E01-4800-91CD-0B3F812256A7', #'NAVY': '3B94DAAE-B7E9-4789-993B-0045AD9149D9', #'NDBC': '828981B0-0039-4360-9788-E788FA6B0875', #'USGS/CMGP': 'C6F11F00-C2BD-4AC6-8E2C-013E16F4932E' } services = { 'SOS': 'urn:x-esri:specification:ServiceType:sos:url', 'WMS': 'urn:x-esri:specification:ServiceType:wms:url', 'WCS': 'urn:x-esri:specification:ServiceType:wcs:url', 'DAP': 'urn:x-esri:specification:ServiceType:odp:url' } endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal c = csw.CatalogueServiceWeb(endpoint, timeout=120) ns = Namespaces() with app.app_context(): for region, uuid in region_map.iteritems(): # Setup uuid filter uuid_filter = fes.PropertyIsEqualTo(propertyname='sys.siteuuid', literal="{%s}" % uuid) # Make CSW request c.getrecords2([uuid_filter], esn='full', maxrecords=999999) for name, record in c.records.iteritems(): # @TODO: unfortunately CSW does not provide us with contact info, so # we must request it manually contact_email = "" metadata_url = None iso_ref = [ x['url'] for x in record.references if x['scheme'] == 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Document' ] if len(iso_ref): metadata_url = iso_ref[0] # Don't query for contact info right now. It takes WAY too long. #r = requests.get(iso_ref[0]) #r.raise_for_status() #node = ET.fromstring(r.content) #safe = nspath_eval("gmd:CI_ResponsibleParty/gmd:contactInfo/gmd:CI_Contact/gmd:address/gmd:CI_Address/gmd:electronicMailAddress/gco:CharacterString", ns.get_namespaces()) #contact_node = node.find(".//" + safe) #if contact_node is not None and contact_node.text != "": # contact_email = contact_node.text # if " or " in contact_email: # contact_email = ",".join(contact_email.split(" or ")) for ref in record.references: # We are only interested in the 'services' if ref["scheme"] in services.values(): url = unicode(ref["url"]) s = db.Service.find_one({ 'data_provider': unicode(region), 'url': url }) if s is None: s = db.Service() s.url = url s.data_provider = unicode(region) s.service_id = unicode(name) s.name = unicode(record.title) s.service_type = unicode( next((k for k, v in services.items() if v == ref["scheme"]))) s.interval = 3600 # 1 hour s.tld = unicode(urlparse(url).netloc) s.updated = datetime.utcnow() s.contact = unicode(contact_email) s.metadata_url = unicode(metadata_url) s.save() s.schedule_harvest()
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["ogc","sa","sml","gml","sos","swe","xlink"]) ns["ows"] = n.get_namespace("ows110") return ns
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["gco","gmd","gml","gml32","gmx","gts","srv","xlink"]) ns[None] = n.get_namespace("gmd") return ns
from __future__ import absolute_import, division, print_function from owslib.namespaces import Namespaces from pyoos.utils.etree import ElementType, etree ns = Namespaces() SML_NS = ns.get_versioned_namespace("sml", "1.0.1") SWE_NS = [ns.get_versioned_namespace("swe", "1.0.1")] class IoosDescribeSensor(object): def __new__(cls, element): if isinstance(element, ElementType): root = element else: root = etree.fromstring(element) sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format( SML_NS ) if hasattr(root, "getroot"): root = root.getroot() # Circular dependencies are bad. consider a reorganization # find the the proper type for the DescribeSensor. from pyoos.parsers.ioos.one.describe_sensor import ( NetworkDS, StationDS, SensorDS,
import warnings import six from owslib.etree import etree from owslib.util import (openURL, ServiceException, testXMLValue, extract_xml_list, xmltag_split, OrderedDict, nspath, bind_url) from owslib.util import nspath from owslib.fgdc import Metadata from owslib.iso import MD_Metadata from owslib.crs import Crs from owslib.namespaces import Namespaces from owslib.map.common import WMSCapabilitiesReader from owslib.util import log n = Namespaces() WMS_NAMESPACE = n.get_namespace("wms") OGC_NAMESPACE = n.get_namespace('ogc') class WebMapService_1_3_0(object): def __getitem__(self, name): ''' check contents dictionary to allow dict like access to service layers ''' if name in self.__getattribute__('contents'): return self.__getattribute__('contents')[name] else: raise KeyError("No content named %s" % name) def __init__(self,
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["gco","gmd","gml","gml32","gmx","gts","srv","xlink"]) ns["che"] = 'http://www.geocat.ch/2008/che' ns[None] = n.get_namespace("gmd") return ns
def get_namespaces(): n = Namespaces() return n.get_namespaces(["sml101", "gml", "xlink", "swe101"])
from owslib.etree import etree from owslib.util import nspath, testXMLValue from owslib.crs import Crs from owslib.feature import WebFeatureService_ from owslib.namespaces import Namespaces #other imports import cgi from cStringIO import StringIO from urllib import urlencode from urllib2 import urlopen import logging from owslib.util import log n = Namespaces() WFS_NAMESPACE = n.get_namespace("wfs20") OWS_NAMESPACE = n.get_namespace("ows110") OGC_NAMESPACE = n.get_namespace("ogc") GML_NAMESPACE = n.get_namespace("gml") FES_NAMESPACE = n.get_namespace("fes") class ServiceException(Exception): pass class WebFeatureService_2_0_0(WebFeatureService_): """Abstraction for OGC Web Feature Service (WFS). Implements IWebFeatureService.
def get_namespaces(): n = Namespaces() ns = n.get_namespaces("dif") ns[None] = n.get_namespace("dif") return ns
def get_namespaces(): ns = Namespaces() return ns.get_namespaces(["swe20", "xlink", "sos20", "om20", "gml32", "xsi", "wml2"])
def get_namespaces(): n = Namespaces() return n.get_namespaces(["sml","gml","xlink"])
from pyoos.utils.etree import etree from owslib.namespaces import Namespaces ns = Namespaces() SML_NS = ns.get_versioned_namespace('sml', '1.0.1') SWE_NS = [ns.get_versioned_namespace('swe', '1.0.1')] class IoosDescribeSensor(object): def __new__(cls, element): if isinstance(element, str): root = etree.fromstring(element) else: root = element sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS) if hasattr(root, 'getroot'): root = root.getroot() # circular dependencies are bad. consider a reorganization # find the the proper type for the DescribeSensor from pyoos.parsers.ioos.one.describe_sensor import (NetworkDS, StationDS, SensorDS) for ds_type, constructor in [('networkID', NetworkDS), ('stationID', StationDS), ('sensorID', SensorDS)]: if root.find(sml_str % ds_type) is not None: return super(IoosDescribeSensor, cls).__new__(constructor) # NOAA CO-OPS sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='urn:ioos:def:identifier:NOAA::networkID']".format(SML_NS) if root.find(sml_str) is not None:
def harvest(self, request): ## harvest (Harvester object, request = [community, source, verb, mdprefix, mdsubset]) # Harvest all files with <mdprefix> and <mdsubset> from <source> via sickle module and store those to hard drive. # # Parameters: # ----------- # (list) request - A list with following items: # 1. community # 2. source (OAI URL) # 3. verb (ListIdentifiers, ListRecords or JSONAPI) # 4. mdprefix (OAI md format as oai_dc, iso etc.) # 5. mdsubset # # Return Values: # -------------- # 1. (integer) is -1 if something went wrong # create a request dictionary: req = { "community": request[0], "url": request[1], "lverb": request[2], "mdprefix": request[3], "mdsubset": request[4] if len(request) > 4 else None } # create dictionary with stats: resKeys = ['count', 'tcount', 'ecount', 'time'] results = dict.fromkeys(resKeys, 0) stats = { "tottcount": 0, # total number of provided datasets "totcount": 0, # total number of successful harvested datasets "totecount": 0, # total number of failed datasets "totdcount": 0, # total number of all deleted datasets "tcount": 0, # number of all provided datasets per subset "count": 0, # number of all successful harvested datasets per subset "ecount": 0, # number of all failed datasets per subset "dcount": 0, # number of all deleted datasets per subset "timestart": time.time(), # start time per subset process } # the gbif api client class GBIF_CLIENT(object): # call action api: ## GBIF.action('package_list',{}) def __init__(self, api_url): ##, api_key): self.api_url = api_url self.logger = logging.getLogger('root') def JSONAPI(self, action, offset, chunklen, key): ## JSONAPI (action) - method return self.__action_api(action, offset, chunklen, key) def __action_api(self, action, offset, chunklen, key): # Make the HTTP request for get datasets from GBIF portal response = '' rvalue = 0 ## offset = 0 limit = chunklen ## None for DataCite-JSON !!! api_url = self.api_url if key: action_url = "{apiurl}/{action}/{key}".format( apiurl=api_url, action=action, key=str(key)) elif offset == None: action_url = "{apiurl}/{action}".format(apiurl=api_url, action=action) else: action_url = "{apiurl}/{action}?offset={offset}&limit={limit}".format( apiurl=api_url, action=action, offset=str(offset), limit=str(limit)) self.logger.debug('action_url: %s' % action_url) try: request = Request(action_url) response = urlopen(request) except HTTPError as e: self.logger.error( '%s : The server %s couldn\'t fulfill the action %s.' % (e.code, self.api_url, action)) if (e.code == 403): self.logger.critical( 'Access forbidden, maybe the API key is not valid?' ) exit(e.code) elif (e.code == 409): self.logger.critical( 'Maybe you have a parameter error?') return {"success": False} elif (e.code == 500): self.logger.critical('Internal server error') exit(e.code) except URLError as e: exit('%s' % e.reason) else: out = json.loads(response.read()) assert response.code >= 200 return out requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # if the number of files in a subset dir is greater than <count_break> # then create a new one with the name <set> + '_' + <count_set> count_break = 5000 count_set = 1 start = time.time() # set subset: mdsubset = req["mdsubset"] if (not mdsubset): subset = 'SET' elif mdsubset.endswith( '_' ): # no OAI subsets, but different OAI-URLs for same community subset = mdsubset[:-1] mdsubset = None elif len(mdsubset) > 2 and mdsubset[-1].isdigit( ) and mdsubset[-2] == '_': subset = mdsubset[:-2] else: subset = mdsubset if req["community"] == "b2share" or re.match( r'http(.*?)b2share(.*?)api(.*?)', req["url"]): setMapFile = '%s/mapfiles/b2share_mapset.json' % (os.getcwd()) elif req["community"] == "dara" and req[ "url"] == "https://www.da-ra.de/oaip/oai": setMapFile = '%s/mapfiles/dara_mapset.json' % (os.getcwd()) else: setMapFile = None if setMapFile: with open(setMapFile) as sm: setMap = json.load(sm) if mdsubset in setMap: mdsubset = setMap[mdsubset] if (self.fromdate): subset = subset + '_f' + self.fromdate self.logger.debug(' |- Subset: \t%s' % subset) # make subset dir: subsetdir = '/'.join([ self.base_outdir, req['community'] + '-' + req['mdprefix'], subset + '_' + str(count_set) ]) noffs = 0 # set to number of record, where harvesting should start stats['tcount'] = noffs fcount = 0 oldperc = 0 ntotrecs = 0 choffset = 0 chunklen = 1000 pageno = 1 records = list() ## JSON-API jsonapi_verbs = ['dataset', 'works', 'records'] if req["lverb"] in jsonapi_verbs: GBIF = GBIF_CLIENT(req['url']) # create GBIF object harvestreq = getattr(GBIF, 'JSONAPI', None) outtypedir = 'hjson' outtypeext = 'json' if mdsubset and req["lverb"] == 'works': haction = 'works?publisher-id=' + mdsubset dresultkey = 'data' elif req["lverb"] == 'records': haction = req["lverb"] if mdsubset: haction += '?q=community:' + mdsubset + '&size=' + str( chunklen) + '&page=' + str(pageno) dresultkey = 'hits' else: haction = req["lverb"] dresultkey = 'results' try: chunk = harvestreq( **{ 'action': haction, 'offset': None, 'chunklen': chunklen, 'key': None }) self.logger.debug(" Got first %d records : chunk['data'] %s " % (chunklen, chunk[dresultkey])) except (HTTPError, ConnectionError, Exception) as e: self.logger.critical( "%s :\n\thaction %s\n\tharvest request %s\n" % (e, haction, req)) return -1 if req["lverb"] == 'dataset': while ('endOfRecords' in chunk and not chunk['endOfRecords']): if 'results' in chunk: records.extend(chunk['results']) choffset += chunklen chunk = harvestreq( **{ 'action': haction, 'offset': choffset, 'chunklen': chunklen, 'key': None }) self.logger.debug( " Got next records [%d,%d] from chunk %s " % (choffset, choffset + chunklen, chunk)) elif req["lverb"] == 'records': records.extend(chunk['hits']['hits']) while ('hits' in chunk and 'next' in chunk['links']): if 'hits' in chunk: records.extend(chunk['hits']['hits']) pageno += 1 chunk = harvestreq( **{ 'action': haction, 'page': pageno, 'size': chunklen, 'key': None }) self.logger.debug( " Got next records [%d,%d] from chunk %s " % (choffset, choffset + chunklen, chunk)) else: if 'data' in chunk: records.extend(chunk['data']) # OAI-PMH (verb = ListRecords/Identifier ) elif req["lverb"].startswith('List'): sickle = Sickle(req['url'], max_retries=3, timeout=300) outtypedir = 'xml' outtypeext = 'xml' harvestreq = getattr(sickle, req["lverb"], None) try: records, rc = tee( harvestreq( **{ 'metadataPrefix': req['mdprefix'], 'set': mdsubset, 'ignore_deleted': True, 'from': self.fromdate })) except (HTTPError, ConnectionError) as err: self.logger.critical("%s during connecting to %s\n" % (err, req['url'])) return -1 except (ImportError, etree.XMLSyntaxError, CannotDisseminateFormat, Exception) as err: self.logger.critical("%s during harvest request %s\n" % (err, req)) return -1 # CSW2.0 elif req["lverb"].startswith('csw'): outtypedir = 'xml' outtypeext = 'xml' startposition = 0 maxrecords = 20 try: src = CatalogueServiceWeb(req['url']) NS = Namespaces() namespaces = NS.get_namespaces() if req['mdprefix'] == 'iso19139' or req['mdprefix'] == 'own': nsp = namespaces['gmd'] else: nsp = namespaces['csw'] harvestreq = getattr(src, 'getrecords2') chunk = harvestreq( **{ 'esn': 'full', 'startposition': choffset, 'maxrecords': maxrecords, 'outputschema': nsp }) chunklist = list(src.records.items()) while (len(chunklist) > 0): records.extend(chunklist) choffset += maxrecords chunk = harvestreq( **{ 'esn': 'full', 'startposition': choffset, 'maxrecords': maxrecords, 'outputschema': nsp }) chunklist = list(src.records.items()) self.logger.debug( " Got next %s records [%d,%d] from chunk " % (nsp, choffset, choffset + chunklen)) except (HTTPError, ConnectionError) as err: self.logger.critical("%s during connecting to %s\n" % (err, req['url'])) return -1 except (ImportError, CannotDisseminateFormat, Exception) as err: self.logger.error("%s : During harvest request %s\n" % (err, req)) ##return -1 # SparQL elif req["lverb"].startswith('Sparql'): outtypedir = 'hjson' outtypeext = 'json' startposition = 0 maxrecords = 1000 try: src = SPARQLWrapper(req['url']) harvestreq = getattr(src, 'query', 'format') ## statement = ''' prefix cpmeta: <http://meta.icos-cp.eu/ontologies/cpmeta/> prefix prov: <http://www.w3.org/ns/prov#> select (str(?submTime) as ?time) ?dobj ?spec ?dataLevel ?fileName ?submitterName where{ ?dobj cpmeta:hasObjectSpec [rdfs:label ?spec ; cpmeta:hasDataLevel ?dataLevel]. ?dobj cpmeta:hasName ?fileName . ?dobj cpmeta:wasSubmittedBy ?submission . ?submission prov:endedAtTime ?submTime . ?submission prov:wasAssociatedWith [cpmeta:hasName ?submitterName]. } order by desc(?submTime) limit 1000 ''' src.setQuery(statement) src.setReturnFormat(JSON) records = harvestreq().convert()['results']['bindings'] except (HTTPError, ConnectionError) as err: self.logger.critical("%s during connecting to %s\n" % (err, req['url'])) return -1 except (ImportError, CannotDisseminateFormat, Exception) as err: self.logger.critical("%s during harvest request %s\n" % (err, req)) return -1 else: self.logger.critical(' Not supported harvest type %s' % req["lverb"]) sys.exit() self.logger.debug(" Harvest method used %s" % req["lverb"]) try: if req["lverb"].startswith('List'): ntotrecs = len(list(rc)) else: ntotrecs = len(records) except Exception as err: self.logger.error('%s Iteratation does not work ?' % (err)) print("\t|- Retrieved %d records in %d sec - write %s files to disc" % (ntotrecs, time.time() - start, outtypeext.upper())) if ntotrecs == 0: self.logger.warning("\t|- Can not access any records to harvest") return -1 self.logger.debug(' | %-4s | %-25s | %-25s |' % ('#', 'OAI Identifier', 'DS Identifier')) start2 = time.time() if (not os.path.isdir(subsetdir + '/' + outtypedir)): os.makedirs(subsetdir + '/' + outtypedir) delete_ids = list() # loop over records for record in records: ## counter and progress bar stats['tcount'] += 1 fcount += 1 if fcount <= noffs: continue if ntotrecs > 0: perc = int(fcount * 100 / ntotrecs) bartags = int(perc / 5) if perc % 10 == 0 and perc != oldperc: oldperc = perc print("\r\t[%-20s] %5d (%3d%%) in %d sec" % ('=' * bartags, fcount, perc, time.time() - start2)) sys.stdout.flush() # Set oai_id and generate a uniquely identifier for this dataset: delete_flag = False if req["lverb"] == 'dataset' or req["lverb"] == 'works' or req[ "lverb"] == 'records': ## Harvest via JSON-API if 'key' in record: oai_id = record['key'] elif 'id' in record: oai_id = record['id'] elif req["lverb"] == 'csw': ## Harvest via CSW2.0 if hasattr(record, 'identifier'): oai_id = record.identifier elif (record): oai_id = record[0] else: self.logger.critical( 'Record %s has no attrribute identifier %s' % record) elif req[ "lverb"] == 'ListIdentifiers': ## OAI-PMH harvesting of XML records if (record.deleted): stats['totdcount'] += 1 delete_flag = True ##HEW-D continue else: oai_id = record.identifier record = sickle.GetRecord( **{ 'metadataPrefix': req['mdprefix'], 'identifier': record.identifier }) elif req["lverb"] == 'ListRecords': if (record.header.deleted): stats['totdcount'] += 1 continue else: oai_id = record.header.identifier elif req["lverb"].startswith('Sparql'): oai_id = record['fileName']['value'] # generate a uniquely identifier and a filename for this dataset: uid = str(uuid.uuid5(uuid.NAMESPACE_DNS, oai_id)) outfile = '%s/%s/%s.%s' % (subsetdir, outtypedir, os.path.basename(uid), outtypeext) if delete_flag: # record marked as deleted on provider site jsonfile = '%s/%s/%s.%s' % (subsetdir, 'json', os.path.basename(uid), 'json') # remove xml and json file: os.remove(xmlfile) os.remove(jsonfile) delete_ids.append(uid) # write record on disc try: self.logger.debug(' | h | %-4d | %-45s | %-45s |' % (stats['count'] + 1, oai_id, uid)) self.logger.debug( 'Try to write the harvested JSON record to %s' % outfile) if outtypeext == 'xml': # get and write the XML content: if req["lverb"] == 'csw': metadata = etree.fromstring(record[1].xml) elif hasattr(record, 'raw'): metadata = etree.fromstring(record.raw) elif hasattr(record, 'xml'): metadata = etree.fromstring(record.xml) if (metadata is not None): try: metadata = etree.tostring( metadata, pretty_print=True).decode('utf-8') except (Exception, UnicodeEncodeError) as e: self.logger.critical('%s : Metadata: %s ...' % (e, metadata[:20])) ##if PY2 : ## try: ## metadata = metadata.encode('utf-8') ## except (Exception,UnicodeEncodeError) as e : ## self.logger.debug('%s : Metadata : %s ...' % (e,metadata[20])) try: f = open(outfile, 'w') f.write(metadata) f.close except (Exception, IOError) as err: self.logger.critical( "%s : Cannot write metadata in xml file %s" % (err, outfile)) stats['ecount'] += 1 continue else: logging.debug('Harvested XML file written to %s' % outfile) stats['count'] += 1 else: stats['ecount'] += 1 self.logger.error('No metadata available for %s' % record) elif outtypeext == 'json': # get the raw json content: if (record is not None): try: with open(outfile, 'w') as f: json.dump(record, f, sort_keys=True, indent=4) except IOError: logging.error( "[ERROR] Cannot write metadata in out file '%s': %s\n" % (outfile)) stats['ecount'] += 1 continue else: stats['count'] += 1 logging.debug('Harvested JSON file written to %s' % outfile) else: stats['ecount'] += 1 logging.warning( ' [WARNING] No metadata available for %s' % record['key']) ##HEW-???' % oai_id) except TypeError as e: logging.error(' [ERROR] TypeError: %s' % e) stats['ecount'] += 1 continue except Exception as e: logging.error(" [ERROR] %s and %s" % (e, traceback.format_exc())) ## logging.debug(metadata) stats['ecount'] += 1 continue # Next or last subset? if (stats['count'] == count_break) or (fcount == ntotrecs): print(' | %d records written to subset directory %s ' % (stats['count'], subsetdir)) # clean up current subset and write ids to remove to delete file for df in os.listdir(subsetdir + '/' + outtypedir): df = os.path.join(subsetdir + '/' + outtypedir, df) logging.debug('File to delete : %s' % df) id = os.path.splitext(os.path.basename(df))[0] jf = os.path.join(subsetdir + '/json/', id + '.json') if os.stat(df).st_mtime < start - 1 * 86400: os.remove(df) logging.warning('File %s is deleted' % df) if os.path.exists(jf): os.remove(jf) logging.warning('File %s is deleted' % jf) delete_ids.append(id) logging.warning('Append Id %s to list delete_ids' % id) stats['dcount'] += 1 print(' | %d records deleted from subset directory %s ' % (stats['dcount'], subsetdir)) if not fcount == ntotrecs: # next subset neded subsetdir = self.save_subset(req, stats, subset, count_set) if (not os.path.isdir(subsetdir + '/' + outtypedir)): os.makedirs(subsetdir + '/' + outtypedir) count_set += 1 # add all subset stats to total stats and reset the temporal subset stats: for key in ['tcount', 'ecount', 'count', 'dcount']: stats['tot' + key] += stats[key] stats[key] = 0 # start with a new time: stats['timestart'] = time.time() logging.debug( ' | %d records written to subset directory %s (if not failed).' % (stats['count'], subsetdir)) # path to the file with all ids to delete: delete_file = '/'.join([ self.base_outdir, 'delete', req['community'] + '-' + req['mdprefix'] + '.del' ]) if len(delete_ids) > 0: with open(delete_file, 'a') as file: for id in delete_ids: file.write(id + '\n') # add all subset stats to total stats and reset the temporal subset stats: for key in ['tcount', 'ecount', 'count', 'dcount']: stats['tot' + key] += stats[key] print( ' \t|- %-10s |@ %-10s |\n\t| Provided | Harvested | Failed | Deleted |\n\t| %8d | %9d | %6d | %6d |' % ('Finished', time.strftime("%H:%M:%S"), stats['tottcount'], stats['totcount'], stats['totecount'], stats['totdcount']))
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["fes","ogc","om","gml32","sml","swe20","swes","xlink"]) ns["ows"] = n.get_namespace("ows110") ns["sos"] = n.get_namespace("sos20") return ns
def get_namespaces(): n = Namespaces() return n.get_namespaces(["gml","ogc","ows","wfs"])
def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["dif","fes","gml","ogc","xs","xsi"]) ns[None] = n.get_namespace("ogc") return ns
from urllib import urlencode import warnings import six from owslib.etree import etree from owslib.util import (openURL, testXMLValue, extract_xml_list, xmltag_split, OrderedDict, ServiceException, bind_url, nspath_eval, Authentication) from owslib.fgdc import Metadata from owslib.iso import MD_Metadata from owslib.map.common import WMSCapabilitiesReader, AbstractContentMetadata from owslib.namespaces import Namespaces n = Namespaces() class CapabilitiesError(Exception): pass class WebMapService_1_1_1(object): """Abstraction for OGC Web Map Service (WMS). Implements IWebMapService. """ def __getitem__(self, name): ''' check contents dictionary to allow dict like access to service layers '''
def get_namespaces(): n = Namespaces() return n.get_namespaces(["gmd", "gml", "gmi", "ogc","ows","wfs"])
class EsaCciCatalogueService: def __init__(self, catalogue_url: str): self._catalogue_url = catalogue_url self._catalogue = None self._catalogue_service = None self._namespaces = Namespaces() def reset(self): self._catalogue = None self._catalogue_service = None def getrecords(self, monitor: Monitor = Monitor.NONE): if not self._catalogue_service: self._init_service() if not self._catalogue: self._build_catalogue(monitor.child(1)) return self._catalogue def _build_catalogue(self, monitor: Monitor = Monitor.NONE): self._catalogue = {} catalogue_metadata = {} start_position = 0 max_records = _CSW_MAX_RESULTS matches = -1 while True: # fetch record metadata self._catalogue_service.getrecords2( esn='full', outputschema=self._namespaces.get_namespace('gmd'), startposition=start_position, maxrecords=max_records) if matches == -1: # set counters, start progress monitor matches = self._catalogue_service.results.get('matches') if matches == 0: break monitor.start(label="Fetching catalogue data... (%d records)" % matches, total_work=ceil(matches / max_records)) catalogue_metadata.update(self._catalogue_service.records) monitor.progress(work=1) # bump counters start_position += max_records if start_position > matches: break self._catalogue = { record.identification.uricode[0]: { 'abstract': record.identification.abstract, 'bbox_minx': record.identification.bbox.minx if record.identification.bbox else None, 'bbox_miny': record.identification.bbox.miny if record.identification.bbox else None, 'bbox_maxx': record.identification.bbox.maxx if record.identification.bbox else None, 'bbox_maxy': record.identification.bbox.maxy if record.identification.bbox else None, 'creation_date': next( iter(e.date for e in record.identification.date if e and e.type == 'creation'), None), 'publication_date': next( iter(e.date for e in record.identification.date if e and e.type == 'publication'), None), 'title': record.identification.title, 'data_sources': record.identification.uricode[1:], 'licences': record.identification.uselimitation, 'temporal_coverage_start': record.identification.temporalextent_start, 'temporal_coverage_end': record.identification.temporalextent_end } for record in catalogue_metadata.values() if record.identification and len(record.identification.uricode) > 0 } monitor.done() def _init_service(self): if self._catalogue: return if not self._catalogue_service: self._catalogue_service = CatalogueServiceWeb( url=self._catalogue_url, timeout=_CSW_TIMEOUT, skip_caps=True)