def process_doc(self, doc): """ Attempt to parse an xml string conforming to either an SOS or SensorML dataset and return the results """ xml_doc = ET.fromstring(doc) if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities": ds = SensorObservationService(None, xml=doc) # SensorObservationService does not store the etree doc root, # so maybe use monkey patching here for now? ds._root = xml_doc elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML": ds = SensorML(xml_doc) else: raise ValueError("Unrecognized XML root element: {}".format(xml_doc.tag)) return ds
def main(): fl = str() for f, val in flags.items(): if val is True: fl += f try: run_command('r.in.sos', flags=fl, **options) except: return 0 if any(value is True and key in ['o', 'v', 'p', 't'] for key, value in flags.items()): return 0 service = SensorObservationService(options['url'], version=options['version'], username=options['username'], password=options['password']) for off in options['offering'].split(','): out = soslib.handle_not_given_options(service, off, options['procedure'], options['observed_properties'], options['event_time']) procedure, observed_properties, event_time = out for observed_property in observed_properties: map_name = '{}_{}_{}'.format(options['output'], off, observed_property) if ':' in map_name: map_name = '_'.join(map_name.split(':')) if '-' in map_name: map_name = '_'.join(map_name.split('-')) if '.' in map_name: map_name = '_'.join(map_name.split('.')) maps_list_file = get_maps(map_name) create_temporal(maps_list_file, map_name) return 0
def run_test_resource(resource_type, url): """tests a CSW service and provides run metrics""" if resource_type not in RESOURCE_TYPES.keys(): msg = 'Invalid resource type: %s' % resource_type LOGGER.error(msg) raise RuntimeError(msg) title = None start_time = datetime.datetime.utcnow() message = None try: if resource_type == 'OGC:WMS': ows = WebMapService(url) elif resource_type == 'OGC:WFS': ows = WebFeatureService(url) elif resource_type == 'OGC:WCS': ows = WebCoverageService(url) elif resource_type == 'OGC:WPS': ows = WebProcessingService(url) elif resource_type == 'OGC:CSW': ows = CatalogueServiceWeb(url) elif resource_type == 'OGC:SOS': ows = SensorObservationService(url) elif resource_type == 'WWW:LINK': ows = urlopen(url) import re title_re = re.compile("<title>(.+?)</title>") title = title_re.search(ows.read()).group(1) success = True if resource_type != 'WWW:LINK': title = ows.identification.title except Exception, err: msg = str(err) LOGGER.exception(msg) message = msg success = False
def getCapabilitiesSOS200(getcap_response): """ Retrieve information from GetCapabilities response, and then show the window displaying it. argument: >>> getcap_response: class 'requests.models.Response' object containing GetCapabilities request response retrieved from HTTP "Get" response. """ global cap_window # Use GetCapabilityWindow() imported from gui module # located in "ui" subdirectory. cap_window = GetCapabilityWindow() sos = SensorObservationService(None,xml=getcap_response.content) sos_id = sos.identification cap_window.title_value.setPlainText(sos_id.title) cap_window.abstract_value.setPlainText(sos_id.abstract) p = sos.provider cap_window.provider_name_value.setPlainText(p.name) cap_window.provider_website_value.setPlainText(p.url) sc = p.contact cap_window.contact_phone_value.setText(sc.phone) cap_window.contact_email_value.setText(sc.email) cap_window.contact_address_value.setText(sc.address) cap_window.contact_city_value.setText(sc.city) cap_window.contact_region_value.setText(sc.region) cap_window.contact_postcode_value.setText(sc.postcode) cap_window.contact_country_value.setText(sc.country) cap_window.show()
def main(): parsed_obs = dict() layerscount = 0 service = SensorObservationService(options['url'], version=options['version'], username=options['username'], password=options['password']) if any(value is True and key in ['o', 'v', 'p', 't'] for key, value in flags.items()): soslib.get_description(service, options, flags) soslib.check_missing_params(options['offering'], options['output']) if options['granularity'] != '': import grass.temporal as tgis tgis.init() seconds_granularity = int( tgis.gran_to_gran(options['granularity'], '1 second', True)) else: seconds_granularity = 1 target = soslib.get_target_crs() run_command('g.remove', 'f', type='vector', name=options['output']) new = VectorTopo(options['output']) for off in options['offering'].split(','): # TODO: Find better way than iteration (at best OWSLib upgrade) out = soslib.handle_not_given_options(service, off, options['procedure'], options['observed_properties'], options['event_time']) procedure, observed_properties, event_time = out if flags['s']: create_maps(_, off, _, new, _, _, service, target, _, procedure) else: try: obs = service.get_observation( offerings=[off], responseFormat=options['response_format'], observedProperties=observed_properties, procedure=procedure, eventTime=event_time, timeout=int(options['timeout']), username=options['username'], password=options['password']) except: # TODO: catch errors properly (e.g. timeout) grass.fatal('Request did not succeed!') try: if options['version'] in ['1.0.0', '1.0'] and str( options['response_format'] ) == 'text/xml;subtype="om/1.0.0"': for prop in observed_properties: parsed_obs.update( {prop: soslib.xml2geojson(obs, prop, flags['i'])}) elif str(options['response_format']) == 'application/json': for prop in observed_properties: parsed_obs.update( {prop: soslib.json2geojson(obs, prop)}) except AttributeError: if sys.version_info[0] >= 3: sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError( 'There is no data for at least one of your procedures, ' 'could you change the time parameter, observed ' 'properties, procedures or offerings') except ValueError as e: if sys.version_info[0] >= 3: sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise e create_maps(parsed_obs, off, layerscount, new, seconds_granularity, event_time, service, target, observed_properties) layerscount += len(parsed_obs) return 0
def main(): parsed_obs = dict() service = SensorObservationService(options['url'], version=options['version'], username=options['username'], password=options['password']) if any(value is True and key in ['o', 'v', 'p', 't'] for key, value in flags.iteritems()): get_description(service, options, flags) if options['offering'] == '' or options['output'] == '': if sys.version >= (3, 0): sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError( "You have to define any flags or use 'output' and 'offering' " "parameters to get the data") if options['granularity'] != '': import grass.temporal as tgis tgis.init() secondsGranularity = int( tgis.gran_to_gran(options['granularity'], '1 second', True)) else: secondsGranularity = 1 if options['resolution'] == '': resolution = None else: resolution = float(options['resolution']) if options['bbox'] != '': bbox = options['bbox'].split(',') run_command('g.region', n=float(bbox[0]), e=float(bbox[1]), s=float(bbox[2]), w=float(bbox[3]), res=resolution) else: grass.warning('You have not setted the bounding box. Bounding box will' ' be automatically based on procedure geometries for ' 'every map.') for off in options['offering'].split(','): # TODO: Find better way than iteration (at best OWSLib upgrade) procedure, observed_properties, event_time = handle_not_given_options( service, off, options['procedure'], options['observed_properties'], options['event_time']) event_time = 'T'.join(event_time.split(' ')) obs = service.get_observation( offerings=[off], responseFormat=options['response_format'], observedProperties=[observed_properties], procedure=procedure, eventTime=event_time, username=options['username'], password=options['password']) try: if options['version'] in ['1.0.0', '1.0'] and \ str(options['response_format']) == 'text/xml;subtype="om/1.0.0"': for property in observed_properties.split(','): parsed_obs.update({property: xml2geojson(obs, property)}) elif str(options['response_format']) == 'application/json': for property in observed_properties.split(','): parsed_obs.update({property: json2geojson(obs, property)}) except AttributeError: if sys.version >= (3, 0): sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError('There is no data for at least one of your ' 'procedures, could you change the time ' 'parameter, observed properties, ' 'procedures or offerings') create_maps(parsed_obs, off, secondsGranularity, resolution) return 0
def _parse_sos(context, repos, record, identifier, version): from owslib.sos import SensorObservationService bboxs = [] recobjs = [] serviceobj = repos.dataset() if version == '1.0.0': schema = 'http://www.opengis.net/sos/1.0' else: schema = 'http://www.opengis.net/sos/2.0' md = SensorObservationService(record, version=version) # generate record of service instance _set(context, serviceobj, 'pycsw:Identifier', identifier) _set(context, serviceobj, 'pycsw:Typename', 'csw:Record') _set(context, serviceobj, 'pycsw:Schema', schema) _set(context, serviceobj, 'pycsw:MdSource', record) _set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now()) _set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(etree.tostring(md._capabilities))) _set(context, serviceobj, 'pycsw:Type', 'service') _set(context, serviceobj, 'pycsw:Title', md.identification.title) _set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract) _set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords)) _set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name) _set(context, serviceobj, 'pycsw:Publisher', md.provider.name) _set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name) _set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name) _set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessconstraints) _set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees) _set(context, serviceobj, 'pycsw:Source', record) _set(context, serviceobj, 'pycsw:Format', md.identification.type) _set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326') _set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees') _set(context, serviceobj, 'pycsw:ServiceType', 'OGC:SOS') _set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version) _set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations])) _set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents))) _set(context, serviceobj, 'pycsw:CouplingType', 'tight') links = [ '%s,OGC-SOS Sensor Observation Service,OGC:SOS,%s' % (identifier, md.url), ] _set(context, serviceobj, 'pycsw:Links', '^'.join(links)) # generate record foreach offering LOGGER.debug('Harvesting %d SOS ObservationOffering\'s ', len(md.contents)) for offering in md.contents: recobj = repos.dataset() identifier2 = '%s-%s' % (identifier, md.contents[offering].id) _set(context, recobj, 'pycsw:Identifier', identifier2) _set(context, recobj, 'pycsw:Typename', 'csw:Record') _set(context, recobj, 'pycsw:Schema', schema) _set(context, recobj, 'pycsw:MdSource', record) _set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now()) _set(context, recobj, 'pycsw:Type', 'dataset') _set(context, recobj, 'pycsw:ParentIdentifier', identifier) _set(context, recobj, 'pycsw:Title', md.contents[offering].description) _set(context, recobj, 'pycsw:Abstract', md.contents[offering].description) _set(context, recobj, 'pycsw:TempExtent_begin', util.datetime2iso8601(md.contents[offering].begin_position)) _set(context, recobj, 'pycsw:TempExtent_end', util.datetime2iso8601(md.contents[offering].end_position)) #For observed_properties that have mmi url or urn, we simply want the observation name. observed_properties = [] for obs in md.contents[offering].observed_properties: #Observation is stored as urn representation: urn:ogc:def:phenomenon:mmisw.org:cf:sea_water_salinity if obs.lower().startswith(('urn:', 'x-urn')): observed_properties.append(obs.rsplit(':', 1)[-1]) #Observation is stored as uri representation: http://mmisw.org/ont/cf/parameter/sea_floor_depth_below_sea_surface elif obs.lower().startswith(('http://', 'https://')): observed_properties.append(obs.rsplit('/', 1)[-1]) else: observed_properties.append(obs) #Build anytext from description and the observed_properties. anytext = [] anytext.append(md.contents[offering].description) anytext.extend(observed_properties) _set(context, recobj, 'pycsw:AnyText', util.get_anytext(anytext)) _set(context, recobj, 'pycsw:Keywords', ','.join(observed_properties)) bbox = md.contents[offering].bbox if bbox is not None: tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3]) wkt_polygon = util.bbox2wktpolygon(tmp) _set(context, recobj, 'pycsw:BoundingBox', wkt_polygon) _set(context, recobj, 'pycsw:CRS', md.contents[offering].bbox_srs.id) _set(context, recobj, 'pycsw:DistanceUOM', 'degrees') bboxs.append(wkt_polygon) _set(context, recobj, 'pycsw:XML', caps2iso(recobj, md, context)) recobjs.append(recobj) # Derive a bbox based on aggregated featuretype bbox values bbox_agg = util.bbox_from_polygons(bboxs) if bbox_agg is not None: _set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg) _set(context, serviceobj, 'pycsw:XML', caps2iso(serviceobj, md, context)) recobjs.insert(0, serviceobj) return recobjs
def test_sos_caps(): sos = SensorObservationService(SERVICE_URL) assert str(sos.contents['81102 - PM10']) == "Offering id: 81102 - PM10, name: Particulate Matter < 10 µm" assert repr(sos.contents['81102 - PM10']) == "<SosObservationOffering 'Particulate Matter < 10 µm'>"
# Accessing ncSOS with OWSLib # <codecell> from owslib.sos import SensorObservationService import pdb from owslib.etree import etree import pandas as pd import datetime as dt # <codecell> # usgs woods hole # buoy data (ADCP) url = 'http://geoport.whoi.edu/thredds/sos/usgs/data2/emontgomery/stellwagen/CF-1.6/HURRIRENE_BB/9141wh-a.nc' usgs = SensorObservationService(url) contents = usgs.contents # <codecell> usgs.contents # <codecell> off = usgs.offerings[1] off.name # <codecell> off.response_formats
def showServerSelectionDialog(self): text, ok = QtGui.QInputDialog.getText(self, 'SOS server selection', 'Enter sos service url:') if ok: # Reset attributes. # # Reset UI attributes. self.statusBar.clearMessage() # Second block of main window attributes. self.select_offering_comboBox.clear() self.select_prop_comboBox.clear() self.starting_time_pushButton.setText("") self.ending_time_pushButton.setText("") self.time_series_starting_time_value.setText("") self.time_series_ending_time_value.setText("") # QGIS related attributes. # Remove station features and get an empty station layer. listOfIds = [ feat.id() for feat in self.stations_layer.getFeatures() ] self.stations_layer.dataProvider().deleteFeatures(listOfIds) # Other attributes. self.getobs_response = '' self.getseries_boolean = False # Set attributes using retrieved SOS server information. # self.sos_service_url = str(text) self.getcap_response = requests.get( self.sos_service_url + '?REQUEST=GetCapabilities' '&SERVICE=SOS&ACCEPTVERSIONS=2.0.0', stream=True) self.sos = SensorObservationService( None, xml=self.getcap_response.content, version="2.0.0") self.WGS84bbox_set = set( WGS84conversion(off) for off in self.sos.offerings) self.WGS84bbox_list = list(self.WGS84bbox_set) # Set UI attributes self.selected_sos_server_lineEdit.setText(self.sos_service_url) if self.WGS84bbox_list == []: # Display error message using QMessageBox. empty_bbox_msg = QtGui.QMessageBox() empty_bbox_msg.setWindowTitle("Error") empty_bbox_msg.setTextFormat(QtCore.Qt.RichText) msg_text = ('Each offering bounding box is empty when using ' 'OWSLib with this SOS 2.0 server') empty_bbox_msg.setText(msg_text) i_text = ( 'This plugin uses the Open Source ' '<a href=\"https://geopython.github.io/OWSLib/\">OWSlib library</a> ' 'to retrieve SOS 2.0 data. When collecting offerings ' 'and their corresponding featureOfInterest bounding ' 'boxes, only empty lists were retrieved.') empty_bbox_msg.setInformativeText(i_text) d_text = ( 'In order to solve the problem, you may want to have ' 'a look at OWSlib documentation ' '(https://geopython.github.io/OWSLib/) and at the ' 'Python source file ' '(https://github.com/geopython/OWSLib/blob/master/owslib/swe/observation/sos200.py/) ' 'containing the offering class, including the ' 'bounding box attribute.') empty_bbox_msg.setDetailedText(d_text) empty_bbox_msg.setIcon(QtGui.QMessageBox.Critical) empty_bbox_msg.exec_() # Reload station layer as selected SOS server has changed. self.stations_layer.triggerRepaint() elif self.WGS84bbox_list == [None]: # Display warning message using QMessageBox. none_bbox_msg = QtGui.QMessageBox() none_bbox_msg.setWindowTitle("Warning") none_bbox_msg.setTextFormat(QtCore.Qt.RichText) msg_text = ( "Each offering has 'None' bounding box when using " "OWSLib with this SOS 2.0 server. This plugin uses " "the Open Source " "<a href=\"https://geopython.github.io/OWSLib/\">OWSlib library</a>" " to retrieve SOS 2.0 data. When collecting the " "featureOfInterest bounding box for each offering, " "only None objects were retrieved. Consequently, " "no feature of interest could be added to the " "'Features of interest' layer generated by the plugin.") none_bbox_msg.setText(msg_text) i_text = ( 'Please select directly an offering in the "Offering" ' 'combobox to unlock the "ObservedPropery" combobox. ' 'You will then be able to select all GetObservation ' 'request parameters and to retrieve desired ' 'time series.') none_bbox_msg.setInformativeText(i_text) d_text = ( 'In order to solve the problem, you may want to have ' 'a look at OWSlib documentation ' '(https://geopython.github.io/OWSLib/) and at the ' 'Python source file ' '(https://github.com/geopython/OWSLib/blob/master/owslib/swe/observation/sos200.py/) ' 'containing the offering class, including the ' 'bounding box attribute.') none_bbox_msg.setDetailedText(d_text) none_bbox_msg.setIcon(QtGui.QMessageBox.Warning) none_bbox_msg.exec_() # Reload station layer as selected SOS server has changed. self.stations_layer.triggerRepaint() # Fill OfferingComboBox with all offerings from stations # which have None bbox. self.selected_station_index = 0 station = self.WGS84bbox_list[self.selected_station_index] for o in GetOfferingsList(self.sos, station).offering_list: self.select_offering_comboBox.addItem(o.id) else: # Stations bbox content is valid. # # For each station... for i, s in enumerate(self.WGS84bbox_list): # Define pairs of coordinates of two points which # generate the bounding box of this stations. if s is not None: xmin = min(s[1], s[3]) xmax = max(s[1], s[3]) ymin = min(s[0], s[2]) ymax = max(s[0], s[2]) # Create new point which represents the station. # Here, we choose to set station location using the point # which has min lat and min long, as several SOS 2.0 # we have tested lead to two points coinciding. # Therefore, we make the assumption xmin==xmax and # ymin==ymax. # When it is not the case, station location is likely # to be incorrect. # Further development is required to solve this issue. new_feature = QgsFeature() new_feature.setGeometry( QgsGeometry.fromPoint(QgsPoint(xmin, ymin))) new_feature.setAttributes([i]) self.stations_layer.dataProvider().addFeatures( [new_feature]) # Update layer and refresh QGIS canvas extent. self.stations_layer.updateExtents() QgsMapLayerRegistry.instance().addMapLayer(self.stations_layer) self.stations_layer.triggerRepaint() canvas = qgis.utils.iface.mapCanvas() canvas.setExtent(self.stations_layer.extent()) # Inform user he/she has to spatially select a station from # newly added 'Features of interest' layer. foi_layer_msg = QtGui.QMessageBox() foi_layer_msg.setWindowTitle("Information") msg_text = ( 'A "Features of interest" layer has been added to the map!' ) foi_layer_msg.setText(msg_text) i_text = ( 'Please select a station of the "Features of interest" ' 'layer using the select features tool of the QGIS ' 'toolbar to unlock the "Offering" combobox and the ' '"ObservedPropery" combobox. You will then be able to ' 'select all GetObservation request parameters and to ' 'retrieve desired time series.') foi_layer_msg.setInformativeText(i_text) foi_layer_msg.setIcon(QtGui.QMessageBox.Information) foi_layer_msg.exec_()
def service(self): if self._service is None: service = SensorObservationService(SOSURL, version=SOSVERSION) self._service = service return self._service
def main(): parsed_obs = dict() service = SensorObservationService(options['url'], version=options['version'], username=options['username'], password=options['password']) if any(value is True and key in ['o', 'v', 'p', 't'] for key, value in flags.iteritems()): get_description(service, options, flags) if options['offering'] == '' or options['output'] == '': if sys.version >= (3, 0): sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError("You have to define any flags or use 'output' and" " 'offering' parameters to get the data") if options['granularity'] != '': import grass.temporal as tgis tgis.init() secondsGranularity = int( tgis.gran_to_gran(options['granularity'], '1 second', True)) else: secondsGranularity = 1 for off in options['offering'].split(','): # TODO: Find better way than iteration (at best OWSLib upgrade) procedure, observed_properties, event_time = handle_not_given_options( service, off, options['procedure'], options['observed_properties'], options['event_time']) event_time = 'T'.join(event_time.split(' ')) obs = service.get_observation( offerings=[off], responseFormat=options['response_format'], observedProperties=observed_properties, procedure=procedure, eventTime=event_time, username=options['username'], password=options['password']) try: if options['version'] in ['1.0.0', '1.0'] and str( options['response_format'] ) == 'text/xml;subtype="om/1.0.0"': for prop in observed_properties: parsed_obs.update({prop: xml2geojson(obs, prop)}) elif str(options['response_format']) == 'application/json': for prop in observed_properties: parsed_obs.update({prop: json2geojson(obs, prop)}) except AttributeError: if sys.version >= (3, 0): sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError('There is no data for at least one of your ' 'procedures, could you change the time ' 'parameter, observed properties, ' 'procedures or offerings') create_maps(parsed_obs, off, secondsGranularity) return 0
def main(): parsed_obs = dict() service = SensorObservationService(options['url'], version=options['version'], username=options['username'], password=options['password']) if any(value is True and key in ['o', 'v', 'p', 't'] for key, value in flags.items()): soslib.get_description(service, options, flags) soslib.check_missing_params(options['offering'], options['output']) if options['granularity'] != '': import grass.temporal as tgis tgis.init() seconds_granularity = int( tgis.gran_to_gran(options['granularity'], '1 second', True)) else: seconds_granularity = 1 if options['resolution'] == '': a = grass.read_command('g.region', flags='gf') resolution = float(a.split('nsres=')[1].split(' ')[0]) run_command('g.message', flags='w', message='No resolution was setted. Using the resolution ' '{} (nres of your current setting).'.format(resolution)) else: resolution = float(options['resolution']) if options['bbox'] != '': bbox = options['bbox'].split(',') run_command('g.region', n=float(bbox[0]), e=float(bbox[1]), s=float(bbox[2]), w=float(bbox[3]), res=resolution) else: grass.warning('You have not setted the bounding box. Bounding box will' ' be automatically based on procedure geometries for ' 'every map.') target = soslib.get_target_crs() for off in options['offering'].split(','): # TODO: Find better way than iteration (at best OWSLib upgrade) out = soslib.handle_not_given_options(service, off, options['procedure'], options['observed_properties'], options['event_time']) procedure, observed_properties, event_time = out if flags['s']: create_maps(_, off, _, resolution, _, service, target, procedure) else: try: obs = service.get_observation( offerings=[off], responseFormat=options['response_format'], observedProperties=observed_properties, procedure=procedure, eventTime=event_time, username=options['username'], password=options['password']) except: # TODO: catch errors properly (e.g. timeout) grass.fatal('Request did not succeed!') try: if options['version'] in ['1.0.0', '1.0'] and \ options['response_format'] == 'text/xml;subtype="om/1.0.0"': for prop in observed_properties: parsed_obs.update( {prop: soslib.xml2geojson(obs, prop)}) elif str(options['response_format']) == 'application/json': for prop in observed_properties: parsed_obs.update( {prop: soslib.json2geojson(obs, prop)}) except AttributeError: if sys.version_info[0] >= 3: sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise AttributeError( 'There is no data for at least one of your procedures, ' 'could you change the time parameter, observed ' 'properties, procedures or offerings') except ValueError as e: if sys.version_info[0] >= 3: sys.tracebacklimit = None else: sys.tracebacklimit = 0 raise e create_maps(parsed_obs, off, seconds_granularity, resolution, event_time, service, target) return 0
def getSeriesSOS200(sos_url, station_number, offering_number, property_number, starting_time_string, ending_time_string): dates = [] values = [] get_cap_resp1 = requests.get(sos_url + '?REQUEST=GetCapabilities&SERVICE=SOS&ACCEPTVERSIONS=2.0.0', stream=True) sos1 = SensorObservationService(None,xml=get_cap_resp1.content, version="2.0.0") ####### ####### DEFINING REQUEST PARAMETERS ####### WGS84bbox_set=set(WGS84conversion(off) for off in sos1.offerings) WGS84bbox_list=list(WGS84bbox_set) # selecting STATION station = WGS84bbox_list[station_number] # selecting OFFERINGS off = GetOfferingsList(sos1, station).offering_list[offering_number] offerings = [off.id] selected_offering = off.id # selecting OBSERVED PROPERTIES prop = off.observed_properties[property_number] observedProperties = [prop] # selecting FORMAT " omFormat = 'http://www.opengis.net/om/2.0' waterml2Format = 'http://www.opengis.net/waterml/2.0' jsonFormat = 'application/json' # namespaces for request namespaces = 'xmlns(om,http://www.opengis.net/om/2.0)' # selecting TIMEPERIOD user_starting_time = dateutil.parser.parse(starting_time_string) user_ending_time = dateutil.parser.parse(ending_time_string) ####### ####### REQUEST PARAMETERS ARE NOW DEFINED ####### period_starting_time = user_starting_time iso_period_starting_time = period_starting_time.isoformat() period_ending_time = period_starting_time + datetime.timedelta(seconds = 43200) unit = '' while period_ending_time <= user_ending_time: iso_period_starting_time = period_starting_time.isoformat() iso_period_ending_time = period_ending_time.isoformat() event_time = "om:phenomenonTime,"+str(iso_period_starting_time)+"/"+str(iso_period_ending_time) response1 = None while response1 is None: try: response1 = sos1.get_observation(offerings=offerings, responseFormat=omFormat, observedProperties=observedProperties, timeout=600, namespaces=namespaces, eventTime=event_time) except: pass xml_tree1 = etree.fromstring(response1) parsed_response1 = owslib.swe.observation.sos200.SOSGetObservationResponse(xml_tree1) unit = parsed_response1.observations[0].get_result().uom for i, obs in enumerate(parsed_response1.observations): if obs.resultTime is not None and obs.get_result().value is not None: dates.append(obs.resultTime) values.append(obs.get_result().value) # print ">> obsr [%d]: "%i, obs.get_result().value, obs.get_result().uom, obs.resultTime, type(obs) else: pass period_starting_time += datetime.timedelta(seconds = 43200) period_ending_time += datetime.timedelta(seconds = 43200) print period_ending_time, user_ending_time return dates, values, selected_offering, prop, unit
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a Pandas Dataframe """ # oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec, # second is to accommodate NDBC). More info here: # http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe oFrmts = [ 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"', 'text/xml;subtype="sensorML/1.0.1"' ] params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"\nError: unable to connect to SOS service: {url} due to HTTP connection error." .format(url=sos_url_params)) self.log.write( u"\nHTTP connection error: {err}.".format(err=str(e))) sys.exit( "\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=str(e))) # vars to store returns from sos_collector.metadata_plus_exceptions function: sml_recs = {} sml_errors = {} describe_sensor_url = {} # leverage Pyoos Collector to query for all available stations and obtain SensorML # (if station subset not passed in --stations param) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns # write out stations in SOS that will be handled: if self.verbose: self.log.write(u"\nStations to process for SOS: {sos}".format( sos=sos_url_params)) print("Stations to process for SOS: {sos}".format( sos=sos_url_params)) for feature in sos_collector.features: self.log.write(u"\n - {feature}".format(feature=feature)) print(" - {feature}".format(feature=feature)) # iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC): # for fmt in reversed(oFrmts): for fmt in oFrmts: try: sml_recs, sml_errors = sos_collector.metadata_plus_exceptions( output_format=fmt, timeout=200) # if no valid SensorML docs returned, try next oFrmt: if not sml_recs: continue else: # assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to # create DescribeSensor URLs for failures to record in logs): for station in sos_collector.features: describe_sensor_url[ station] = self.generate_describe_sensor_url( sosgc, procedure=station, oFrmt=fmt) # report on errors returned from metadata_plus_exceptions: if sml_errors: if self.verbose: for station, msg in iteritems(sml_errors): self.log.write( u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) print( "SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) else: self.log.write( u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) print( "Success, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) break # ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt: except ServiceException as e: continue station_recs = [] failures = [] # generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings: for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: # iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS): for fmt in oFrmts: try: describe_sensor_url[ station_urn] = self.generate_describe_sensor_url( sosgc, procedure=station_urn, oFrmt=fmt) sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=fmt, timeout=200) break except ServiceException as e: sml_errors[station_urn] = str(e) continue sml = SensorML(sml_str) else: # process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting): try: sml = sml_recs[station_urn] except KeyError: self.log.write( u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) failures.append(station_urn) continue if self.sos_type.lower() == 'ndbc': # later: add an error check sosgc_station_offering = sosgc.contents[ 'station-' + station_urn.split(':')[-1]] else: sosgc_station_offering = None try: ds = IoosDescribeSensor(sml._root) except AttributeError: self.log.write( u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) station = OrderedDict() # debug: if self.verbose: self.log.write(u"\n\nProcessing station: {station}".format( station=station_urn)) print("Processing station: {station}".format( station=station_urn)) self.log.write("\n" + etree.tostring(sml._root).decode('utf-8')) # assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found): try: pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \ if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \ else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates'))) station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) except AttributeError as e: station['lon'] = None station['lat'] = None system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document # obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList): contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ else system_el.findall(self.nsp('sml:contact')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact # verify a 'publisher' Contact exists (template expects one): if "publisher" not in contacts_dct.keys(): self.log.write( u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) print( "Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) failures.append(station_urn) continue sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available, otherwise use current date # bc ISO requires date value in output location in template: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except (AttributeError, TypeError) as e: begin_service_date = datetime.now(pytz.utc) station['station_urn'] = station_urn station['sos_url'] = sos_url_params station['describesensor_url'] = describe_sensor_url[station_urn] station['shortName'] = ds.shortName station['longName'] = ds.longName if self.sos_type.lower() == 'ndbc': station['wmoID'] = station_urn.split(':')[-1] else: station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appropriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station['starting'] = sosgc_station_offering.begin_position station['ending'] = sosgc_station_offering.end_position else: station['starting'] = ds.starting station['ending'] = ds.ending if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station[ 'variable_uris'] = sosgc_station_offering.observed_properties station['variables'] = [ var.split('/')[-1] for var in sosgc_station_offering.observed_properties ] station['parameter_uris'] = ','.join(station['variable_uris']) station['parameters'] = ','.join(station['variables']) else: station['variable_uris'] = ds.variables station['variables'] = [ var.split('/')[-1] for var in ds.variables ] station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) if self.verbose: for var in station['variable_uris']: self.log.write(u"\nvariable: {var}".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' vals from SensorML: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() # if sosgc.content.name == station_urn ] # match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to # populate 'download_formats' list, that is then used to generate GetObservation requests for the template: # (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' ) download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"\nresponseFormat: {format}".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"\ndownloadFormats: {format}".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}" utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else '' if station['starting'] is not None and station[ 'ending'] is not None: event_time = event_time_formatstr.format( begin=station['ending'] - timedelta(hours=self.getobs_req_hours), end=station['ending'], utc_code=utc_code) if self.verbose: self.log.write( u"\nUsing starting/ending times from SensorML for eventTime" ) print( "Using starting/ending times from SensorML for eventTime" ) self.log.write( u"\nobservationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) print( "observationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = event_time_formatstr.format(begin=then, end=now, utc_code=utc_code) if self.verbose: self.log.write( u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) print( "No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"\neventTime: {time}".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type # and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in station['variable_uris']: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP.get(format, format), 'format_name': self.RESPONSE_FORMAT_NAME_MAP.get(format, format) } if self.verbose: self.log.write( u"\ngetobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) print( "getobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) # extra debug for failed stations in verbose mode: if self.verbose: self.log.write( u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:" ) print("SOS DescribeSensor request errors recap. Failed requests:") for station_fail, msg in iteritems(sml_errors): self.log.write( u"\n{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace( "&", "&"))) print("{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace("&", "&"))) if failures: self.log.write( u"\nStations in 'failures' list (should match DescribeSensor errors):" ) print( "Stations in 'failures' list (should match DescribeSensor errors):" ) for station_fail in failures: self.log.write(u"\n{station}".format(station=station_fail)) print("{station}".format(station=station_fail)) if station_recs: stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df else: return None
def _parse_sos(context, repos, record, identifier, version): from owslib.sos import SensorObservationService bboxs = [] recobjs = [] serviceobj = repos.dataset() if version == '1.0.0': schema = 'http://www.opengis.net/sos/1.0' else: schema = 'http://www.opengis.net/sos/2.0' md = SensorObservationService(record, version=version) # generate record of service instance _set(context, serviceobj, 'pycsw:Identifier', identifier) _set(context, serviceobj, 'pycsw:Typename', 'csw:Record') _set(context, serviceobj, 'pycsw:Schema', schema) _set(context, serviceobj, 'pycsw:MdSource', record) _set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now()) _set(context, serviceobj, 'pycsw:XML', etree.tostring(md._capabilities)) _set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(etree.tostring(md._capabilities))) _set(context, serviceobj, 'pycsw:Type', 'service') _set(context, serviceobj, 'pycsw:Title', md.identification.title) _set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract) _set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords)) _set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name) _set(context, serviceobj, 'pycsw:Publisher', md.provider.name) _set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name) _set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name) _set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessconstraints) _set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees) _set(context, serviceobj, 'pycsw:Source', record) _set(context, serviceobj, 'pycsw:Format', md.identification.type) _set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326') _set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees') _set(context, serviceobj, 'pycsw:ServiceType', md.identification.type) _set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version) _set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations])) _set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents))) _set(context, serviceobj, 'pycsw:CouplingType', 'tight') links = [ '%s,OGC-SOS Sensor Observation Service,OGC:SOS,%s' % (identifier, md.url), '%s,OGC-SOS Capabilities service (ver %s),OGC:SOS-%s-http-get-capabilities,%s' % (identifier, version, version, build_get_url(md.url, { 'service': 'SOS', 'version': version, 'request': 'GetCapabilities' })), ] _set(context, serviceobj, 'pycsw:Links', '^'.join(links)) # generate record foreach offering LOGGER.debug('Harvesting %d SOS ObservationOffering\'s ', len(md.contents)) for offering in md.contents: recobj = repos.dataset() identifier2 = '%s-%s' % (identifier, md.contents[offering].id) _set(context, recobj, 'pycsw:Identifier', identifier2) _set(context, recobj, 'pycsw:Typename', 'csw:Record') _set(context, recobj, 'pycsw:Schema', schema) _set(context, recobj, 'pycsw:MdSource', record) _set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now()) _set(context, recobj, 'pycsw:XML', etree.tostring(md._capabilities)) _set(context, recobj, 'pycsw:Type', 'dataset') _set(context, recobj, 'pycsw:ParentIdentifier', identifier) _set(context, recobj, 'pycsw:Title', md.contents[offering].description) _set(context, recobj, 'pycsw:Abstract', md.contents[offering].description) _set(context, recobj, 'pycsw:TempExtent_begin', util.datetime2iso8601(md.contents[offering].begin_position)) _set(context, recobj, 'pycsw:TempExtent_end', util.datetime2iso8601(md.contents[offering].end_position)) _set(context, recobj, 'pycsw:AnyText', util.get_anytext([md.contents[offering].description])) bbox = md.contents[offering].bbox if bbox is not None: tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3]) wkt_polygon = util.bbox2wktpolygon(tmp) _set(context, recobj, 'pycsw:BoundingBox', wkt_polygon) _set(context, recobj, 'pycsw:CRS', md.contents[offering].bbox_srs.id) _set(context, recobj, 'pycsw:DistanceUOM', 'degrees') bboxs.append(wkt_polygon) recobjs.append(recobj) # Derive a bbox based on aggregated featuretype bbox values bbox_agg = util.bbox_from_polygons(bboxs) if bbox_agg is not None: _set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg) recobjs.insert(0, serviceobj) return recobjs
def get_capabilities(self): cap = self.get_cache('capabilities') if not cap: self.sos_service = SensorObservationService(self.service_url, version=self.version) cap = self.set_cache('capabilities', self.sos_service) return cap
def sniff_test_resource(config, resource_type, url): """tests a service and provides run metrics""" if resource_type not in RESOURCE_TYPES.keys(): msg = gettext('Invalid resource type') msg2 = '%s: %s' % (msg, resource_type) LOGGER.error(msg2) raise RuntimeError(msg2) title = None start_time = datetime.datetime.utcnow() message = None try: if resource_type == 'OGC:WMS': ows = WebMapService(url) elif resource_type == 'OGC:WMTS': ows = WebMapTileService(url) elif resource_type == 'OSGeo:TMS': ows = TileMapService(url) elif resource_type == 'OGC:WFS': ows = WebFeatureService(url) elif resource_type == 'OGC:WCS': ows = WebCoverageService(url, version='1.0.0') elif resource_type == 'OGC:WPS': ows = WebProcessingService(url) elif resource_type == 'OGC:CSW': ows = CatalogueServiceWeb(url) elif resource_type == 'OGC:SOS': ows = SensorObservationService(url) elif resource_type == 'OGC:STA': ows = urlopen(url) elif resource_type in ['WWW:LINK', 'urn:geoss:waf']: ows = urlopen(url) if resource_type == 'WWW:LINK': content_type = ows.info().getheader('Content-Type') # Check content if the response is not an image if 'image/' not in content_type: content = ows.read() import re try: title_re = re.compile("<title>(.+?)</title>") title = title_re.search(content).group(1) except: title = url # Optional check for any OGC-Exceptions in Response if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']: exception_text = None try: except_re = re.compile( "ServiceException>|ExceptionReport>") exception_text = except_re.search(content).group(0) except: # No Exception in Response text pass if exception_text: # Found OGC-Exception in Response text raise Exception("Exception in response: %s" % exception_text) del content elif resource_type == 'urn:geoss:waf': title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname) elif resource_type == 'FTP': ows = urlopen(url) title = urlparse(url).hostname success = True if resource_type.startswith(('OGC:', 'OSGeo')): if resource_type == 'OGC:STA': title = 'OGC STA' else: title = ows.identification.title if title is None: title = '%s %s %s' % (resource_type, gettext('for'), url) except Exception as err: msg = str(err) LOGGER.exception(msg) message = msg success = False end_time = datetime.datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) return [title, success, response_time, message, start_time]
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a GeoDataFrame """ # LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) sos_url_params_esc = sos_url_params.replace("&", "&") # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n" .format(url=sos_url_params)) self.log.write( u"HTTP connection error: {err}.\n".format(err=e.message)) sys.exit( "Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=e.message)) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns sml_lst = sos_collector.metadata(timeout=200) station_recs = [] for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=oFrmt, timeout=200) sml = SensorML(sml_str) else: sml = sml_lst[station_idx] # debug: # if self.verbose: # self.log.write(unicode(etree.tostring(sml._root))) ds = IoosDescribeSensor(sml._root) pos = testXMLValue( ds.system.location.find(self.nsp('gml:Point/gml:pos'))) system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document contacts = system_el.findall( self.nsp('sml:contact/sml:ContactList/sml:member')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except AttributeError as e: begin_service_date = None station = OrderedDict() station['station_urn'] = station_urn station['sos_url'] = sos_url_params_esc station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) station['shortName'] = ds.shortName station['longName'] = ds.longName station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appopriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct # MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used # in the template code currently in favor of 'contacts_dct' # station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont) # station['operator_org'] = contacts_dct['operator'].organization # station['operator_country'] = contacts_dct['operator'].country # station['operator_url'] = contacts_dct['operator'].url # station['operator_email'] = contacts_dct['operator'].email # station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont) # station['publisher_org'] = contacts_dct['publisher'].organization # station['publisher_url'] = contacts_dct['publisher'].url # station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress station['starting'] = ds.starting station['ending'] = ds.ending # station['starting_isostr'] = datetime.isoformat(ds.starting) # station['ending_isostr'] = datetime.isoformat(ds.ending) station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) station['variables'] = [var.split('/')[-1] for var in ds.variables] # debug: if self.verbose: self.log.write(u"\nProcessing station: {station}\n".format( station=station_urn)) print("\nProcessing station: {station}".format( station=station_urn)) for var in ds.variables: self.log.write(u"variable: {var}\n".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' values and populate list: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ] # subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter # (or 'application/json,application/zip; subtype=x-netcdf' by default): download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"responseFormat: {format}\n".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"downloadFormats: {format}\n".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: if ds.starting is not None and ds.ending is not None: event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=ds.ending - timedelta(hours=self.getobs_req_hours), end=ds.ending) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=then, end=now) if self.verbose: self.log.write( u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n" .format(then=then, now=now)) print( "then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"eventTime: {time}\n".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in ds.variables: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_request_url_esc = getobs_request_url.replace( "&", "&") getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url_esc, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format], 'format_name': self.RESPONSE_FORMAT_NAME_MAP[format] } if self.verbose: self.log.write( u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) print( "getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df