def process_doc(self, doc):
"""
Attempt to parse an xml string conforming to either an SOS or SensorML
dataset and return the results
"""
xml_doc = ET.fromstring(doc)
if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
ds = SensorObservationService(None, xml=doc)
# SensorObservationService does not store the etree doc root,
# so maybe use monkey patching here for now?
ds._root = xml_doc
elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
ds = SensorML(xml_doc)
else:
raise ValueError("Unrecognized XML root element: {}".format(xml_doc.tag))
return ds
def main():
fl = str()
for f, val in flags.items():
if val is True:
fl += f
try:
run_command('r.in.sos', flags=fl, **options)
except:
return 0
if any(value is True and key in ['o', 'v', 'p', 't']
for key, value in flags.items()):
return 0
service = SensorObservationService(options['url'],
version=options['version'],
username=options['username'],
password=options['password'])
for off in options['offering'].split(','):
out = soslib.handle_not_given_options(service, off,
options['procedure'],
options['observed_properties'],
options['event_time'])
procedure, observed_properties, event_time = out
for observed_property in observed_properties:
map_name = '{}_{}_{}'.format(options['output'], off,
observed_property)
if ':' in map_name:
map_name = '_'.join(map_name.split(':'))
if '-' in map_name:
map_name = '_'.join(map_name.split('-'))
if '.' in map_name:
map_name = '_'.join(map_name.split('.'))
maps_list_file = get_maps(map_name)
create_temporal(maps_list_file, map_name)
return 0
def run_test_resource(resource_type, url):
"""tests a CSW service and provides run metrics"""
if resource_type not in RESOURCE_TYPES.keys():
msg = 'Invalid resource type: %s' % resource_type
LOGGER.error(msg)
raise RuntimeError(msg)
title = None
start_time = datetime.datetime.utcnow()
message = None
try:
if resource_type == 'OGC:WMS':
ows = WebMapService(url)
elif resource_type == 'OGC:WFS':
ows = WebFeatureService(url)
elif resource_type == 'OGC:WCS':
ows = WebCoverageService(url)
elif resource_type == 'OGC:WPS':
ows = WebProcessingService(url)
elif resource_type == 'OGC:CSW':
ows = CatalogueServiceWeb(url)
elif resource_type == 'OGC:SOS':
ows = SensorObservationService(url)
elif resource_type == 'WWW:LINK':
ows = urlopen(url)
import re
title_re = re.compile("<title>(.+?)</title>")
title = title_re.search(ows.read()).group(1)
success = True
if resource_type != 'WWW:LINK':
title = ows.identification.title
except Exception, err:
msg = str(err)
LOGGER.exception(msg)
message = msg
success = False
def getCapabilitiesSOS200(getcap_response):
"""
Retrieve information from GetCapabilities response,
and then show the window displaying it.
argument:
>>> getcap_response:
class 'requests.models.Response' object containing GetCapabilities
request response retrieved from HTTP "Get" response.
"""
global cap_window
# Use GetCapabilityWindow() imported from gui module
# located in "ui" subdirectory.
cap_window = GetCapabilityWindow()
sos = SensorObservationService(None,xml=getcap_response.content)
sos_id = sos.identification
cap_window.title_value.setPlainText(sos_id.title)
cap_window.abstract_value.setPlainText(sos_id.abstract)
p = sos.provider
cap_window.provider_name_value.setPlainText(p.name)
cap_window.provider_website_value.setPlainText(p.url)
sc = p.contact
cap_window.contact_phone_value.setText(sc.phone)
cap_window.contact_email_value.setText(sc.email)
cap_window.contact_address_value.setText(sc.address)
cap_window.contact_city_value.setText(sc.city)
cap_window.contact_region_value.setText(sc.region)
cap_window.contact_postcode_value.setText(sc.postcode)
cap_window.contact_country_value.setText(sc.country)
cap_window.show()
def main():
parsed_obs = dict()
layerscount = 0
service = SensorObservationService(options['url'],
version=options['version'],
username=options['username'],
password=options['password'])
if any(value is True and key in ['o', 'v', 'p', 't']
for key, value in flags.items()):
soslib.get_description(service, options, flags)
soslib.check_missing_params(options['offering'], options['output'])
if options['granularity'] != '':
import grass.temporal as tgis
tgis.init()
seconds_granularity = int(
tgis.gran_to_gran(options['granularity'], '1 second', True))
else:
seconds_granularity = 1
target = soslib.get_target_crs()
run_command('g.remove', 'f', type='vector', name=options['output'])
new = VectorTopo(options['output'])
for off in options['offering'].split(','):
# TODO: Find better way than iteration (at best OWSLib upgrade)
out = soslib.handle_not_given_options(service, off,
options['procedure'],
options['observed_properties'],
options['event_time'])
procedure, observed_properties, event_time = out
if flags['s']:
create_maps(_, off, _, new, _, _, service, target, _, procedure)
else:
try:
obs = service.get_observation(
offerings=[off],
responseFormat=options['response_format'],
observedProperties=observed_properties,
procedure=procedure,
eventTime=event_time,
timeout=int(options['timeout']),
username=options['username'],
password=options['password'])
except:
# TODO: catch errors properly (e.g. timeout)
grass.fatal('Request did not succeed!')
try:
if options['version'] in ['1.0.0', '1.0'] and str(
options['response_format']
) == 'text/xml;subtype="om/1.0.0"':
for prop in observed_properties:
parsed_obs.update(
{prop: soslib.xml2geojson(obs, prop, flags['i'])})
elif str(options['response_format']) == 'application/json':
for prop in observed_properties:
parsed_obs.update(
{prop: soslib.json2geojson(obs, prop)})
except AttributeError:
if sys.version_info[0] >= 3:
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError(
'There is no data for at least one of your procedures, '
'could you change the time parameter, observed '
'properties, procedures or offerings')
except ValueError as e:
if sys.version_info[0] >= 3:
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise e
create_maps(parsed_obs, off, layerscount, new, seconds_granularity,
event_time, service, target, observed_properties)
layerscount += len(parsed_obs)
return 0
def main():
parsed_obs = dict()
service = SensorObservationService(options['url'],
version=options['version'],
username=options['username'],
password=options['password'])
if any(value is True and key in ['o', 'v', 'p', 't']
for key, value in flags.iteritems()):
get_description(service, options, flags)
if options['offering'] == '' or options['output'] == '':
if sys.version >= (3, 0):
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError(
"You have to define any flags or use 'output' and 'offering' "
"parameters to get the data")
if options['granularity'] != '':
import grass.temporal as tgis
tgis.init()
secondsGranularity = int(
tgis.gran_to_gran(options['granularity'], '1 second', True))
else:
secondsGranularity = 1
if options['resolution'] == '':
resolution = None
else:
resolution = float(options['resolution'])
if options['bbox'] != '':
bbox = options['bbox'].split(',')
run_command('g.region',
n=float(bbox[0]),
e=float(bbox[1]),
s=float(bbox[2]),
w=float(bbox[3]),
res=resolution)
else:
grass.warning('You have not setted the bounding box. Bounding box will'
' be automatically based on procedure geometries for '
'every map.')
for off in options['offering'].split(','):
# TODO: Find better way than iteration (at best OWSLib upgrade)
procedure, observed_properties, event_time = handle_not_given_options(
service, off, options['procedure'], options['observed_properties'],
options['event_time'])
event_time = 'T'.join(event_time.split(' '))
obs = service.get_observation(
offerings=[off],
responseFormat=options['response_format'],
observedProperties=[observed_properties],
procedure=procedure,
eventTime=event_time,
username=options['username'],
password=options['password'])
try:
if options['version'] in ['1.0.0', '1.0'] and \
str(options['response_format']) == 'text/xml;subtype="om/1.0.0"':
for property in observed_properties.split(','):
parsed_obs.update({property: xml2geojson(obs, property)})
elif str(options['response_format']) == 'application/json':
for property in observed_properties.split(','):
parsed_obs.update({property: json2geojson(obs, property)})
except AttributeError:
if sys.version >= (3, 0):
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError('There is no data for at least one of your '
'procedures, could you change the time '
'parameter, observed properties, '
'procedures or offerings')
create_maps(parsed_obs, off, secondsGranularity, resolution)
return 0
def _parse_sos(context, repos, record, identifier, version):
from owslib.sos import SensorObservationService
bboxs = []
recobjs = []
serviceobj = repos.dataset()
if version == '1.0.0':
schema = 'http://www.opengis.net/sos/1.0'
else:
schema = 'http://www.opengis.net/sos/2.0'
md = SensorObservationService(record, version=version)
# generate record of service instance
_set(context, serviceobj, 'pycsw:Identifier', identifier)
_set(context, serviceobj, 'pycsw:Typename', 'csw:Record')
_set(context, serviceobj, 'pycsw:Schema', schema)
_set(context, serviceobj, 'pycsw:MdSource', record)
_set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(etree.tostring(md._capabilities)))
_set(context, serviceobj, 'pycsw:Type', 'service')
_set(context, serviceobj, 'pycsw:Title', md.identification.title)
_set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords))
_set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:Publisher', md.provider.name)
_set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessconstraints)
_set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees)
_set(context, serviceobj, 'pycsw:Source', record)
_set(context, serviceobj, 'pycsw:Format', md.identification.type)
_set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees')
_set(context, serviceobj, 'pycsw:ServiceType', 'OGC:SOS')
_set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version)
_set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations]))
_set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents)))
_set(context, serviceobj, 'pycsw:CouplingType', 'tight')
links = [
'%s,OGC-SOS Sensor Observation Service,OGC:SOS,%s' % (identifier, md.url),
]
_set(context, serviceobj, 'pycsw:Links', '^'.join(links))
# generate record foreach offering
LOGGER.debug('Harvesting %d SOS ObservationOffering\'s ', len(md.contents))
for offering in md.contents:
recobj = repos.dataset()
identifier2 = '%s-%s' % (identifier, md.contents[offering].id)
_set(context, recobj, 'pycsw:Identifier', identifier2)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', schema)
_set(context, recobj, 'pycsw:MdSource', record)
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:Type', 'dataset')
_set(context, recobj, 'pycsw:ParentIdentifier', identifier)
_set(context, recobj, 'pycsw:Title', md.contents[offering].description)
_set(context, recobj, 'pycsw:Abstract', md.contents[offering].description)
_set(context, recobj, 'pycsw:TempExtent_begin', util.datetime2iso8601(md.contents[offering].begin_position))
_set(context, recobj, 'pycsw:TempExtent_end', util.datetime2iso8601(md.contents[offering].end_position))
#For observed_properties that have mmi url or urn, we simply want the observation name.
observed_properties = []
for obs in md.contents[offering].observed_properties:
#Observation is stored as urn representation: urn:ogc:def:phenomenon:mmisw.org:cf:sea_water_salinity
if obs.lower().startswith(('urn:', 'x-urn')):
observed_properties.append(obs.rsplit(':', 1)[-1])
#Observation is stored as uri representation: http://mmisw.org/ont/cf/parameter/sea_floor_depth_below_sea_surface
elif obs.lower().startswith(('http://', 'https://')):
observed_properties.append(obs.rsplit('/', 1)[-1])
else:
observed_properties.append(obs)
#Build anytext from description and the observed_properties.
anytext = []
anytext.append(md.contents[offering].description)
anytext.extend(observed_properties)
_set(context, recobj, 'pycsw:AnyText', util.get_anytext(anytext))
_set(context, recobj, 'pycsw:Keywords', ','.join(observed_properties))
bbox = md.contents[offering].bbox
if bbox is not None:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
wkt_polygon = util.bbox2wktpolygon(tmp)
_set(context, recobj, 'pycsw:BoundingBox', wkt_polygon)
_set(context, recobj, 'pycsw:CRS', md.contents[offering].bbox_srs.id)
_set(context, recobj, 'pycsw:DistanceUOM', 'degrees')
bboxs.append(wkt_polygon)
_set(context, recobj, 'pycsw:XML', caps2iso(recobj, md, context))
recobjs.append(recobj)
# Derive a bbox based on aggregated featuretype bbox values
bbox_agg = util.bbox_from_polygons(bboxs)
if bbox_agg is not None:
_set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg)
_set(context, serviceobj, 'pycsw:XML', caps2iso(serviceobj, md, context))
recobjs.insert(0, serviceobj)
return recobjs
def test_sos_caps():
sos = SensorObservationService(SERVICE_URL)
assert str(sos.contents['81102 - PM10']) == "Offering id: 81102 - PM10, name: Particulate Matter < 10 µm"
assert repr(sos.contents['81102 - PM10']) == "<SosObservationOffering 'Particulate Matter < 10 µm'>"
# Accessing ncSOS with OWSLib # <codecell> from owslib.sos import SensorObservationService import pdb from owslib.etree import etree import pandas as pd import datetime as dt # <codecell> # usgs woods hole # buoy data (ADCP) url = 'http://geoport.whoi.edu/thredds/sos/usgs/data2/emontgomery/stellwagen/CF-1.6/HURRIRENE_BB/9141wh-a.nc' usgs = SensorObservationService(url) contents = usgs.contents # <codecell> usgs.contents # <codecell> off = usgs.offerings[1] off.name # <codecell> off.response_formats
def showServerSelectionDialog(self):
text, ok = QtGui.QInputDialog.getText(self, 'SOS server selection',
'Enter sos service url:')
if ok:
# Reset attributes.
#
# Reset UI attributes.
self.statusBar.clearMessage()
# Second block of main window attributes.
self.select_offering_comboBox.clear()
self.select_prop_comboBox.clear()
self.starting_time_pushButton.setText("")
self.ending_time_pushButton.setText("")
self.time_series_starting_time_value.setText("")
self.time_series_ending_time_value.setText("")
# QGIS related attributes.
# Remove station features and get an empty station layer.
listOfIds = [
feat.id() for feat in self.stations_layer.getFeatures()
]
self.stations_layer.dataProvider().deleteFeatures(listOfIds)
# Other attributes.
self.getobs_response = ''
self.getseries_boolean = False
# Set attributes using retrieved SOS server information.
#
self.sos_service_url = str(text)
self.getcap_response = requests.get(
self.sos_service_url + '?REQUEST=GetCapabilities'
'&SERVICE=SOS&ACCEPTVERSIONS=2.0.0',
stream=True)
self.sos = SensorObservationService(
None, xml=self.getcap_response.content, version="2.0.0")
self.WGS84bbox_set = set(
WGS84conversion(off) for off in self.sos.offerings)
self.WGS84bbox_list = list(self.WGS84bbox_set)
# Set UI attributes
self.selected_sos_server_lineEdit.setText(self.sos_service_url)
if self.WGS84bbox_list == []:
# Display error message using QMessageBox.
empty_bbox_msg = QtGui.QMessageBox()
empty_bbox_msg.setWindowTitle("Error")
empty_bbox_msg.setTextFormat(QtCore.Qt.RichText)
msg_text = ('Each offering bounding box is empty when using '
'OWSLib with this SOS 2.0 server')
empty_bbox_msg.setText(msg_text)
i_text = (
'This plugin uses the Open Source '
'<a href=\"https://geopython.github.io/OWSLib/\">OWSlib library</a> '
'to retrieve SOS 2.0 data. When collecting offerings '
'and their corresponding featureOfInterest bounding '
'boxes, only empty lists were retrieved.')
empty_bbox_msg.setInformativeText(i_text)
d_text = (
'In order to solve the problem, you may want to have '
'a look at OWSlib documentation '
'(https://geopython.github.io/OWSLib/) and at the '
'Python source file '
'(https://github.com/geopython/OWSLib/blob/master/owslib/swe/observation/sos200.py/) '
'containing the offering class, including the '
'bounding box attribute.')
empty_bbox_msg.setDetailedText(d_text)
empty_bbox_msg.setIcon(QtGui.QMessageBox.Critical)
empty_bbox_msg.exec_()
# Reload station layer as selected SOS server has changed.
self.stations_layer.triggerRepaint()
elif self.WGS84bbox_list == [None]:
# Display warning message using QMessageBox.
none_bbox_msg = QtGui.QMessageBox()
none_bbox_msg.setWindowTitle("Warning")
none_bbox_msg.setTextFormat(QtCore.Qt.RichText)
msg_text = (
"Each offering has 'None' bounding box when using "
"OWSLib with this SOS 2.0 server. This plugin uses "
"the Open Source "
"<a href=\"https://geopython.github.io/OWSLib/\">OWSlib library</a>"
" to retrieve SOS 2.0 data. When collecting the "
"featureOfInterest bounding box for each offering, "
"only None objects were retrieved. Consequently, "
"no feature of interest could be added to the "
"'Features of interest' layer generated by the plugin.")
none_bbox_msg.setText(msg_text)
i_text = (
'Please select directly an offering in the "Offering" '
'combobox to unlock the "ObservedPropery" combobox. '
'You will then be able to select all GetObservation '
'request parameters and to retrieve desired '
'time series.')
none_bbox_msg.setInformativeText(i_text)
d_text = (
'In order to solve the problem, you may want to have '
'a look at OWSlib documentation '
'(https://geopython.github.io/OWSLib/) and at the '
'Python source file '
'(https://github.com/geopython/OWSLib/blob/master/owslib/swe/observation/sos200.py/) '
'containing the offering class, including the '
'bounding box attribute.')
none_bbox_msg.setDetailedText(d_text)
none_bbox_msg.setIcon(QtGui.QMessageBox.Warning)
none_bbox_msg.exec_()
# Reload station layer as selected SOS server has changed.
self.stations_layer.triggerRepaint()
# Fill OfferingComboBox with all offerings from stations
# which have None bbox.
self.selected_station_index = 0
station = self.WGS84bbox_list[self.selected_station_index]
for o in GetOfferingsList(self.sos, station).offering_list:
self.select_offering_comboBox.addItem(o.id)
else:
# Stations bbox content is valid.
#
# For each station...
for i, s in enumerate(self.WGS84bbox_list):
# Define pairs of coordinates of two points which
# generate the bounding box of this stations.
if s is not None:
xmin = min(s[1], s[3])
xmax = max(s[1], s[3])
ymin = min(s[0], s[2])
ymax = max(s[0], s[2])
# Create new point which represents the station.
# Here, we choose to set station location using the point
# which has min lat and min long, as several SOS 2.0
# we have tested lead to two points coinciding.
# Therefore, we make the assumption xmin==xmax and
# ymin==ymax.
# When it is not the case, station location is likely
# to be incorrect.
# Further development is required to solve this issue.
new_feature = QgsFeature()
new_feature.setGeometry(
QgsGeometry.fromPoint(QgsPoint(xmin, ymin)))
new_feature.setAttributes([i])
self.stations_layer.dataProvider().addFeatures(
[new_feature])
# Update layer and refresh QGIS canvas extent.
self.stations_layer.updateExtents()
QgsMapLayerRegistry.instance().addMapLayer(self.stations_layer)
self.stations_layer.triggerRepaint()
canvas = qgis.utils.iface.mapCanvas()
canvas.setExtent(self.stations_layer.extent())
# Inform user he/she has to spatially select a station from
# newly added 'Features of interest' layer.
foi_layer_msg = QtGui.QMessageBox()
foi_layer_msg.setWindowTitle("Information")
msg_text = (
'A "Features of interest" layer has been added to the map!'
)
foi_layer_msg.setText(msg_text)
i_text = (
'Please select a station of the "Features of interest" '
'layer using the select features tool of the QGIS '
'toolbar to unlock the "Offering" combobox and the '
'"ObservedPropery" combobox. You will then be able to '
'select all GetObservation request parameters and to '
'retrieve desired time series.')
foi_layer_msg.setInformativeText(i_text)
foi_layer_msg.setIcon(QtGui.QMessageBox.Information)
foi_layer_msg.exec_()
def service(self):
if self._service is None:
service = SensorObservationService(SOSURL, version=SOSVERSION)
self._service = service
return self._service
def main():
parsed_obs = dict()
service = SensorObservationService(options['url'],
version=options['version'],
username=options['username'],
password=options['password'])
if any(value is True and key in ['o', 'v', 'p', 't']
for key, value in flags.iteritems()):
get_description(service, options, flags)
if options['offering'] == '' or options['output'] == '':
if sys.version >= (3, 0):
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError("You have to define any flags or use 'output' and"
" 'offering' parameters to get the data")
if options['granularity'] != '':
import grass.temporal as tgis
tgis.init()
secondsGranularity = int(
tgis.gran_to_gran(options['granularity'], '1 second', True))
else:
secondsGranularity = 1
for off in options['offering'].split(','):
# TODO: Find better way than iteration (at best OWSLib upgrade)
procedure, observed_properties, event_time = handle_not_given_options(
service, off, options['procedure'], options['observed_properties'],
options['event_time'])
event_time = 'T'.join(event_time.split(' '))
obs = service.get_observation(
offerings=[off],
responseFormat=options['response_format'],
observedProperties=observed_properties,
procedure=procedure,
eventTime=event_time,
username=options['username'],
password=options['password'])
try:
if options['version'] in ['1.0.0', '1.0'] and str(
options['response_format']
) == 'text/xml;subtype="om/1.0.0"':
for prop in observed_properties:
parsed_obs.update({prop: xml2geojson(obs, prop)})
elif str(options['response_format']) == 'application/json':
for prop in observed_properties:
parsed_obs.update({prop: json2geojson(obs, prop)})
except AttributeError:
if sys.version >= (3, 0):
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError('There is no data for at least one of your '
'procedures, could you change the time '
'parameter, observed properties, '
'procedures or offerings')
create_maps(parsed_obs, off, secondsGranularity)
return 0
def main():
parsed_obs = dict()
service = SensorObservationService(options['url'],
version=options['version'],
username=options['username'],
password=options['password'])
if any(value is True and key in ['o', 'v', 'p', 't']
for key, value in flags.items()):
soslib.get_description(service, options, flags)
soslib.check_missing_params(options['offering'], options['output'])
if options['granularity'] != '':
import grass.temporal as tgis
tgis.init()
seconds_granularity = int(
tgis.gran_to_gran(options['granularity'], '1 second', True))
else:
seconds_granularity = 1
if options['resolution'] == '':
a = grass.read_command('g.region', flags='gf')
resolution = float(a.split('nsres=')[1].split(' ')[0])
run_command('g.message',
flags='w',
message='No resolution was setted. Using the resolution '
'{} (nres of your current setting).'.format(resolution))
else:
resolution = float(options['resolution'])
if options['bbox'] != '':
bbox = options['bbox'].split(',')
run_command('g.region',
n=float(bbox[0]),
e=float(bbox[1]),
s=float(bbox[2]),
w=float(bbox[3]),
res=resolution)
else:
grass.warning('You have not setted the bounding box. Bounding box will'
' be automatically based on procedure geometries for '
'every map.')
target = soslib.get_target_crs()
for off in options['offering'].split(','):
# TODO: Find better way than iteration (at best OWSLib upgrade)
out = soslib.handle_not_given_options(service, off,
options['procedure'],
options['observed_properties'],
options['event_time'])
procedure, observed_properties, event_time = out
if flags['s']:
create_maps(_, off, _, resolution, _, service, target, procedure)
else:
try:
obs = service.get_observation(
offerings=[off],
responseFormat=options['response_format'],
observedProperties=observed_properties,
procedure=procedure,
eventTime=event_time,
username=options['username'],
password=options['password'])
except:
# TODO: catch errors properly (e.g. timeout)
grass.fatal('Request did not succeed!')
try:
if options['version'] in ['1.0.0', '1.0'] and \
options['response_format'] == 'text/xml;subtype="om/1.0.0"':
for prop in observed_properties:
parsed_obs.update(
{prop: soslib.xml2geojson(obs, prop)})
elif str(options['response_format']) == 'application/json':
for prop in observed_properties:
parsed_obs.update(
{prop: soslib.json2geojson(obs, prop)})
except AttributeError:
if sys.version_info[0] >= 3:
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise AttributeError(
'There is no data for at least one of your procedures, '
'could you change the time parameter, observed '
'properties, procedures or offerings')
except ValueError as e:
if sys.version_info[0] >= 3:
sys.tracebacklimit = None
else:
sys.tracebacklimit = 0
raise e
create_maps(parsed_obs, off, seconds_granularity, resolution,
event_time, service, target)
return 0
def getSeriesSOS200(sos_url, station_number, offering_number, property_number, starting_time_string, ending_time_string):
dates = []
values = []
get_cap_resp1 = requests.get(sos_url + '?REQUEST=GetCapabilities&SERVICE=SOS&ACCEPTVERSIONS=2.0.0', stream=True)
sos1 = SensorObservationService(None,xml=get_cap_resp1.content, version="2.0.0")
#######
####### DEFINING REQUEST PARAMETERS
#######
WGS84bbox_set=set(WGS84conversion(off) for off in sos1.offerings)
WGS84bbox_list=list(WGS84bbox_set)
# selecting STATION
station = WGS84bbox_list[station_number]
# selecting OFFERINGS
off = GetOfferingsList(sos1, station).offering_list[offering_number]
offerings = [off.id]
selected_offering = off.id
# selecting OBSERVED PROPERTIES
prop = off.observed_properties[property_number]
observedProperties = [prop]
# selecting FORMAT "
omFormat = 'http://www.opengis.net/om/2.0'
waterml2Format = 'http://www.opengis.net/waterml/2.0'
jsonFormat = 'application/json'
# namespaces for request
namespaces = 'xmlns(om,http://www.opengis.net/om/2.0)'
# selecting TIMEPERIOD
user_starting_time = dateutil.parser.parse(starting_time_string)
user_ending_time = dateutil.parser.parse(ending_time_string)
#######
####### REQUEST PARAMETERS ARE NOW DEFINED
#######
period_starting_time = user_starting_time
iso_period_starting_time = period_starting_time.isoformat()
period_ending_time = period_starting_time + datetime.timedelta(seconds = 43200)
unit = ''
while period_ending_time <= user_ending_time:
iso_period_starting_time = period_starting_time.isoformat()
iso_period_ending_time = period_ending_time.isoformat()
event_time = "om:phenomenonTime,"+str(iso_period_starting_time)+"/"+str(iso_period_ending_time)
response1 = None
while response1 is None:
try:
response1 = sos1.get_observation(offerings=offerings, responseFormat=omFormat, observedProperties=observedProperties, timeout=600, namespaces=namespaces, eventTime=event_time)
except:
pass
xml_tree1 = etree.fromstring(response1)
parsed_response1 = owslib.swe.observation.sos200.SOSGetObservationResponse(xml_tree1)
unit = parsed_response1.observations[0].get_result().uom
for i, obs in enumerate(parsed_response1.observations):
if obs.resultTime is not None and obs.get_result().value is not None:
dates.append(obs.resultTime)
values.append(obs.get_result().value)
# print ">> obsr [%d]: "%i, obs.get_result().value, obs.get_result().uom, obs.resultTime, type(obs)
else:
pass
period_starting_time += datetime.timedelta(seconds = 43200)
period_ending_time += datetime.timedelta(seconds = 43200)
print period_ending_time, user_ending_time
return dates, values, selected_offering, prop, unit
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a Pandas Dataframe
"""
# oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec,
# second is to accommodate NDBC). More info here:
# http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe
oFrmts = [
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
'text/xml;subtype="sensorML/1.0.1"'
]
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"\nError: unable to connect to SOS service: {url} due to HTTP connection error."
.format(url=sos_url_params))
self.log.write(
u"\nHTTP connection error: {err}.".format(err=str(e)))
sys.exit(
"\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=str(e)))
# vars to store returns from sos_collector.metadata_plus_exceptions function:
sml_recs = {}
sml_errors = {}
describe_sensor_url = {}
# leverage Pyoos Collector to query for all available stations and obtain SensorML
# (if station subset not passed in --stations param)
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
# write out stations in SOS that will be handled:
if self.verbose:
self.log.write(u"\nStations to process for SOS: {sos}".format(
sos=sos_url_params))
print("Stations to process for SOS: {sos}".format(
sos=sos_url_params))
for feature in sos_collector.features:
self.log.write(u"\n - {feature}".format(feature=feature))
print(" - {feature}".format(feature=feature))
# iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC):
# for fmt in reversed(oFrmts):
for fmt in oFrmts:
try:
sml_recs, sml_errors = sos_collector.metadata_plus_exceptions(
output_format=fmt, timeout=200)
# if no valid SensorML docs returned, try next oFrmt:
if not sml_recs:
continue
else:
# assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to
# create DescribeSensor URLs for failures to record in logs):
for station in sos_collector.features:
describe_sensor_url[
station] = self.generate_describe_sensor_url(
sosgc, procedure=station, oFrmt=fmt)
# report on errors returned from metadata_plus_exceptions:
if sml_errors:
if self.verbose:
for station, msg in iteritems(sml_errors):
self.log.write(
u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
print(
"SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
else:
self.log.write(
u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
print(
"Success, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
break
# ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt:
except ServiceException as e:
continue
station_recs = []
failures = []
# generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings:
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
# iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS):
for fmt in oFrmts:
try:
describe_sensor_url[
station_urn] = self.generate_describe_sensor_url(
sosgc, procedure=station_urn, oFrmt=fmt)
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=fmt,
timeout=200)
break
except ServiceException as e:
sml_errors[station_urn] = str(e)
continue
sml = SensorML(sml_str)
else:
# process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting):
try:
sml = sml_recs[station_urn]
except KeyError:
self.log.write(
u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
failures.append(station_urn)
continue
if self.sos_type.lower() == 'ndbc':
# later: add an error check
sosgc_station_offering = sosgc.contents[
'station-' + station_urn.split(':')[-1]]
else:
sosgc_station_offering = None
try:
ds = IoosDescribeSensor(sml._root)
except AttributeError:
self.log.write(
u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
station = OrderedDict()
# debug:
if self.verbose:
self.log.write(u"\n\nProcessing station: {station}".format(
station=station_urn))
print("Processing station: {station}".format(
station=station_urn))
self.log.write("\n" +
etree.tostring(sml._root).decode('utf-8'))
# assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found):
try:
pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \
if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \
else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates')))
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
except AttributeError as e:
station['lon'] = None
station['lat'] = None
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
# obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList):
contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
else system_el.findall(self.nsp('sml:contact'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
# verify a 'publisher' Contact exists (template expects one):
if "publisher" not in contacts_dct.keys():
self.log.write(
u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
print(
"Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
failures.append(station_urn)
continue
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available, otherwise use current date
# bc ISO requires date value in output location in template:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except (AttributeError, TypeError) as e:
begin_service_date = datetime.now(pytz.utc)
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params
station['describesensor_url'] = describe_sensor_url[station_urn]
station['shortName'] = ds.shortName
station['longName'] = ds.longName
if self.sos_type.lower() == 'ndbc':
station['wmoID'] = station_urn.split(':')[-1]
else:
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appropriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station['starting'] = sosgc_station_offering.begin_position
station['ending'] = sosgc_station_offering.end_position
else:
station['starting'] = ds.starting
station['ending'] = ds.ending
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station[
'variable_uris'] = sosgc_station_offering.observed_properties
station['variables'] = [
var.split('/')[-1]
for var in sosgc_station_offering.observed_properties
]
station['parameter_uris'] = ','.join(station['variable_uris'])
station['parameters'] = ','.join(station['variables'])
else:
station['variable_uris'] = ds.variables
station['variables'] = [
var.split('/')[-1] for var in ds.variables
]
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
if self.verbose:
for var in station['variable_uris']:
self.log.write(u"\nvariable: {var}".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' vals from SensorML:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items()
# if sosgc.content.name == station_urn ]
# match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to
# populate 'download_formats' list, that is then used to generate GetObservation requests for the template:
# (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' )
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"\nresponseFormat: {format}".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"\ndownloadFormats: {format}".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}"
utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else ''
if station['starting'] is not None and station[
'ending'] is not None:
event_time = event_time_formatstr.format(
begin=station['ending'] -
timedelta(hours=self.getobs_req_hours),
end=station['ending'],
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nUsing starting/ending times from SensorML for eventTime"
)
print(
"Using starting/ending times from SensorML for eventTime"
)
self.log.write(
u"\nobservationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
print(
"observationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = event_time_formatstr.format(begin=then,
end=now,
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
print(
"No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"\neventTime: {time}".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type
# and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in station['variable_uris']:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_req_dct[variable + '-' + format] = {
'variable':
variable,
'url':
getobs_request_url,
'format_type':
self.RESPONSE_FORMAT_TYPE_MAP.get(format, format),
'format_name':
self.RESPONSE_FORMAT_NAME_MAP.get(format, format)
}
if self.verbose:
self.log.write(
u"\ngetobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
# extra debug for failed stations in verbose mode:
if self.verbose:
self.log.write(
u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:"
)
print("SOS DescribeSensor request errors recap. Failed requests:")
for station_fail, msg in iteritems(sml_errors):
self.log.write(
u"\n{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace(
"&", "&")))
print("{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace("&",
"&")))
if failures:
self.log.write(
u"\nStations in 'failures' list (should match DescribeSensor errors):"
)
print(
"Stations in 'failures' list (should match DescribeSensor errors):"
)
for station_fail in failures:
self.log.write(u"\n{station}".format(station=station_fail))
print("{station}".format(station=station_fail))
if station_recs:
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
else:
return None
def _parse_sos(context, repos, record, identifier, version):
from owslib.sos import SensorObservationService
bboxs = []
recobjs = []
serviceobj = repos.dataset()
if version == '1.0.0':
schema = 'http://www.opengis.net/sos/1.0'
else:
schema = 'http://www.opengis.net/sos/2.0'
md = SensorObservationService(record, version=version)
# generate record of service instance
_set(context, serviceobj, 'pycsw:Identifier', identifier)
_set(context, serviceobj, 'pycsw:Typename', 'csw:Record')
_set(context, serviceobj, 'pycsw:Schema', schema)
_set(context, serviceobj, 'pycsw:MdSource', record)
_set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, serviceobj, 'pycsw:XML', etree.tostring(md._capabilities))
_set(context, serviceobj, 'pycsw:AnyText',
util.get_anytext(etree.tostring(md._capabilities)))
_set(context, serviceobj, 'pycsw:Type', 'service')
_set(context, serviceobj, 'pycsw:Title', md.identification.title)
_set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, serviceobj, 'pycsw:Keywords',
','.join(md.identification.keywords))
_set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:Publisher', md.provider.name)
_set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:OrganizationName',
md.provider.contact.name)
_set(context, serviceobj, 'pycsw:AccessConstraints',
md.identification.accessconstraints)
_set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees)
_set(context, serviceobj, 'pycsw:Source', record)
_set(context, serviceobj, 'pycsw:Format', md.identification.type)
_set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees')
_set(context, serviceobj, 'pycsw:ServiceType', md.identification.type)
_set(context, serviceobj, 'pycsw:ServiceTypeVersion',
md.identification.version)
_set(context, serviceobj, 'pycsw:Operation',
','.join([d.name for d in md.operations]))
_set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents)))
_set(context, serviceobj, 'pycsw:CouplingType', 'tight')
links = [
'%s,OGC-SOS Sensor Observation Service,OGC:SOS,%s' %
(identifier, md.url),
'%s,OGC-SOS Capabilities service (ver %s),OGC:SOS-%s-http-get-capabilities,%s'
% (identifier, version, version,
build_get_url(md.url, {
'service': 'SOS',
'version': version,
'request': 'GetCapabilities'
})),
]
_set(context, serviceobj, 'pycsw:Links', '^'.join(links))
# generate record foreach offering
LOGGER.debug('Harvesting %d SOS ObservationOffering\'s ', len(md.contents))
for offering in md.contents:
recobj = repos.dataset()
identifier2 = '%s-%s' % (identifier, md.contents[offering].id)
_set(context, recobj, 'pycsw:Identifier', identifier2)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', schema)
_set(context, recobj, 'pycsw:MdSource', record)
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:XML', etree.tostring(md._capabilities))
_set(context, recobj, 'pycsw:Type', 'dataset')
_set(context, recobj, 'pycsw:ParentIdentifier', identifier)
_set(context, recobj, 'pycsw:Title', md.contents[offering].description)
_set(context, recobj, 'pycsw:Abstract',
md.contents[offering].description)
_set(context, recobj, 'pycsw:TempExtent_begin',
util.datetime2iso8601(md.contents[offering].begin_position))
_set(context, recobj, 'pycsw:TempExtent_end',
util.datetime2iso8601(md.contents[offering].end_position))
_set(context, recobj, 'pycsw:AnyText',
util.get_anytext([md.contents[offering].description]))
bbox = md.contents[offering].bbox
if bbox is not None:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
wkt_polygon = util.bbox2wktpolygon(tmp)
_set(context, recobj, 'pycsw:BoundingBox', wkt_polygon)
_set(context, recobj, 'pycsw:CRS',
md.contents[offering].bbox_srs.id)
_set(context, recobj, 'pycsw:DistanceUOM', 'degrees')
bboxs.append(wkt_polygon)
recobjs.append(recobj)
# Derive a bbox based on aggregated featuretype bbox values
bbox_agg = util.bbox_from_polygons(bboxs)
if bbox_agg is not None:
_set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg)
recobjs.insert(0, serviceobj)
return recobjs
def get_capabilities(self):
cap = self.get_cache('capabilities')
if not cap:
self.sos_service = SensorObservationService(self.service_url, version=self.version)
cap = self.set_cache('capabilities', self.sos_service)
return cap
def sniff_test_resource(config, resource_type, url):
"""tests a service and provides run metrics"""
if resource_type not in RESOURCE_TYPES.keys():
msg = gettext('Invalid resource type')
msg2 = '%s: %s' % (msg, resource_type)
LOGGER.error(msg2)
raise RuntimeError(msg2)
title = None
start_time = datetime.datetime.utcnow()
message = None
try:
if resource_type == 'OGC:WMS':
ows = WebMapService(url)
elif resource_type == 'OGC:WMTS':
ows = WebMapTileService(url)
elif resource_type == 'OSGeo:TMS':
ows = TileMapService(url)
elif resource_type == 'OGC:WFS':
ows = WebFeatureService(url)
elif resource_type == 'OGC:WCS':
ows = WebCoverageService(url, version='1.0.0')
elif resource_type == 'OGC:WPS':
ows = WebProcessingService(url)
elif resource_type == 'OGC:CSW':
ows = CatalogueServiceWeb(url)
elif resource_type == 'OGC:SOS':
ows = SensorObservationService(url)
elif resource_type == 'OGC:STA':
ows = urlopen(url)
elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
ows = urlopen(url)
if resource_type == 'WWW:LINK':
content_type = ows.info().getheader('Content-Type')
# Check content if the response is not an image
if 'image/' not in content_type:
content = ows.read()
import re
try:
title_re = re.compile("<title>(.+?)</title>")
title = title_re.search(content).group(1)
except:
title = url
# Optional check for any OGC-Exceptions in Response
if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
exception_text = None
try:
except_re = re.compile(
"ServiceException>|ExceptionReport>")
exception_text = except_re.search(content).group(0)
except:
# No Exception in Response text
pass
if exception_text:
# Found OGC-Exception in Response text
raise Exception("Exception in response: %s" %
exception_text)
del content
elif resource_type == 'urn:geoss:waf':
title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
elif resource_type == 'FTP':
ows = urlopen(url)
title = urlparse(url).hostname
success = True
if resource_type.startswith(('OGC:', 'OSGeo')):
if resource_type == 'OGC:STA':
title = 'OGC STA'
else:
title = ows.identification.title
if title is None:
title = '%s %s %s' % (resource_type, gettext('for'), url)
except Exception as err:
msg = str(err)
LOGGER.exception(msg)
message = msg
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
return [title, success, response_time, message, start_time]
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a GeoDataFrame
"""
# LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST
oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
sos_url_params_esc = sos_url_params.replace("&", "&")
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n"
.format(url=sos_url_params))
self.log.write(
u"HTTP connection error: {err}.\n".format(err=e.message))
sys.exit(
"Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=e.message))
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
sml_lst = sos_collector.metadata(timeout=200)
station_recs = []
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=oFrmt,
timeout=200)
sml = SensorML(sml_str)
else:
sml = sml_lst[station_idx]
# debug:
# if self.verbose:
# self.log.write(unicode(etree.tostring(sml._root)))
ds = IoosDescribeSensor(sml._root)
pos = testXMLValue(
ds.system.location.find(self.nsp('gml:Point/gml:pos')))
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
contacts = system_el.findall(
self.nsp('sml:contact/sml:ContactList/sml:member'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except AttributeError as e:
begin_service_date = None
station = OrderedDict()
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params_esc
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
station['shortName'] = ds.shortName
station['longName'] = ds.longName
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appopriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
# MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used
# in the template code currently in favor of 'contacts_dct'
# station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont)
# station['operator_org'] = contacts_dct['operator'].organization
# station['operator_country'] = contacts_dct['operator'].country
# station['operator_url'] = contacts_dct['operator'].url
# station['operator_email'] = contacts_dct['operator'].email
# station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont)
# station['publisher_org'] = contacts_dct['publisher'].organization
# station['publisher_url'] = contacts_dct['publisher'].url
# station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress
station['starting'] = ds.starting
station['ending'] = ds.ending
# station['starting_isostr'] = datetime.isoformat(ds.starting)
# station['ending_isostr'] = datetime.isoformat(ds.ending)
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
station['variables'] = [var.split('/')[-1] for var in ds.variables]
# debug:
if self.verbose:
self.log.write(u"\nProcessing station: {station}\n".format(
station=station_urn))
print("\nProcessing station: {station}".format(
station=station_urn))
for var in ds.variables:
self.log.write(u"variable: {var}\n".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' values and populate list:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ]
# subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter
# (or 'application/json,application/zip; subtype=x-netcdf' by default):
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"responseFormat: {format}\n".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"downloadFormats: {format}\n".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
if ds.starting is not None and ds.ending is not None:
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=ds.ending - timedelta(hours=self.getobs_req_hours),
end=ds.ending)
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=then, end=now)
if self.verbose:
self.log.write(
u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n"
.format(then=then, now=now))
print(
"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"eventTime: {time}\n".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in ds.variables:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_request_url_esc = getobs_request_url.replace(
"&", "&")
getobs_req_dct[variable + '-' + format] = {
'variable': variable,
'url': getobs_request_url_esc,
'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format],
'format_name': self.RESPONSE_FORMAT_NAME_MAP[format]
}
if self.verbose:
self.log.write(
u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
