def __init__(self, element):
super(DescribeSensor, self).__init__(element=element)
""" Common things between all describe sensor requests """
self.ioos_version = "1.0"
self.system = SensorML(element).members[0]
self.shortName = get_named_by_definition(self.system.get_identifiers_by_name("shortName"), "http://mmisw.org/ont/ioos/definition/shortName")
self.longName = get_named_by_definition(self.system.get_identifiers_by_name("longName"), "http://mmisw.org/ont/ioos/definition/longName")
self.keywords = map(unicode, self.system.keywords)
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError):
self.starting = None
self.ending = None
class DescribeSensor(IoosDescribeSensor):
def __init__(self, element):
super(DescribeSensor, self).__init__(element=element)
""" Common things between all describe sensor requests """
self.ioos_version = "1.0"
self.system = SensorML(element).members[0]
self.shortName = get_named_by_definition(
self.system.get_identifiers_by_name("shortName"), "http://mmisw.org/ont/ioos/definition/shortName"
)
self.longName = get_named_by_definition(
self.system.get_identifiers_by_name("longName"), "http://mmisw.org/ont/ioos/definition/longName"
)
self.keywords = map(unicode, self.system.keywords)
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(
self.system.get_capabilities_by_name("observationTimeRange")[0].find(
".//" + nsp("swe101:TimeRange/swe101:value")
)
).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError):
self.starting = None
self.ending = None
def metadata(self,
output_format=None,
feature_name_callback=None,
**kwargs):
"""
Gets SensorML objects for all procedures in your filtered features.
You should override the default output_format for servers that do not
respond properly.
"""
callback = feature_name_callback or str
if output_format is None:
output_format = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
responses = []
if self.features is not None:
for feature in self.features:
ds_kwargs = kwargs.copy()
ds_kwargs.update({
'outputFormat': output_format,
'procedure': callback(feature)
})
responses.append(
SensorML(self.server.describe_sensor(**ds_kwargs)))
return responses
def metadata_plus_exceptions(self,
output_format=None,
feature_name_callback=None,
**kwargs):
"""
Gets SensorML objects for all procedures in your filtered features.
Return two dictionaries for service responses keyed by 'feature':
responses: values are SOS DescribeSensor response text
response_failures: values are exception text content furnished from ServiceException, ExceptionReport
You should override the default output_format for servers that do not
respond properly.
"""
callback = feature_name_callback or str
if output_format is None:
output_format = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
responses = {}
response_failures = {}
if self.features is not None:
for feature in self.features:
ds_kwargs = kwargs.copy()
ds_kwargs.update({
'outputFormat': output_format,
'procedure': callback(feature)
})
try:
responses[feature] = (SensorML(
self.server.describe_sensor(**ds_kwargs)))
except (ServiceException, ExceptionReport) as e:
response_failures[feature] = str(e)
return (responses, response_failures)
def load_dataset(self, ds_str):
"""
Helper method to load a dataset or SOS GC/DS url.
"""
ds = None
# try to figure out if this is a local NetCDF Dataset, a remote one, or an SOS GC/DS url
doc = None
pr = urlparse(ds_str)
if pr.netloc: # looks like a remote url
rhead = requests.head(ds_str)
# if we get a 400 here, it's likely a Dataset openable OpenDAP url
if rhead.status_code == 400:
pass
elif rhead.status_code == 200 and rhead.headers[
'content-type'] == 'text/xml':
# probably interesting, grab it
r = requests.get(ds_str)
r.raise_for_status()
doc = r.text
else:
raise StandardError(
"Could not understand response code %s and content-type %s"
% (rhead.status_code,
rhead.headers.get('content-type', 'none')))
else:
# do a cheap imitation of libmagic
# http://stackoverflow.com/a/7392391/84732
textchars = ''.join(
map(chr, [7, 8, 9, 10, 12, 13, 27] + range(0x20, 0x100)))
is_binary_string = lambda bytes: bool(
bytes.translate(None, textchars))
with open(ds_str, 'rb') as f:
first_chunk = f.read(1024)
if is_binary_string(first_chunk):
# likely netcdf file
pass
else:
f.seek(0)
doc = "".join(f.readlines())
if doc is not None:
xml_doc = ET.fromstring(str(doc))
if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
ds = SensorObservationService(ds_str, xml=str(doc))
elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
ds = SensorML(xml_doc)
else:
raise StandardError("Unrecognized XML root element: %s" %
xml_doc.tag)
else:
# no doc? try the dataset constructor
ds = Dataset(ds_str)
return ds
def __init__(self, element, nsmap=extension_sos_utils.nsmap):
self._root = element
element = element.find(f".//{nspath_eval('sml:SensorML', nsmap)}")
self.sensor_ml = SensorML(element)
if hasattr(self.sensor_ml, 'systems') and self.sensor_ml.systems:
self.sensor = self.sensor_ml.systems[0]
self.id = self.sensor.identifiers['uniqueID'].value
self.name = self.sensor.identifiers['longName'].value
class DescribeSensor(IoosDescribeSensor):
@classmethod
def get_named_by_definition(cls, element_list, string_def):
"""Attempts to get an IOOS definition from a list of xml elements"""
try:
return next((st.value for st in element_list
if st.definition == string_def))
except:
return None
def get_ioos_def(self, ident, elem_type, ont):
"""Gets a definition given an identifier and where to search for it"""
if elem_type == 'identifier':
getter_fn = self.system.get_identifiers_by_name
elif elem_type == 'classifier':
getter_fn = self.system.get_classifiers_by_name
else:
raise ValueError("Unknown element type '{}'".format(elem_type))
return DescribeSensor.get_named_by_definition(getter_fn(ident),
urljoin(ont, ident))
def __init__(self, element):
""" Common things between all describe sensor requests """
if isinstance(element, ElementType):
root = element
else:
root = etree.fromstring(element)
sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS)
# TODO: make this cleaner
if hasattr(root, 'getroot'):
root = root.getroot()
self.system = SensorML(element).members[0]
self.ioos_version = testXMLValue(root.find(".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS)))
if self.ioos_version != "1.0":
warnings.warn("Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version)
self.shortName = self.get_ioos_def('shortName', 'identifier', ont)
self.longName = self.get_ioos_def('longName', 'identifier', ont)
self.keywords = list(map(str, self.system.keywords))
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError, IndexError):
self.starting = None
self.ending = None
def __init__(self, element, nsmap=extension_sos_utils.nsmap):
self._root = element
# self.procedure_description_format = testXMLValue(self._root.find(nspath_eval('swes:procedureDescriptionFormat', nsmap)))
# sensor_ml = SensorML(self._root.find(nspath_eval('sml:SensorML', nsmap)))
self.sensor_ml = SensorML(element)
if hasattr(self.sensor_ml, 'systems') and self.sensor_ml.systems:
self.sensor = self.sensor_ml.systems[0]
self.id = self.sensor.identifiers['uniqueID'].value
self.name = self.sensor.identifiers['longName'].value
class DescribeSensor(IoosDescribeSensor):
@classmethod
def get_named_by_definition(cls, element_list, string_def):
"""Attempts to get an IOOS definition from a list of xml elements"""
try:
return next((st.value for st in element_list
if st.definition == string_def))
except:
return None
def get_ioos_def(self, ident, elem_type, ont):
"""Gets a definition given an identifier and where to search for it"""
if elem_type == 'identifier':
getter_fn = self.system.get_identifiers_by_name
elif elem_type == 'classifier':
getter_fn = self.system.get_classifiers_by_name
else:
raise ValueError("Unknown element type '{}'".format(elem_type))
return DescribeSensor.get_named_by_definition(getter_fn(ident),
urljoin(ont, ident))
def __init__(self, element):
""" Common things between all describe sensor requests """
if isinstance(element, str):
root = etree.fromstring(element)
else:
root = element
sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS)
# TODO: make this cleaner
if hasattr(root, 'getroot'):
root = root.getroot()
self.system = SensorML(element).members[0]
self.ioos_version = testXMLValue(root.find(".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS)))
if self.ioos_version != "1.0":
warnings.warn("Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version)
self.shortName = self.get_ioos_def('shortName', 'identifier', ont)
self.longName = self.get_ioos_def('longName', 'identifier', ont)
self.keywords = map(unicode, self.system.keywords)
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError, IndexError):
self.starting = None
self.ending = None
def __init__(self, element):
""" Common things between all describe sensor requests """
if isinstance(element, ElementType):
root = element
else:
root = etree.fromstring(element)
# sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS)
# TODO: make this cleaner
if hasattr(root, "getroot"):
root = root.getroot()
self.system = SensorML(element).members[0]
self.ioos_version = testXMLValue(
root.find(
".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value"
% (SWE_NS, SWE_NS, SWE_NS)))
if self.ioos_version != "1.0":
warnings.warn(
"Warning: Unsupported IOOS version (%s). Supported: [1.0]" %
self.ioos_version)
self.shortName = self.get_ioos_def("shortName", "identifier", ont)
self.longName = self.get_ioos_def("longName", "identifier", ont)
self.keywords = list(map(str, self.system.keywords))
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(
self.system.get_capabilities_by_name("observationTimeRange")
[0].find(".//" +
nsp("swe101:TimeRange/swe101:value"))).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError, IndexError):
self.starting = None
self.ending = None
def syncvesk():
sensors = {}
service = SensorObservationService(SOSURL, version=SOSVERSION)
get_obs = service.get_operation_by_name('GetObservation')
describe_sensor = service.get_operation_by_name('DescribeSensor')
sensor_ids = describe_sensor.parameters['procedure']['values']
for sensor_id in sensor_ids:
response = service.describe_sensor(
outputFormat='http://www.opengis.net/sensorML/1.0.1',
procedure=sensor_id)
tr = etree.fromstring(response)
element = tr.find('.//' + nspath_eval('sml:SensorML', namespaces))
ds = SensorML(element)
name = ds.members[0].name
sensors[sensor_id] = name
offerings = [off.id for off in service.offerings]
_observed_properties = list(
set(op for off in service.offerings for op in off.observed_properties))
observed_properties = SOS_PARAMS.keys()
end_time = datetime.now() # - timedelta(hours=120*10)
start_time = end_time - timedelta(hours=12)
temporalFilter = "om:phenomenonTime,{}/{}".format(start_time.isoformat(),
end_time.isoformat())
response_format = 'application/json'
_namespaces = "xmlns(om,http://www.opengis.net/om/2.0)"
_response = service.get_observation(offerings=offerings,
responseFormat=response_format,
observedProperties=observed_properties,
namespaces=_namespaces,
temporalFilter=temporalFilter)
response = json.loads(_response)
for obs in response['observations']:
location = obs['featureOfInterest']['name']['value']
_parameter = obs['observableProperty']
parameter = SOS_PARAMS.get(_parameter)
timestamp = obs['phenomenonTime']
network = 'CNR'
value = float(obs['result']['value'])
rjson = {
'location': location,
'parameter': parameter,
"timestamp": timestamp,
'network': network,
"value": value
}
_write([rjson])
pass
pass
def process_doc(doc):
"""
Attempt to parse an xml string conforming to either an SOS or SensorML
dataset and return the results
"""
xml_doc = ET.fromstring(str(doc))
if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
ds = SensorObservationService(ds_str, xml=str(doc))
elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
ds = SensorML(xml_doc)
else:
raise ValueError("Unrecognized XML root element: %s" % xml_doc.tag)
return ds
def __init__(self, element):
""" Common things between all describe sensor requests """
if isinstance(element, ElementType):
root = element
else:
root = etree.fromstring(element)
# sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS)
# TODO: make this cleaner
if hasattr(root, "getroot"):
root = root.getroot()
self.system = SensorML(element).members[0]
self.ioos_version = testXMLValue(
root.find(
".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value"
% (SWE_NS, SWE_NS, SWE_NS)
)
)
if self.ioos_version != "1.0":
warnings.warn(
"Warning: Unsupported IOOS version (%s). Supported: [1.0]"
% self.ioos_version
)
self.shortName = self.get_ioos_def("shortName", "identifier", ont)
self.longName = self.get_ioos_def("longName", "identifier", ont)
self.keywords = list(map(str, self.system.keywords))
# Location
try:
self.location = self.system.location[0]
except (TypeError, IndexError): # No location exists
self.location = None
# Timerange
try:
timerange = testXMLValue(
self.system.get_capabilities_by_name("observationTimeRange")[
0
].find(".//" + nsp("swe101:TimeRange/swe101:value"))
).split(" ")
self.starting = parser.parse(timerange[0])
self.ending = parser.parse(timerange[1])
except (AttributeError, TypeError, ValueError, IndexError):
self.starting = None
self.ending = None
def process_doc(self, doc):
"""
Attempt to parse an xml string conforming to either an SOS or SensorML
dataset and return the results
"""
xml_doc = ET.fromstring(doc)
if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
ds = SensorObservationService(None, xml=doc)
# SensorObservationService does not store the etree doc root,
# so maybe use monkey patching here for now?
ds._root = xml_doc
elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
ds = SensorML(xml_doc)
else:
raise ValueError("Unrecognized XML root element: %s" % xml_doc.tag)
return ds
def sensors(self):
if self._sensors is None:
sensors = {}
get_obs = self.service.get_operation_by_name('GetObservation')
describe_sensor = self.service.get_operation_by_name(
'DescribeSensor')
sensor_ids = describe_sensor.parameters['procedure']['values']
for sensor_id in sensor_ids:
response = self.service.describe_sensor(
outputFormat='http://www.opengis.net/sensorML/1.0.1',
procedure=sensor_id)
tr = etree.fromstring(response)
element = tr.find('.//' +
nspath_eval('sml:SensorML', namespaces))
ds = SensorML(element)
name = ds.members[0].name
sensors[sensor_id] = name
self._sensors = sensors
return self._sensors
def get_coops_metadata(station):
"""
Get longName and sensorName for specific station from COOPS SOS using
DescribeSensor and owslib.swe.sensor.sml.SensorML.
Examples
--------
>>> long_name, station_id = get_coops_metadata(8651370)
>>> long_name
'Duck, NC'
>>> station_id
'urn:ioos:station:NOAA.NOS.CO-OPS:8651370'
"""
from owslib.swe.sensor.sml import SensorML
url = ('opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
'service=SOS&'
'request=DescribeSensor&version=1.0.0&'
'outputFormat=text/xml;'
'subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"&'
'procedure=urn:ioos:station:NOAA.NOS.CO-OPS:%s') % station
url = parse_url(url)
xml = etree.parse(urlopen(url))
root = SensorML(xml)
if not root.members or len(root.members) > 1:
msg = "Expected 1 member, got {}".format
raise ValueError(msg(len(root.members)))
system = root.members[0]
# NOTE: Some metadata of interest.
# system.description
# short_name = _get_value(system.identifiers, name='shortName')
# [c.values() for c in system.components]
long_name = _get_value(system.identifiers, name='longName')
long_name = long_name.split('station, ')[-1].strip()
station_id = _get_value(system.identifiers, name='stationID')
return long_name, station_id
def create_maps(parsed_obs, offering, layer, new, secondsGranularity,
event_time, service):
"""
Add layers representing offerings and observed properties to the vector map
:param parsed_obs: Observations for a given offering in geoJSON format
:param offering: A collection of sensors used to conveniently group them up
:param layer: Count of yet existing layers in vector map
:param new: Given vector map which should be updated with new layers
:param secondsGranularity: Granularity in seconds
"""
i = layer + 1
points = dict()
freeCat = 1
if flags['s']:
# Set target projection of current LOCATION
target_crs = grass.read_command('g.proj',
flags='fj').rstrip(os.linesep)
target = osr.SpatialReference(target_crs)
target.ImportFromProj4(target_crs)
if target == 'XY location (unprojected)':
grass.fatal("Sorry, XY locations are not supported!")
# The following is work in progress
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'varchar'),
(u'description', 'varchar'), (u'keywords', 'varchar'),
(u'sensor_type', 'varchar'), (u'system_type', 'varchar'),
(u'crs', 'INTEGER'), (u'x', 'DOUBLE'), (u'y', 'DOUBLE'),
(u'z', 'DOUBLE')]
# new = Vector(new)
if new.is_open() is False:
new.open('w', tab_name=options['output'], tab_cols=cols)
offs = [o.id for o in service.offerings]
off_idx = offs.index(offering)
outputFormat = service.get_operation_by_name(
'DescribeSensor').parameters['outputFormat']['values'][0]
procedures = service.offerings[off_idx].procedures
for proc in procedures:
response = service.describe_sensor(procedure=proc,
outputFormat=outputFormat)
root = SensorML(response)
system = root.members[0]
name = system.name
desc = system.description
keywords = ','.join(system.keywords)
sensType = system.classifiers['Sensor Type'].value
sysType = system.classifiers['System Type'].value
crs = int(system.location[0].attrib['srsName'].split(':')[1])
coords = system.location[0][0].text.replace('\n', '')
sx = float(coords.split(',')[0])
sy = float(coords.split(',')[1])
sz = float(coords.split(',')[2])
# Set source projection from SOS
source = osr.SpatialReference()
source.ImportFromEPSG(crs)
transform = osr.CoordinateTransformation(source, target)
point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format(
sx, sy, sz))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
z = point.GetZ()
if name not in points.keys():
points.update({name: freeCat})
point = Point(x, y, z)
new.write(point,
cat=freeCat,
attrs=(
u'{}'.format(system.name.decode('utf-8')),
system.description,
','.join(system.keywords),
system.classifiers['Sensor Type'].value,
system.classifiers['System Type'].value,
crs,
float(coords.split(',')[0]),
float(coords.split(',')[1]),
float(coords.split(',')[2]),
))
freeCat += 1
new.table.conn.commit()
new.close(build=True)
else:
timestampPattern = '%Y-%m-%dT%H:%M:%S' # TODO: Timezone
startTime = event_time.split('+')[0]
epochS = int(time.mktime(time.strptime(startTime, timestampPattern)))
endTime = event_time.split('+')[1].split('/')[1]
epochE = int(time.mktime(time.strptime(endTime, timestampPattern)))
for key, observation in parsed_obs.iteritems():
tableName = '{}_{}_{}'.format(options['output'], offering, key)
if ':' in tableName:
tableName = '_'.join(tableName.split(':'))
if '-' in tableName:
tableName = '_'.join(tableName.split('-'))
if '.' in tableName:
tableName = '_'.join(tableName.split('.'))
data = json.loads(observation)
intervals = {}
for secondsStamp in range(epochS, epochE + 1, secondsGranularity):
intervals.update({secondsStamp: dict()})
timestampPattern = 't%Y%m%dT%H%M%S' # TODO: Timezone
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'VARCHAR')]
for a in data['features']:
name = a['properties']['name']
if name not in points.keys():
if new.is_open() is False:
new.open('w')
points.update({name: freeCat})
new.write(Point(*a['geometry']['coordinates']),
cat=freeCat)
freeCat += 1
for timestamp, value in a['properties'].iteritems():
if timestamp != 'name':
observationStartTime = timestamp[:-4]
secondsTimestamp = int(
time.mktime(
time.strptime(observationStartTime,
timestampPattern)))
for interval in intervals.keys():
if secondsTimestamp >= interval \
and secondsTimestamp < (
interval + secondsGranularity):
if name in intervals[interval].keys():
intervals[interval][name].append(
float(value))
else:
timestamp2 = datetime.datetime.fromtimestamp(
interval).strftime('t%Y%m%dT%H%M%S')
intervals[interval].update(
{name: [float(value)]})
if (u'%s' % timestamp2,
'DOUBLE') not in cols:
cols.append(
(u'%s' % timestamp2, 'DOUBLE'))
break
if len(cols) > 2000:
grass.warning(
'Recommended number of columns is less than 2000, you have '
'reached {}\nYou should set an event_time with a smaller range'
' or recompile SQLite limits as described at '
'https://sqlite.org/limits.html'.format(len(cols)))
link = Link(
layer=i,
name=tableName,
table=tableName,
key='cat',
database='$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db',
driver='sqlite')
if new.is_open():
new.close()
new.open('rw')
new.dblinks.add(link)
new.table = new.dblinks[i - 1].table()
new.table.create(cols)
inserts = dict()
for interval in intervals.keys():
if len(intervals[interval]) != 0:
timestamp = datetime.datetime.fromtimestamp(
interval).strftime('t%Y%m%dT%H%M%S')
for name, values in intervals[interval].iteritems():
if options['method'] == 'average':
aggregatedValue = sum(values) / len(values)
elif options['method'] == 'sum':
aggregatedValue = sum(values)
if name not in inserts.keys():
insert = [None] * len(cols)
insert[0] = points[name]
insert[1] = name
insert[cols.index(
(timestamp, 'DOUBLE'))] = aggregatedValue
inserts.update({name: insert})
else:
inserts[name][cols.index(
(timestamp, 'DOUBLE'))] = aggregatedValue
for insert in inserts.values():
new.table.insert(tuple(insert))
new.table.conn.commit()
new.close(build=False)
run_command('v.build', quiet=True, map=options['output'])
i += 1
def maps_without_observations(offering, resolution, service, procedures,
target):
"""Import just pixels/sensors without their observations.
:param offering: A collection of sensors used to conveniently group them up
:param resolution: 2D grid resolution for rasterization
:param service: SensorObservationService() type object of request
:param procedures: List of queried procedures (observation providors)
:param target:
"""
offs = [o.id for o in service.offerings]
off_idx = offs.index(offering)
output_format = service.get_operation_by_name(
'DescribeSensor').parameters['outputFormat']['values'][0]
if procedures:
procedures = procedures.split(',')
else:
procedures = service.offerings[off_idx].procedures
tempfile_path = grass.tempfile()
n = None
s = None
e = None
w = None
with open(tempfile_path, 'w') as tempFile:
for proc in procedures:
response = service.describe_sensor(procedure=proc,
output_format=output_format)
root = SensorML(response)
system = root.members[0]
crs = int(system.location[0].attrib['srsName'].split(':')[-1])
coords = system.location[0][0].text.replace('\n', '')
sx = float(coords.split(',')[0])
sy = float(coords.split(',')[1])
sz = float(coords.split(',')[2])
transform = soslib.get_transformation(crs, target)
point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format(
sx, sy, sz))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
z = point.GetZ()
tempFile.write('{} {} {}\n'.format(x, y, z))
if not n:
n = y + resolution / 2
s = y - resolution / 2
e = x + resolution / 2
w = x - resolution / 2
else:
if y >= n:
n = y + resolution / 2
if y <= s:
s = y - resolution / 2
if x >= e:
e = x + resolution / 2
if x <= w:
w = x - resolution / 2
run_command('g.region', n=n, s=s, w=w, e=e, res=resolution)
run_command('r.in.xyz',
input=tempfile_path,
separator='space',
output='{}_{}'.format(options['output'], offering))
def process_station(self, uid, offering):
""" Makes a DescribeSensor request based on a 'uid' parameter being a
station procedure. Also pass along an offering with
getCapabilities information for items such as temporal extent"""
GML_NS = "http://www.opengis.net/gml"
XLINK_NS = "http://www.w3.org/1999/xlink"
with app.app_context():
app.logger.info("process_station: %s", uid)
desc_sens = self._describe_sensor(uid, timeout=1200)
# FIXME: add some kind of notice saying the station failed
if desc_sens is None:
app.logger.warn(
"Could not get a valid describeSensor response")
return
metadata_value = etree.fromstring(desc_sens)
sensor_ml = SensorML(metadata_value)
try:
station_ds = IoosDescribeSensor(metadata_value)
# if this doesn't conform to IOOS SensorML sub, fall back to
# manually picking apart the SensorML
except ows.ExceptionReport:
station_ds = process_sensorml(sensor_ml.members[0])
unique_id = station_ds.id
if unique_id is None:
app.logger.warn(
"Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'"
)
return
dataset = db.Dataset.find_one({'uid': unicode(unique_id)})
if dataset is None:
dataset = db.Dataset()
dataset.uid = unicode(unique_id)
dataset['active'] = True
# Find service reference in Dataset.services and remove (to replace it)
tmp = dataset.services[:]
for d in tmp:
if d['service_id'] == self.service.get('_id'):
dataset.services.remove(d)
# Parsing messages
messages = []
# NAME
name = unicode_or_none(station_ds.shortName)
if name is None:
messages.append(
u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'"
)
# DESCRIPTION
description = unicode_or_none(station_ds.longName)
if description is None:
messages.append(
u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'"
)
# PLATFORM TYPE
asset_type = unicode_or_none(
getattr(station_ds, 'platformType', None))
if asset_type is None:
messages.append(
u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'"
)
# LOCATION is in GML
gj = None
loc = station_ds.location
if loc is not None and loc.tag == "{%s}Point" % GML_NS:
pos_element = loc.find("{%s}pos" % GML_NS)
# some older responses may uses the deprecated coordinates
# element
if pos_element is None:
# if pos not found use deprecated coordinates element
pos_element = loc.find("{%s}coordinates" % GML_NS)
# strip out points
positions = map(float, pos_element.text.split(" "))
for el in [pos_element, loc]:
srs_name = testXMLAttribute(el, "srsName")
if srs_name:
crs = Crs(srs_name)
if crs.axisorder == "yx":
gj = json.loads(
geojson.dumps(
geojson.Point([positions[1],
positions[0]])))
else:
gj = json.loads(
geojson.dumps(
geojson.Point([positions[0],
positions[1]])))
break
else:
if positions:
messages.append(
u"Position(s) found but could not parse SRS: %s, %s"
% (positions, srs_name))
else:
messages.append(
u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s"
% loc)
meta_str = unicode(etree.tostring(metadata_value)).strip()
if len(meta_str) > 4000000:
messages.append(
u'Metadata document was too large to store (len: %s)' %
len(meta_str))
meta_str = u''
service = {
# Reset service
'name': name,
'description': description,
'service_type': self.service.get('service_type'),
'service_id': ObjectId(self.service.get('_id')),
'data_provider': self.service.get('data_provider'),
'metadata_type': u'sensorml',
'metadata_value': u'',
'time_min': getattr(offering, 'begin_position', None),
'time_max': getattr(offering, 'end_position', None),
'messages': map(unicode, messages),
'keywords': map(unicode, sorted(station_ds.keywords)),
'variables': map(unicode, sorted(station_ds.variables)),
'asset_type': get_common_name(asset_type),
'geojson': gj,
'updated': datetime.utcnow()
}
dataset.services.append(service)
dataset.updated = datetime.utcnow()
dataset.save()
# do compliance checker / metadata now
scores = self.ccheck_station(sensor_ml)
metamap = self.metamap_station(sensor_ml)
try:
self.save_ccheck_station('ioos', dataset._id, scores, metamap)
except Exception as e:
app.logger.warn(
"could not save compliancecheck/metamap information: %s",
e)
return "Harvest Successful"
def maps_without_observations(offering, new, service, procedures, target):
"""Import just vector points/sensors without their observations.
:param offering: A collection of sensors used to conveniently group them up
:param new: Given vector map which should be updated with new layers
:param service: SensorObservationService() type object of request
:param procedures: List of queried procedures (observation providors)
:param target:
"""
points = dict()
free_cat = 1
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'varchar'),
(u'description', 'varchar'), (u'keywords', 'varchar'),
(u'sensor_type', 'varchar'), (u'system_type', 'varchar'),
(u'crs', 'INTEGER'), (u'x', 'DOUBLE'), (u'y', 'DOUBLE'),
(u'z', 'DOUBLE')]
# new = Vector(new)
if new.is_open() is False:
new.open('w', tab_name=options['output'], tab_cols=cols)
offs = [o.id for o in service.offerings]
off_idx = offs.index(offering)
output_format = service.get_operation_by_name(
'DescribeSensor').parameters['outputFormat']['values'][0]
if procedures:
procedures = procedures.split(',')
else:
procedures = service.offerings[off_idx].procedures
for proc in procedures:
response = service.describe_sensor(procedure=proc,
outputFormat=output_format)
root = SensorML(response)
system = root.members[0]
name = system.name
desc = system.description
keywords = ','.join(system.keywords)
sens_type = system.classifiers['Sensor Type'].value
sys_type = system.classifiers['System Type'].value
crs = int(system.location[0].attrib['srsName'].split(':')[-1])
coords = system.location[0][0].text.replace('\n', '')
sx = float(coords.split(',')[0])
sy = float(coords.split(',')[1])
sz = float(coords.split(',')[2])
# Set source projection from SOS
transform = soslib.get_transformation(crs, target)
point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format(
sx, sy, sz))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
z = point.GetZ()
if name not in points.keys():
points.update({name: free_cat})
point = Point(x, y, z)
new.write(point,
cat=free_cat,
attrs=(
u'{}'.format(system.name),
system.description,
','.join(system.keywords),
system.classifiers['Sensor Type'].value,
system.classifiers['System Type'].value,
crs,
float(coords.split(',')[0]),
float(coords.split(',')[1]),
float(coords.split(',')[2]),
))
free_cat += 1
new.table.conn.commit()
new.close(build=True)
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a Pandas Dataframe
"""
# oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec,
# second is to accommodate NDBC). More info here:
# http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe
oFrmts = [
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
'text/xml;subtype="sensorML/1.0.1"'
]
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"\nError: unable to connect to SOS service: {url} due to HTTP connection error."
.format(url=sos_url_params))
self.log.write(
u"\nHTTP connection error: {err}.".format(err=str(e)))
sys.exit(
"\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=str(e)))
# vars to store returns from sos_collector.metadata_plus_exceptions function:
sml_recs = {}
sml_errors = {}
describe_sensor_url = {}
# leverage Pyoos Collector to query for all available stations and obtain SensorML
# (if station subset not passed in --stations param)
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
# write out stations in SOS that will be handled:
if self.verbose:
self.log.write(u"\nStations to process for SOS: {sos}".format(
sos=sos_url_params))
print("Stations to process for SOS: {sos}".format(
sos=sos_url_params))
for feature in sos_collector.features:
self.log.write(u"\n - {feature}".format(feature=feature))
print(" - {feature}".format(feature=feature))
# iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC):
# for fmt in reversed(oFrmts):
for fmt in oFrmts:
try:
sml_recs, sml_errors = sos_collector.metadata_plus_exceptions(
output_format=fmt, timeout=200)
# if no valid SensorML docs returned, try next oFrmt:
if not sml_recs:
continue
else:
# assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to
# create DescribeSensor URLs for failures to record in logs):
for station in sos_collector.features:
describe_sensor_url[
station] = self.generate_describe_sensor_url(
sosgc, procedure=station, oFrmt=fmt)
# report on errors returned from metadata_plus_exceptions:
if sml_errors:
if self.verbose:
for station, msg in iteritems(sml_errors):
self.log.write(
u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
print(
"SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
else:
self.log.write(
u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
print(
"Success, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
break
# ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt:
except ServiceException as e:
continue
station_recs = []
failures = []
# generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings:
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
# iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS):
for fmt in oFrmts:
try:
describe_sensor_url[
station_urn] = self.generate_describe_sensor_url(
sosgc, procedure=station_urn, oFrmt=fmt)
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=fmt,
timeout=200)
break
except ServiceException as e:
sml_errors[station_urn] = str(e)
continue
sml = SensorML(sml_str)
else:
# process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting):
try:
sml = sml_recs[station_urn]
except KeyError:
self.log.write(
u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
failures.append(station_urn)
continue
if self.sos_type.lower() == 'ndbc':
# later: add an error check
sosgc_station_offering = sosgc.contents[
'station-' + station_urn.split(':')[-1]]
else:
sosgc_station_offering = None
try:
ds = IoosDescribeSensor(sml._root)
except AttributeError:
self.log.write(
u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
station = OrderedDict()
# debug:
if self.verbose:
self.log.write(u"\n\nProcessing station: {station}".format(
station=station_urn))
print("Processing station: {station}".format(
station=station_urn))
self.log.write("\n" +
etree.tostring(sml._root).decode('utf-8'))
# assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found):
try:
pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \
if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \
else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates')))
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
except AttributeError as e:
station['lon'] = None
station['lat'] = None
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
# obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList):
contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
else system_el.findall(self.nsp('sml:contact'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
# verify a 'publisher' Contact exists (template expects one):
if "publisher" not in contacts_dct.keys():
self.log.write(
u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
print(
"Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
failures.append(station_urn)
continue
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available, otherwise use current date
# bc ISO requires date value in output location in template:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except (AttributeError, TypeError) as e:
begin_service_date = datetime.now(pytz.utc)
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params
station['describesensor_url'] = describe_sensor_url[station_urn]
station['shortName'] = ds.shortName
station['longName'] = ds.longName
if self.sos_type.lower() == 'ndbc':
station['wmoID'] = station_urn.split(':')[-1]
else:
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appropriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station['starting'] = sosgc_station_offering.begin_position
station['ending'] = sosgc_station_offering.end_position
else:
station['starting'] = ds.starting
station['ending'] = ds.ending
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station[
'variable_uris'] = sosgc_station_offering.observed_properties
station['variables'] = [
var.split('/')[-1]
for var in sosgc_station_offering.observed_properties
]
station['parameter_uris'] = ','.join(station['variable_uris'])
station['parameters'] = ','.join(station['variables'])
else:
station['variable_uris'] = ds.variables
station['variables'] = [
var.split('/')[-1] for var in ds.variables
]
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
if self.verbose:
for var in station['variable_uris']:
self.log.write(u"\nvariable: {var}".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' vals from SensorML:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items()
# if sosgc.content.name == station_urn ]
# match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to
# populate 'download_formats' list, that is then used to generate GetObservation requests for the template:
# (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' )
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"\nresponseFormat: {format}".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"\ndownloadFormats: {format}".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}"
utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else ''
if station['starting'] is not None and station[
'ending'] is not None:
event_time = event_time_formatstr.format(
begin=station['ending'] -
timedelta(hours=self.getobs_req_hours),
end=station['ending'],
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nUsing starting/ending times from SensorML for eventTime"
)
print(
"Using starting/ending times from SensorML for eventTime"
)
self.log.write(
u"\nobservationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
print(
"observationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = event_time_formatstr.format(begin=then,
end=now,
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
print(
"No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"\neventTime: {time}".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type
# and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in station['variable_uris']:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_req_dct[variable + '-' + format] = {
'variable':
variable,
'url':
getobs_request_url,
'format_type':
self.RESPONSE_FORMAT_TYPE_MAP.get(format, format),
'format_name':
self.RESPONSE_FORMAT_NAME_MAP.get(format, format)
}
if self.verbose:
self.log.write(
u"\ngetobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
# extra debug for failed stations in verbose mode:
if self.verbose:
self.log.write(
u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:"
)
print("SOS DescribeSensor request errors recap. Failed requests:")
for station_fail, msg in iteritems(sml_errors):
self.log.write(
u"\n{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace(
"&", "&")))
print("{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace("&",
"&")))
if failures:
self.log.write(
u"\nStations in 'failures' list (should match DescribeSensor errors):"
)
print(
"Stations in 'failures' list (should match DescribeSensor errors):"
)
for station_fail in failures:
self.log.write(u"\n{station}".format(station=station_fail))
print("{station}".format(station=station_fail))
if station_recs:
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
else:
return None
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a GeoDataFrame
"""
# LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST
oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
sos_url_params_esc = sos_url_params.replace("&", "&")
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n"
.format(url=sos_url_params))
self.log.write(
u"HTTP connection error: {err}.\n".format(err=e.message))
sys.exit(
"Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=e.message))
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
sml_lst = sos_collector.metadata(timeout=200)
station_recs = []
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=oFrmt,
timeout=200)
sml = SensorML(sml_str)
else:
sml = sml_lst[station_idx]
# debug:
# if self.verbose:
# self.log.write(unicode(etree.tostring(sml._root)))
ds = IoosDescribeSensor(sml._root)
pos = testXMLValue(
ds.system.location.find(self.nsp('gml:Point/gml:pos')))
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
contacts = system_el.findall(
self.nsp('sml:contact/sml:ContactList/sml:member'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except AttributeError as e:
begin_service_date = None
station = OrderedDict()
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params_esc
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
station['shortName'] = ds.shortName
station['longName'] = ds.longName
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appopriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
# MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used
# in the template code currently in favor of 'contacts_dct'
# station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont)
# station['operator_org'] = contacts_dct['operator'].organization
# station['operator_country'] = contacts_dct['operator'].country
# station['operator_url'] = contacts_dct['operator'].url
# station['operator_email'] = contacts_dct['operator'].email
# station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont)
# station['publisher_org'] = contacts_dct['publisher'].organization
# station['publisher_url'] = contacts_dct['publisher'].url
# station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress
station['starting'] = ds.starting
station['ending'] = ds.ending
# station['starting_isostr'] = datetime.isoformat(ds.starting)
# station['ending_isostr'] = datetime.isoformat(ds.ending)
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
station['variables'] = [var.split('/')[-1] for var in ds.variables]
# debug:
if self.verbose:
self.log.write(u"\nProcessing station: {station}\n".format(
station=station_urn))
print("\nProcessing station: {station}".format(
station=station_urn))
for var in ds.variables:
self.log.write(u"variable: {var}\n".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' values and populate list:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ]
# subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter
# (or 'application/json,application/zip; subtype=x-netcdf' by default):
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"responseFormat: {format}\n".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"downloadFormats: {format}\n".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
if ds.starting is not None and ds.ending is not None:
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=ds.ending - timedelta(hours=self.getobs_req_hours),
end=ds.ending)
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=then, end=now)
if self.verbose:
self.log.write(
u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n"
.format(then=then, now=now))
print(
"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"eventTime: {time}\n".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in ds.variables:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_request_url_esc = getobs_request_url.replace(
"&", "&")
getobs_req_dct[variable + '-' + format] = {
'variable': variable,
'url': getobs_request_url_esc,
'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format],
'format_name': self.RESPONSE_FORMAT_NAME_MAP[format]
}
if self.verbose:
self.log.write(
u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
