class CoopsSosTest(unittest.TestCase):
def setUp(self):
self.c = CoopsSos()
def test_coops_server_id(self):
assert self.c.server.identification.title == "NOAA.NOS.CO-OPS SOS"
assert self.c.server.identification.service == 'OGC:SOS'
assert self.c.server.identification.version == '1.0.0'
assert self.c.server.identification.abstract == 'NOAA.NOS.CO-OPS Sensor Observation Service (SOS) Server'
assert self.c.server.identification.keywords == [
'Air Temperature', 'Barometric Pressure', 'Conductivity',
'Currents', 'Datums', 'Rain Fall', 'Relative Humidity',
'Harmonic Constituents', 'Salinity', 'Water Level',
'Water Level Predictions', 'Water Temperature', 'Winds'
]
assert self.c.server.identification.fees == 'NONE'
assert self.c.server.identification.accessconstraints == 'NONE'
def test_coops_describe_sensor(self):
procedure = self.c.server.offerings[1].procedures[0]
outputFormat = self.c.server.get_operation_by_name(
'DescribeSensor').parameters['outputFormat']['values'][0]
response = self.c.get_metadata(procedure=procedure,
outputFormat=outputFormat)
assert isinstance(response.systems[0].id, str)
def test_coops_get_observation(self):
# COOPS does not support empty eventtime parameters
eventTime = "2012-10-01T00:00:00Z/2012-10-01T23:59:00Z"
station = None
station_names = list(set(['station-9052000']))
for offering in self.c.server.offerings:
if offering.id in station_names:
station = offering
break
response = res = self.c.get_raw_data(
offerings=[station.name],
responseFormat='text/xml;schema="ioos/0.6.1"',
observedProperties=[
'http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'
],
eventTime=eventTime,
dataType='VerifiedSixMinute')
assert isinstance(response, str)
def get_coops(start, end, sos_name, units, bbox, verbose=False):
collector = CoopsSos()
collector.set_bbox(bbox)
collector.end_time = end
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
config = dict(units=units, sos_name=sos_name,)
data = collector2table(
collector=collector,
config=config,
col="{} ({})".format(sos_name, units.format(cf_units.UT_ISO_8859_1)),
)
# Clean the table.
table = dict(
station_name=[s._metadata.get("station_name") for s in data],
station_code=[s._metadata.get("station_code") for s in data],
sensor=[s._metadata.get("sensor") for s in data],
lon=[s._metadata.get("lon") for s in data],
lat=[s._metadata.get("lat") for s in data],
depth=[s._metadata.get("depth", "NA") for s in data],
)
table = pd.DataFrame(table).set_index("station_name")
if verbose:
print("Collector offerings")
print("{}: {} offerings".format(title, len(ofrs)))
return data, table
class CoopsSosTest(unittest.TestCase):
def setUp(self):
self.c = CoopsSos()
def test_coops_server_id(self):
assert self.c.server.identification.title == "NOAA.NOS.CO-OPS SOS"
assert self.c.server.identification.service == 'OGC:SOS'
assert self.c.server.identification.version == '1.0.0'
assert self.c.server.identification.abstract == 'NOAA.NOS.CO-OPS Sensor Observation Service (SOS) Server'
assert self.c.server.identification.keywords == ['Air Temperature', 'Barometric Pressure', 'Conductivity', 'Currents', 'Datums', 'Rain Fall', 'Relative Humidity', 'Harmonic Constituents', 'Salinity', 'Water Level', 'Water Level Predictions', 'Water Temperature', 'Winds']
assert self.c.server.identification.fees == 'NONE'
assert self.c.server.identification.accessconstraints == 'NONE'
def test_coops_describe_sensor(self):
procedure = self.c.server.offerings[1].procedures[0]
outputFormat = self.c.server.get_operation_by_name('DescribeSensor').parameters['outputFormat']['values'][0]
response = self.c.get_metadata(procedure=procedure,
outputFormat=outputFormat)
assert isinstance(response.systems[0].id, str)
def test_coops_get_observation(self):
# COOPS does not support empty eventtime parameters
eventTime = "2012-10-01T00:00:00Z/2012-10-01T23:59:00Z"
station = None
station_names = list(set(['station-9052000']))
for offering in self.c.server.offerings:
if offering.id in station_names:
station = offering
break
response = res = self.c.get_raw_data(
offerings=[station.name],
responseFormat='text/xml;schema="ioos/0.6.1"',
observedProperties=['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'],
eventTime=eventTime,
dataType='VerifiedSixMinute')
assert isinstance(response, str)
def get_coops(start, end, sos_name, units, bbox, verbose=False):
"""
function to read COOPS data
We need to retry in case of failure b/c the server cannot handle
the high traffic during hurricane season.
"""
print(' >> Get CO-OPS information')
collector = CoopsSos()
collector.set_bbox(bbox)
collector.end_time = end
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
config = dict(
units=units,
sos_name=sos_name,
datum='MSL', ###Saeed added
)
data = collector2table(
collector=collector,
config=config,
col='{} ({})'.format(sos_name, units.format(cf_units.UT_ISO_8859_1))
)
# Clean the table.
table = dict(
station_name = [s._metadata.get('station_name') for s in data],
station_code = [s._metadata.get('station_code') for s in data],
sensor = [s._metadata.get('sensor') for s in data],
lon = [s._metadata.get('lon') for s in data],
lat = [s._metadata.get('lat') for s in data],
depth = [s._metadata.get('depth', 'NA') for s in data],
)
table = pd.DataFrame(table).set_index('station_name')
if verbose:
print('Collector offerings')
print('{}: {} offerings'.format(title, len(ofrs)))
return data, table
# Print out all the SOS Data URL endpoints
# <codecell>
sos_urls = service_urls(csw.records,
service='sos:url')
print("\n".join(sos_urls))
# <markdowncell>
# ## 1. Get observations from SOS
# Here we are using a custom class from pyoos to read the CO-OPS SOS. This is definitely unsavory, as the whole point of using a standard is avoid the need for custom classes for each service. Need to examine the consequences of removing this and just going with straight SOS service using OWSLib.
# <codecell>
collector = CoopsSos()
collector.set_datum('NAVD') # MSL
collector.server.identification.title
collector.start_time = jd_start
collector.end_time = jd_stop
collector.variables = [sos_name]
# <codecell>
ofrs = collector.server.offerings
print(len(ofrs))
for p in ofrs[700:710]:
print(p)
# <markdowncell>
# <markdowncell>
# ### SOS Requirements
# #### Use Pyoos SOS collector to obtain Observation data from COOPS.
# <codecell>
start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M')
# <codecell>
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
collector = CoopsSos()
collector.set_datum('NAVD')
collector.server.identification.title
collector.start_time = start_time
collector.end_time = end_time
collector.variables = [data_dict["water"]["sos_name"]]
# <codecell>
print("Date: %s to %s" % (iso_start, iso_end))
box_str = ','.join(str(e) for e in box)
print("Lat/Lon Box: %s" % box_str)
# <codecell>
# Grab the sos url and use it for the service.
from pyoos.collectors.coops.coops_sos import CoopsSos
from pyoos.collectors.ndbc.ndbc_sos import NdbcSos
from owslib.ows import ExceptionReport
from datetime import timedelta
from copy import copy
from StringIO import StringIO
sos_dfs = []
for sos in sos_servers:
if "co-ops" in sos.lower() or "ndbc" in sos.lower():
# CSV Output
if "co-ops" in sos.lower():
# Use the COOPS collector
collector = CoopsSos()
elif "ndbc" in sos.lower():
# Use the NDBC collector
collector = NdbcSos()
for v in variables_to_query:
collector.filter(variables=[v])
collector.filter(bbox=bounding_box)
new_start = copy(start_date)
new_end = copy(start_date)
# Hold dataframe for periodic concat
v_frame = None
while new_end < end_date:
new_end = min(end_date, new_start + timedelta(days=1))
collector.filter(start=new_start)
class CoopsSosTest(unittest.TestCase):
def setUp(self):
self.c = CoopsSos()
def test_coops_server_id(self):
assert self.c.server.identification.title == "NOAA.NOS.CO-OPS SOS"
assert self.c.server.identification.service == 'OGC:SOS'
assert self.c.server.identification.version == '1.0.0'
assert self.c.server.identification.abstract == 'NOAA.NOS.CO-OPS Sensor Observation Service (SOS) Server'
assert self.c.server.identification.keywords == [
'Air Temperature', 'Barometric Pressure', 'Conductivity',
'Currents', 'Datums', 'Rain Fall', 'Relative Humidity',
'Harmonic Constituents', 'Salinity', 'Visibility', 'Water Level',
'Water Level Predictions', 'Water Temperature', 'Winds'
]
assert self.c.server.identification.fees == 'NONE'
assert self.c.server.identification.accessconstraints == 'NONE'
def test_coops_describe_sensor(self):
self.c.features = ['8454000']
response = self.c.metadata(
output_format='text/xml;subtype="sensorML/1.0.1"')
assert isinstance(response[0], SensorML)
def test_raw_coops_get_observation(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = [
'http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'
]
response = self.c.raw(responseFormat="text/csv")
assert isinstance(response, basestring)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:A1,41.8071,-71.4012,2012-10-01T00:00:00Z,1.465,urn:ioos:def:datum:noaa::MLLW,1.064
"""
data = list(csv.DictReader(StringIO.StringIO(response)))
assert data[0][
'station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]['date_time'] == "2012-10-01T00:00:00Z"
assert data[0][
'water_surface_height_above_reference_datum (m)'] == "1.465"
assert data[0]['vertical_position (m)'] == "1.064"
def test_raw_coops_get_observation_with_dataType(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = [
'http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'
]
self.c.dataType = "VerifiedHighLow"
response = self.c.raw(responseFormat="text/csv")
assert isinstance(response, basestring)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,1.617,urn:ioos:def:datum:noaa::MLLW,1.064
"""
data = list(csv.DictReader(StringIO.StringIO(response)))
assert data[0][
'station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]['date_time'] == "2012-10-01T01:00:00Z"
assert data[0][
'water_surface_height_above_reference_datum (m)'] == "1.617"
assert data[0]['vertical_position (m)'] == "1.064"
def test_raw_coops_get_observation_with_datum(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = [
'http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'
]
self.c.dataType = "VerifiedHighLow"
self.c.datum = "NAVD"
response = self.c.raw(responseFormat="text/csv")
assert isinstance(response, basestring)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,0.863,urn:ogc:def:datum:epsg::5103,1.818
"""
data = list(csv.DictReader(StringIO.StringIO(response)))
assert len(data) == 4
assert data[0][
'station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ogc:def:datum:epsg::5103"
assert data[0]['date_time'] == "2012-10-01T01:00:00Z"
assert data[0][
'water_surface_height_above_reference_datum (m)'] == "0.863"
assert data[0]['vertical_position (m)'] == "1.818"
name = s
names.append(name)
ndbc['name'] = names
ndbc.set_index('name', inplace=True)
to_html(ndbc.head())
# ### CoopsSoS
# In[9]:
from pyoos.collectors.coops.coops_sos import CoopsSos
collector_coops = CoopsSos()
collector_coops.set_bbox(bbox)
collector_coops.end_time = stop
collector_coops.start_time = start
collector_coops.variables = [sos_name]
ofrs = collector_coops.server.offerings
title = collector_coops.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[10]:
from utilities import get_coops_metadata
bbox = [-87.40, 24.25, -74.70, 36.70]
sos_name = "water_surface_height_above_reference_datum"
In this example we will use only the `CoopsSos`,
but it is worth mentioning that `pyoos` has other collectors like IOOS SWE,
NcSOS, 52N, NERRS, NDBC, etc.
Pyoos' usage is quite simple, all we have to do is:
- create an instance of the collector we will use and,
- feed the instance with the data for the collection.
from pyoos.collectors.coops.coops_sos import CoopsSos
collector = CoopsSos()
collector.set_bbox(bbox)
collector.end_time = end_time
collector.start_time = start_time
collector.variables = [sos_name]
Let's check we we got with the search above.
ofrs = collector.server.offerings
title = collector.server.identification.title
print("Collector offerings")
print("{}: {} offerings".format(title, len(ofrs)))
OK... That is quite misleading. We did not find 1113 stations with that search.
class CoopsSosTest(unittest.TestCase):
def setUp(self):
self.c = CoopsSos()
def test_coops_server_id(self):
assert self.c.server.identification.title == "NOAA.NOS.CO-OPS SOS"
assert self.c.server.identification.service == 'OGC:SOS'
assert self.c.server.identification.version == '1.0.0'
assert self.c.server.identification.abstract == 'NOAA.NOS.CO-OPS Sensor Observation Service (SOS) Server'
assert self.c.server.identification.keywords == ['Air Temperature', 'Barometric Pressure', 'Conductivity', 'Currents', 'Datum', 'Harmonic Constituents', 'Rain Fall', 'Relative Humidity', 'Salinity', 'Visibility', 'Water Level', 'Water Level Predictions', 'Water Temperature', 'Winds']
assert self.c.server.identification.fees == 'NONE'
assert self.c.server.identification.accessconstraints == 'NONE'
def test_coops_describe_sensor(self):
self.c.features = ['8454000']
response = self.c.metadata(output_format='text/xml;subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"')
assert isinstance(response[0], SensorML)
def test_raw_coops_get_observation(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = ['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum'] # noqa
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:A1,41.8071,-71.4012,2012-10-01T00:00:00Z,1.465,urn:ioos:def:datum:noaa::MLLW,1.064
"""
data = list(csv.DictReader(io.StringIO(response)))
assert data[0]['station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]['date_time'] == "2012-10-01T00:00:00Z"
assert data[0]['water_surface_height_above_reference_datum (m)'] == "1.465"
assert data[0]['vertical_position (m)'] == "1.064"
def test_raw_coops_get_observation_with_dataType(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = ['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum']
self.c.dataType = "VerifiedHighLow"
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,1.617,urn:ioos:def:datum:noaa::MLLW,1.064
"""
data = list(csv.DictReader(io.StringIO(response)))
assert data[0]['station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]['date_time'] == "2012-10-01T01:00:00Z"
assert data[0]['water_surface_height_above_reference_datum (m)'] == "1.617"
assert data[0]['vertical_position (m)'] == "1.064"
def test_raw_coops_get_observation_with_datum(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ['8454000']
self.c.variables = ['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum']
self.c.dataType = "VerifiedHighLow"
self.c.datum = "NAVD"
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)"
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,0.863,urn:ogc:def:datum:epsg::5103,1.818
"""
data = list(csv.DictReader(io.StringIO(response)))
assert len(data) == 4
assert data[0]['station_id'] == 'urn:ioos:station:NOAA.NOS.CO-OPS:8454000'
assert data[0]['datum_id'] == "urn:ogc:def:datum:epsg::5103"
assert data[0]['date_time'] == "2012-10-01T01:00:00Z"
assert data[0]['water_surface_height_above_reference_datum (m)'] == "0.863"
assert data[0]['vertical_position (m)'] == "1.818"
def get_nos_data(self, site, observations, begin_date, units_coverter,
db_obj):
start_time = time.time()
logger = logging.getLogger(self.__class__.__name__)
logger.debug("Starting get_nos_data")
row_entry_date = datetime.now()
utc_tz = timezone('UTC')
eastern_tz = timezone('US/Eastern')
platform_handle = 'nos.%s.met' % (site)
if db_obj.platformExists(platform_handle) is None:
obs_list = []
for obs_setup in observations:
if site in obs_setup['sites']:
for xenia_obs in obs_setup['xenia_obs']:
obs_list.append({
'obs_name': xenia_obs['xenia_name'],
'uom_name': xenia_obs['xenia_units'],
's_order': 1
})
db_obj.buildMinimalPlatform(platform_handle, obs_list)
#Build sensor_id and m_type_id list.
for obs_setup in observations:
if site in obs_setup['sites']:
for xenia_obs in obs_setup['xenia_obs']:
m_type_id = db_obj.mTypeExists(xenia_obs['xenia_name'],
xenia_obs['xenia_units'])
sensor_id = db_obj.sensorExists(xenia_obs['xenia_name'],
xenia_obs['xenia_units'],
platform_handle, 1)
xenia_obs['m_type_id'] = m_type_id
xenia_obs['sensor_id'] = sensor_id
sos_query = CoopsSos()
#dates.sort(reverse=True)
logger.debug("Query site: %s for date: %s" % (site, begin_date))
sos_query.clear()
#utc_end_date = begin_date.astimezone(utc_tz) + timedelta(hours=24)
utc_end_date = begin_date.astimezone(utc_tz)
start_date = begin_date.astimezone(utc_tz) - timedelta(hours=24)
for obs_setup in observations:
if site in obs_setup['sites']:
date_ndx = None
value_ndx = None
lat_ndx = None
lon_ndx = None
depth_ndx = None
sos_query.filter(features=[site],
start=start_date,
end=utc_end_date,
variables=[obs_setup['sos_obs_query']])
try:
# results = nos_query.collect()
response = sos_query.raw(responseFormat="text/csv")
except Exception as e:
logger.exception(e)
else:
csv_reader = csv.reader(
response.decode('utf-8').split('\n'), delimiter=',')
line_cnt = 0
for row in csv_reader:
for xenia_obs_setup in obs_setup['xenia_obs']:
obs_type = xenia_obs_setup['xenia_name']
uom_type = xenia_obs_setup['xenia_units']
s_order = 1
if line_cnt > 0 and len(row):
obs_date = datetime.strptime(
row[date_ndx], '%Y-%m-%dT%H:%M:%SZ')
try:
obs_val = float(row[value_ndx])
except ValueError as e:
logger.exception(e)
obs_val = 0.0
try:
if depth_ndx is not None:
depth = float(row[depth_ndx])
else:
depth = 0
except ValueError as e:
logger.exception(e)
depth = 0
try:
if lat_ndx is not None:
latitude = float(row[lat_ndx])
if lon_ndx is not None:
longitude = float(row[lon_ndx])
except ValueError as e:
logger.exception(e)
latitude = 0.0
longitude = 0.0
obs_rec = multi_obs(
row_entry_date=row_entry_date,
platform_handle=platform_handle,
sensor_id=xenia_obs_setup['sensor_id'],
m_type_id=xenia_obs_setup['m_type_id'],
m_date=obs_date.strftime(
'%Y-%m-%dT%H:%M:%S'),
m_lon=longitude,
m_lat=latitude,
m_z=depth,
m_value=obs_val,
)
rec_id = db_obj.addRec(obs_rec, True)
if rec_id is not None:
logger.debug("Adding obs: %s(%s) Date: %s Value: %s S_Order: %d" % \
(obs_type, uom_type, obs_date, obs_val, s_order))
else:
logger.error("Failed adding obs: %s(%s) Date: %s Value: %s S_Order: %d" % \
(obs_type, uom_type, obs_date, obs_val, s_order))
else:
if value_ndx is None:
for ndx, val in enumerate(row):
if val.lower().find(
xenia_obs_setup['sos_obs_name']
) != -1:
value_ndx = ndx
if val.lower().find('date_time') != -1:
date_ndx = ndx
if val.lower().find('latitude') != -1:
lat_ndx = ndx
if val.lower().find('longitude') != -1:
lon_ndx = ndx
if val.lower().find('depth') != -1:
depth_ndx = ndx
line_cnt += 1
"""
for single_obs in nos_obs:
obs_type = nos_obs[single_obs]['xenia_name']
uom_type = nos_obs[single_obs]['xenia_units']
s_order = 1
nos_query.filter(features=[site], start=start_date, end=utc_end_date, variables=[single_obs])
try:
#results = nos_query.collect()
response = nos_query.raw(responseFormat="text/csv")
except Exception as e:
logger.exception(e)
else:
csv_reader = csv.reader(response.split('\n'), delimiter=',')
line_cnt = 0
for row in csv_reader:
if line_cnt > 0 and len(row):
obs_date = datetime.strptime(row[4], '%Y-%m-%dT%H:%M:%SZ')
obs_val = float(row[5])
logger.debug("Adding obs: %s(%s) Date: %s Value: %s S_Order: %d" %\
(obs_type, uom_type, obs_date, obs_val, s_order))
line_cnt += 1
"""
logger.debug("Finished get_nos_data in %f seconds" %
(time.time() - start_time))
return
def setUp(self):
self.c = CoopsSos()
# coding: utf-8 # In[1]: from pyoos.collectors.coops.coops_sos import CoopsSos collector = CoopsSos() # In[3]: get_ipython().system(u'conda list pyoos') # In[ ]:
def get_nos_data(site, dates, units_coverter, db_obj):
start_time = time.time()
logger = logging.getLogger(__name__)
logger.debug("Starting get_nos_data")
row_entry_date = datetime.now()
utc_tz = timezone('UTC')
eastern_tz = timezone('US/Eastern')
platform_handle = 'nos.%s.met' % (site)
if db_obj.platformExists(platform_handle) == -1:
obs_list = []
for single_obs in nos_obs:
obs_list.append({
'obs_name': nos_obs[single_obs]['xenia_name'],
'uom_name': nos_obs[single_obs]['xenia_units'],
's_order': 1
})
db_obj.buildMinimalPlatform(platform_handle, obs_list)
nos_query = CoopsSos()
#dates.sort(reverse=True)
for rec_date in dates:
logger.debug("Query site: %s for date: %s" % (site, rec_date))
nos_query.clear()
utc_end_date = rec_date.astimezone(utc_tz) + timedelta(hours=24)
start_date = rec_date.astimezone(utc_tz) - timedelta(hours=24)
for single_obs in nos_obs:
obs_type = nos_obs[single_obs]['xenia_name']
uom_type = nos_obs[single_obs]['xenia_units']
s_order = 1
nos_query.filter(features=[site],
start=start_date,
end=utc_end_date,
variables=[single_obs])
try:
#results = nos_query.collect()
response = nos_query.raw(responseFormat="text/csv")
except Exception as e:
logger.exception(e)
else:
csv_reader = csv.reader(response.split('\n'), delimiter=',')
line_cnt = 0
for row in csv_reader:
if line_cnt > 0 and len(row):
obs_date = datetime.strptime(row[4],
'%Y-%m-%dT%H:%M:%SZ')
obs_val = float(row[5])
logger.debug("Adding obs: %s(%s) Date: %s Value: %s S_Order: %d" %\
(obs_type, uom_type, obs_date, obs_val, s_order))
if not db_obj.addMeasurement(
obs_type,
uom_type,
platform_handle,
obs_date.strftime('%Y-%m-%dT%H:%M:%S'),
float(row[2]),
float(row[3]),
0, [obs_val],
sOrder=s_order,
autoCommit=True,
rowEntryDate=row_entry_date):
logger.error(db_obj.lastErrorMsg)
line_cnt += 1
logger.debug("Finished get_nos_data in %f seconds" %
(time.time() - start_time))
return
except (IOError, OSError, RuntimeError, TimeoutError) as e:
print("Could not access URL {}.html\n{!r}".format(url, e))
dap_urls = non_stations
print(fmt(" Filtered DAP "))
for url in dap_urls:
print("{}.html".format(url))
Now we have a nice list of all the models available in the catalog for the domain we specified.
We still need to find the observations for the same domain.
To accomplish that we will use the `pyoos` library and search the [SOS CO-OPS](https://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/) services using the virtually the same configuration options from the catalog search.
from pyoos.collectors.coops.coops_sos import CoopsSos
collector_coops = CoopsSos()
collector_coops.set_bbox(config["region"]["bbox"])
collector_coops.end_time = config["date"]["stop"]
collector_coops.start_time = config["date"]["start"]
collector_coops.variables = [config["sos_name"]]
ofrs = collector_coops.server.offerings
title = collector_coops.server.identification.title
print(fmt(" Collector offerings "))
print("{}: {} offerings".format(title, len(ofrs)))
To make it easier to work with the data we extract the time-series as pandas tables and interpolate them to a common 1-hour interval index.
import pandas as pd
from ioos_tools.ioos import collector2table
for rec, item in csw.records.items():
log.info('{}'.format(item.title))
log.info(fmt(' DAP '))
for url in dap_urls:
log.info('{}.html'.format(url))
log.info(fmt(' SOS '))
for url in sos_urls:
log.info('{}'.format(url))
# In[9]:
from pyoos.collectors.coops.coops_sos import CoopsSos
collector = CoopsSos()
sos_name = 'water_surface_height_above_reference_datum'
datum = 'NAVD'
collector.set_datum(datum)
collector.end_time = stop
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[10]:
print "\n".join(sos_urls)
# <markdowncell>
# ### SOS Requirements
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# <codecell>
collector = CoopsSos()
collector.start_time = start_time
collector.end_time = end_time
collector.variables = data_dict["winds"]["sos_name"]
collector.server.identification.title
print collector.start_time,":", collector.end_time
ofrs = collector.server.offerings
# <markdowncell>
# ###Find all SOS stations within the bounding box and time extent
# <codecell>
print "Date: ",iso_start," to ", iso_end
box_str=','.join(str(e) for e in bounding_box)
log.info('{}'.format(item.title))
log.info(fmt(' DAP URLs '))
for url in dap_urls:
log.info('{}.html'.format(url))
log.info(fmt(' SOS URLs '))
for url in sos_urls:
log.info('{}'.format(url))
# In[ ]:
from pyoos.collectors.coops.coops_sos import CoopsSos
collector = CoopsSos()
sos_name = 'water_surface_height_above_reference_datum'
datum = 'NAVD'
collector.set_datum(datum)
collector.end_time = stop
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[ ]:
st_list[station_name] = {}
st_list[station_name]["lat"] = lat_data[i]
st_list[station_name]["source"] = source
st_list[station_name]["lon"] = lon_data[i]
print station_name
print "number of stations in bbox",len(st_list.keys())
return st_list
# <markdowncell>
# #COOPS Station Locations
# <codecell>
coops_collector = CoopsSos()
coops_collector.start_time = start_time
coops_collector.end_time = end_time
coops_collector.variables = data_dict["currents"]["sos_name"]
coops_collector.server.identification.title
print coops_collector.start_time,":", coops_collector.end_time
ofrs = coops_collector.server.offerings
print(len(ofrs))
# <codecell>
print "Date: ",iso_start," to ", iso_end
box_str=','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ",box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
print "Total SOS:",len(sos_urls)
print "\n".join(sos_urls)
# <markdowncell>
# ###Get most recent observations from NOAA-COOPS stations in bounding box
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# Define the Coops collector
collector = CoopsSos()
print collector.server.identification.title
collector.variables = data_dict["temp"]["sos_name"]
collector.server.identification.title
# Don't specify start and end date in the filter and the most recent observation will be returned
collector.filter(bbox=bounding_box,
variables=data_dict["temp"]["sos_name"])
response = collector.raw(responseFormat="text/csv")
obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')),
parse_dates=True,
index_col='date_time')
# Now let's specify start and end times
collector.start_time = start_time
# Print out all the SOS Data URL endpoints
# <codecell>
sos_urls = service_urls(csw.records, service='sos:url')
print("\n".join(sos_urls))
# <markdowncell>
# ## 1. Get observations from SOS
# Here we are using a custom class from pyoos to read the CO-OPS SOS. This is definitely unsavory, as the whole point of using a standard is avoid the need for custom classes for each service. Need to examine the consequences of removing this and just going with straight SOS service using OWSLib.
# <codecell>
collector = CoopsSos()
collector.set_datum('NAVD') # MSL
collector.server.identification.title
collector.start_time = jd_start
collector.end_time = jd_stop
collector.variables = [sos_name]
# <codecell>
ofrs = collector.server.offerings
print(len(ofrs))
for p in ofrs[700:710]:
print(p)
# <markdowncell>
class CoopsSosTest(unittest.TestCase):
def setUp(self):
self.c = CoopsSos()
def test_coops_server_id(self):
assert self.c.server.identification.title == "NOAA.NOS.CO-OPS SOS"
assert self.c.server.identification.service == "OGC:SOS"
assert self.c.server.identification.version == "1.0.0"
assert (
self.c.server.identification.abstract
== "NOAA.NOS.CO-OPS Sensor Observation Service (SOS) Server"
)
assert self.c.server.identification.keywords == [
"Air Temperature",
"Barometric Pressure",
"Conductivity",
"Currents",
"Datum",
"Harmonic Constituents",
"Rain Fall",
"Relative Humidity",
"Salinity",
"Visibility",
"Water Level",
"Water Level Predictions",
"Water Temperature",
"Winds",
]
assert self.c.server.identification.fees == "NONE"
assert self.c.server.identification.accessconstraints == "NONE"
def test_coops_describe_sensor(self):
self.c.features = ["8454000"]
response = self.c.metadata(
output_format='text/xml;subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
)
assert isinstance(response[0], SensorML)
def test_raw_coops_get_observation(self):
self.c.start_time = datetime.strptime("2018-10-07", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2018-10-12", "%Y-%m-%d")
self.c.features = ["8728690"]
self.c.variables = [
"water_surface_height_above_reference_datum"
] # noqa
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
data = list(csv.DictReader(io.StringIO(response)))
assert (
data[0]["station_id"] == "urn:ioos:station:NOAA.NOS.CO-OPS:8728690"
)
assert data[0]["datum_id"] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]["date_time"] == "2018-10-07T00:00:00Z"
assert (
data[0]["water_surface_height_above_reference_datum (m)"]
== "0.385"
)
assert data[0]["vertical_position (m)"] == "1.307"
def test_raw_coops_get_observation_with_dataType(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ["8454000"]
self.c.variables = [
"http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum"
]
self.c.dataType = "VerifiedHighLow"
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)" # noqa
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,1.617,urn:ioos:def:datum:noaa::MLLW,1.064
"""
data = list(csv.DictReader(io.StringIO(response)))
assert (
data[0]["station_id"] == "urn:ioos:station:NOAA.NOS.CO-OPS:8454000"
)
assert data[0]["datum_id"] == "urn:ioos:def:datum:noaa::MLLW"
assert data[0]["date_time"] == "2012-10-01T01:00:00Z"
assert (
data[0]["water_surface_height_above_reference_datum (m)"]
== "1.617"
)
assert data[0]["vertical_position (m)"] == "1.064"
def test_raw_coops_get_observation_with_datum(self):
self.c.start_time = datetime.strptime("2012-10-01", "%Y-%m-%d")
self.c.end_time = datetime.strptime("2012-10-02", "%Y-%m-%d")
self.c.features = ["8454000"]
self.c.variables = [
"http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum"
]
self.c.dataType = "VerifiedHighLow"
self.c.datum = "NAVD"
response = self.c.raw(responseFormat="text/csv").decode()
assert isinstance(response, string_types)
"""
station_id,sensor_id,"latitude (degree)","longitude (degree)",date_time,"water_surface_height_above_reference_datum (m)",datum_id,"vertical_position (m)" # noqa
urn:ioos:station:NOAA.NOS.CO-OPS:8454000,urn:ioos:sensor:NOAA.NOS.CO-OPS:8454000:W3,41.8071,-71.4012,2012-10-01T01:00:00Z,0.863,urn:ogc:def:datum:epsg::5103,1.818
"""
data = list(csv.DictReader(io.StringIO(response)))
assert len(data) == 4
assert (
data[0]["station_id"] == "urn:ioos:station:NOAA.NOS.CO-OPS:8454000"
)
assert data[0]["datum_id"] == "urn:ogc:def:datum:epsg::5103"
assert data[0]["date_time"] == "2012-10-01T01:00:00Z"
assert (
data[0]["water_surface_height_above_reference_datum (m)"]
== "0.863"
)
assert data[0]["vertical_position (m)"] == "1.818"
print "Total SOS:",len(sos_urls)
print "\n".join(sos_urls[0:5])
# <markdowncell>
# ###Get most recent observations from NOAA COOPS stations in bounding box
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# Define the Coops collector
collector = CoopsSos()
print collector.server.identification.title
collector.variables = data_dict["temp"]["sos_name"]
collector.server.identification.title
# Don't specify start and end date in the filter and the most recent observation will be returned
collector.filter(bbox=bounding_box,
variables=data_dict["temp"]["sos_name"])
response = collector.raw(responseFormat="text/csv")
obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')),
parse_dates=True,
index_col='date_time')
# Now let's specify start and end times
collector.start_time = start_time
def process_nos8661070_data(platform_handle,
units_converter,
xenia_db,
unique_dates):
logger = logging.getLogger(__name__)
utc_tz = timezone('UTC')
eastern_tz= timezone('US/Eastern')
row_entry_date = datetime.now()
platform_name_parts = platform_handle.split('.')
"""
Create a data collection object.
Contructor parameters are:
url - THe SWE endpoint we're interested in
version - Optional default is '1.0.0' The SWE version the endpoint.
xml - Optional default is None - The XML response from a GetCapabilities query of the server.
"""
dataCollector = CoopsSos()
"""
obs_list = ['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum',
'http://mmisw.org/ont/cf/parameter/sea_water_temperature',
'http://mmisw.org/ont/cf/parameter/wind_speed',
'http://mmisw.org/ont/cf/parameter/wind_from_direction']
obs_list = [('water_surface_height_above_reference_datum', 'm'),
('sea_water_temperature', 'celsius'),
('wind_speed', 'm_s-1'),
('wind_from_direction', 'degrees_true')]
"""
nos_to_xenia = {
"water_surface_height_above_reference_datum": {
"units": "m",
"xenia_name": "water_level",
"xenia_units": "m"
},
"sea_water_temperature": {
"units": "celsius",
"xenia_name": "water_temperature",
"xenia_units": "celsius"
},
"wind_speed": {
"units": "m_s-1",
"xenia_name": "wind_speed",
"xenia_units": "m_s-1"
},
"wind_from_direction": {
"units": "degrees_true",
"xenia_name": "wind_from_direction",
"xenia_units": "degrees_true"
}
}
#nos_obs = nos_to_xenia.keys()
nos_obs = ['sea_water_temperature']
if xenia_db.platformExists(platform_handle) == -1:
s_order = 1
obs_list = []
for obs_key in nos_to_xenia:
obs_info = nos_to_xenia[obs_key]
obs_list.append({'obs_name': obs_info['xenia_name'],
'uom_name': obs_info['xenia_units'],
's_order': s_order})
xenia_db.buildMinimalPlatform(platform_handle, obs_list)
for start_date in unique_dates:
utc_start_date = (eastern_tz.localize(datetime.strptime(start_date, '%Y-%m-%d'))).astimezone(utc_tz)
start_date = utc_start_date - timedelta(hours=24)
logger.debug("Platform: %s Begin Date: %s End Date: %s" % (platform_handle, start_date, utc_start_date))
for single_obs in nos_obs:
obs_type = nos_to_xenia[single_obs]['xenia_name']
uom_type = nos_to_xenia[single_obs]['xenia_units']
s_order = 1
dataCollector.filter(features=['8661070'],
variables=[single_obs],
start=start_date,
end=utc_start_date)
try:
response = dataCollector.raw(responseFormat="text/csv")
except Exception as e:
logger.exception(e)
else:
csv_reader = csv.reader(response.split('\n'), delimiter=',')
line_cnt = 0
for row in csv_reader:
if line_cnt > 0 and len(row):
obs_date = datetime.strptime(row[4], '%Y-%m-%dT%H:%M:%SZ')
obs_val = float(row[5])
logger.debug("Adding obs: %s(%s) Date: %s Value: %s S_Order: %d" %\
(single_obs, uom_type, obs_date, obs_val, s_order))
if not xenia_db.addMeasurement(obs_type,
uom_type,
platform_handle,
obs_date.strftime('%Y-%m-%dT%H:%M:%S'),
float(row[2]),
float(row[3]),
0,
[obs_val],
sOrder=s_order,
autoCommit=True,
rowEntryDate=row_entry_date ):
logger.error(xenia_db.lastErrorMsg)
line_cnt += 1
# <markdowncell>
# ### SOS Requirements
# #### Use Pyoos SOS collector to obtain Observation data from COOPS.
# <codecell>
start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M')
# <codecell>
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
collector = CoopsSos()
collector.set_datum('NAVD')
collector.server.identification.title
collector.start_time = start_time
collector.end_time = end_time
collector.variables = [data_dict["water"]["sos_name"]]
# <codecell>
print("Date: %s to %s" % (iso_start, iso_end))
box_str = ','.join(str(e) for e in box)
print("Lat/Lon Box: %s" % box_str)
# <codecell>
# Grab the opendap sos url and use it for the service.
st_list[station_name]["lat"] = lat_data[i]
st_list[station_name]["source"] = source
st_list[station_name]["lon"] = lon_data[i]
print station_name
print "number of stations in bbox", len(st_list.keys())
return st_list
# <markdowncell>
# #COOPS Station Locations
# <codecell>
coops_collector = CoopsSos()
coops_collector.start_time = start_time
coops_collector.end_time = end_time
coops_collector.variables = data_dict["currents"]["sos_name"]
coops_collector.server.identification.title
print coops_collector.start_time, ":", coops_collector.end_time
ofrs = coops_collector.server.offerings
print(len(ofrs))
# <codecell>
print "Date: ", iso_start, " to ", iso_end
box_str = ','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ", box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
def ind2ij(a,index):
"""
returns a[j,i] for a.ravel()[index]
"""
n,m = shape(lon)
j = ceil(index/m).astype(int)
i = remainder(index,m)
return i,j
# <markdowncell>
# ## 1. Get observations from SOS
# <codecell>
collector = CoopsSos()
# <codecell>
collector.server.identification.title
# <codecell>
collector.start_time = jd_start
collector.end_time = jd_stop
collector.variables = [sos_name]
# <codecell>
ofrs = collector.server.offerings
def setUp(self):
self.c = CoopsSos()
def process_nos8661070_data(platform_handle, units_converter, xenia_db,
unique_dates):
logger = logging.getLogger(__name__)
utc_tz = timezone('UTC')
eastern_tz = timezone('US/Eastern')
row_entry_date = datetime.now()
platform_name_parts = platform_handle.split('.')
"""
Create a data collection object.
Contructor parameters are:
url - THe SWE endpoint we're interested in
version - Optional default is '1.0.0' The SWE version the endpoint.
xml - Optional default is None - The XML response from a GetCapabilities query of the server.
"""
dataCollector = CoopsSos()
"""
obs_list = ['http://mmisw.org/ont/cf/parameter/water_surface_height_above_reference_datum',
'http://mmisw.org/ont/cf/parameter/sea_water_temperature',
'http://mmisw.org/ont/cf/parameter/wind_speed',
'http://mmisw.org/ont/cf/parameter/wind_from_direction']
obs_list = [('water_surface_height_above_reference_datum', 'm'),
('sea_water_temperature', 'celsius'),
('wind_speed', 'm_s-1'),
('wind_from_direction', 'degrees_true')]
"""
nos_to_xenia = {
"water_surface_height_above_reference_datum": {
"units": "m",
"xenia_name": "water_level",
"xenia_units": "m"
},
"sea_water_temperature": {
"units": "celsius",
"xenia_name": "water_temperature",
"xenia_units": "celsius"
},
"wind_speed": {
"units": "m_s-1",
"xenia_name": "wind_speed",
"xenia_units": "m_s-1"
},
"wind_from_direction": {
"units": "degrees_true",
"xenia_name": "wind_from_direction",
"xenia_units": "degrees_true"
}
}
#nos_obs = nos_to_xenia.keys()
nos_obs = ['sea_water_temperature']
if xenia_db.platformExists(platform_handle) == -1:
s_order = 1
obs_list = []
for obs_key in nos_to_xenia:
obs_info = nos_to_xenia[obs_key]
obs_list.append({
'obs_name': obs_info['xenia_name'],
'uom_name': obs_info['xenia_units'],
's_order': s_order
})
xenia_db.buildMinimalPlatform(platform_handle, obs_list)
for start_date in unique_dates:
utc_start_date = (eastern_tz.localize(
datetime.strptime(start_date, '%Y-%m-%d'))).astimezone(utc_tz)
start_date = utc_start_date - timedelta(hours=24)
logger.debug("Platform: %s Begin Date: %s End Date: %s" %
(platform_handle, start_date, utc_start_date))
for single_obs in nos_obs:
obs_type = nos_to_xenia[single_obs]['xenia_name']
uom_type = nos_to_xenia[single_obs]['xenia_units']
s_order = 1
dataCollector.filter(features=['8661070'],
variables=[single_obs],
start=start_date,
end=utc_start_date)
try:
response = dataCollector.raw(responseFormat="text/csv")
except Exception as e:
logger.exception(e)
else:
csv_reader = csv.reader(response.split('\n'), delimiter=',')
line_cnt = 0
for row in csv_reader:
if line_cnt > 0 and len(row):
obs_date = datetime.strptime(row[4],
'%Y-%m-%dT%H:%M:%SZ')
obs_val = float(row[5])
logger.debug("Adding obs: %s(%s) Date: %s Value: %s S_Order: %d" %\
(single_obs, uom_type, obs_date, obs_val, s_order))
if not xenia_db.addMeasurement(
obs_type,
uom_type,
platform_handle,
obs_date.strftime('%Y-%m-%dT%H:%M:%S'),
float(row[2]),
float(row[3]),
0, [obs_val],
sOrder=s_order,
autoCommit=True,
rowEntryDate=row_entry_date):
logger.error(xenia_db.lastErrorMsg)
line_cnt += 1
