def __init__(self, wqx_metadata, wqx_data):
if not isinstance(wqx_metadata, WqxOutbound):
wqx_metadata = WqxOutbound(wqx_metadata)
if not isinstance(wqx_data, WqxOutbound):
wqx_data = WqxOutbound(wqx_data)
if wqx_data.failed or wqx_metadata.failed:
self.feature = None
else:
stations = []
station_lookup = []
# Create a station for every MonitoringLocation
for org in wqx_metadata.organizations:
for ml in org.locations:
s = Station()
s.uid = ml.id
s.name = ml.name
s.set_property("station_type", ml.type)
s.set_property("location_description", ml.description)
s.set_property("huc", ml.huc)
s.set_property("county", ml.county)
s.set_property("state", ml.state)
s.set_property("country", ml.country)
s.set_property("organization_id", org.description.id)
s.set_property("organization_name", org.description.name)
s.set_property("vertical_units", ml.vertical_measure_units)
s.set_property("horizontal_crs", ml.horizontal_crs_name)
s.set_property("vertical_crs", ml.vertical_crs_name)
# Now set the station's location
vertical = 0
try:
vertical = float(ml.vertical_measure_value)
except:
pass
# convert the vertical to meters if it is ft (which it always is)
if ml.vertical_measure_units == "ft":
vertical /= 3.28084
s.set_property("vertical_units", "m")
s.location = sPoint(float(ml.longitude), float(ml.latitude), vertical)
stations.append(s)
station_lookup.append(s.uid)
for org in wqx_data.organizations:
for a in org.activities:
p = Point()
p.time = a.start_time
for r in a.results:
p.add_member(Member(value=r.value, unit=r.units, name=r.name, description=r.short_name, standard=None, quality=r.quality, method_id=a.method_id, method_name=a.method_name))
# Assign data to the correct station pulled from the metadata
station = stations[station_lookup.index(a.location_id)]
p.location = station.location
station.add_element(p)
self.feature = StationCollection(elements=stations)
def __init__(self, response_list, nerrs_stations=None):
assert isinstance(response_list, dict)
if nerrs_stations is None:
from pyoos.collectors.nerrs.nerrs_soap import NerrsSoap
nerrs_stations = NerrsSoap().stations
def get_station(feature):
for s in nerrs_stations:
if s['Station_Code'].lower() == feature.lower():
return s
skip_tags = ["DateTimeStamp", "utcStamp", "data", "MaxWSpdT"]
stations = []
for feature, response in response_list.iteritems():
if not isinstance(response, etree._Element):
response = etree.fromstring(response)
feature = get_station(feature)
s = Station()
s.uid = feature['Station_Code']
s.name = feature['Station_Name']
s.location = sPoint(float(feature['Longitude']),
float(feature['Latitude']), 0)
s.set_property("state", feature['State'])
s.set_property("siteid", feature['NERR_Site_ID'])
s.set_property("horizontal_crs", "EPSG:4326")
s.set_property("vertical_units", "m")
s.set_property("vertical_crs", "EPSG:4297")
s.set_property("location_description", feature['Reserve_Name'])
for data in response.findall(".//data"):
p = Point()
t = AsaTime.parse(testXMLValue(data.find("utcStamp")))
t = t.replace(tzinfo=pytz.utc)
p.time = t
p.location = s.location
for child in data:
if child.tag not in skip_tags:
try:
val = float(child.text)
p.add_member(
Member(value=val,
unit=units(child.tag),
name=child.tag,
description=child.tag,
standard=standard(child.tag)))
except TypeError:
# Value was None
pass
s.add_element(p)
stations.append(s)
self.feature = StationCollection(elements=stations)
def parse_response(self,response): if response is None: return None xml = etree.fromstring(response) time_series = self._read_xml(xml) if len(time_series) > 1: # retval is a StationCollection retval = StationCollection() for ts in time_series: st = self._timeseries_to_station(ts) retval.add_element(st) else: # retval is a Station retval = self._timeseries_to_station(time_series[0]) return retval
def parse_station_collection(self, cb_func, cb_args, desired_stations):
"""
Function to group Stations into a StationCollection
- cb_func: the get_station function to call from the nerr collector
- cb_args: arguments to pass along into the get_station function
- desired_stations: list of stations that are to be added into the collections
returns a StationCollection object
"""
if not isinstance(cb_args, dict):
return None
if not isinstance(desired_stations, list) or len(desired_stations) < 1:
return None
retval = StationCollection()
for ds in desired_stations:
elem = cb_func(ds.code, site_id=ds.id, metadata=ds, **cb_args)
retval.add_element(elem)
return retval
def parse_station_collection(self, cb_func, cb_args, desired_stations): """ Function to group Stations into a StationCollection - cb_func: the get_station function to call from the nerr collector - cb_args: arguments to pass along into the get_station function - desired_stations: list of stations that are to be added into the collections returns a StationCollection object """ if not isinstance(cb_args, dict): return None if not isinstance(desired_stations, list) or len(desired_stations) < 1: return None retval = StationCollection() for ds in desired_stations: elem = cb_func(ds.code, site_id=ds.id, metadata=ds, **cb_args) retval.add_element(elem) return retval
def _build_station_collection(self, parsed_metadata, parsed_data):
stations = []
for station_code, station_metadata in parsed_metadata.items():
s = Station()
s.uid = station_code
s.name = station_metadata['nwsli']
s.location = sPoint(station_metadata['longitude'],
station_metadata['latitude'],
0) # hads always vertically zero
s.set_property("location_description",
station_metadata['location_text'])
s.set_property("state", station_metadata['state'])
s.set_property("country", "USA") # @TODO
s.set_property("vertical_units", "ft")
s.set_property("horizontal_crs", "EPSG:4326")
s.set_property("vertical_crs", None)
s.set_property("hsa", station_metadata['hsa'])
s.set_property("init_transmit", station_metadata['init_transmit'])
s.set_property("manufacturer", station_metadata['manufacturer'])
s.set_property("owner", station_metadata['owner'])
s.set_property("channel", station_metadata['channel'])
stations.append(s)
# data
# possibility no data for this station, or vars filtered all out
if station_code not in parsed_data:
continue
# need to group into distinct time/z value pairs
# create a keyfunc (creates string of <z>-<timestamp>)
zandtime = lambda x: str(x[3]) + "-" + str(
time.mktime(x[1].timetuple()))
# annotate data with z values, sort, group by keyfunc (z/time)
grouped_data = groupby(
sorted([(x[0], x[1], x[2], parsed_metadata[station_code]
['variables'][x[0]]['base_elevation'])
for x in parsed_data[station_code]],
key=zandtime), zandtime)
for _, group in grouped_data:
# group is an iterator, turn it into a list (it will have at least one item)
groupvals = list(group)
p = Point()
p.time = groupvals[0][1]
p.location = sPoint(station_metadata['longitude'],
station_metadata['latitude'],
groupvals[0][3])
for val in groupvals:
std_var = self.get_variable_info(val[0])
if std_var is None:
print("Unknown PE Code, ignoring: {} (station: {}).".
format(val[0], station_code))
continue
p.add_member(
Member(value=val[2],
standard=std_var[0],
unit=std_var[1],
name=std_var[2],
description=std_var[3]))
s.add_element(p)
return StationCollection(elements=stations)
def __init__(self, waterml_data):
if isinstance(waterml_data, str) or isinstance(waterml_data, unicode):
try:
self._root = etree.fromstring(str(waterml_data))
except ValueError:
# Strip out the XML header due to UTF8 encoding declaration
self._root = etree.fromstring(waterml_data[56:])
else:
self._root = waterml_data
response = WaterML_1_1(self._root).response
stations = []
station_lookup = []
for timeseries in response.time_series:
station_code = sorted(timeseries.source_info.site_codes)[0]
# Create station if we have not seen it
if station_code not in station_lookup:
s = Station()
s.uid = station_code
info = timeseries.source_info
s.name = info.site_name
s.set_property("station_type", info.site_types)
s.set_property("huc", info.site_properties.get("hucCd"))
s.set_property("county", info.site_properties.get("countyCd"))
s.set_property("state", info.site_properties.get("stateCd"))
# Now set the station's location
vertical = info.elevation
if vertical is None:
vertical = 0
try:
location = info.location.geo_coords[0]
srs = info.location.srs[0]
except:
print "Could not find a location for %s... skipping station" % s.uid
continue
s.location = sPoint(float(location[0]), float(location[1]),
vertical)
s.set_property("horizontal_crs", srs)
s.set_property("vertical_units", "m")
s.set_property("vertical_crs", info.vertical_datum)
s.set_property("location_description", info.location.notes)
stations.append(s)
station_lookup.append(s.uid)
times = {}
variable = timeseries.variable
for variable_timeseries in timeseries.values:
for r in variable_timeseries:
dt = r.date_time
if dt.tzinfo is None:
dt = dt.replace(tzinfo=pytz.utc)
dt = dt.astimezone(pytz.utc)
if dt not in times.keys():
times[dt] = []
times[dt].append(
Member(value=r.value,
unit=variable.unit.code,
name=variable.variable_name,
description=variable.variable_description,
standard=variable.variable_code))
station = stations[station_lookup.index(station_code)]
for dts, members in times.iteritems():
p = Point()
p.time = dts
p.location = station.location
p.members = members
station.add_element(p)
self.feature = StationCollection(elements=stations)
def __init__(self, awc_list):
for awc_data in awc_list:
self._root = etree.fromstring(awc_data.encode())
'''Code to get station iterator goes here.'''
stations = []
station_lookup = []
times = []
for metar in self._root.iter('METAR'):
uid = metar.find("station_id").text
if uid not in station_lookup:
s = Station()
s.uid = uid
vertical = metar.find("elevation_m").text
if vertical is None:
vertical = 0
s.location = sPoint(float(metar.find("latitude").text),
float(metar.find("longitude").text),
float(vertical))
s.set_property("metar_type", metar.find("metar_type").text)
s.set_property("horizontal_crs", "GCS")
s.set_property("vertical_units", "m")
s.set_property("vertical_crs", "AGL")
stations.append(s)
station_lookup.append(s.uid)
times.append({})
else:
s = stations[station_lookup.index(uid)]
variables = [
"elevation_m", "raw_text", "temp_c", "dewpoint_c",
"wind_dir_degrees", "wind_speed_kt",
"visibility_statute_mi", "altim_in_hg", "wx_string",
"sky_condition", "flight_category"
]
for variable in variables:
time_string = metar.find("observation_time").text
dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ")
if dt.tzinfo is None:
dt = dt.replace(tzinfo=pytz.utc)
dt = dt.astimezone(pytz.utc)
if dt not in list(times[station_lookup.index(
s.uid)].keys()):
times[station_lookup.index(s.uid)][dt] = []
if metar.find(variable) != None:
if variable in set(
["raw_text", "flight_category", "wx_string"]):
times[station_lookup.index(s.uid)][dt].append(
Member(value=metar.find(variable).text,
unit=None,
name=variable,
description=variable,
standard=None))
elif variable == "sky_condition":
sky_condition = []
for cond in metar.findall(variable):
cover = cond.attrib["sky_cover"]
try:
alt = cond.attrib["cloud_base_ft_agl"]
except:
alt = None
sky_condition.append({
"sky_cover": cover,
"cloud_base_ft_agl": alt
})
times[station_lookup.index(s.uid)][dt].append(
Member(value=sky_condition,
unit="ft_above_ground_level",
name=variable,
description=variable,
standard=None))
else:
times[station_lookup.index(s.uid)][dt].append(
Member(value=float(metar.find(variable).text),
unit=variable.split("_")[-1],
name=variable,
description=variable,
standard=None))
for time_dict, station in zip(times, stations):
for dts, members in time_dict.items():
p = Point()
p.time = dts
p.location = station.location
p.members = members
station.add_element(p)
self.feature = StationCollection(elements=stations)
def __init__(self, element):
record = DataRecord(element)
stations_field = record.get_by_name("stations")
stations = {}
sensors = {}
for station in stations_field.content.field:
s = StationProfile()
s.name = station.name
s.uid = station.content.get_by_name("stationID").content.value
# Location
vector = station.content.get_by_name("platformLocation").content
srss = vector.referenceFrame.split("&")
hsrs = None
try:
hsrs = Crs(srss[0])
except ValueError:
pass
vsrs = None
try:
vsrs = Crs(srss[-1].replace("2=http:", "http:"))
except ValueError:
pass
s.set_property("horizontal_srs", hsrs)
s.set_property("vertical_srs", vsrs)
s.set_property("localFrame", vector.localFrame)
lat = vector.get_by_name("latitude").content.value
lon = vector.get_by_name("longitude").content.value
z = vector.get_by_name("height").content.value
loc = [lon, lat]
if z:
loc.append(z)
s.location = sPoint(*loc)
# sensors
for sensor in station.content.get_by_name("sensors").content.field:
name = sensor.name
uri = sensor.content.get_by_name("sensorID").content.value
sensors[name] = {'station' : s.uid,
'name' : name,
'uri' : uri}
# orientation
ori_el = sensor.content.get_by_name("sensorOrientation")
if ori_el:
orientation = self._parse_sensor_orientation(ori_el)
sensors[name]['sensor_orientation'] = orientation
# location
loc_el = sensor.content.get_by_name("sensorLocation")
if loc_el:
location = self._parse_location(loc_el, s.location)
sensors[name]['location'] = location
# profile bins
profbins_el = sensor.content.get_by_name("profileBins")
if profbins_el:
profile_bins = self._parse_profile_bins(profbins_el)
sensors[name]['profile_bins'] = profile_bins
# OR profile heights
profheights_el = sensor.content.get_by_name('profileHeights')
if profheights_el:
profile_heights = self._parse_profile_heights(profheights_el)
sensors[name]['profile_heights'] = profile_heights
s.sensors = sensors
stations[s.uid] = s
sensor_data = self._parse_sensor_data(record.get_by_name('observationData'), sensors)
# sensor data is dict of station id -> profile collection
for station_id, sensor_profile_data in sensor_data.iteritems():
stations[station_id].elements.extend(sensor_profile_data._elements)
if len(stations) > 1:
self.feature = StationCollection(elements=stations)
elif len(stations) == 1:
self.feature = next(stations.itervalues())
else:
print "No stations found!"
self.feature = None
def __init__(self, element):
record = DataRecord(element)
# Top level org structure
stations_field = record.get_by_name("stations")
stations = {}
sensors = {}
for station in stations_field.content.field:
s = Station()
s.name = station.name
s.uid = station.content.get_by_name("stationID").content.value
# Location
vector = station.content.get_by_name("platformLocation").content
srss = vector.referenceFrame.split("&")
hsrs = None
try:
hsrs = Crs(srss[0])
except ValueError:
pass
vsrs = None
try:
vsrs = Crs(srss[-1].replace("2=http:", "http:"))
except ValueError:
pass
s.set_property("horizontal_srs", hsrs)
s.set_property("vertical_srs", vsrs)
s.set_property("localFrame", vector.localFrame)
lat = vector.get_by_name("latitude").content.value
lon = vector.get_by_name("longitude").content.value
z = vector.get_by_name("height").content.value
loc = [lon, lat]
if z:
loc.append(z)
s.location = sPoint(*loc)
# Sensors
for sensor in station.content.get_by_name("sensors").content.field:
name = sensor.name
uri = sensor.content.get_by_name("sensorID").content.value
height = None
location_quantity = sensor.content.get_by_name(
"height").content
if location_quantity.referenceFrame == "#%s_frame" % s.name:
# Uses the station as reference frame
if location_quantity.value and z:
height = z + location_quantity.value
horizontal_srs = s.get_property("horizontal_srs")
vertical_srs = s.get_property("vertical_srs")
else:
# Uses its own height
if location_quantity.value:
height = location_quantity.value
horizontal_srs = None
vertical_srs = None
if hasattr(sensor, 'referenceFrame'):
srss = sensor.referenceFrame.split("&")
try:
horizontal_srs = Crs(srss[0])
except ValueError:
pass
try:
vertical_srs = Crs(srss[-1].replace(
"2=http:", "http:"))
except ValueError:
pass
loc = [s.location.x, s.location.y]
if height:
loc.append(height)
location = sPoint(*loc)
sensors[name] = {
'station': s.uid,
'name': name,
'uri': uri,
'horizontal_srs': horizontal_srs,
'vertical_srs': vertical_srs,
'location': location,
'columns': [], # Array of Members representing the columns
'values': [] # Array of Points (the actual data)
}
stations[s.uid] = s
# Start building the column structure
data_array = record.get_by_name("observationData").content
data_record = data_array.elementType.content
columns = []
# Data outside of the <field name="sensors"> DataChoice element
for f in data_record.field:
columns.append(f)
# Data inside of DataChoice
sensor_data = data_record.get_by_name("sensor")
for sendata in sensor_data.content.item:
if sendata.content is not None:
sensors[sendata.name]['columns'] = []
sensors[sendata.name]['values'] = []
for f in sendata.content.field:
# Create a model Member for each column that will be copied and filled with data from each row
sensors[sendata.name]['columns'].append(f)
decimalSeparator = data_array.encoding.decimalSeparator
tokenSeparator = data_array.encoding.tokenSeparator
blockSeparator = data_array.encoding.blockSeparator
collapseWhiteSpaces = data_array.encoding.collapseWhiteSpaces
data_values = data_array.values
self.raw_data = copy(data_values)
for row in filter(lambda x: x != "",
data_values.split(blockSeparator)):
pt = None
members = []
values = row.split(tokenSeparator)
sensor_key = None
i = 0
for x in columns:
if isinstance(
x.content, Time
) and x.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
pt = Point()
pt.time = parser.parse(values[i])
elif isinstance(x.content, DataChoice):
sensor_key = values[i]
dc_cols = sensors[sensor_key]['columns']
for j, c in enumerate(dc_cols):
if isinstance(c.content, AbstractSimpleComponent):
m = Member(units=c.content.uom,
name=c.name,
standard=c.content.definition,
value=float(values[i + 1]))
members.append(m)
elif isinstance(
c.content, Time
) and c.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
pt = Point()
pt.time = parser.parse(values[i])
# For each data column
i += 1
elif isinstance(x.content, AbstractSimpleComponent):
m = Member(units=x.content.uom,
name=x.name,
standard=x.content.definition,
value=float(values[i]))
members.append(m)
else:
print "WHAT AM I"
i += 1
pt.members = members
pt.location = stations[sensors[sensor_key]['station']].location
sensors[sensor_key]['values'].append(pt)
for k, v in stations.iteritems():
for sk, sv in sensors.iteritems():
# Match on station uid
if sv['station'] == k:
v.elements = self._merge_points(v.elements or [],
sv['values'])
if len(stations) > 1:
self.feature = StationCollection(elements=stations)
elif len(stations) == 1:
self.feature = next(stations.itervalues())
else:
print "No stations found!"
