def __init__(self, waterml_data): if isinstance(waterml_data, str) or isinstance(waterml_data, unicode): try: self._root = etree.fromstring(str(waterml_data)) except ValueError: # Strip out the XML header due to UTF8 encoding declaration self._root = etree.fromstring(waterml_data[56:]) else: self._root = waterml_data response = WaterML_1_1(self._root).response stations = [] station_lookup = [] for timeseries in response.time_series: station_code = sorted(timeseries.source_info.site_codes)[0] # Create station if we have not seen it if station_code not in station_lookup: s = Station() s.uid = station_code info = timeseries.source_info s.name = info.site_name s.set_property("station_type", info.site_types) s.set_property("huc", info.site_properties.get("hucCd")) s.set_property("county", info.site_properties.get("countyCd")) s.set_property("state", info.site_properties.get("stateCd")) # Now set the station's location vertical = info.elevation if vertical is None: vertical = 0 try: location = info.location.geo_coords[0] srs = info.location.srs[0] except: print "Could not find a location for %s... skipping station" % s.uid continue s.location = sPoint(float(location[0]), float(location[1]), vertical) s.set_property("horizontal_crs", srs) s.set_property("vertical_units", "m") s.set_property("vertical_crs", info.vertical_datum) s.set_property("location_description", info.location.notes) stations.append(s) station_lookup.append(s.uid) times = {} variable = timeseries.variable for variable_timeseries in timeseries.values: for r in variable_timeseries: dt = r.date_time if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.utc) dt = dt.astimezone(pytz.utc) if dt not in times.keys(): times[dt] = [] times[dt].append(Member(value=r.value, unit=variable.unit.code, name=variable.variable_name, description=variable.variable_description, standard=variable.variable_code)) station = stations[station_lookup.index(station_code)] for dts,members in times.iteritems(): p = Point() p.time = dts p.location = station.location p.members = members station.add_element(p) self.feature = StationCollection(elements=stations)
def __init__(self, element): record = DataRecord(element) # Top level org structure stations_field = record.get_by_name("stations") stations = {} sensors = {} for station in stations_field.content.field: s = Station() s.name = station.name s.uid = station.content.get_by_name("stationID").content.value # Location vector = station.content.get_by_name("platformLocation").content srss = vector.referenceFrame.split("&") hsrs = None try: hsrs = Crs(srss[0]) except ValueError: pass vsrs = None try: vsrs = Crs(srss[-1].replace("2=http:", "http:")) except ValueError: pass s.set_property("horizontal_srs", hsrs) s.set_property("vertical_srs", vsrs) s.set_property("localFrame", vector.localFrame) lat = vector.get_by_name("latitude").content.value lon = vector.get_by_name("longitude").content.value z = vector.get_by_name("height").content.value loc = [lon, lat] if z: loc.append(z) s.location = sPoint(*loc) # Sensors for sensor in station.content.get_by_name("sensors").content.field: name = sensor.name uri = sensor.content.get_by_name("sensorID").content.value height = None location_quantity = sensor.content.get_by_name("height").content if location_quantity.referenceFrame == "#%s_frame" % s.name: # Uses the station as reference frame if location_quantity.value and z: height = z + location_quantity.value horizontal_srs = s.get_property("horizontal_srs") vertical_srs = s.get_property("vertical_srs") else: # Uses its own height if location_quantity.value: height = location_quantity.value horizontal_srs = None vertical_srs = None if hasattr(sensor, 'referenceFrame'): srss = sensor.referenceFrame.split("&") try: horizontal_srs = Crs(srss[0]) except ValueError: pass try: vertical_srs = Crs(srss[-1].replace("2=http:", "http:")) except ValueError: pass loc = [s.location.x, s.location.y] if height: loc.append(height) location = sPoint(*loc) sensors[name] = { 'station' : s.uid, 'name' : name, 'uri' : uri, 'horizontal_srs' : horizontal_srs, 'vertical_srs' : vertical_srs, 'location' : location, 'columns' : [], # Array of Members representing the columns 'values' : [] # Array of Points (the actual data) } stations[s.uid] = s # Start building the column structure data_array = record.get_by_name("observationData").content data_record = data_array.elementType.content columns = [] # Data outside of the <field name="sensors"> DataChoice element for f in data_record.field: columns.append(f) # Data inside of DataChoice sensor_data = data_record.get_by_name("sensor") for sendata in sensor_data.content.item: if sendata.content is not None: sensors[sendata.name]['columns'] = [] sensors[sendata.name]['values'] = [] for f in sendata.content.field: # Create a model Member for each column that will be copied and filled with data from each row sensors[sendata.name]['columns'].append(f) decimalSeparator = data_array.encoding.decimalSeparator tokenSeparator = data_array.encoding.tokenSeparator blockSeparator = data_array.encoding.blockSeparator collapseWhiteSpaces = data_array.encoding.collapseWhiteSpaces data_values = data_array.values self.raw_data = copy(data_values) for row in filter(lambda x: x != "", data_values.split(blockSeparator)): pt = None members = [] values = row.split(tokenSeparator) sensor_key = None i = 0 for x in columns: if isinstance(x.content, Time) and x.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime": pt = Point() pt.time = parser.parse(values[i]) elif isinstance(x.content, DataChoice): sensor_key = values[i] dc_cols = sensors[sensor_key]['columns'] for j,c in enumerate(dc_cols): if isinstance(c.content, AbstractSimpleComponent): m = Member( units=c.content.uom, name=c.name, standard=c.content.definition, value=float(values[i+1])) members.append(m) elif isinstance(c.content, Time) and c.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime": pt = Point() pt.time = parser.parse(v) # For each data column i += 1 elif isinstance(x.content, AbstractSimpleComponent): m = Member( units=x.content.uom, name=x.name, standard=x.content.definition, value=float(values[i])) members.append(m) else: print "WHAT AM I" i += 1 pt.members = members pt.location = stations[sensors[sensor_key]['station']].location sensors[sensor_key]['values'].append(pt) for k,v in stations.iteritems(): for sk, sv in sensors.iteritems(): # Match on station uid if sv['station'] == k: v.elements = self._merge_points(v.elements or [], sv['values']) if len(stations) > 1: self.feature = StationCollection(elements=stations) elif len(stations) == 1: self.feature = next(stations.itervalues()) else: print "No stations found!"
def __init__(self, waterml_data): if isinstance(waterml_data, str) or isinstance(waterml_data, unicode): try: self._root = etree.fromstring(str(waterml_data)) except ValueError: # Strip out the XML header due to UTF8 encoding declaration self._root = etree.fromstring(waterml_data[56:]) else: self._root = waterml_data response = WaterML_1_1(self._root).response stations = [] station_lookup = [] for timeseries in response.time_series: station_code = sorted(timeseries.source_info.site_codes)[0] # Create station if we have not seen it if station_code not in station_lookup: s = Station() s.uid = station_code info = timeseries.source_info s.name = info.site_name s.set_property("station_type", info.site_types) s.set_property("huc", info.site_properties.get("hucCd")) s.set_property("county", info.site_properties.get("countyCd")) s.set_property("state", info.site_properties.get("stateCd")) # Now set the station's location vertical = info.elevation if vertical is None: vertical = 0 try: location = info.location.geo_coords[0] srs = info.location.srs[0] except: print "Could not find a location for %s... skipping station" % s.uid continue s.location = sPoint(float(location[0]), float(location[1]), vertical) s.set_property("horizontal_crs", srs) s.set_property("vertical_units", "m") s.set_property("vertical_crs", info.vertical_datum) s.set_property("location_description", info.location.notes) stations.append(s) station_lookup.append(s.uid) times = {} variable = timeseries.variable for variable_timeseries in timeseries.values: for r in variable_timeseries: dt = r.date_time if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.utc) dt = dt.astimezone(pytz.utc) if dt not in times.keys(): times[dt] = [] times[dt].append( Member(value=r.value, unit=variable.unit.code, name=variable.variable_name, description=variable.variable_description, standard=variable.variable_code)) station = stations[station_lookup.index(station_code)] for dts, members in times.iteritems(): p = Point() p.time = dts p.location = station.location p.members = members station.add_element(p) self.feature = StationCollection(elements=stations)
def __init__(self, awc_list): for awc_data in awc_list: self._root = etree.fromstring(awc_data.encode()) '''Code to get station iterator goes here.''' stations = [] station_lookup = [] times = [] for metar in self._root.iter('METAR'): uid = metar.find("station_id").text if uid not in station_lookup: s = Station() s.uid = uid vertical = metar.find("elevation_m").text if vertical is None: vertical = 0 s.location = sPoint(float(metar.find("latitude").text), float(metar.find("longitude").text), float(vertical)) s.set_property("metar_type", metar.find("metar_type").text) s.set_property("horizontal_crs", "GCS") s.set_property("vertical_units", "m") s.set_property("vertical_crs", "AGL") stations.append(s) station_lookup.append(s.uid) times.append({}) else: s = stations[station_lookup.index(uid)] variables = [ "elevation_m", "raw_text", "temp_c", "dewpoint_c", "wind_dir_degrees", "wind_speed_kt", "visibility_statute_mi", "altim_in_hg", "wx_string", "sky_condition", "flight_category" ] for variable in variables: time_string = metar.find("observation_time").text dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ") if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.utc) dt = dt.astimezone(pytz.utc) if dt not in list(times[station_lookup.index( s.uid)].keys()): times[station_lookup.index(s.uid)][dt] = [] if metar.find(variable) != None: if variable in set( ["raw_text", "flight_category", "wx_string"]): times[station_lookup.index(s.uid)][dt].append( Member(value=metar.find(variable).text, unit=None, name=variable, description=variable, standard=None)) elif variable == "sky_condition": sky_condition = [] for cond in metar.findall(variable): cover = cond.attrib["sky_cover"] try: alt = cond.attrib["cloud_base_ft_agl"] except: alt = None sky_condition.append({ "sky_cover": cover, "cloud_base_ft_agl": alt }) times[station_lookup.index(s.uid)][dt].append( Member(value=sky_condition, unit="ft_above_ground_level", name=variable, description=variable, standard=None)) else: times[station_lookup.index(s.uid)][dt].append( Member(value=float(metar.find(variable).text), unit=variable.split("_")[-1], name=variable, description=variable, standard=None)) for time_dict, station in zip(times, stations): for dts, members in time_dict.items(): p = Point() p.time = dts p.location = station.location p.members = members station.add_element(p) self.feature = StationCollection(elements=stations)
def __init__(self, awc_list): for awc_data in awc_list: if isinstance(awc_data, str) or isinstance(awc_data, unicode): try: self._root = etree.fromstring(str(awc_data)) except ValueError: # Strip out the XML header due to UTF8 encoding declaration self._root = etree.fromstring(awc_data[56:]) else: raise ValueError("Cannot parse response into ElementTree xml object") '''Code to get station iterator goes here ''' stations = [] station_lookup = [] times = [] for metar in self._root.iter('METAR'): uid = metar.find("station_id").text if uid not in station_lookup: s = Station() s.uid = uid vertical = metar.find("elevation_m").text if vertical is None: vertical = 0 s.location = sPoint(float(metar.find("latitude").text), float(metar.find("longitude").text), float(vertical)) s.set_property("metar_type", metar.find("metar_type").text) s.set_property("horizontal_crs", "GCS") s.set_property("vertical_units", "m") s.set_property("vertical_crs", "AGL") stations.append(s) station_lookup.append(s.uid) times.append({}) else: s = stations[station_lookup.index(uid)] variables = ["elevation_m", "raw_text", "temp_c", "dewpoint_c", "wind_dir_degrees", "wind_speed_kt", "visibility_statute_mi", "altim_in_hg", "wx_string", "sky_condition", "flight_category"] for variable in variables: time_string = metar.find("observation_time").text dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ") if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.utc) dt = dt.astimezone(pytz.utc) if dt not in times[station_lookup.index(s.uid)].keys(): times[station_lookup.index(s.uid)][dt] = [] if metar.find(variable) != None: if variable in set(["raw_text", "flight_category", "wx_string"]): times[station_lookup.index(s.uid)][dt].append(Member(value=metar.find(variable).text, unit=None, name=variable, description=variable, standard=None)) elif variable == "sky_condition": sky_condition = [] for cond in metar.findall(variable): cover = cond.attrib["sky_cover"] try: alt = cond.attrib["cloud_base_ft_agl"] except: alt = None sky_condition.append({"sky_cover" : cover, "cloud_base_ft_agl" : alt}) times[station_lookup.index(s.uid)][dt].append(Member(value=sky_condition, unit="ft_above_ground_level", name=variable, description=variable, standard=None)) else: times[station_lookup.index(s.uid)][dt].append(Member(value=float(metar.find(variable).text), unit=variable.split("_")[-1], name=variable, description=variable, standard=None)) for time_dict, station in zip(times, stations): for dts, members in time_dict.iteritems(): p = Point() p.time = dts p.location = station.location p.members = members station.add_element(p) self.feature = StationCollection(elements=stations)
def __init__(self, element): record = DataRecord(element) # Top level org structure stations_field = record.get_by_name("stations") stations = {} sensors = {} for station in stations_field.content.field: s = Station() s.name = station.name s.uid = station.content.get_by_name("stationID").content.value # Location vector = station.content.get_by_name("platformLocation").content srss = vector.referenceFrame.split("&") hsrs = None try: hsrs = Crs(srss[0]) except ValueError: pass vsrs = None try: vsrs = Crs(srss[-1].replace("2=http:", "http:")) except ValueError: pass s.set_property("horizontal_srs", hsrs) s.set_property("vertical_srs", vsrs) s.set_property("localFrame", vector.localFrame) lat = vector.get_by_name("latitude").content.value lon = vector.get_by_name("longitude").content.value z = vector.get_by_name("height").content.value loc = [lon, lat] if z: loc.append(z) s.location = sPoint(*loc) # Sensors for sensor in station.content.get_by_name("sensors").content.field: name = sensor.name uri = sensor.content.get_by_name("sensorID").content.value height = None location_quantity = sensor.content.get_by_name( "height").content if location_quantity.referenceFrame == "#%s_frame" % s.name: # Uses the station as reference frame if location_quantity.value and z: height = z + location_quantity.value horizontal_srs = s.get_property("horizontal_srs") vertical_srs = s.get_property("vertical_srs") else: # Uses its own height if location_quantity.value: height = location_quantity.value horizontal_srs = None vertical_srs = None if hasattr(sensor, 'referenceFrame'): srss = sensor.referenceFrame.split("&") try: horizontal_srs = Crs(srss[0]) except ValueError: pass try: vertical_srs = Crs(srss[-1].replace( "2=http:", "http:")) except ValueError: pass loc = [s.location.x, s.location.y] if height: loc.append(height) location = sPoint(*loc) sensors[name] = { 'station': s.uid, 'name': name, 'uri': uri, 'horizontal_srs': horizontal_srs, 'vertical_srs': vertical_srs, 'location': location, 'columns': [], # Array of Members representing the columns 'values': [] # Array of Points (the actual data) } stations[s.uid] = s # Start building the column structure data_array = record.get_by_name("observationData").content data_record = data_array.elementType.content columns = [] # Data outside of the <field name="sensors"> DataChoice element for f in data_record.field: columns.append(f) # Data inside of DataChoice sensor_data = data_record.get_by_name("sensor") for sendata in sensor_data.content.item: if sendata.content is not None: sensors[sendata.name]['columns'] = [] sensors[sendata.name]['values'] = [] for f in sendata.content.field: # Create a model Member for each column that will be copied and filled with data from each row sensors[sendata.name]['columns'].append(f) decimalSeparator = data_array.encoding.decimalSeparator tokenSeparator = data_array.encoding.tokenSeparator blockSeparator = data_array.encoding.blockSeparator collapseWhiteSpaces = data_array.encoding.collapseWhiteSpaces data_values = data_array.values self.raw_data = copy(data_values) for row in filter(lambda x: x != "", data_values.split(blockSeparator)): pt = None members = [] values = row.split(tokenSeparator) sensor_key = None i = 0 for x in columns: if isinstance( x.content, Time ) and x.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime": pt = Point() pt.time = parser.parse(values[i]) elif isinstance(x.content, DataChoice): sensor_key = values[i] dc_cols = sensors[sensor_key]['columns'] for j, c in enumerate(dc_cols): if isinstance(c.content, AbstractSimpleComponent): m = Member(units=c.content.uom, name=c.name, standard=c.content.definition, value=float(values[i + 1])) members.append(m) elif isinstance( c.content, Time ) and c.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime": pt = Point() pt.time = parser.parse(values[i]) # For each data column i += 1 elif isinstance(x.content, AbstractSimpleComponent): m = Member(units=x.content.uom, name=x.name, standard=x.content.definition, value=float(values[i])) members.append(m) else: print "WHAT AM I" i += 1 pt.members = members pt.location = stations[sensors[sensor_key]['station']].location sensors[sensor_key]['values'].append(pt) for k, v in stations.iteritems(): for sk, sv in sensors.iteritems(): # Match on station uid if sv['station'] == k: v.elements = self._merge_points(v.elements or [], sv['values']) if len(stations) > 1: self.feature = StationCollection(elements=stations) elif len(stations) == 1: self.feature = next(stations.itervalues()) else: print "No stations found!"
def __init__(self, awc_list): for awc_data in awc_list: try: self._root = etree.fromstring(awc_data.encode()) except Exception as e: warnings.warn( "Could not read from {!r}, got {}".format(awc_data, e) ) continue """Code to get station iterator goes here.""" stations = [] station_lookup = [] times = [] for metar in self._root.iter("METAR"): uid = metar.find("station_id").text if uid not in station_lookup: s = Station() s.uid = uid vertical = metar.find("elevation_m").text if vertical is None: vertical = 0 s.location = sPoint( float(metar.find("latitude").text), float(metar.find("longitude").text), float(vertical), ) s.set_property("metar_type", metar.find("metar_type").text) s.set_property("horizontal_crs", "GCS") s.set_property("vertical_units", "m") s.set_property("vertical_crs", "AGL") stations.append(s) station_lookup.append(s.uid) times.append({}) else: s = stations[station_lookup.index(uid)] variables = [ "elevation_m", "raw_text", "temp_c", "dewpoint_c", "wind_dir_degrees", "wind_speed_kt", "visibility_statute_mi", "altim_in_hg", "wx_string", "sky_condition", "flight_category", ] for variable in variables: time_string = metar.find("observation_time").text dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ") if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.utc) dt = dt.astimezone(pytz.utc) if dt not in list( times[station_lookup.index(s.uid)].keys() ): times[station_lookup.index(s.uid)][dt] = [] if metar.find(variable): if variable in {"raw_text", "flight_category", "wx_string"}: times[station_lookup.index(s.uid)][dt].append( Member( value=metar.find(variable).text, unit=None, name=variable, description=variable, standard=None, ) ) elif variable == "sky_condition": sky_condition = [] for cond in metar.findall(variable): cover = cond.attrib["sky_cover"] try: alt = cond.attrib["cloud_base_ft_agl"] except Exception: alt = None sky_condition.append( { "sky_cover": cover, "cloud_base_ft_agl": alt, } ) times[station_lookup.index(s.uid)][dt].append( Member( value=sky_condition, unit="ft_above_ground_level", name=variable, description=variable, standard=None, ) ) else: times[station_lookup.index(s.uid)][dt].append( Member( value=float(metar.find(variable).text), unit=variable.split("_")[-1], name=variable, description=variable, standard=None, ) ) for time_dict, station in zip(times, stations): for dts, members in time_dict.items(): p = Point() p.time = dts p.location = station.location p.members = members station.add_element(p) self.feature = StationCollection(elements=stations)