Exemple #1
0
    def __init__(self, waterml_data):
        
        if isinstance(waterml_data, str) or isinstance(waterml_data, unicode):
            try:
                self._root = etree.fromstring(str(waterml_data))
            except ValueError:
                # Strip out the XML header due to UTF8 encoding declaration
                self._root = etree.fromstring(waterml_data[56:])
        else:
            self._root = waterml_data

        response = WaterML_1_1(self._root).response

        stations = []
        station_lookup = []

        for timeseries in response.time_series:
            station_code = sorted(timeseries.source_info.site_codes)[0]
            # Create station if we have not seen it
            if station_code not in station_lookup:
                s = Station()
                s.uid = station_code

                info = timeseries.source_info
                s.name = info.site_name
                s.set_property("station_type", info.site_types)
                s.set_property("huc", info.site_properties.get("hucCd"))
                s.set_property("county", info.site_properties.get("countyCd"))
                s.set_property("state", info.site_properties.get("stateCd"))

                # Now set the station's location
                vertical = info.elevation
                if vertical is None:
                    vertical = 0

                try:
                    location = info.location.geo_coords[0]
                    srs = info.location.srs[0]
                except:
                    print "Could not find a location for %s... skipping station" % s.uid
                    continue

                s.location = sPoint(float(location[0]), float(location[1]), vertical)
                s.set_property("horizontal_crs", srs)
                s.set_property("vertical_units", "m")
                s.set_property("vertical_crs", info.vertical_datum)
                s.set_property("location_description", info.location.notes)

                stations.append(s)
                station_lookup.append(s.uid)


            times = {}
            variable = timeseries.variable
            for variable_timeseries in timeseries.values:
                for r in variable_timeseries:
                    dt = r.date_time
                    if dt.tzinfo is None:
                        dt = dt.replace(tzinfo=pytz.utc)
                    dt = dt.astimezone(pytz.utc)
                    
                    if dt not in times.keys():
                        times[dt] = []

                    times[dt].append(Member(value=r.value, unit=variable.unit.code, name=variable.variable_name, description=variable.variable_description, standard=variable.variable_code))

            station = stations[station_lookup.index(station_code)]
            for dts,members in times.iteritems():
                p = Point()
                p.time = dts
                p.location = station.location
                p.members = members
                station.add_element(p)
        
        self.feature = StationCollection(elements=stations)
Exemple #2
0
    def __init__(self, element):

        record = DataRecord(element)

        # Top level org structure
        stations_field = record.get_by_name("stations")
        stations = {}
        sensors = {}
        for station in stations_field.content.field:
            s = Station()
            s.name  = station.name
            s.uid   = station.content.get_by_name("stationID").content.value

            # Location
            vector  = station.content.get_by_name("platformLocation").content
            srss = vector.referenceFrame.split("&")
            hsrs = None
            try:
                hsrs = Crs(srss[0])
            except ValueError:
                pass

            vsrs = None
            try:
                vsrs = Crs(srss[-1].replace("2=http:", "http:"))
            except ValueError:
                pass

            s.set_property("horizontal_srs", hsrs)
            s.set_property("vertical_srs",   vsrs)
            s.set_property("localFrame",     vector.localFrame)

            lat = vector.get_by_name("latitude").content.value
            lon = vector.get_by_name("longitude").content.value
            z   = vector.get_by_name("height").content.value

            loc = [lon, lat]
            if z:
                loc.append(z)

            s.location = sPoint(*loc)

            # Sensors
            for sensor in station.content.get_by_name("sensors").content.field:
                name        = sensor.name
                uri         = sensor.content.get_by_name("sensorID").content.value
                height      = None
                location_quantity = sensor.content.get_by_name("height").content
                if location_quantity.referenceFrame == "#%s_frame" % s.name:
                    # Uses the station as reference frame
                    if location_quantity.value and z:
                        height      = z + location_quantity.value
                    horizontal_srs  = s.get_property("horizontal_srs")
                    vertical_srs    = s.get_property("vertical_srs")
                else:
                    # Uses its own height
                    if location_quantity.value:
                        height      = location_quantity.value
                    horizontal_srs  = None
                    vertical_srs    = None
                    if hasattr(sensor, 'referenceFrame'):
                        srss            = sensor.referenceFrame.split("&")
                        try:
                            horizontal_srs = Crs(srss[0])
                        except ValueError:
                            pass
                        try:
                            vertical_srs = Crs(srss[-1].replace("2=http:", "http:"))
                        except ValueError:
                            pass

                loc = [s.location.x, s.location.y]
                if height:
                    loc.append(height)

                location            = sPoint(*loc)

                sensors[name] = {   'station'           : s.uid,
                                    'name'              : name,
                                    'uri'               : uri,
                                    'horizontal_srs'    : horizontal_srs,
                                    'vertical_srs'      : vertical_srs,
                                    'location'          : location,
                                    'columns'           : [], # Array of Members representing the columns
                                    'values'            : []  # Array of Points (the actual data)
                                }

            stations[s.uid] = s


        # Start building the column structure
        data_array = record.get_by_name("observationData").content
        data_record = data_array.elementType.content

        columns = []
        # Data outside of the <field name="sensors"> DataChoice element
        for f in data_record.field:
            columns.append(f)

        # Data inside of DataChoice
        sensor_data = data_record.get_by_name("sensor")
        for sendata in sensor_data.content.item:
            if sendata.content is not None:
                sensors[sendata.name]['columns'] = []
                sensors[sendata.name]['values'] = []
                for f in sendata.content.field:
                    # Create a model Member for each column that will be copied and filled with data from each row
                    sensors[sendata.name]['columns'].append(f)

        decimalSeparator    = data_array.encoding.decimalSeparator
        tokenSeparator      = data_array.encoding.tokenSeparator
        blockSeparator      = data_array.encoding.blockSeparator
        collapseWhiteSpaces = data_array.encoding.collapseWhiteSpaces

        data_values = data_array.values
        self.raw_data = copy(data_values)

        for row in filter(lambda x: x != "", data_values.split(blockSeparator)):

            pt = None
            members     = []
            values      = row.split(tokenSeparator)
            sensor_key  = None
            i           = 0

            for x in columns:

                if isinstance(x.content, Time) and x.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
                    pt      = Point()
                    pt.time = parser.parse(values[i])

                elif isinstance(x.content, DataChoice):
                    sensor_key = values[i]
                    dc_cols = sensors[sensor_key]['columns']

                    for j,c in enumerate(dc_cols):
                        if isinstance(c.content, AbstractSimpleComponent):
                            m = Member( units=c.content.uom,
                                        name=c.name,
                                        standard=c.content.definition,
                                        value=float(values[i+1]))
                            members.append(m)

                        elif isinstance(c.content, Time) and c.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
                            pt      = Point()
                            pt.time = parser.parse(v)

                        # For each data column
                        i += 1
                
                elif isinstance(x.content, AbstractSimpleComponent):
                    m = Member( units=x.content.uom,
                                name=x.name,
                                standard=x.content.definition,
                                value=float(values[i]))
                    members.append(m)

                else:
                    print "WHAT AM I"

                i += 1

            pt.members = members
            pt.location = stations[sensors[sensor_key]['station']].location
            sensors[sensor_key]['values'].append(pt)

        for k,v in stations.iteritems():
            for sk, sv in sensors.iteritems():
                # Match on station uid
                if sv['station'] == k:
                    v.elements = self._merge_points(v.elements or [], sv['values'])

        if len(stations) > 1:
            self.feature = StationCollection(elements=stations)
        elif len(stations) == 1:
            self.feature = next(stations.itervalues())
        else:
            print "No stations found!"
Exemple #3
0
    def __init__(self, waterml_data):

        if isinstance(waterml_data, str) or isinstance(waterml_data, unicode):
            try:
                self._root = etree.fromstring(str(waterml_data))
            except ValueError:
                # Strip out the XML header due to UTF8 encoding declaration
                self._root = etree.fromstring(waterml_data[56:])
        else:
            self._root = waterml_data

        response = WaterML_1_1(self._root).response

        stations = []
        station_lookup = []

        for timeseries in response.time_series:
            station_code = sorted(timeseries.source_info.site_codes)[0]
            # Create station if we have not seen it
            if station_code not in station_lookup:
                s = Station()
                s.uid = station_code

                info = timeseries.source_info
                s.name = info.site_name
                s.set_property("station_type", info.site_types)
                s.set_property("huc", info.site_properties.get("hucCd"))
                s.set_property("county", info.site_properties.get("countyCd"))
                s.set_property("state", info.site_properties.get("stateCd"))

                # Now set the station's location
                vertical = info.elevation
                if vertical is None:
                    vertical = 0

                try:
                    location = info.location.geo_coords[0]
                    srs = info.location.srs[0]
                except:
                    print "Could not find a location for %s... skipping station" % s.uid
                    continue

                s.location = sPoint(float(location[0]), float(location[1]),
                                    vertical)
                s.set_property("horizontal_crs", srs)
                s.set_property("vertical_units", "m")
                s.set_property("vertical_crs", info.vertical_datum)
                s.set_property("location_description", info.location.notes)

                stations.append(s)
                station_lookup.append(s.uid)

            times = {}
            variable = timeseries.variable
            for variable_timeseries in timeseries.values:
                for r in variable_timeseries:
                    dt = r.date_time
                    if dt.tzinfo is None:
                        dt = dt.replace(tzinfo=pytz.utc)
                    dt = dt.astimezone(pytz.utc)

                    if dt not in times.keys():
                        times[dt] = []

                    times[dt].append(
                        Member(value=r.value,
                               unit=variable.unit.code,
                               name=variable.variable_name,
                               description=variable.variable_description,
                               standard=variable.variable_code))

            station = stations[station_lookup.index(station_code)]
            for dts, members in times.iteritems():
                p = Point()
                p.time = dts
                p.location = station.location
                p.members = members
                station.add_element(p)

        self.feature = StationCollection(elements=stations)
Exemple #4
0
    def __init__(self, awc_list):
        for awc_data in awc_list:
            self._root = etree.fromstring(awc_data.encode())
            '''Code to get station iterator goes here.'''
            stations = []
            station_lookup = []
            times = []

            for metar in self._root.iter('METAR'):
                uid = metar.find("station_id").text
                if uid not in station_lookup:
                    s = Station()
                    s.uid = uid
                    vertical = metar.find("elevation_m").text
                    if vertical is None:
                        vertical = 0
                    s.location = sPoint(float(metar.find("latitude").text),
                                        float(metar.find("longitude").text),
                                        float(vertical))
                    s.set_property("metar_type", metar.find("metar_type").text)
                    s.set_property("horizontal_crs", "GCS")
                    s.set_property("vertical_units", "m")
                    s.set_property("vertical_crs", "AGL")

                    stations.append(s)
                    station_lookup.append(s.uid)
                    times.append({})
                else:
                    s = stations[station_lookup.index(uid)]

                variables = [
                    "elevation_m", "raw_text", "temp_c", "dewpoint_c",
                    "wind_dir_degrees", "wind_speed_kt",
                    "visibility_statute_mi", "altim_in_hg", "wx_string",
                    "sky_condition", "flight_category"
                ]
                for variable in variables:
                    time_string = metar.find("observation_time").text
                    dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ")
                    if dt.tzinfo is None:
                        dt = dt.replace(tzinfo=pytz.utc)
                    dt = dt.astimezone(pytz.utc)

                    if dt not in list(times[station_lookup.index(
                            s.uid)].keys()):
                        times[station_lookup.index(s.uid)][dt] = []

                    if metar.find(variable) != None:
                        if variable in set(
                            ["raw_text", "flight_category", "wx_string"]):
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(value=metar.find(variable).text,
                                       unit=None,
                                       name=variable,
                                       description=variable,
                                       standard=None))
                        elif variable == "sky_condition":
                            sky_condition = []
                            for cond in metar.findall(variable):
                                cover = cond.attrib["sky_cover"]
                                try:
                                    alt = cond.attrib["cloud_base_ft_agl"]
                                except:
                                    alt = None
                                sky_condition.append({
                                    "sky_cover": cover,
                                    "cloud_base_ft_agl": alt
                                })
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(value=sky_condition,
                                       unit="ft_above_ground_level",
                                       name=variable,
                                       description=variable,
                                       standard=None))
                        else:
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(value=float(metar.find(variable).text),
                                       unit=variable.split("_")[-1],
                                       name=variable,
                                       description=variable,
                                       standard=None))

            for time_dict, station in zip(times, stations):
                for dts, members in time_dict.items():
                    p = Point()
                    p.time = dts
                    p.location = station.location
                    p.members = members
                    station.add_element(p)

        self.feature = StationCollection(elements=stations)
Exemple #5
0
    def __init__(self, awc_list):
        for awc_data in awc_list:
            if isinstance(awc_data, str) or isinstance(awc_data, unicode):
                try:
                    self._root = etree.fromstring(str(awc_data))
                except ValueError:
                    # Strip out the XML header due to UTF8 encoding declaration
                    self._root = etree.fromstring(awc_data[56:])
            else:
                raise ValueError("Cannot parse response into ElementTree xml object")

            '''Code to get station iterator goes here
            '''
            stations = []
            station_lookup = []
            times = []

            for metar in self._root.iter('METAR'):
                uid = metar.find("station_id").text
                if uid not in station_lookup:
                    s = Station()
                    s.uid = uid
                    vertical = metar.find("elevation_m").text
                    if vertical is None:
                        vertical = 0
                    s.location = sPoint(float(metar.find("latitude").text), float(metar.find("longitude").text), float(vertical))
                    s.set_property("metar_type", metar.find("metar_type").text)
                    s.set_property("horizontal_crs", "GCS")
                    s.set_property("vertical_units", "m")
                    s.set_property("vertical_crs", "AGL")

                    stations.append(s)
                    station_lookup.append(s.uid)
                    times.append({})
                else:
                    s = stations[station_lookup.index(uid)]

                variables = ["elevation_m", "raw_text", "temp_c", "dewpoint_c", "wind_dir_degrees", "wind_speed_kt", "visibility_statute_mi", "altim_in_hg", "wx_string", "sky_condition", "flight_category"]
                for variable in variables:
                    time_string = metar.find("observation_time").text
                    dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ")
                    if dt.tzinfo is None:
                        dt = dt.replace(tzinfo=pytz.utc)
                    dt = dt.astimezone(pytz.utc)

                    if dt not in times[station_lookup.index(s.uid)].keys():
                        times[station_lookup.index(s.uid)][dt] = []

                    if metar.find(variable) != None:
                        if variable in set(["raw_text", "flight_category", "wx_string"]):
                            times[station_lookup.index(s.uid)][dt].append(Member(value=metar.find(variable).text, unit=None, name=variable, description=variable, standard=None))
                        elif variable == "sky_condition":
                            sky_condition = []
                            for cond in metar.findall(variable):
                                cover = cond.attrib["sky_cover"]
                                try:
                                    alt = cond.attrib["cloud_base_ft_agl"]
                                except:
                                    alt = None
                                sky_condition.append({"sky_cover" : cover, "cloud_base_ft_agl" : alt})
                            times[station_lookup.index(s.uid)][dt].append(Member(value=sky_condition, unit="ft_above_ground_level", name=variable, description=variable, standard=None))
                        else:
                            times[station_lookup.index(s.uid)][dt].append(Member(value=float(metar.find(variable).text), unit=variable.split("_")[-1], name=variable, description=variable, standard=None))

            for time_dict, station in zip(times, stations):
                for dts, members in time_dict.iteritems():
                    p = Point()
                    p.time = dts
                    p.location = station.location
                    p.members = members
                    station.add_element(p)

        self.feature = StationCollection(elements=stations)
    def __init__(self, element):

        record = DataRecord(element)

        # Top level org structure
        stations_field = record.get_by_name("stations")
        stations = {}
        sensors = {}
        for station in stations_field.content.field:
            s = Station()
            s.name = station.name
            s.uid = station.content.get_by_name("stationID").content.value

            # Location
            vector = station.content.get_by_name("platformLocation").content
            srss = vector.referenceFrame.split("&amp;")
            hsrs = None
            try:
                hsrs = Crs(srss[0])
            except ValueError:
                pass

            vsrs = None
            try:
                vsrs = Crs(srss[-1].replace("2=http:", "http:"))
            except ValueError:
                pass

            s.set_property("horizontal_srs", hsrs)
            s.set_property("vertical_srs", vsrs)
            s.set_property("localFrame", vector.localFrame)

            lat = vector.get_by_name("latitude").content.value
            lon = vector.get_by_name("longitude").content.value
            z = vector.get_by_name("height").content.value

            loc = [lon, lat]
            if z:
                loc.append(z)

            s.location = sPoint(*loc)

            # Sensors
            for sensor in station.content.get_by_name("sensors").content.field:
                name = sensor.name
                uri = sensor.content.get_by_name("sensorID").content.value
                height = None
                location_quantity = sensor.content.get_by_name(
                    "height").content
                if location_quantity.referenceFrame == "#%s_frame" % s.name:
                    # Uses the station as reference frame
                    if location_quantity.value and z:
                        height = z + location_quantity.value
                    horizontal_srs = s.get_property("horizontal_srs")
                    vertical_srs = s.get_property("vertical_srs")
                else:
                    # Uses its own height
                    if location_quantity.value:
                        height = location_quantity.value
                    horizontal_srs = None
                    vertical_srs = None
                    if hasattr(sensor, 'referenceFrame'):
                        srss = sensor.referenceFrame.split("&amp;")
                        try:
                            horizontal_srs = Crs(srss[0])
                        except ValueError:
                            pass
                        try:
                            vertical_srs = Crs(srss[-1].replace(
                                "2=http:", "http:"))
                        except ValueError:
                            pass

                loc = [s.location.x, s.location.y]
                if height:
                    loc.append(height)

                location = sPoint(*loc)

                sensors[name] = {
                    'station': s.uid,
                    'name': name,
                    'uri': uri,
                    'horizontal_srs': horizontal_srs,
                    'vertical_srs': vertical_srs,
                    'location': location,
                    'columns': [],  # Array of Members representing the columns
                    'values': []  # Array of Points (the actual data)
                }

            stations[s.uid] = s

        # Start building the column structure
        data_array = record.get_by_name("observationData").content
        data_record = data_array.elementType.content

        columns = []
        # Data outside of the <field name="sensors"> DataChoice element
        for f in data_record.field:
            columns.append(f)

        # Data inside of DataChoice
        sensor_data = data_record.get_by_name("sensor")
        for sendata in sensor_data.content.item:
            if sendata.content is not None:
                sensors[sendata.name]['columns'] = []
                sensors[sendata.name]['values'] = []
                for f in sendata.content.field:
                    # Create a model Member for each column that will be copied and filled with data from each row
                    sensors[sendata.name]['columns'].append(f)

        decimalSeparator = data_array.encoding.decimalSeparator
        tokenSeparator = data_array.encoding.tokenSeparator
        blockSeparator = data_array.encoding.blockSeparator
        collapseWhiteSpaces = data_array.encoding.collapseWhiteSpaces

        data_values = data_array.values
        self.raw_data = copy(data_values)

        for row in filter(lambda x: x != "",
                          data_values.split(blockSeparator)):

            pt = None
            members = []
            values = row.split(tokenSeparator)
            sensor_key = None
            i = 0

            for x in columns:

                if isinstance(
                        x.content, Time
                ) and x.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
                    pt = Point()
                    pt.time = parser.parse(values[i])

                elif isinstance(x.content, DataChoice):
                    sensor_key = values[i]
                    dc_cols = sensors[sensor_key]['columns']

                    for j, c in enumerate(dc_cols):
                        if isinstance(c.content, AbstractSimpleComponent):
                            m = Member(units=c.content.uom,
                                       name=c.name,
                                       standard=c.content.definition,
                                       value=float(values[i + 1]))
                            members.append(m)

                        elif isinstance(
                                c.content, Time
                        ) and c.content.definition == "http://www.opengis.net/def/property/OGC/0/SamplingTime":
                            pt = Point()
                            pt.time = parser.parse(values[i])

                        # For each data column
                        i += 1

                elif isinstance(x.content, AbstractSimpleComponent):
                    m = Member(units=x.content.uom,
                               name=x.name,
                               standard=x.content.definition,
                               value=float(values[i]))
                    members.append(m)

                else:
                    print "WHAT AM I"

                i += 1

            pt.members = members
            pt.location = stations[sensors[sensor_key]['station']].location
            sensors[sensor_key]['values'].append(pt)

        for k, v in stations.iteritems():
            for sk, sv in sensors.iteritems():
                # Match on station uid
                if sv['station'] == k:
                    v.elements = self._merge_points(v.elements or [],
                                                    sv['values'])

        if len(stations) > 1:
            self.feature = StationCollection(elements=stations)
        elif len(stations) == 1:
            self.feature = next(stations.itervalues())
        else:
            print "No stations found!"
Exemple #7
0
    def __init__(self, awc_list):
        for awc_data in awc_list:
            try:
                self._root = etree.fromstring(awc_data.encode())
            except Exception as e:
                warnings.warn(
                    "Could not read from {!r}, got {}".format(awc_data, e)
                )
                continue

            """Code to get station iterator goes here."""
            stations = []
            station_lookup = []
            times = []

            for metar in self._root.iter("METAR"):
                uid = metar.find("station_id").text
                if uid not in station_lookup:
                    s = Station()
                    s.uid = uid
                    vertical = metar.find("elevation_m").text
                    if vertical is None:
                        vertical = 0
                    s.location = sPoint(
                        float(metar.find("latitude").text),
                        float(metar.find("longitude").text),
                        float(vertical),
                    )
                    s.set_property("metar_type", metar.find("metar_type").text)
                    s.set_property("horizontal_crs", "GCS")
                    s.set_property("vertical_units", "m")
                    s.set_property("vertical_crs", "AGL")

                    stations.append(s)
                    station_lookup.append(s.uid)
                    times.append({})
                else:
                    s = stations[station_lookup.index(uid)]

                variables = [
                    "elevation_m",
                    "raw_text",
                    "temp_c",
                    "dewpoint_c",
                    "wind_dir_degrees",
                    "wind_speed_kt",
                    "visibility_statute_mi",
                    "altim_in_hg",
                    "wx_string",
                    "sky_condition",
                    "flight_category",
                ]
                for variable in variables:
                    time_string = metar.find("observation_time").text
                    dt = datetime.strptime(time_string, "%Y-%m-%dT%H:%M:%SZ")
                    if dt.tzinfo is None:
                        dt = dt.replace(tzinfo=pytz.utc)
                    dt = dt.astimezone(pytz.utc)

                    if dt not in list(
                        times[station_lookup.index(s.uid)].keys()
                    ):
                        times[station_lookup.index(s.uid)][dt] = []

                    if metar.find(variable):
                        if variable in {"raw_text", "flight_category", "wx_string"}:
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(
                                    value=metar.find(variable).text,
                                    unit=None,
                                    name=variable,
                                    description=variable,
                                    standard=None,
                                )
                            )
                        elif variable == "sky_condition":
                            sky_condition = []
                            for cond in metar.findall(variable):
                                cover = cond.attrib["sky_cover"]
                                try:
                                    alt = cond.attrib["cloud_base_ft_agl"]
                                except Exception:
                                    alt = None
                                sky_condition.append(
                                    {
                                        "sky_cover": cover,
                                        "cloud_base_ft_agl": alt,
                                    }
                                )
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(
                                    value=sky_condition,
                                    unit="ft_above_ground_level",
                                    name=variable,
                                    description=variable,
                                    standard=None,
                                )
                            )
                        else:
                            times[station_lookup.index(s.uid)][dt].append(
                                Member(
                                    value=float(metar.find(variable).text),
                                    unit=variable.split("_")[-1],
                                    name=variable,
                                    description=variable,
                                    standard=None,
                                )
                            )

            for time_dict, station in zip(times, stations):
                for dts, members in time_dict.items():
                    p = Point()
                    p.time = dts
                    p.location = station.location
                    p.members = members
                    station.add_element(p)

        self.feature = StationCollection(elements=stations)