예제 #1
0
class IoosSweSosTest(unittest.TestCase):
    def setUp(self):
        # we have to pick SOS to test against, NANOOS it is:
        self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
        # self.c.features = [urn.name for urn in self.c.server.offerings]

    @pytest.mark.xfail
    def test_server_id(self):
        assert (
            self.c.server.identification.title
            == "NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server"
        )
        assert self.c.server.identification.service == "OGC:SOS"
        assert self.c.server.identification.version == "1.0.0"
        assert self.c.server.identification.fees == "NONE"
        assert self.c.server.identification.accessconstraints == "NONE"

    @pytest.mark.xfail
    def test_metdata(self):
        self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
        response = self.c.metadata(
            output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
        )
        assert isinstance(response, list)
        assert isinstance(response[0], SensorML)

    @pytest.mark.xfail
    def test_metadata_plus_exceptions(self):
        self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
        response, failures = self.c.metadata_plus_exceptions(
            output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
        )
        assert isinstance(response, dict)
        assert isinstance(failures, dict)
        assert isinstance(response[self.c.features[0]], SensorML)
예제 #2
0
class IoosSweSosTest(unittest.TestCase):
    def setUp(self):
        # we have to pick SOS to test against, NANOOS it is:
        self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
        # self.c.features = [urn.name for urn in self.c.server.offerings]

    @pytest.mark.xfail
    def test_server_id(self):
        assert (
            self.c.server.identification.title ==
            "NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server"
        )
        assert self.c.server.identification.service == "OGC:SOS"
        assert self.c.server.identification.version == "1.0.0"
        assert self.c.server.identification.fees == "NONE"
        assert self.c.server.identification.accessconstraints == "NONE"

    @pytest.mark.xfail
    def test_metdata(self):
        self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
        response = self.c.metadata(
            output_format=
            'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"')
        assert isinstance(response, list)
        assert isinstance(response[0], SensorML)

    @pytest.mark.xfail
    def test_metadata_plus_exceptions(self):
        self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
        response, failures = self.c.metadata_plus_exceptions(
            output_format=
            'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"')
        assert isinstance(response, dict)
        assert isinstance(failures, dict)
        assert isinstance(response[self.c.features[0]], SensorML)
예제 #3
0
 def setUp(self):
     # we have to pick SOS to test against, NANOOS it is:
     self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
예제 #4
0
                    else:
                        v_frame = pd.concat([v_frame, new_frame])
                        v_frame = v_frame.drop_duplicates()
                except ExceptionReport as e:
                    print "  [-] Error obtaining {!s} from {!s} - Message from server: {!s}".format(v, sos, e)
                    continue
                finally:
                    new_start = new_end

            if v_frame is not None:
                sos_dfs.append(v_frame)
    
    else:
        # Use the generic IOOS SWE collector
        try:
            collector = IoosSweSos(sos)
        except BaseException as e:
            print "[-]  Could not process {!s}.  Reason: {!s}".format(sos, e)
            continue
            
        for v in variables_to_query:
            collector.filter(variables=[v])
            collector.filter(bbox=bounding_box)
            collector.filter(start=start_date)
            collector.filter(end=end_date)
            collector.filter(variables=[v])
            try:
                data = collector.collect()
                print data
            except BaseException as e:
                print "[-]  Could not process {!s}.  Reason: {!s}".format(sos, e)
예제 #5
0
 def setUp(self):
     # we have to pick SOS to test against, NANOOS it is:
     self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
예제 #6
0
def main():

  #The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature...
  obsList = ['water_temperature','salinity']
  """
  Create a data collection object.
  Contructor parameters are:
    url - THe SWE endpoint we're interested in
    version - Optional default is '1.0.0' The SWE version the endpoint.
    xml - Optional default is None - The XML response from a GetCapabilities query of the server.
  """
  dataCollector = IoosSweSos(url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc', xml=None)

  #Loop through the offerings from the server.
  #Offerings are the stations/platforms/data device that the server house data from.
  offerings = dataCollector.server.offerings
  for offer in offerings:
    #We skip over the 'all' offering.
    #all "*network*" offerings represent aggregations of stations;
    # unlike all other offerings we've dealt with basically correspond to one station.
    if(offer.name.split(':')[-1] != 'all'):

      print "Offering: %s %s Obs: %s"\
            % (offer.name, offer.description, offer.observed_properties)


      #Check to see if the station offered has the observation we're interested in.
      #Loop through the observed_properties attribute and compare.
      obsFilterList = []
      #We want to query just the observations n the obsOfInterest list.
      #The observer_properties should be coded with an MMI href, however due
      #to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds
      #a URN pattern. Regardless, an end user should just need to know the observation
      #name and not the MMI url, so this tries to do a simple
      #string find of our desired observation in the observed_property.
      for obsOfInterest in obsList:
        for observedProperty in offer.observed_properties:
          if(observedProperty.find(obsOfInterest) != -1):
            #Because of the ncSOS bug 107, we don't add the observedProperty. When
            #that bug gets addressed, we would add the observedProperty.
            #obsFilterList.append(observedProperty)
            obsFilterList.append(obsOfInterest)
            break

      if(len(obsFilterList)):
        #We create a filter based on the observations of interest,
        # and a time frame of the 12 hours of data from yesterday.
        #There is an ncSOS bug that affects getting the current data
        #Issue: https://github.com/asascience-open/ncSOS/issues/112
        """
        Filter function params:
        bbox - Bounding box, not implemented for SOS calls currently.
        start  - start date for records
        end - end date for records
        features - I assume it's a list of the stations of interest. Not sure why this is here
          and then the offerings parameter in the collect() function.
        variables - Poorly named parameter to house the observed_properties
        """
        try:
          dataCollector.filter(
                                 variables=obsFilterList,
                                 start=datetime.utcnow() - timedelta(hours=24),
                                 end=datetime.utcnow()  - timedelta(hours=12))
          """
          collect() params:
          offering - Station list

          responseFormat - The desired return format for the SOS call.
          eventTime - If the start/end times are not provided in the filter(), this param
            can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date.

          returns a list of OmObservation objects. OmObservation object has no member functions.
          """
          response = dataCollector.collect(offerings=[offer.name])
          for obsRec in response:
            stationRec = obsRec.feature
            print "Station: %s Location: %s" % (stationRec.name, stationRec.get_location())

            #The elements are a list of the observed_properties returned wrapped in a Point object.
            for obsProp in stationRec.get_elements():
              print "Observation Date/Time: %s" % (obsProp.get_time())
              #print "Member names: %s" % (obsProp.get_member_names())
              for member in obsProp.get_members():
                for key,value in member.iteritems():
                  print "%s = %s" % (key, value)

        except ows.ExceptionReport,e:
          traceback.print_exc(e)

        except Exception,e:
          traceback.print_exc(e)
예제 #7
0
    def get_stations_df(self, sos_url, station_urns_sel=None):
        """ Returns a Pandas Dataframe
        """
        # oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec,
        # second is to accommodate NDBC).  More info here:
        # http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe
        oFrmts = [
            'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
            'text/xml;subtype="sensorML/1.0.1"'
        ]
        params = {
            'service': 'SOS',
            'request': 'GetCapabilities',
            'acceptVersions': '1.0.0'
        }
        sos_url_params = sos_url + '?' + urlencode(params)
        # sos_url_params_quoted = quote(sos_url_params,"/=:")
        # sos_url_params_unquoted = unquote(sos_url_params)

        try:
            sosgc = SensorObservationService(sos_url_params)
        except (ConnectionError, ReadTimeout) as e:
            self.log.write(
                u"\nError: unable to connect to SOS service: {url} due to HTTP connection error."
                .format(url=sos_url_params))
            self.log.write(
                u"\nHTTP connection error: {err}.".format(err=str(e)))
            sys.exit(
                "\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
                .format(url=sos_url_params, err=str(e)))

        # vars to store returns from sos_collector.metadata_plus_exceptions function:
        sml_recs = {}
        sml_errors = {}
        describe_sensor_url = {}

        # leverage Pyoos Collector to query for all available stations and obtain SensorML
        # (if station subset not passed in --stations param)
        if station_urns_sel is not None:
            station_urns = station_urns_sel
        else:
            sos_collector = IoosSweSos(sos_url)
            station_urns = [
                urn.name for urn in sos_collector.server.offerings
                if 'network' not in urn.name.split(':')
            ]
            sos_collector.features = station_urns

            # write out stations in SOS that will be handled:
            if self.verbose:
                self.log.write(u"\nStations to process for SOS: {sos}".format(
                    sos=sos_url_params))
                print("Stations to process for SOS: {sos}".format(
                    sos=sos_url_params))
                for feature in sos_collector.features:
                    self.log.write(u"\n - {feature}".format(feature=feature))
                    print(" - {feature}".format(feature=feature))

            # iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC):
            # for fmt in reversed(oFrmts):
            for fmt in oFrmts:
                try:
                    sml_recs, sml_errors = sos_collector.metadata_plus_exceptions(
                        output_format=fmt, timeout=200)
                    # if no valid SensorML docs returned, try next oFrmt:
                    if not sml_recs:
                        continue
                    else:
                        # assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to
                        # create DescribeSensor URLs for failures to record in logs):
                        for station in sos_collector.features:
                            describe_sensor_url[
                                station] = self.generate_describe_sensor_url(
                                    sosgc, procedure=station, oFrmt=fmt)

                        # report on errors returned from metadata_plus_exceptions:
                        if sml_errors:
                            if self.verbose:
                                for station, msg in iteritems(sml_errors):
                                    self.log.write(
                                        u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
                                        .format(station=station, msg=msg))
                                    print(
                                        "SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
                                        .format(station=station, msg=msg))
                        else:
                            self.log.write(
                                u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}"
                                .format(sos=sos_url_params))
                            print(
                                "Success, no errors returned from DescribeSensor requests in service: {sos}"
                                .format(sos=sos_url_params))
                    break
                # ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt:
                except ServiceException as e:
                    continue

        station_recs = []
        failures = []
        # generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings:
        for station_idx, station_urn in enumerate(station_urns):
            if station_urns_sel is not None:
                # iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS):
                for fmt in oFrmts:
                    try:
                        describe_sensor_url[
                            station_urn] = self.generate_describe_sensor_url(
                                sosgc, procedure=station_urn, oFrmt=fmt)
                        sml_str = sosgc.describe_sensor(procedure=station_urn,
                                                        outputFormat=fmt,
                                                        timeout=200)
                        break
                    except ServiceException as e:
                        sml_errors[station_urn] = str(e)
                        continue
                sml = SensorML(sml_str)

            else:
                # process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting):
                try:
                    sml = sml_recs[station_urn]
                except KeyError:
                    self.log.write(
                        u"\n\nStation: {station} failed (no SensorML in sml_recs dict).  URL: {ds}"
                        .format(station=station_urn,
                                ds=describe_sensor_url[station_urn].replace(
                                    "&", "&")))
                    print(
                        "Station: {station} failed (no SensorML in sml_recs dict).  URL: {ds}"
                        .format(station=station_urn,
                                ds=describe_sensor_url[station_urn].replace(
                                    "&", "&")))
                    failures.append(station_urn)
                    continue

            if self.sos_type.lower() == 'ndbc':
                # later: add an error check
                sosgc_station_offering = sosgc.contents[
                    'station-' + station_urn.split(':')[-1]]
            else:
                sosgc_station_offering = None

            try:
                ds = IoosDescribeSensor(sml._root)
            except AttributeError:
                self.log.write(
                    u"\nInvalid SensorML passed to IoosDescribeSensor.  Check DescribeSensor request for : {station}, URL: "
                    .format(station=station,
                            ds=describe_sensor_url[station_urn].replace(
                                "&", "&")))
                print(
                    "Invalid SensorML passed to IoosDescribeSensor.  Check DescribeSensor request for : {station}, URL: "
                    .format(station=station,
                            ds=describe_sensor_url[station_urn].replace(
                                "&", "&")))

            station = OrderedDict()
            # debug:
            if self.verbose:
                self.log.write(u"\n\nProcessing station: {station}".format(
                    station=station_urn))
                print("Processing station: {station}".format(
                    station=station_urn))
                self.log.write("\n" +
                               etree.tostring(sml._root).decode('utf-8'))

            # assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found):
            try:
                pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \
                    if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \
                    else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates')))
                station['lon'] = float(pos.split()[1])
                station['lat'] = float(pos.split()[0])
            except AttributeError as e:
                station['lon'] = None
                station['lat'] = None

            system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
                self.nsp('sml:System'))

            # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
            # errors but there is not enough information in SensorML for alternatives)
            # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
            documents = system_el.findall(
                self.nsp('sml:documentation/sml:DocumentList/sml:member'))
            documents_dct = {}
            for d in documents:
                document = Documentation(d)
                name = testXMLAttribute(d, "name")
                # url = document.documents[0].url
                documents_dct[name] = document

            # obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of  ContactList):
            contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
                if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
                else system_el.findall(self.nsp('sml:contact'))
            contacts_dct = {}
            for c in contacts:
                contact = Contact(c)
                role = contact.role.split('/')[-1]
                contacts_dct[role] = contact

            # verify a 'publisher' Contact exists (template expects one):
            if "publisher" not in contacts_dct.keys():
                self.log.write(
                    u"\n\nStation: {station} skipped.  No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required.  Roles defined: [{roles}]"
                    .format(station=station_urn,
                            roles=", ".join(contacts_dct.keys())))
                print(
                    "Station: {station} skipped.  No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required.  Roles defined: [{roles}]"
                    .format(station=station_urn,
                            roles=", ".join(contacts_dct.keys())))
                failures.append(station_urn)
                continue

            sweQuants = system_el.findall(
                self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
            quant_lst = [
                sweQuant.attrib['definition'] for sweQuant in sweQuants
            ]
            parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]

            # attempt to read beginPosition, if available, otherwise use current date
            # bc ISO requires date value in output location in template:
            beginPosition = testXMLValue(
                system_el.find(
                    self.nsp(
                        'sml:validTime/gml:TimePeriod/gml:beginPosition')))
            try:
                begin_service_date = parser.parse(beginPosition)
            except (AttributeError, TypeError) as e:
                begin_service_date = datetime.now(pytz.utc)

            station['station_urn'] = station_urn
            station['sos_url'] = sos_url_params
            station['describesensor_url'] = describe_sensor_url[station_urn]

            station['shortName'] = ds.shortName
            station['longName'] = ds.longName
            if self.sos_type.lower() == 'ndbc':
                station['wmoID'] = station_urn.split(':')[-1]
            else:
                station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
            station['serverName'] = self.server_name

            # Some capabilities-level metadata:
            station['title'] = sosgc.identification.title
            station['abstract'] = sosgc.identification.abstract
            station['keywords'] = sosgc.identification.keywords
            station['begin_service_date'] = begin_service_date

            # Beware that a station can have >1 classifier of the same type
            # This code does not accommodate that possibility
            station['platformType'] = ds.platformType
            station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
                                                       'classifier', ont)
            station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)

            # store some nested dictionaries in 'station' for appropriate SensorML sources:
            station['contacts_dct'] = contacts_dct
            station['documents_dct'] = documents_dct

            if self.sos_type.lower(
            ) == 'ndbc' and sosgc_station_offering is not None:
                station['starting'] = sosgc_station_offering.begin_position
                station['ending'] = sosgc_station_offering.end_position
            else:
                station['starting'] = ds.starting
                station['ending'] = ds.ending

            if self.sos_type.lower(
            ) == 'ndbc' and sosgc_station_offering is not None:
                station[
                    'variable_uris'] = sosgc_station_offering.observed_properties
                station['variables'] = [
                    var.split('/')[-1]
                    for var in sosgc_station_offering.observed_properties
                ]
                station['parameter_uris'] = ','.join(station['variable_uris'])
                station['parameters'] = ','.join(station['variables'])
            else:
                station['variable_uris'] = ds.variables
                station['variables'] = [
                    var.split('/')[-1] for var in ds.variables
                ]
                station['parameter_uris'] = ','.join(quant_lst)
                station['parameters'] = ','.join(parameter_lst)

            if self.verbose:
                for var in station['variable_uris']:
                    self.log.write(u"\nvariable: {var}".format(var=var))
                    print("variable: {var}".format(var=var))

            # print(sosgc.contents)
            # for id, offering in sosgc.contents.iteritems():
            #    print("sosgc.contents: {item}".format(item=id))

            # parse 'responseFormat' vals from SensorML:
            # response_formats = sosgc.contents[station_urn].response_formats
            response_formats = []
            for id, sosgc.content in sosgc.contents.items():
                if sosgc.content.name == station_urn:
                    response_formats = sosgc.content.response_formats
            # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items()
            #                      if sosgc.content.name == station_urn ]

            # match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to
            # populate 'download_formats' list, that is then used to generate GetObservation requests for the template:
            # (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' )
            download_formats = [
                response_format for response_format in response_formats
                if response_format in self.response_formats
            ]
            station['response_formats'] = response_formats
            station['download_formats'] = download_formats

            if self.verbose:
                for format in response_formats:
                    self.log.write(
                        u"\nresponseFormat: {format}".format(format=format))
                    print("responseFormat: {format}".format(format=format))
                for format in download_formats:
                    self.log.write(
                        u"\ndownloadFormats: {format}".format(format=format))
                    print("downloadFormats: {format}".format(format=format))

            # calculate event_time using self.getobs_req_hours:
            event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}"
            utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else ''
            if station['starting'] is not None and station[
                    'ending'] is not None:
                event_time = event_time_formatstr.format(
                    begin=station['ending'] -
                    timedelta(hours=self.getobs_req_hours),
                    end=station['ending'],
                    utc_code=utc_code)
                if self.verbose:
                    self.log.write(
                        u"\nUsing starting/ending times from SensorML for eventTime"
                    )
                    print(
                        "Using starting/ending times from SensorML for eventTime"
                    )
                    self.log.write(
                        u"\nobservationTimeRange: starting: {start}, ending: {end}"
                        .format(start=station['starting'],
                                end=station['ending']))
                    print(
                        "observationTimeRange: starting: {start}, ending: {end}"
                        .format(start=station['starting'],
                                end=station['ending']))
            else:
                now = datetime.now(pytz.utc)
                then = now - timedelta(hours=self.getobs_req_hours)
                event_time = event_time_formatstr.format(begin=then,
                                                         end=now,
                                                         utc_code=utc_code)
                if self.verbose:
                    self.log.write(
                        u"\nNo 'observationTimeRange' present in SensorML.  Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
                        .format(then=then, now=now))
                    print(
                        "No 'observationTimeRange' present in SensorML.  Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
                        .format(then=then, now=now))

            if self.verbose:
                self.log.write(u"\neventTime: {time}".format(time=event_time))
                print("eventTime: {time}".format(time=event_time))

            # create a dict to store parameters for valid example GetObservation requests for station:
            getobs_req_dct = {}
            # populate a parameters dictionary for download links for each 'observedProperty' type
            # and secondly for each 'responseFormat' per observedProperty:
            getobs_params_base = {
                'service': 'SOS',
                'request': 'GetObservation',
                'version': '1.0.0',
                'offering': station_urn,
                'eventTime': event_time
            }
            for variable in station['variable_uris']:
                getobs_params = getobs_params_base.copy()
                getobs_params['observedProperty'] = variable
                variable = variable.split('/')[-1]
                for format in download_formats:
                    getobs_params['responseFormat'] = format
                    getobs_request_url_encoded = sos_url + '?' + urlencode(
                        getobs_params)
                    getobs_request_url = unquote(getobs_request_url_encoded)
                    getobs_req_dct[variable + '-' + format] = {
                        'variable':
                        variable,
                        'url':
                        getobs_request_url,
                        'format_type':
                        self.RESPONSE_FORMAT_TYPE_MAP.get(format, format),
                        'format_name':
                        self.RESPONSE_FORMAT_NAME_MAP.get(format, format)
                    }
                    if self.verbose:
                        self.log.write(
                            u"\ngetobs_request_url (var: {variable}): {getobs_request_url}"
                            .format(variable=variable.split("/")[-1],
                                    getobs_request_url=getobs_request_url))
                        print(
                            "getobs_request_url (var: {variable}): {getobs_request_url}"
                            .format(variable=variable.split("/")[-1],
                                    getobs_request_url=getobs_request_url))

            # ToDo: finish adding the 'getobs_req_dct' to the output template
            station['getobs_req_dct'] = getobs_req_dct

            station_recs.append(station)

        # extra debug for failed stations in verbose mode:
        if self.verbose:
            self.log.write(
                u"\n\n\nSOS DescribeSensor request errors recap.  Failed requests:"
            )
            print("SOS DescribeSensor request errors recap.  Failed requests:")
            for station_fail, msg in iteritems(sml_errors):
                self.log.write(
                    u"\n{station} - {msg}.  DescribeSensor URL: {ds}".format(
                        station=station_fail,
                        msg=msg,
                        ds=describe_sensor_url[station_fail].replace(
                            "&", "&")))
                print("{station} - {msg}.  DescribeSensor URL: {ds}".format(
                    station=station_fail,
                    msg=msg,
                    ds=describe_sensor_url[station_fail].replace("&",
                                                                 "&")))
            if failures:
                self.log.write(
                    u"\nStations in 'failures' list (should match DescribeSensor errors):"
                )
                print(
                    "Stations in 'failures' list (should match DescribeSensor errors):"
                )
                for station_fail in failures:
                    self.log.write(u"\n{station}".format(station=station_fail))
                    print("{station}".format(station=station_fail))

        if station_recs:
            stations_df = pd.DataFrame.from_records(station_recs,
                                                    columns=station.keys())
            stations_df.index = stations_df['station_urn']
            return stations_df
        else:
            return None
예제 #8
0
def main():

    #The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature...
    obsList = ['water_temperature', 'salinity']
    """
  Create a data collection object.
  Contructor parameters are:
    url - THe SWE endpoint we're interested in
    version - Optional default is '1.0.0' The SWE version the endpoint.
    xml - Optional default is None - The XML response from a GetCapabilities query of the server.
  """
    dataCollector = IoosSweSos(
        url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc',
        xml=None)

    #Loop through the offerings from the server.
    #Offerings are the stations/platforms/data device that the server house data from.
    offerings = dataCollector.server.offerings
    for offer in offerings:
        #We skip over the 'all' offering.
        #all "*network*" offerings represent aggregations of stations;
        # unlike all other offerings we've dealt with basically correspond to one station.
        if (offer.name.split(':')[-1] != 'all'):

            print "Offering: %s %s Obs: %s"\
                  % (offer.name, offer.description, offer.observed_properties)

            #Check to see if the station offered has the observation we're interested in.
            #Loop through the observed_properties attribute and compare.
            obsFilterList = []
            #We want to query just the observations n the obsOfInterest list.
            #The observer_properties should be coded with an MMI href, however due
            #to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds
            #a URN pattern. Regardless, an end user should just need to know the observation
            #name and not the MMI url, so this tries to do a simple
            #string find of our desired observation in the observed_property.
            for obsOfInterest in obsList:
                for observedProperty in offer.observed_properties:
                    if (observedProperty.find(obsOfInterest) != -1):
                        #Because of the ncSOS bug 107, we don't add the observedProperty. When
                        #that bug gets addressed, we would add the observedProperty.
                        #obsFilterList.append(observedProperty)
                        obsFilterList.append(obsOfInterest)
                        break

            if (len(obsFilterList)):
                #We create a filter based on the observations of interest,
                # and a time frame of the 12 hours of data from yesterday.
                #There is an ncSOS bug that affects getting the current data
                #Issue: https://github.com/asascience-open/ncSOS/issues/112
                """
        Filter function params:
        bbox - Bounding box, not implemented for SOS calls currently.
        start  - start date for records
        end - end date for records
        features - I assume it's a list of the stations of interest. Not sure why this is here
          and then the offerings parameter in the collect() function.
        variables - Poorly named parameter to house the observed_properties
        """
                try:
                    dataCollector.filter(
                        variables=obsFilterList,
                        start=datetime.utcnow() - timedelta(hours=24),
                        end=datetime.utcnow() - timedelta(hours=12))
                    """
          collect() params:
          offering - Station list

          responseFormat - The desired return format for the SOS call.
          eventTime - If the start/end times are not provided in the filter(), this param
            can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date.

          returns a list of OmObservation objects. OmObservation object has no member functions.
          """
                    response = dataCollector.collect(offerings=[offer.name])
                    for obsRec in response:
                        stationRec = obsRec.feature
                        print "Station: %s Location: %s" % (
                            stationRec.name, stationRec.get_location())

                        #The elements are a list of the observed_properties returned wrapped in a Point object.
                        for obsProp in stationRec.get_elements():
                            print "Observation Date/Time: %s" % (
                                obsProp.get_time())
                            #print "Member names: %s" % (obsProp.get_member_names())
                            for member in obsProp.get_members():
                                for key, value in member.iteritems():
                                    print "%s = %s" % (key, value)

                except ows.ExceptionReport, e:
                    traceback.print_exc(e)

                except Exception, e:
                    traceback.print_exc(e)
예제 #9
0
    def get_stations_df(self, sos_url, station_urns_sel=None):
        """ Returns a GeoDataFrame
        """
        # LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST
        oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
        params = {
            'service': 'SOS',
            'request': 'GetCapabilities',
            'acceptVersions': '1.0.0'
        }
        sos_url_params = sos_url + '?' + urlencode(params)
        sos_url_params_esc = sos_url_params.replace("&", "&")
        # sos_url_params_quoted = quote(sos_url_params,"/=:")
        # sos_url_params_unquoted = unquote(sos_url_params)

        try:
            sosgc = SensorObservationService(sos_url_params)
        except (ConnectionError, ReadTimeout) as e:
            self.log.write(
                u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n"
                .format(url=sos_url_params))
            self.log.write(
                u"HTTP connection error: {err}.\n".format(err=e.message))
            sys.exit(
                "Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
                .format(url=sos_url_params, err=e.message))

        if station_urns_sel is not None:
            station_urns = station_urns_sel
        else:
            sos_collector = IoosSweSos(sos_url)
            station_urns = [
                urn.name for urn in sos_collector.server.offerings
                if 'network' not in urn.name.split(':')
            ]
            sos_collector.features = station_urns
            sml_lst = sos_collector.metadata(timeout=200)

        station_recs = []
        for station_idx, station_urn in enumerate(station_urns):
            if station_urns_sel is not None:
                sml_str = sosgc.describe_sensor(procedure=station_urn,
                                                outputFormat=oFrmt,
                                                timeout=200)
                sml = SensorML(sml_str)

            else:
                sml = sml_lst[station_idx]

            # debug:
            # if self.verbose:
            #    self.log.write(unicode(etree.tostring(sml._root)))

            ds = IoosDescribeSensor(sml._root)

            pos = testXMLValue(
                ds.system.location.find(self.nsp('gml:Point/gml:pos')))

            system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
                self.nsp('sml:System'))

            # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
            # errors but there is not enough information in SensorML for alternatives)
            # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
            documents = system_el.findall(
                self.nsp('sml:documentation/sml:DocumentList/sml:member'))
            documents_dct = {}
            for d in documents:
                document = Documentation(d)
                name = testXMLAttribute(d, "name")
                # url = document.documents[0].url
                documents_dct[name] = document

            contacts = system_el.findall(
                self.nsp('sml:contact/sml:ContactList/sml:member'))
            contacts_dct = {}
            for c in contacts:
                contact = Contact(c)
                role = contact.role.split('/')[-1]
                contacts_dct[role] = contact

            sweQuants = system_el.findall(
                self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
            quant_lst = [
                sweQuant.attrib['definition'] for sweQuant in sweQuants
            ]
            parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]

            # attempt to read beginPosition, if available:
            beginPosition = testXMLValue(
                system_el.find(
                    self.nsp(
                        'sml:validTime/gml:TimePeriod/gml:beginPosition')))
            try:
                begin_service_date = parser.parse(beginPosition)
            except AttributeError as e:
                begin_service_date = None

            station = OrderedDict()
            station['station_urn'] = station_urn
            station['sos_url'] = sos_url_params_esc
            station['lon'] = float(pos.split()[1])
            station['lat'] = float(pos.split()[0])

            station['shortName'] = ds.shortName
            station['longName'] = ds.longName
            station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
            station['serverName'] = self.server_name

            # Some capabilities-level metadata:
            station['title'] = sosgc.identification.title
            station['abstract'] = sosgc.identification.abstract
            station['keywords'] = sosgc.identification.keywords
            station['begin_service_date'] = begin_service_date

            # Beware that a station can have >1 classifier of the same type
            # This code does not accommodate that possibility
            station['platformType'] = ds.platformType
            station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
                                                       'classifier', ont)
            station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)

            # store some nested dictionaries in 'station' for appopriate SensorML sources:
            station['contacts_dct'] = contacts_dct
            station['documents_dct'] = documents_dct

            # MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used
            # in the template code currently in favor of 'contacts_dct'
            # station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont)
            # station['operator_org'] = contacts_dct['operator'].organization
            # station['operator_country'] = contacts_dct['operator'].country
            # station['operator_url'] = contacts_dct['operator'].url
            # station['operator_email'] = contacts_dct['operator'].email

            # station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont)
            # station['publisher_org'] = contacts_dct['publisher'].organization
            # station['publisher_url'] = contacts_dct['publisher'].url
            # station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress

            station['starting'] = ds.starting
            station['ending'] = ds.ending
            # station['starting_isostr'] = datetime.isoformat(ds.starting)
            # station['ending_isostr'] = datetime.isoformat(ds.ending)

            station['parameter_uris'] = ','.join(quant_lst)
            station['parameters'] = ','.join(parameter_lst)
            station['variables'] = [var.split('/')[-1] for var in ds.variables]

            # debug:
            if self.verbose:
                self.log.write(u"\nProcessing station: {station}\n".format(
                    station=station_urn))
                print("\nProcessing station: {station}".format(
                    station=station_urn))
                for var in ds.variables:
                    self.log.write(u"variable: {var}\n".format(var=var))
                    print("variable: {var}".format(var=var))

            # print(sosgc.contents)
            # for id, offering in sosgc.contents.iteritems():
            #    print("sosgc.contents: {item}".format(item=id))

            # parse 'responseFormat' values and populate list:
            # response_formats = sosgc.contents[station_urn].response_formats
            response_formats = []
            for id, sosgc.content in sosgc.contents.items():
                if sosgc.content.name == station_urn:
                    response_formats = sosgc.content.response_formats
            # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ]

            # subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter
            # (or 'application/json,application/zip; subtype=x-netcdf' by default):
            download_formats = [
                response_format for response_format in response_formats
                if response_format in self.response_formats
            ]
            station['response_formats'] = response_formats
            station['download_formats'] = download_formats

            if self.verbose:
                for format in response_formats:
                    self.log.write(
                        u"responseFormat: {format}\n".format(format=format))
                    print("responseFormat: {format}".format(format=format))
                for format in download_formats:
                    self.log.write(
                        u"downloadFormats: {format}\n".format(format=format))
                    print("downloadFormats: {format}".format(format=format))

            # calculate event_time using self.getobs_req_hours:
            if ds.starting is not None and ds.ending is not None:
                event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
                    begin=ds.ending - timedelta(hours=self.getobs_req_hours),
                    end=ds.ending)
            else:
                now = datetime.now(pytz.utc)
                then = now - timedelta(hours=self.getobs_req_hours)
                event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
                    begin=then, end=now)
                if self.verbose:
                    self.log.write(
                        u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n"
                        .format(then=then, now=now))
                    print(
                        "then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
                        .format(then=then, now=now))

            if self.verbose:
                self.log.write(u"eventTime: {time}\n".format(time=event_time))
                print("eventTime: {time}".format(time=event_time))

            # create a dict to store parameters for valid example GetObservation requests for station:
            getobs_req_dct = {}
            # populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty:
            getobs_params_base = {
                'service': 'SOS',
                'request': 'GetObservation',
                'version': '1.0.0',
                'offering': station_urn,
                'eventTime': event_time
            }
            for variable in ds.variables:
                getobs_params = getobs_params_base.copy()
                getobs_params['observedProperty'] = variable
                variable = variable.split('/')[-1]
                for format in download_formats:
                    getobs_params['responseFormat'] = format
                    getobs_request_url_encoded = sos_url + '?' + urlencode(
                        getobs_params)
                    getobs_request_url = unquote(getobs_request_url_encoded)
                    getobs_request_url_esc = getobs_request_url.replace(
                        "&", "&")
                    getobs_req_dct[variable + '-' + format] = {
                        'variable': variable,
                        'url': getobs_request_url_esc,
                        'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format],
                        'format_name': self.RESPONSE_FORMAT_NAME_MAP[format]
                    }
                    if self.verbose:
                        self.log.write(
                            u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n"
                            .format(
                                variable=variable.split("/")[-1],
                                getobs_request_url=getobs_request_url,
                                getobs_request_url_esc=getobs_request_url_esc))
                        print(
                            "getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}"
                            .format(
                                variable=variable.split("/")[-1],
                                getobs_request_url=getobs_request_url,
                                getobs_request_url_esc=getobs_request_url_esc))

            # ToDo: finish adding the 'getobs_req_dct' to the output template
            station['getobs_req_dct'] = getobs_req_dct

            station_recs.append(station)

        stations_df = pd.DataFrame.from_records(station_recs,
                                                columns=station.keys())
        stations_df.index = stations_df['station_urn']

        return stations_df