Esempio n. 1
0
    def getSOSProcedureObservations(self, name, begin, end, qi=False):
        """
            Execute a getObservation
            
            > Return an array observations.  
        
        """
        begin1 = ""
        end1 = ""

        # Checking dates format
        if isinstance(begin, datetime):
            # Check tz
            if begin.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            begin1 = begin.isoformat()
        elif isinstance(begin, str):
            tmp = iso.parse_datetime(begin)
            if tmp.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            begin1 = tmp.isoformat()

        if isinstance(end, datetime):
            # Check tz
            if end.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            end1 = end.isoformat()
        elif isinstance(end, str):
            tmp = iso.parse_datetime(end)
            if tmp.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            end1 = tmp.isoformat()

        # Executing request
        res = req.get("%s/%s" % (self.host, self.service),
                      params={
                          'service': 'SOS',
                          'version': '1.0.0',
                          'request': 'GetObservation',
                          'offering': 'temporary',
                          'responseFormat': 'application/json',
                          'procedure': name,
                          'qualityIndex': qi,
                          'eventTime': "%s/%s" % (begin1, end1),
                          'observedProperty': ":"
                      },
                      auth=self.auth)

        json = res.json()

        return json['ObservationCollection']['member'][0]['result'][
            'DataArray']['values']
Esempio n. 2
0
    def extractSamplingFromGOJson(self, jsonRes):

        if "beginPosition" in jsonRes["samplingTime"]:

            begin = jsonRes["samplingTime"]["beginPosition"]
            end = jsonRes["samplingTime"]["endPosition"]

            return [iso.parse_datetime(begin), iso.parse_datetime(end)]

        else:
            return [None, None]
Esempio n. 3
0
    def getObservation(self, name, begin=None, end=None, qi=False):
        begin1 = ""
        end1 = ""

        # Checking dates format
        if isinstance(begin, datetime):
            # Check tz
            if begin.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            begin1 = begin.isoformat()

        elif isinstance(begin, str):
            tmp = iso.parse_datetime(begin)
            if tmp.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            begin1 = tmp.isoformat()

        if isinstance(end, datetime):
            # Check tz
            if end.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            end1 = end.isoformat()

        elif isinstance(end, str):
            tmp = iso.parse_datetime(end)
            if tmp.tzinfo is None:
                raise Exception(
                    "Time Zone (tzinfo) is mandatory in datetime objects")
            end1 = tmp.isoformat()

        params = {
            'service': 'SOS',
            'version': '1.0.0',
            'request': 'GetObservation',
            'offering': 'temporary',
            'responseFormat': 'application/json',
            'procedure': name,
            'qualityIndex': qi,
            'observedProperty': ":"
        }
        if begin:
            params['eventTime'] = "%s/%s" % (begin1, end1)
        # Executing request
        res = req.get("%s/%s" % (self.host, self.service),
                      params=params,
                      auth=self.auth)
        jsonRes = res.json()
        return jsonRes
Esempio n. 4
0
 def getSOSProcedureObservations(self, name, begin, end, qi = False):
     """
         Execute a getObservation
         
         > Return an array observations.  
     
     """
     begin1 = ""
     end1 = ""
     
     
     # Checking dates format
     if isinstance(begin, datetime):
         # Check tz
         if begin.tzinfo is None:
             raise Exception("Time Zone (tzinfo) is mandatory in datetime objects")
         begin1 = begin.isoformat()
     elif isinstance(begin, str):
         tmp = iso.parse_datetime(begin)
         if tmp.tzinfo is None:
             raise Exception("Time Zone (tzinfo) is mandatory in datetime objects")
         begin1 = tmp.isoformat()
         
         
     if isinstance(end, datetime):
         # Check tz
         if end.tzinfo is None:
             raise Exception("Time Zone (tzinfo) is mandatory in datetime objects")
         end1 = end.isoformat()
     elif isinstance(end, str):
         tmp = iso.parse_datetime(end)
         if tmp.tzinfo is None:
             raise Exception("Time Zone (tzinfo) is mandatory in datetime objects")
         end1 = tmp.isoformat()
         
     # Executing request
     res = req.get("%s/%s" % (self.host, self.service), params={
         'service': 'SOS', 
         'version': '1.0.0',
         'request': 'GetObservation',
         'offering': 'temporary',
         'responseFormat': 'application/json',
         'procedure': name,
         'qualityIndex': qi,
         'eventTime': "%s/%s" % (begin1,end1),
         'observedProperty': ":"
     }, auth=self.auth)
     
     json = res.json()
     
     return json['ObservationCollection']['member'][0]['result']['DataArray']['values']
Esempio n. 5
0
 def extractSamplingFromGOJson(self, json):
     
     if "beginPosition" in json["samplingTime"]:
         
         begin = json["samplingTime"]["beginPosition"]
         end = json["samplingTime"]["endPosition"]
         
     
         return [
             iso.parse_datetime(begin),
             iso.parse_datetime(end)
         ]
     
     else:
     
         print "%s: %s - %s" % (name, None, None) 
         return [None,None]
Esempio n. 6
0
 def executePut(self):
     """
     Method for executing a PUT requests that rename a SOS service
                       
         @note: This method renames:
             1. create a new service folder, 
             2. copy content from old to new service configuration file
             3. rename the databse schema
             4. delete old service files
         
         The POST must be in Json format with mandatory service key
                 
         >>> {
                 "service" : "service_name"
             } 
     """
     
     if self.service == "default":
         raise Exception("offerings PUT operation can not be done for 'default' service instance.")
         
     try:                
         self.offering = self.pathinfo[(self.pathinfo.index("offerings")+1)]
         
         if self.offering == 'temporary' and self.offering != self.json["name"]:
             raise Exception("'temporary' offering name cannot be updated")
                 
         servicedb = databaseManager.PgDB(
                         self.serviceconf.connection["user"],
                         self.serviceconf.connection["password"],
                         self.serviceconf.connection["dbname"],
                         self.serviceconf.connection["host"],
                         self.serviceconf.connection["port"]
                         )
     
         sql  = "UPDATE %s.offerings" % self.service
         sql += " SET name_off = %s, desc_off = %s, expiration_off = %s , active_off = %s "
         sql += " WHERE name_off = %s"
     
         name = self.json["name"]
         desc = self.json["description"]
         
         try:
             exp = isodate.parse_datetime(self.json["expiration"])
         except:
             exp = None
         try:
             act = True if (self.json.has_key("active") and self.json["active"]=='on') else False
         except:
             act = False
         
         pars = (name,desc,exp,act,self.offering)
         
         servicedb.execute(sql,pars)
         self.setMessage("offering successfully updated")
         
     except Exception as e:
         self.setException("Error in updating an offering: %s" % e)
Esempio n. 7
0
    def executePut(self):
        """
        Method for executing a PUT requests that rename a SOS service
                          
            @note: This method renames:
                1. create a new service folder, 
                2. copy content from old to new service configuration file
                3. rename the databse schema
                4. delete old service files
            
            The POST must be in Json format with mandatory service key
                    
            >>> {
                    "service" : "service_name"
                } 
        """

        if self.service == "default":
            raise Exception("offerings PUT operation can not be done for 'default' service instance.")

        try:
            self.offering = self.pathinfo[(self.pathinfo.index("offerings") + 1)]

            if self.offering == "temporary" and self.offering != self.json["name"]:
                raise Exception("'temporary' offering name cannot be updated")

            servicedb = databaseManager.PgDB(
                self.serviceconf.connection["user"],
                self.serviceconf.connection["password"],
                self.serviceconf.connection["dbname"],
                self.serviceconf.connection["host"],
                self.serviceconf.connection["port"],
            )

            sql = "UPDATE %s.offerings" % self.service
            sql += " SET name_off = %s, desc_off = %s, expiration_off = %s , active_off = %s "
            sql += " WHERE name_off = %s"

            name = self.json["name"]
            desc = self.json["description"]

            try:
                exp = isodate.parse_datetime(self.json["expiration"])
            except:
                exp = None
            try:
                act = True if (self.json.has_key("active") and self.json["active"] == "on") else False
            except:
                act = False

            pars = (name, desc, exp, act, self.offering)

            servicedb.execute(sql, pars)
            self.setMessage("offering successfully updated")

        except Exception as e:
            self.setException("Error in updating an offering: %s" % e)
Esempio n. 8
0
    def executePost(self, db=True):
        """
        Method for executing a POST requests that initialize a new SOS service
                      
        @note: This method creates a new istSOS offering
        
        The POST must be in Json format with mandatory offering key:
                
        >>> {
                "offering" : "meteorology", 
                "description" : "meteo information"
                "expiration" : "2012-12-30T12:00"
                "active" : "sos_db"
            } 
        """
        servicedb = databaseManager.PgDB(
            self.serviceconf.connection["user"],
            self.serviceconf.connection["password"],
            self.serviceconf.connection["dbname"],
            self.serviceconf.connection["host"],
            self.serviceconf.connection["port"],
        )

        # insert new offering in db
        try:
            sql = "INSERT INTO %s.offerings" % self.service
            sql += " (name_off,desc_off,expiration_off,active_off)"
            sql += " VALUES (%s, %s, %s, %s)"

            name = self.json["name"]
            try:
                desc = self.json["description"]
            except:
                desc = None
            try:
                exp = isodate.parse_datetime(self.json["expiration"])
            except:
                exp = None
            try:
                act = self.json["active"]
            except:
                act = False

            pars = (name, desc, exp, act)

            servicedb.execute(sql, pars)
            self.setMessage("new offering successfully added")

        except Exception as e:
            self.setException("Error in adding new offering: %s" % e)
Esempio n. 9
0
 def executePost(self,db=True):
     """
     Method for executing a POST requests that initialize a new SOS service
                   
     @note: This method creates a new istSOS offering
     
     The POST must be in Json format with mandatory offering key:
             
     >>> {
             "offering" : "meteorology", 
             "description" : "meteo information"
             "expiration" : "2012-12-30T12:00"
             "active" : "sos_db"
         } 
     """
     servicedb = databaseManager.PgDB(
                     self.serviceconf.connection["user"],
                     self.serviceconf.connection["password"],
                     self.serviceconf.connection["dbname"],
                     self.serviceconf.connection["host"],
                     self.serviceconf.connection["port"]
                     )
     
     #insert new offering in db
     try:
         sql  = "INSERT INTO %s.offerings" % self.service
         sql += " (name_off,desc_off,expiration_off,active_off)"
         sql += " VALUES (%s, %s, %s, %s)"
     
         name = self.json["name"]
         try:
             desc = self.json["description"]
         except:
             desc = None
         try:
             exp = isodate.parse_datetime(self.json["expiration"])
         except:
             exp = None
         try:
             act = self.json["active"]
         except:
             act = False
         
         pars = (name,desc,exp,act)
         
         servicedb.execute(sql,pars)
         self.setMessage("new offering successfully added")            
         
     except Exception as e:
         self.setException("Error in adding new offering: %s" % e)
Esempio n. 10
0
def makeFile(res, procedure, op, path):
    text = res.text
    text = text.replace("%s," % procedure, "")
    lines = text.split('\n')
    if lines[-1] == '':
        del lines[-1]
    tmpOp = op.replace("x-ist::","x-istsos:1.0:")
    lines[0] = "%s,%s,%s:qualityIndex" % (isoop, tmpOp, tmpOp)    
    if len(lines)>1:
        datenumber = iso.parse_datetime(lines[-1].split(",")[0])       
        print "File: %s/%s_%s.dat" % (path, procedure, datetime.datetime.strftime(datenumber, "%Y%m%d%H%M%S%f"))
        out_file = open("%s/%s_%s.dat" % (path, procedure, datetime.datetime.strftime(datenumber, "%Y%m%d%H%M%S%f")),"w")
        out_file.write("\n".join(lines))
        #print "\n".join(lines)
        out_file.close()
Esempio n. 11
0
def makeFile(res, procedure, op, path):
    text = res.text
    text = text.replace("%s," % procedure, "")
    lines = text.split('\n')
    if lines[-1] == '':
        del lines[-1]
    tmpOp = op.replace("x-ist::", "x-istsos:1.0:")
    lines[0] = "%s,%s,%s:qualityIndex" % (isoop, tmpOp, tmpOp)
    if len(lines) > 1:
        datenumber = iso.parse_datetime(lines[-1].split(",")[0])
        print "File: %s/%s_%s.dat" % (path, procedure,
                                      datetime.datetime.strftime(
                                          datenumber, "%Y%m%d%H%M%S%f"))
        out_file = open(
            "%s/%s_%s.dat" %
            (path, procedure,
             datetime.datetime.strftime(datenumber, "%Y%m%d%H%M%S%f")), "w")
        out_file.write("\n".join(lines))
        out_file.close()
Esempio n. 12
0
    def setData(self,pgdb,o,filter):
        """get data according to request filters"""
        # @todo mettere da qualche altra parte
            
        #SET FOI OF PROCEDURE
        #=========================================
        sqlFoi  = "SELECT name_fty, name_foi, ST_AsGml(ST_Transform(geom_foi,%s)) as gml, st_x(geom_foi) as x, st_y(geom_foi) as y " %(filter.srsName)
        sqlFoi += " FROM %s.procedures, %s.foi, %s.feature_type" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        sqlFoi += " WHERE id_foi_fk=id_foi AND id_fty_fk=id_fty AND id_prc=%s" %(o["id_prc"])
        try:
            resFoi = pgdb.select(sqlFoi)
        except:
            raise Exception("SQL: %s"%(sqlFoi))
        
        self.featureOfInterest = resFoi[0]["name_foi"]
        self.foi_urn = filter.sosConfig.urn["feature"] + resFoi[0]["name_fty"] + ":" + resFoi[0]["name_foi"]
        srs = filter.srsName or filter.sosConfig.istsosepsg
        if resFoi[0]["gml"].find("srsName")<0:
            self.foiGml = resFoi[0]["gml"][:resFoi[0]["gml"].find(">")] + " srsName=\"EPSG:%s\"" % srs + resFoi[0]["gml"][resFoi[0]["gml"].find(">"):]
        else:
            self.foiGml = resFoi[0]["gml"]
        
        self.srs = srs
        self.x = resFoi[0]["x"]
        self.y = resFoi[0]["y"]
        
        #SET INFORMATION ABOUT OBSERVED_PROPERTIES
        #=========================================       
        sqlObsPro = "SELECT id_pro, id_opr, name_opr, def_opr, name_uom FROM %s.observed_properties, %s.proc_obs, %s.uoms" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        sqlObsPro += " WHERE id_opr_fk=id_opr AND id_uom_fk=id_uom AND id_prc_fk=%s" %(o["id_prc"])
        sqlObsPro += " AND ("
        #sqlObsPro += " OR ".join(["def_opr='" + str(i) + "'" for i in filter.observedProperty])
        sqlObsPro += " OR ".join(["def_opr SIMILAR TO '%(:|)" + str(i) + "(:|)%'" for i in filter.observedProperty])
        sqlObsPro += " ) ORDER BY def_opr ASC"
        try:
            obspr_res = pgdb.select(sqlObsPro)
        except:
            raise Exception("SQL: %s"%(sqlObsPro))
            
        self.observedProperty = []
        self.observedPropertyName = []
        self.opr_urn = []
        self.uom = []
        self.qualityIndex = filter.qualityIndex
        
        for row in obspr_res:
            self.observedProperty += [str(row["def_opr"])]
            self.observedPropertyName +=[str(row["name_opr"])]
            self.opr_urn += [str(row["def_opr"])]
            try:
                #self.uom += [str(row["name_uom"]).encode('utf-8')]
                self.uom += [row["name_uom"]]
            except:
                self.uom += ["n/a"]
            if self.qualityIndex==True:
                self.observedProperty += [str(row["def_opr"])+":qualityIndex"]
                self.observedPropertyName += [str(row["name_opr"])+":qualityIndex"]
                self.opr_urn += [str(row["def_opr"] +":qualityIndex")]
                self.uom += ["-"]
        
        #SET DATA
        #=========================================getSampligTime
        #CASE "insitu-fixed-point" or "insitu-mobile-point"
        #-----------------------------------------
        if self.procedureType in ["insitu-fixed-point","insitu-mobile-point"]:
            sqlSel = "SELECT et.time_eti as t," 
            joinar=[]
            cols=[]

            aggrCols=[]
            aggrNotNull=[]

            valeFieldName = []
            for idx, obspr_row in enumerate(obspr_res):
                if self.qualityIndex==True:
                    #cols.append("C%s.val_msr as c%s_v, C%s.id_qi_fk as c%s_qi" %(idx,idx,idx,idx))
                    cols.append("C%s.val_msr as c%s_v, COALESCE(C%s.id_qi_fk,%s) as c%s_qi" %(idx,idx,idx,filter.aggregate_nodata_qi,idx))
                    valeFieldName.append("c%s_v" %(idx))
                    valeFieldName.append("c%s_qi" %(idx))
                else:
                    cols.append("C%s.val_msr as c%s_v" %(idx,idx))
                    valeFieldName.append("c%s_v" %(idx))

                # If Aggregatation funtion is set
                #---------------------------------
                if filter.aggregate_interval != None:
                    # This can be usefull with string values
                    '''aggrCols.append("CASE WHEN %s(dt.c%s_v) is NULL THEN '%s' ELSE '' || %s(dt.c%s_v) END as c%s_v\n" % ( 
                        filter.aggregate_function, idx, filter.aggregate_nodata, filter.aggregate_function, idx, idx)
                    )'''
                    # This accept only numeric results
                    aggrCols.append("COALESCE(%s(dt.c%s_v),'%s') as c%s_v\n" %(filter.aggregate_function,idx,filter.aggregate_nodata,idx))
                    if self.qualityIndex==True:
                        #raise sosException.SOSException(3,"QI: %s"%(self.qualityIndex))
                        aggrCols.append("COALESCE(MIN(dt.c%s_qi),%s) as c%s_qi\n" %( idx, filter.aggregate_nodata_qi, idx ))
                    aggrNotNull.append(" c%s_v > -900 " %(idx))
                
                # Set SQL JOINS
                #---------------
                join_txt  = " left join (\n"
                join_txt += " SELECT distinct A%s.id_msr, A%s.val_msr, A%s.id_eti_fk\n" %(idx,idx,idx)

                if self.qualityIndex==True:
                    join_txt += ",A%s.id_qi_fk\n" %(idx)
                join_txt += "   FROM %s.measures A%s, %s.event_time B%s\n" %(filter.sosConfig.schema,idx,filter.sosConfig.schema,idx)
                join_txt += " WHERE A%s.id_eti_fk = B%s.id_eti\n" %(idx,idx)
                join_txt += " AND A%s.id_pro_fk=%s\n" %(idx,obspr_row["id_pro"])
                join_txt += " AND B%s.id_prc_fk=%s\n" %(idx,o["id_prc"])
                
                # if qualityIndex has filter
                #------------------------------
                #if filter.qualityIndex and filter.qualityIndex.__class__.__name__=='str':
                #    join_txt += " AND %s\n" %(filter.qualityIndex)

                # ATTENTION: HERE -999 VALUES ARE EXCLUDED WHEN ASKING AN AGGREAGATE FUNCTION
                if filter.aggregate_interval != None: # >> Should be removed because measures data is not inserted if there is a nodata value
                    join_txt += " AND A%s.val_msr > -900 " % idx
                
                # If eventTime is set add to JOIN part
                #--------------------------------------
                if filter.eventTime:
                    join_txt += " AND ("
                    etf=[]
                    for ft in filter.eventTime:
                        if len(ft)==2:
                            etf.append("B%s.time_eti > timestamptz '%s' AND B%s.time_eti <= timestamptz '%s' \n" %(idx,ft[0],idx,ft[1]))
                        elif len(ft)==1:
                            etf.append("B%s.time_eti = timestamptz '%s' \n" %(idx,ft[0]))
                        else:
                            raise Exception("error in time filter")
                    join_txt += " OR ".join(etf)
                    join_txt +=  ")\n"
                else:
                    join_txt += " AND B%s.time_eti = (SELECT max(time_eti) FROM %s.event_time WHERE id_prc_fk=%s) \n" %(idx,filter.sosConfig.schema,o["id_prc"])
                
                # close SQL JOINS
                #-----------------
                join_txt += " ) as C%s\n" %(idx)
                join_txt += " on C%s.id_eti_fk = et.id_eti" %(idx)
                joinar.append(join_txt)
            
            
            #If MOBILE PROCEDURE
            #--------------------
            if self.procedureType=="insitu-mobile-point":
                join_txt  = " left join (\n"
                join_txt += " SELECT distinct Ax.id_pos, X(ST_Transform(Ax.geom_pos,%s)) as x,Y(ST_Transform(Ax.geom_pos,%s)) as y,Z(ST_Transform(Ax.geom_pos,%s)) as z, Ax.id_eti_fk\n" %(filter.srsName,filter.srsName,filter.srsName)
                if self.qualityIndex==True:
                    join_txt += ", Ax.id_qi_fk as posqi\n"
                join_txt += "   FROM %s.positions Ax, %s.event_time Bx\n" %(filter.sosConfig.schema,filter.sosConfig.schema)
                join_txt += " WHERE Ax.id_eti_fk = Bx.id_eti"
                join_txt += " AND Bx.id_prc_fk=%s" %(o["id_prc"])
                
                if filter.eventTime:
                    join_txt += " AND ("
                    etf=[]
                    for ft in filter.eventTime:
                        if len(ft)==2:
                            etf.append("Bx.time_eti > timestamptz '%s' AND Bx.time_eti <= timestamptz '%s' " %(ft[0],ft[1]))
                        elif len(ft)==1:
                            etf.append("Bx.time_eti = timestamptz '%s' " %(ft[0]))                       
                        else:
                            raise Exception("error in time filter")
                    join_txt += " OR ".join(etf)
                    join_txt +=  ")\n"
                else:
                    join_txt += " AND Bx.time_eti = (SELECT max(time_eti) FROM %s.event_time WHERE id_prc_fk=%s) " %(filter.sosConfig.schema,o["id_prc"])
                
                join_txt += " ) as Cx on Cx.id_eti_fk = et.id_eti\n"
                sqlSel += " Cx.x as x, Cx.y as y, Cx.z as z, "
                if self.qualityIndex==True:
                    #sqlSel += "COALESCE(Cx.posqi,%s) as posqi, " % filter.aggregate_nodata_qi
                    sqlSel += "Cx.posqi, "
                joinar.append(join_txt)
            
            
            # Set FROM CLAUSE
            #-----------------    
            sqlSel += ", ".join(cols)
            sqlSel += " FROM %s.event_time et\n" %(filter.sosConfig.schema)

            #====================            
            # Set WHERE CLAUSES
            #====================
            sqlData = " ".join(joinar)
            sqlData += " WHERE et.id_prc_fk=%s\n" %(o["id_prc"]) 

            # Set FILTER ON RESULT (OGC:COMPARISON) -
            #----------------------------------------
            if filter.result:
                for ind, ov in enumerate(self.observedProperty):
                    if ov.find(filter.result[0])>0:
                        sqlData += " AND C%s.val_msr %s" %(ind,filter.result[1])
                #sqlData += " AND C%s.val_msr %s" %(self.observedProperty.index(filter.result[0]),filter.result[1])
                
            # Set FILTER ON EVENT-TIME -
            #---------------------------
            if filter.eventTime:
                sqlData += " AND ("
                etf=[]
                for ft in filter.eventTime:
                    if len(ft)==2:
                        etf.append("et.time_eti > timestamptz '%s' AND et.time_eti <= timestamptz '%s' " %(ft[0],ft[1]))
                    elif len(ft)==1:
                        etf.append("et.time_eti = timestamptz '%s' " %(ft[0]))                        
                    else:
                        raise Exception("error in time filter")
                sqlData += " OR ".join(etf)
                sqlData +=  ")"
            else:
                sqlData += " AND et.time_eti = (SELECT max(time_eti) FROM %s.event_time WHERE id_prc_fk=%s) " %(filter.sosConfig.schema,o["id_prc"])

            sqlData += " ORDER by et.time_eti"

            sql = sqlSel+sqlData
            
            #
            if filter.aggregate_interval != None:
                self.aggregate_function = filter.aggregate_function.upper()
                '''
                for i in range(0,len(self.observedProperty)):
                    self.observedProperty[i] = "%s:%s" % (self.observedProperty[i], filter.aggregate_function)

                for ob in self.observedProperty:
                    ob = "%s:%s" % (ob, filter.aggregate_function)'''
                
                # Interval preparation
                # Converting ISO 8601 duration
                isoInt = iso.parse_duration(filter.aggregate_interval)
                sqlInt = ""

                if isinstance(isoInt, timedelta):
                
                    if isoInt.days>0:
                        sqlInt += "%s days " % isoInt.days
                    if isoInt.seconds>0:
                        sqlInt += "%s seconds " % isoInt.seconds
                        
                elif isinstance(isoInt, iso.Duration): 
                    if isoInt.years>0:
                        sqlInt += "%s years " % isoInt.years
                    if isoInt.months>0:
                        isoInt.months = int(isoInt.months)
                        sqlInt += "%s months " % isoInt.months
                    if isoInt.days>0:
                        sqlInt += "%s days " % isoInt.days
                    if isoInt.seconds>0:
                        sqlInt += "%s seconds " % isoInt.seconds

                
                # @todo improve this part
                # calculate how many step are included in the asked interval.
                hopBefore = 1
                hop = 0
                tmpStart = iso.parse_datetime(filter.eventTime[0][0])
                tmpEnd = self.samplingTime[1]
                
                while (tmpStart+isoInt)<=tmpEnd and (tmpStart+isoInt)<=iso.parse_datetime(filter.eventTime[0][1]):
                    
                    if   tmpStart <  self.samplingTime[0]:
                        hopBefore+=1
                        hop+=1

                    elif (tmpStart >= self.samplingTime[0]) and ((tmpStart+isoInt)<=self.samplingTime[1]):
                        hop+=1
                        
                    tmpStart=tmpStart+isoInt

                aggregationSQL = "SELECT ts.sint  as t, %s\n"
                aggregationSQL += "FROM\n"
                aggregationSQL += "    (\n" # Generating time series here
                aggregationSQL += "        select\n"
                aggregationSQL += "        (('%s'::TIMESTAMP WITH TIME ZONE)  \n"
                aggregationSQL += "            + s.a * '%s'::interval)::TIMESTAMP WITH TIME ZONE as sint\n"
                aggregationSQL += "        from generate_series(%s, %s) as s(a)\n"
                aggregationSQL += "    ) as ts LEFT JOIN ( \n\n"
                aggregationSQL += "    %s \n\n"
                aggregationSQL += "    ) as dt\n"
                aggregationSQL += "    ON (\n"
                aggregationSQL += "        dt.t > (ts.sint-'%s'::interval)\n"
                aggregationSQL += "        AND\n"
                aggregationSQL += "        dt.t <= (ts.sint) \n"
                aggregationSQL += "    )\n"
                aggregationSQL += "    GROUP BY ts.sint\n"
                aggregationSQL += "    ORDER BY ts.sint"
                sql = aggregationSQL % (", ".join(aggrCols), filter.eventTime[0][0], sqlInt, hopBefore, hop, sql, sqlInt)
                
            else:
                self.aggregate_function = None
                
            #print sql.replace('\n','')
            
            try:
                data_res = pgdb.select(sql)
            except:
                raise Exception("SQL: %s"%(sql))
            

            #------------------------------------            
            #--------- APPEND DATA IN ARRAY -----
            #------------------------------------            
            #append data
            for line in data_res:
                if self.procedureType=="insitu-fixed-point":
                    data_array = [line["t"]]
                elif self.procedureType=="insitu-mobile-point":
                    if self.qualityIndex==True:
                        data_array = [line["t"],line["x"],line["y"],line["z"],line["posqi"]]
                    else:
                        data_array = [line["t"],line["x"],line["y"],line["z"]]
                data_array.extend([line[field] for field in valeFieldName])
                self.data.append(data_array)
            
        #-----------------------------------------                
        #CASE "virtual"
        #-----------------------------------------       
        elif self.procedureType in ["virtual"]:
            
            
            self.aggregate_function = filter.aggregate_function
            self.aggregate_interval = filter.aggregate_interval
            self.aggregate_nodata = filter.aggregate_nodata
            self.aggregate_nodata_qi = filter.aggregate_nodata_qi
                        
            vpFolder = os.path.join(os.path.join(filter.sosConfig.virtual_processes_folder,self.name))
            
            if not os.path.isfile("%s/%s.py" % (vpFolder,self.name)):
                raise Exception("Virtual procedure folder does not contain any Virtual Procedure code for %s" % self.name)
                
            #----- VIRTUAL PROCESS LOADING -----
            try:
                sys.path.append(vpFolder)
            except:
                raise Exception("error in loading virtual procedure path")
            #import procedure process
            exec "import %s as vproc" %(self.name)
            
            # Initialization of virtual procedure will load the source data
            vp = vproc.istvp()
            vp._configure(filter, pgdb)
            # Calculate virtual procedure data
            vp.calculateObservations(self)
Esempio n. 13
0
def applyFunction(ob, filter):
    import copy
    try:
        # Create array container
        begin = iso.parse_datetime(filter.eventTime[0][0])
        end = iso.parse_datetime(filter.eventTime[0][1])
        duration = iso.parse_duration(filter.aggregate_interval)
        result = {}        
        dt = begin
        fields = len(ob.observedProperty)# + 1 # +1 timestamp field not mentioned in the observedProperty array
        
        while dt < end:
            dt2 = dt + duration
            result[dt2]=[]
            for c in range(fields):
                result[dt2].append([])
            
            d = 0
            data = copy.copy(ob.data)
            while len(data) > 0:
                tmp = data.pop(d)
                if dt < tmp[0] and tmp[0] <= dt2:
                    ob.data.pop(d)
                    for c in range(fields):
                        result[dt2][c].append(float(tmp[c+1]))
                elif dt > tmp[0]:
                    ob.data.pop(d)
                elif dt2 < tmp[0]:
                    break
                    
            dt = dt2
            
        data = []
        
        for r in sorted(result):
            record = [r]
            for v in range(len(result[r])):
                if ob.observedProperty[v].split(":")[-1]=="qualityIndex":
                    if len(result[r][v])==0:
                        record.append(filter.aggregate_nodata_qi)
                    else:
                        record.append(int(min(result[r][v])))
                else:
                    val = None
                    if len(result[r][v])==0:
                        val = filter.aggregate_nodata
                    elif filter.aggregate_function.upper() == 'SUM':
                        val = sum(result[r][v])
                    elif filter.aggregate_function.upper() == 'MAX':
                        val = max(result[r][v])
                    elif filter.aggregate_function.upper() == 'MIN':
                        val = min(result[r][v])
                    elif filter.aggregate_function.upper() == 'AVG':
                        val = round(sum(result[r][v])/len(result[r][v]),4)
                    elif filter.aggregate_function.upper() == 'COUNT':
                        val = len(result[r][v])
                    record.append(val)
            data.append(record)
                
        ob.data = data
        
    except Exception as e:
        raise Exception("Error while applying aggregate function on virtual procedures: %s" % (e))
Esempio n. 14
0
 def setDischargeCurves(self):
     "method for setting h-q tranformation tables/curves"       
     #set requested period
     #================================================
     hqFile = os.path.join(
                     self.filter.sosConfig.virtual_processes_folder,
                     self.filter.procedure[0],
                     self.filter.procedure[0]+".rcv"
             )
     tp=[]
     if self.filter.eventTime == None:
         tp = [None,None]
     else:
         for t in self.filter.eventTime:
             if len(t) == 2:
                 if t[0].find('+')==-1:
                     t[0] += "+00:00"
                 if t[1].find('+')==-1:
                     t[1] += "+00:00"    
                 tp.append(iso.parse_datetime(t[0]))
                 tp.append(iso.parse_datetime(t[1]))
             if len(t)==1:
                 if t[0].find('+')==-1:
                     t[0] += "+00:00"
                 tp.append(iso.parse_datetime(t[0]))
     period = (min(tp),max(tp))
     #get required parameters
     #==================================================
     try:        
         hq_fh = open(hqFile,'r')
     except Exception as e:
         raise Exception("Unable to open hq rating curve file at: %s" % hqFile)
     lines = hq_fh.readlines()
     #read header
     hqs = {'from':[],'to':[],'low':[],'up': [],'A':[],'B':[],'C':[],'K':[]}
     head = lines[0].strip().split("|")
     try:
         fromt = head.index('from')  #from time
         tot = head.index('to')      #to time
         low = head.index('low_val') #if value is bigger than
         up = head.index('up_val')   #and is lower than
         A = head.index('A')         #use this A
         B = head.index('B')         #use this B
         C = head.index('C')         #use this C
         K = head.index('K')         #use this K
     except Exception as e:
         raise Exception("setDischargeCurves: FILE %s ,%s error in header.\n %s" %(hqFile,head,e))
     
     #get equations
     if not period[0] == None:
         for l in range(1,len(lines)):
             line = lines[l].split("|")
             if iso.parse_datetime(line[1]) > period[0] or iso.parse_datetime(line[0]) <= period[1]:
                 hqs['from'].append(iso.parse_datetime(line[fromt]))
                 hqs['to'].append(iso.parse_datetime(line[tot]))
                 hqs['low'].append(float(line[low]))
                 hqs['up'].append(float(line[up]))
                 hqs['A'].append(float(line[A]))
                 hqs['B'].append(float(line[B]))
                 hqs['C'].append(float(line[C]))
                 hqs['K'].append(float(line[K]))
     else:
         for l in [-1,-2]:
             try:
                 line = lines[l].split("|")
                 hqs['from'].append(iso.parse_datetime(line[fromt]))
                 hqs['to'].append(iso.parse_datetime(line[tot]))
                 hqs['low'].append(float(line[low]))
                 hqs['up'].append(float(line[up]))
                 hqs['A'].append(float(line[A]))
                 hqs['B'].append(float(line[B]))
                 hqs['C'].append(float(line[C]))
                 hqs['K'].append(float(line[K]))
             except:
                 pass
     #raise sosException.SOSException(3,"%s" %(hqs))
     self.hqCurves = hqs
Esempio n. 15
0
 def getDSEndPosition(self):
     if u'constraint' in self.describe['outputs'][0]:
         return iso.parse_datetime(
             self.describe['outputs'][0]['constraint']['interval'][1])
     return None
Esempio n. 16
0
    def __init__(self,sosRequest,method,requestObject,sosConfig):
        f.sosFilter.__init__(self,sosRequest,method,requestObject,sosConfig)
        # @TODO Declare attribute first!
        # self.offering = None
        # etc..

        #**************************
        if method == "GET":
            #---------- THE OFFERING
            if requestObject.has_key("offering"):
                self.offering = get_name_from_urn(requestObject["offering"],"offering",sosConfig)
            else:
                raise sosException.SOSException(1,"Parameter \"offering\" is mandatory with multiplicity 1")
                
            #---------- THE OBSERVED PROPERTY
            if requestObject.has_key("observedProperty"):
                self.observedProperty = []
                oprs = requestObject["observedProperty"].split(",")
                for opr in oprs:
                    # get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
                    # eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
                    #oprName = get_name_from_urn(opr,"property")
                    oprName = opr
                    self.observedProperty.append(oprName) # one-many ID 
            else:
                raise sosException.SOSException(1,"Parameter \"observedProperty\" is mandatory with multiplicity N")
                
            #---------- RESPONSE FORMAT
            if requestObject.has_key("responseFormat"):
                if not requestObject["responseFormat"] in sosConfig.parameters["GO_responseFormat"]:   
                    raise sosException.SOSException(2,"Parameter \"responseFormat\" sent with invalid value : use one of %s" % "; ".join(sosConfig.parameters["GO_responseFormat"]))
                else:
                    self.responseFormat = requestObject["responseFormat"]
            else:
                raise sosException.SOSException(1,"Parameter \"responseFormat\" is mandatory with multiplicity 1") #one
                
            #OPTIONAL request parameters
            #---------- SRS FILTER                
            if requestObject.has_key("srsName"):
                self.srsName = get_name_from_urn(requestObject["srsName"],"refsystem",sosConfig)         
                if not self.srsName in sosConfig.parameters["GO_srs"]:
                    raise sosException.SOSException(2,"srsName \"%s\" not supported, use one of: %s" %(self.srsName,",".join(sosConfig.parameters["GO_srs"])))
            else:
                self.srsName = sosConfig.parameters["GO_srs"][0]

            #---------- TIME FILTER                
            if requestObject.has_key('eventTime'):
                self.eventTime = []
                for i in requestObject["eventTime"].replace(" ","+").split(","):
                    if len(i.split("/")) < 3:
                        self.eventTime.append(i.split("/"))
                    else:
                        raise sosException.SOSException(2,"Parameter \"eventTime\" bad formatted")
                
                tp=[]
                for t in self.eventTime:
                    if len(t) == 2:
                        tp.append(iso.parse_datetime(t[0]))
                        tp.append(iso.parse_datetime(t[1]))
                    if len(t)==1:
                        tp.append(iso.parse_datetime(t[0]))
                
                # Checking if some event limitation is reached
                #if sosConfig["maxGoPeriod"]:
                if int(sosConfig.maxGoPeriod) > 0:
                    from datetime import timedelta
                    d = timedelta(hours=int(sosConfig.maxGoPeriod))
                    userPeriod = max(tp)-min(tp)
                    if d < userPeriod:
                        raise sosException.SOSException(2,"You are requesting data for a period of [%s hours], but you are not permitted to ask for a period longer than: %s hours" % (userPeriod,d))
                
            else:
                self.eventTime = None
            
            #---------- PROCEDURES FILTER
            if requestObject.has_key("procedure"):
                self.procedure = []
                prcs = requestObject["procedure"].split(",")
                for prc in prcs:
                    prcName = get_name_from_urn(prc,"procedure",sosConfig)
                    self.procedure.append(prcName)
            else:
                self.procedure = None
            
            #---------- FEATURES OF INTEREST FILTER
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None
            if requestObject.has_key("featureOfInterest"):
                foi = requestObject["featureOfInterest"]
                if foi.find("<ogc:")>=0 and foi.find("<gml:")>=0:
                    #raise sosException.SOSException(3,"FOI SPATIAL: %s" %(foi))
                    self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(foi,'geom_foi',sosConfig.istsosepsg)
                else:
                    self.featureOfInterest = get_name_from_urn(foi,"feature",sosConfig)
                    
                #fois = requestObject["featureOfInterest"].split(",")
                #for foi in fois:
                #    foiName = get_name_from_urn(foi,"feature")
                #    self.featureOfInterest.append(foiName)
            
            #---------- FILTERS FOR QUERY NOT SUPPORTED YET            
            if requestObject.has_key("result"):
                #raise sosException.SOSException(3,"Parameter \"result\" not yet supported")
                self.result = sosUtils.ogcCompCons2PostgisSql(requestObject["result"])
            else:
                self.result = None #zero-one optional
            
            #---------- RESULT MODEL
            if requestObject.has_key("resultModel"):
                if requestObject["resultModel"] in sosConfig.parameters["GO_resultModel"]:
                    self.resultModel = requestObject["resultModel"]                    
                else:
                    raise sosException.SOSException(2,"Parameter \"resultModel\" sent with invalid value: supported values are: %s" %",".join(sosConfig.parameters["GO_resultModel"]))                    
            else:
                self.resultModel = sosConfig.parameters["GO_resultModel"][0]
            
            #---------- RESPONSE MODE
            if requestObject.has_key("responseMode"):
                if requestObject["responseMode"] in sosConfig.parameters["GO_responseMode"]:
                    self.responseMode = requestObject["responseMode"]
                else:
                    raise sosException.SOSException(2,"Parameter \"responseMode\" sent with invalid value, supported values are: %s" %(",".join(sosConfig.parameters["GO_responseMode"])))
                    
            else:
                self.responseMode = sosConfig.parameters["GO_responseMode"][0]

            ###########################
            # NON STANDARD PARAMETERS #
            ###########################
            #---------- AGGREGATE INTERVAL
            # In ISO 8601 duration format
            if requestObject.has_key("aggregateInterval"):
                # Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime)!=1 or len(self.eventTime[0])!=2:
                    raise sosException.SOSException(2,exeMsg)
                self.aggregate_interval = requestObject["aggregateInterval"]
                try:
                    iso.parse_duration(self.aggregate_interval)
                except Exception as ex:
                    raise sosException.SOSException(2,"Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s" % ex)
            else:
                self.aggregate_interval = None

            #---------- AGGREGATE FUNCTION
            # sum,avg,max,min
            if requestObject.has_key("aggregateFunction"):
                if self.aggregate_interval==None:
                    raise sosException.SOSException(2,"Using aggregate functions parameters \"aggregateInterval\" and \"aggregateFunction\" are both mandatory")
                self.aggregate_function = requestObject["aggregateFunction"]
                if not (self.aggregate_function.upper() in ["AVG","COUNT","MAX","MIN","SUM"]):
                    raise sosException.SOSException(2,"Available aggregation functions: avg, count, max, min, sum.")
            else:
                self.aggregate_function = None
                
            #---------- AGGREGATE NODATA
            if requestObject.has_key("aggregateNodata"):
                if self.aggregate_interval==None or self.aggregate_function==None:
                    raise sosException.SOSException(2,"Using aggregateNodata parameter requires both \"aggregateInterval\" and \"aggregateFunction\"")
                self.aggregate_nodata = requestObject["aggregateNodata"]
            else:
                self.aggregate_nodata = sosConfig.aggregate_nodata
                
            #---------- AGGREGATE NODATA QUALITY INDEX
            if requestObject.has_key("aggregateNodataQi"):
                if self.aggregate_interval==None or self.aggregate_function==None:
                    raise sosException.SOSException(2,"Using aggregateNodataQi parameter requires both \"aggregateInterval\" and \"aggregateFunction\"")
                self.aggregate_nodata_qi = requestObject["aggregateNodataQi"]
            else:
                self.aggregate_nodata_qi = sosConfig.aggregate_nodata_qi
                
            #------------ QUALITY INDEX
            self.qualityIndex=False
            if requestObject.has_key("qualityIndex"):
                if requestObject["qualityIndex"].upper() == "TRUE":
                    self.qualityIndex = True
                elif requestObject["qualityIndex"].upper() == "FALSE":
                    self.qualityIndex = False
                else:
                    raise sosException.SOSException(2,"qualityIndex can only be True or False!")
                #    self.qualityIndex = sosUtils.CQLvalueFilter2PostgisSql("id_qi_fk",requestObject["qualityIndex"])

                
                
        #**********************
        if method == "POST":
            from xml.dom import minidom
            #---------- THE OFFERING
            offs = requestObject.getElementsByTagName('offering')
            if len(offs) == 1:
                val = offs[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.offering = get_name_from_urn(str(val.data),"offering",sosConfig)
                else:
                    err_txt = "XML parsing error (get value: offering)"
                    raise sosException.SOSException(1,err_txt)
            else:
                err_txt = "Parameter \"offering\" is mandatory with multiplicity 1"
                raise sosException.SOSException(1,err_txt)
            
            
            #---------- THE OBSERVED PROPERTY
            obsProps = requestObject.getElementsByTagName('observedProperty')
            self.observedProperty = []
            if len(obsProps) > 0:
                for obsProp in obsProps:
                    val = obsProp.firstChild
                    if val.nodeType == val.TEXT_NODE:                    
                        # get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
                        # eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
                        #self.observedProperty.append(get_name_from_urn(str(val.data),"property"))
                        self.observedProperty.append(str(val.data))
                    else:
                        err_txt = "XML parsing error (get value: observedProperty)"
                        raise sosException.SOSException(1,err_txt)
            else:
                err_txt = "Parameter \"observedProperty\" is mandatory with multiplicity N"
                raise sosException.SOSException(1,err_txt)
            
            #---------- RESPONSE FORMAT
            respF = requestObject.getElementsByTagName('responseFormat')
            if len(respF) == 1:
                val = respF[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.responseFormat = str(val.data)
                    if self.responseFormat not in sosConfig.parameters["GO_responseFormat"]:   
                        raise sosException.SOSException(2,"Parameter \"responseFormat\" sent with invalid value: use one of %s" % "; ".join(sosConfig.parameters["GO_responseFormat"]))
                else:
                    err_txt = "XML parsing error (get value: responseFormat)"
                    raise sosException.SOSException(1,err_txt)
            else:
                err_txt = "Parameter \"responseFormat\" is mandatory with multiplicity 1"
                raise sosException.SOSException(1,err_txt)
            
            #OPTIONAL request parameters
            #---------- SRS OF RETURNED GML FEATURES
            srss = requestObject.getElementsByTagName('srsName')
            if len(srss) > 0:
                if len(srss) < 2:
                    val = srss[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.srsName = get_name_from_urn(str(val.data),"refsystem",sosConfig)
                    else:
                        err_txt = "XML parsing error (get value: srsName)"
                        raise sosException.SOSException(1,err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"srsName\""
                    raise sosException.SOSException(1,err_txt)
            else:
                self.srsName = sosConfig.parameters["GO_srs"][0]
            
            #---------- TIME FILTER  
            evtms = requestObject.getElementsByTagName('eventTime')
            self.eventTime = []
            if len(evtms) > 0:
                for evtm in evtms:
                    tps = evtm.getElementsByTagName('gml:TimePeriod')
                    for tp in tps:
                        begin = tp.getElementsByTagName('gml:beginPosition')
                        end = tp.getElementsByTagName('gml:endPosition')
                        if len(begin)==1 and len(end)==1:
                            Bval = begin[0].firstChild
                            Eval = end[0].firstChild
                            #raise sosException.SOSException(1,end[0].toprettyxml())
                            if Bval.nodeType == Bval.TEXT_NODE and Eval.nodeType == Eval.TEXT_NODE:
                                self.eventTime.append([str(Bval.data).replace(" ","+"),str(Eval.data).replace(" ","+")])
                                #raise sosException.SOSException(1,str(self.eventTime))
                            else:
                                err_txt = "XML parsing error (get value: TimePeriod)"
                                raise sosException.SOSException(1,err_txt)
                            
                    tis = evtm.getElementsByTagName('gml:TimeInstant')
                    for ti in tis:
                        instant = ti.getElementsByTagName('gml:timePosition')
                        if len(instant)>0 and len(instant)<2:
                            Ival = instant[0].firstChild
                            if Ival.nodeType == Ival.TEXT_NODE:
                                self.eventTime.append([str(Ival.data).replace(" ","+")])
                            else:
                                err_txt = "XML parsing error (get value: Timeinstant)"
                                raise sosException.SOSException(1,err_txt)
            else:
                self.eventTime = None
                
            #---------- PROCEDURES FILTER
            procs = requestObject.getElementsByTagName('procedure')
            if len(procs) > 0:
                self.procedure=[]
                for proc in procs:
                    if "xlink:href" in proc.attributes.keys():
                        self.procedure.append(str(proc.getAttribute("xlink:href")))
                    elif proc.hasChildNodes():
                        val = proc.firstChild
                        if val.nodeType == val.TEXT_NODE:
                            self.procedure.append(get_name_from_urn(str(val.data),"procedure",sosConfig))
                    else:
                        err_txt = "XML parsing error (get value: procedure)"
                        raise sosException.SOSException(1,err_txt)
            else:
                self.procedure = None
            
            #---------- FEATURES OF INTEREST FILTER
            fets = requestObject.getElementsByTagName('featureOfInterest')
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None
            if len(fets)>0:
                if len(fets)<2:
                    elements = [e for e in fets[0].childNodes if e.nodeType == e.ELEMENT_NODE]
                    if len(elements)==1:
                        self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(elements[0],'geom_foi',sosConfig.istsosepsg)
                    else:
                        if "xlink:href" in fets[0].attributes.keys():
                            self.featureOfInterest = str(fets[0].getAttribute("xlink:href"))
                        elif fets[0].hasChildNodes():
                            val = fets[0].firstChild
                            if val.nodeType == val.TEXT_NODE:
                                self.featureOfInterest = get_name_from_urn(str(val.data),"feature",sosConfig)
                        else:
                            err_txt = "XML parsing error (get value: featureOfInterest)"
                            raise sosException.SOSException(1,err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"featureOfInterest\""
                    raise sosException.SOSException(1,err_txt)
            
            #---------- FILTERS FOR QUERY NOT SUPPORTED YET            
            ress = requestObject.getElementsByTagName('result')
            if len(ress)>0:
                raise sosException.SOSException(3,"Parameter \"result\" not yet supported")
            else:
                self.result = None #zero-one optional
            
            #---------- RESULT MODEL
            mods = requestObject.getElementsByTagName('resultModel')
            if len(mods)>0:
                if len(mods)<2:
                    val = mods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.resultModel = str(val.data)
                        if self.resultModel not in sosConfig.parameters["GO_resultModel"]:
                            raise sosException.SOSException(2,"Parameter \"resultModel\" sent with invalid value")
                    else:
                        err_txt = "XML parsing error (get value: resultModel)"
                        raise sosException.SOSException(1,err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"resultModel\""
                    raise sosException.SOSException(1,err_txt)
            else:
                self.resultModel = None
            
            #---------- RESPONSE MODE
            rsmods = requestObject.getElementsByTagName('responseMode')
            if len(rsmods)>0:
                if len(rsmods)<2:
                    val = rsmods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.responseMode = str(val.data)
                        if self.responseMode not in sosConfig.parameters["GO_responseMode"]:
                            raise sosException.SOSException(2,"Parameter \"responseMode\" sent with invalid value")
                    else:
                        err_txt = "XML parsing error (get value: responseMode)"
                        raise sosException.SOSException(1,err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"responseMode\""
                    raise sosException.SOSException(1,err_txt)
            else:
                self.responseMode = sosConfig.parameters["GO_responseMode"][0]
        

            #-------------- AGGREGATE INTERVAL & FUNCTION
            self.aggregate_interval = None
            self.aggregate_function = None
            aggint = requestObject.getElementsByTagName('aggregateInterval')
            aggfun = requestObject.getElementsByTagName('aggregateFunction')
            aggnodata = requestObject.getElementsByTagName('aggregateNodata')
            
            if len(aggint)==1 and len(aggfun)==1:
                #-----------------------
                # -- aggregate_interval
                #-----------------------
                # Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime)!=1 or len(self.eventTime[0])!=2:
                    raise sosException.SOSException(2,exeMsg)
                val = aggint[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_interval = str(val.data)
                    try:
                        iso.parse_duration(self.aggregate_interval)
                    except Exception as ex:
                        raise sosException.SOSException(2,"Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s" % ex)
                else:
                    err_txt = "cannot get ISO8601 duration value in \"aggregateInterval\""
                    raise sosException.SOSException(1,err_txt)
                #-----------------------
                # -- aggregate_function
                #-----------------------
                val = aggfun[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_function = str(val.data)
                    if not (self.aggregate_function.upper() in ["AVG","COUNT","MAX","MIN","SUM"]):
                        raise sosException.SOSException(2,"Available aggregation functions: avg, count, max, min, sum.")
                
                
                #-----------------------------------
                # -- aggregate_no_data default value
                #-----------------------------------
                if len(aggnodata)==1:
                    val = aggnodata[0].firstChild
                    self.aggregate_nodata = str(val.data)
                else:
                    self.aggregate_nodata = sosConfig.aggregate_nodata

           #================================
           #MISSING AGGREGATE QUALITY INDEX
           #================================         
                    
            elif len(aggint)==0 and len(aggfun)==0:
                pass
            else:
                err_txt = "\"aggregateInterval\" and \"aggregate_function\" are both required with multiplicity 1"
                raise sosException.SOSException(1,err_txt)

            #------------ QUALITY INDEX
            self.qualityIndex=False
            qidx = requestObject.getElementsByTagName('qualityIndex')
            if len(qidx)>0:
                if len(qidx)<2:
                    val = qidx[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.qualityIndex = str(val.data)
                        if self.qualityIndex.upper() == "TRUE":
                            self.qualityIndex=True
                        elif self.qualityIndex.upper() == "FALSE":
                            pass
                        else:
                            raise sosException.SOSException(2,"qualityIndex can only be \'True\' or \'False\'")
            elif len(qidx)==0:
                pass
            else:
                err_txt = "\"qualityIndex\" is allowed with multiplicity 1 only"
                raise sosException.SOSException(1,err_txt)
Esempio n. 17
0
    def executePost(self, db=True):
        if self.procedurename is None:
            raise Exception(
                "POST action without procedure name not allowed")

        # Create data array
        data = self.waEnviron['wsgi_input'].split(";")

        # Assigned id always in the first position
        assignedid = data[0]

        if len(data) == 4:  # regular time series
            mode = self.MODE_REGULAR
            start = iso.parse_datetime(data[1])
            step = iso.parse_duration(data[2])
            tmp_data = []
            data = data[3].split("@")
            for idx in range(0, len(data)):
                tmp_data.append([
                    (start + (step * idx)).isoformat()
                ] + data[idx].split(","))
            data = tmp_data

        elif len(data) == 2:  # irregular time series
            mode = self.MODE_IRREGULAR
            data = [i.split(",") for i in data[1].split("@")]

        else:
            raise Exception(
                "Body content wrongly formatted. Please read the docs.")

        try:
            conn = databaseManager.PgDB(
                self.serviceconf.connection['user'],
                self.serviceconf.connection['password'],
                self.serviceconf.connection['dbname'],
                self.serviceconf.connection['host'],
                self.serviceconf.connection['port']
            )
            sql = """
                SELECT
                    procedures.id_prc,
                    proc_obs.id_pro,
                    proc_obs.constr_pro,
                    procedures.stime_prc,
                    procedures.etime_prc
                FROM
                    %s.procedures,
                    %s.proc_obs
                WHERE
                    proc_obs.id_prc_fk = procedures.id_prc
            """ % (self.servicename, self.servicename)
            sql += """
              AND
                assignedid_prc = %s
              ORDER BY
                proc_obs.id_pro ASC;
            """
            rows = conn.select(sql, (assignedid,))

            if len(rows) == 0:
                raise Exception(
                    "Procedure with aid %s not found." % assignedid)

            # check if procedure observations length is ok
            if len(rows) != (len(data[0])-1):
                raise Exception(
                    "Array length missmatch with procedures "
                    "observation number")

            insertEventTime = """
                INSERT INTO %s.event_time (id_prc_fk, time_eti)
            """ % (self.servicename)
            insertEventTime += """
                VALUES (%s, %s::TIMESTAMPTZ) RETURNING id_eti;
            """

            deleteEventTime = """
                DELETE FROM %s.event_time
            """ % (self.servicename)
            deleteEventTime += """
                WHERE id_prc_fk = %s
                AND time_eti = %s::TIMESTAMPTZ
            """

            insertMeasure = """
                INSERT INTO %s.measures(
                    id_eti_fk,
                    id_qi_fk,
                    id_pro_fk,
                    val_msr
                )
            """ % (self.servicename)
            insertMeasure += """
                VALUES (%s, 100, %s, %s);
            """

            updateBeginPosition = """
                UPDATE %s.procedures""" % (self.servicename)
            updateBeginPosition += """
                SET stime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
            """
            updateEndPosition = """
                UPDATE %s.procedures""" % (self.servicename)
            updateEndPosition += """
                SET etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
            """

            id_prc = rows[0][0]
            bp = rows[0][3]
            bpu = False
            ep = rows[0][4]
            epu = False

            for observation in data:
                id_eti = conn.executeInTransaction(
                    insertEventTime, (
                        id_prc, observation[0]))

                for idx in range(0, len(rows)):
                    conn.executeInTransaction(
                        insertMeasure, (
                            int(id_eti[0][0]),  # id_eti
                            int(rows[idx][1]),  # id_pro
                            float(observation[(idx+1)])))

                if (bp is None) or (bp == '') or (
                        iso.parse_datetime(observation[0]) < bp):
                    bp = iso.parse_datetime(observation[0])
                    bpu = True

                if (ep is None) or (ep == '') or (
                        iso.parse_datetime(observation[0]) > ep):
                    ep = iso.parse_datetime(observation[0])
                    epu = True

            if bpu:
                conn.executeInTransaction(
                    updateBeginPosition, (bp.isoformat(), id_prc))

            if epu:
                conn.executeInTransaction(
                    updateEndPosition, (ep.isoformat(), id_prc))

            conn.commitTransaction()

            # self.setData(ret)
            self.setMessage("Thanks for data")

        except Exception as e:
            traceback.print_exc(file=sys.stderr)
            conn.rollbackTransaction()
            raise Exception(
                "Error in fast insert (%s): %s" % (type(e), e))
Esempio n. 18
0
    def __init__(self, filter, pgdb):

        # get procedure information
        sql = """
            SELECT
                id_prc,
                name_prc,
                name_oty,
                name_foi,
                stime_prc,
                etime_prc
            FROM
                %s.procedures,
                %s.obs_type,
                %s.foi""" % (filter.sosConfig.schema, filter.sosConfig.schema,
                             filter.sosConfig.schema)
        sql += """
            WHERE
                id_oty=id_oty_fk
            AND
                id_foi=id_foi_fk
            AND
                assignedid_prc=%s"""

        params = (filter.assignedSensorId, )
        try:
            prc = pgdb.select(sql, params)[0]
        except:
            raise sosException.SOSException(
                "InvalidParameterValue", "assignedSensorId",
                "assignedSensorId '%s' is not valid!" %
                (filter.assignedSensorId))

        # check requested procedure name exists
        if not prc["name_prc"] == filter.procedure:
            raise sosException.SOSException(
                "NoApplicableCode", None,
                "procedure '%s' not associated with provided "
                "assignedSensorId!" % (filter.procedure))

        # check requested  foi name exists
        if not filter.foiName == prc["name_foi"]:
            raise sosException.SOSException(
                "NoApplicableCode", None,
                "featureOfInterest '%s' not associated with "
                "provided assignedSensorId" % (filter.foiName))

        # check provided samplingTime and upadate
        #  begin/end time procedure if necessary
        #  (samplingTime=period or istant of provided
        #  observations defined by samplingTime filter)
        #=============================================
        if filter.samplingTime:
            stime = filter.samplingTime.split("/")
            #
            if len(stime) == 2:  # is a TimePeriod
                start = iso.parse_datetime(stime[0])
                end = iso.parse_datetime(stime[1])

            elif len(stime) == 1:  # is a TimeInstant
                start = end = iso.parse_datetime(stime[0])

            else:
                raise Exception("filter samplingTime error! given '%s'" %
                                (filter.samplingTime))

            if start > end:
                raise Exception(
                    "endPosition (%s) must be after beginPosition (%s)" %
                    (end, start))

            # check samplingTime
            #  > verify procedure begin/end exist
            if not (prc["stime_prc"].__class__.__name__ == "NoneType" and
                    (prc["etime_prc"].__class__.__name__ == "NoneType")):

                # check eventTime interval and update begin/end position when
                #  force flas is active
                if filter.forceInsert:
                    # update begin time of procedure
                    if start < prc["stime_prc"]:
                        sql = """
                            UPDATE
                                %s.procedures""" % (filter.sosConfig.schema)
                        sql += """
                            SET
                                stime_prc=%s::TIMESTAMPTZ
                            WHERE
                                id_prc=%s"""
                        params = (stime[0], prc["id_prc"])
                        try:
                            a = pgdb.executeInTransaction(sql, params)
                            com = True

                        except:
                            raise Exception("SQL: %s" %
                                            (pgdb.mogrify(sql, params)))

                    # update end time of procedure
                    if end > prc["etime_prc"]:
                        sql = """
                            UPDATE
                                %s.procedures""" % (filter.sosConfig.schema)
                        sql += """
                            SET
                                etime_prc=%s::TIMESTAMPTZ
                            WHERE
                                id_prc=%s"""
                        params = (stime[1], prc["id_prc"])
                        try:
                            b = pgdb.executeInTransaction(sql, params)
                            com = True

                        except Exception as err:
                            raise Exception(
                                "SQL: %s - %s" %
                                (pgdb.mogrify(sql, params), err.pgerror))

                # check eventTime interval and update begin/end position when
                #  force flag is off
                else:
                    if filter.sosConfig.sequential:
                        sql = """
                            SELECT
                                max(time_eti) as max_time_eti
                            FROM
                                %s.event_time""" % (filter.sosConfig.schema)
                        sql += """
                            WHERE
                                id_prc_fk = %s
                            GROUP BY
                                id_prc_fk"""
                        params = (prc["id_prc"], )
                        try:
                            lastMsr = pgdb.select(sql,
                                                  params)[0]["max_time_eti"]
                        except:
                            lastMsr = None

                        if lastMsr is not None:
                            # verify begin observation is minor/equal then end
                            #  time procedure and later then last observation
                            if not (end >= prc["etime_prc"] and
                                    (start <= prc["etime_prc"]) and
                                    (start >= lastMsr)):
                                raise Exception(
                                    "begin observation (%s) must be between last "
                                    "observation (%s) and end procedure (%s); end"
                                    " observation (%s) must be after end "
                                    "procedure (%s)" %
                                    (start, lastMsr, prc["etime_prc"], end,
                                     prc["etime_prc"]))
                        else:
                            # verify begin observation is minor/equal then end
                            #  time procedure and later then first observation
                            if not (end >= prc["etime_prc"] and
                                    (start <= prc["etime_prc"]) and
                                    (start >= prc["stime_prc"])):
                                raise Exception(
                                    "begin observation (%s) must be between start "
                                    "procedure (%s) and end procedure (%s); end "
                                    "observation (%s) must be after end procedure "
                                    "(%s)" %
                                    (start, prc["stime_prc"], prc["etime_prc"],
                                     end, prc["etime_prc"]))

                    #-- update end time of procedure
                    sql = """
                        UPDATE
                            %s.procedures""" % (filter.sosConfig.schema)
                    sql += """
                        SET
                            etime_prc=%s::TIMESTAMPTZ
                        WHERE
                            id_prc=%s"""
                    params = (str(stime[1]), int(prc["id_prc"]))
                    try:
                        b = pgdb.executeInTransaction(sql, params)
                        com = True
                    except Exception as err:
                        raise Exception(
                            "SQL: %s - %s" %
                            (pgdb.mogrify(sql, params), err.pgerror))

            else:
                sql = """
                    UPDATE
                        %s.procedures""" % (filter.sosConfig.schema)
                sql += """
                    SET
                        stime_prc=%s::TIMESTAMPTZ,
                        etime_prc=%s::TIMESTAMPTZ
                    WHERE
                        id_prc=%s"""
                params = (str(stime[0]), str(stime[1]), int(prc["id_prc"]))
                try:
                    b = pgdb.executeInTransaction(sql, params)
                    com = True
                except:
                    raise Exception("SQL: %s" % (pgdb.mogrify(sql, params)))

        # check data definition and uom (compare registered
        #  observed properties with provided observations)
        #==================================================
        # get values for provided data: UOM, NAME, URN, ID
        #--------------------------------------------------
        sql = """
            SELECT
                id_pro,
                id_opr,
                def_opr,
                name_uom,
                constr_opr,
                constr_pro
            FROM
                %s.observed_properties,
                %s.proc_obs,
                %s.uoms""" % (filter.sosConfig.schema, filter.sosConfig.schema,
                              filter.sosConfig.schema)
        sql += """
            WHERE
                id_uom_fk = id_uom
            AND
                id_opr_fk = id_opr
            AND
                id_prc_fk = %s"""
        params = (prc["id_prc"], )
        try:
            opr = pgdb.select(sql, params)
        except Exception as err:
            raise Exception("SQL2: %s -%s" %
                            (pgdb.mogrify(sql, params), err.pgerror))

        # get list of available ObservedProperty, unit of measure, property
        #  id for this procedure
        oprNames = []
        oprUoms = []
        oprIds = []  # to be removed ????
        proIds = []
        obsPropConstr = []
        procConstr = []
        '''
        Building a matrix:

        oprNames = ["urn:ogc:def:parameter:x-istsos:1.0:" +
                    "meteo:air:temperature" , ...]
        oprUoms = ["mm" , ...]
        oprIds = [id_opr , ...]
        proIds = [id_pro , ...]
        obsPropConstr = [{
            "interval": ["-40", "50"],
            "role": "urn:ogc:def:classifiers:x-istsos:1.0:" +
                    "qualityIndex:check:acceptable"} , ...]
        procConstr = [{
            "max": "100",
            "role": "urn:ogc:def:classifiers:x-istsos:1.0:" +
                    "qualityIndex:check:reasonable"} , ...]
        '''

        for row in opr:

            oprNames.append(row["def_opr"])
            oprUoms.append(row["name_uom"])
            oprIds.append(row["id_opr"])
            proIds.append(row["id_pro"])

            if not row["constr_opr"] in [None, '']:
                obsPropConstr.append(json.loads(row["constr_opr"]))
            else:
                obsPropConstr.append(None)

            if not row["constr_pro"] in [None, '']:
                procConstr.append(json.loads(row["constr_pro"]))
            else:
                procConstr.append(None)

        # get ordered list of observed properties in data----
        dataKeys = [key for key in filter.data.keys()]

        # get ordered list of unit of measures provided with data-------
        dataUoms = []
        for key in filter.data.keys():
            if "uom" in filter.data[key].keys():
                dataUoms.append(filter.data[key]["uom"])
            else:
                dataUoms.append('None')

        # verify that all the properties observed by this procedure
        #  are provided with the correct data definition and uom
        for i, opr in enumerate(oprNames):
            try:
                k = dataKeys.index(opr)
            except:
                raise sosException.SOSException(
                    "NoApplicableCode", None,
                    "parameter '%s' not observed by RegisteredSensor "
                    "%s - %s" % (opr, oprNames, dataKeys))

            if not dataUoms[k] == oprUoms[i]:
                raise sosException.SOSException(
                    "NoApplicableCode", None,
                    "parameter '%s' not observed with provided unit of "
                    "measure" % (opr))

        # verify if time and coordinates are passed as data parameters
        #  and create the parameters list and parameters ID
        xobs = yobs = zobs = tpar = None
        pars = []  # Observed parameters
        parsId = []
        parsConsObs = []
        parsConsPro = []

        # urn of different parameters
        for i, dn in enumerate(dataKeys):
            if dn.split(":")[-1] in filter.sosConfig.parGeom["x"]:
                xobs = dataKeys[i]
            elif dn.split(":")[-1] in filter.sosConfig.parGeom["y"]:
                yobs = dataKeys[i]
            elif dn.split(":")[-1] in filter.sosConfig.parGeom["z"]:
                zobs = dataKeys[i]
            elif dn.find("iso8601") >= 0:
                tpar = dataKeys[i]
            else:
                if dn.split(":")[-1] != "qualityIndex":
                    pars.append(dn)
                    try:
                        parsId.append(proIds[oprNames.index(dn)])
                        parsConsObs.append(obsPropConstr[oprNames.index(dn)])
                        parsConsPro.append(procConstr[oprNames.index(dn)])
                    except:
                        raise Exception(
                            "parameter %s not observed by this sensor "
                            "%s - %s" % (dn, pars, oprNames))

        # set default quality index if not provided
        for par in pars:
            try:
                dataKeys.index(par + ":qualityIndex")
            except:
                filter.data[par + ":qualityIndex"] = {
                    "vals": [filter.sosConfig.default_qi] *
                    len(filter.data[par]["vals"])
                }

        # verify that mobile sensors provide coordinates as X,Y,Z
        if (xobs is False and yobs is False and
                zobs is False) and (prc["name_oty"] == "insitu-mobile-point"):
            raise Exception("Mobile sensors require x, y, z parameters")

        # verify that time parameter is provided
        if not tpar:
            raise Exception(
                "parameter 'time:iso8601' is required for InsertObservation")

        # verify that eventime are in provided samplingTime
        if len(filter.data[tpar]["vals"]) > 0:
            maxDate = iso.parse_datetime(max(filter.data[tpar]["vals"]))
            minDate = iso.parse_datetime(min(filter.data[tpar]["vals"]))
            if not maxDate <= end and minDate >= start:
                raise Exception(
                    "provided data (min: %s, max:%s) are not included in "
                    "provided <samplingTime> period (%s / %s) for "
                    "procedure %s" %
                    (minDate.isoformat(), maxDate.isoformat(),
                     start.isoformat(), end.isoformat(), prc["name_prc"]))

        # insert observation
        #  delete existing observations if force flag is active
        if filter.forceInsert:
            sql = """
                DELETE FROM
                    %s.event_time""" % (filter.sosConfig.schema)
            sql += """
                WHERE
                    id_prc_fk = %s
                AND
                    time_eti >= %s::TIMESTAMPTZ
                AND
                    time_eti <= %s::TIMESTAMPTZ"""
            params = (prc["id_prc"], stime[0], stime[1])
            try:
                b = pgdb.executeInTransaction(sql, params)
                com = True
            except:
                raise Exception("SQL: %s" % (pgdb.mogrify(sql, params)))

        # CASE I: observations list is void
        if len(filter.data[tpar]["vals"]) == 0:
            self.assignedId = ""
            ids_eti = []

        # CASE I: observations list contains data
        elif len(filter.data[tpar]["vals"]) > 0:
            # insert event times
            ids_eti = []
            params = []
            sql = """
                INSERT INTO
                    %s.event_time (id_prc_fk,time_eti)""" % (
                filter.sosConfig.schema)
            sql += """
                VALUES (
                    %s, %s::TIMESTAMPTZ)
                RETURNING
                    id_eti"""
            for val in filter.data[tpar]["vals"]:
                try:
                    ids_eti.append(
                        pgdb.executeInTransaction(
                            sql, (prc["id_prc"], val))[0]['id_eti'])
                    com = True
                except Exception as e:
                    raise Exception("Error inserting event times for %s: %s" %
                                    (prc["name_prc"], e))

            for i, par in enumerate(pars):
                params = []
                ids_msr = []
                sql = """
                    INSERT INTO %s.measures (
                        id_pro_fk,
                        id_eti_fk,
                        id_qi_fk,
                        val_msr)""" % (filter.sosConfig.schema)
                sql += """
                    VALUES
                        (%s, %s, %s, %s)
                    RETURNING
                        id_msr"""

                pco = parsConsObs[i]
                pcp = parsConsPro[i]
                for ii, id_et in enumerate(ids_eti):
                    if not filter.data[par]["vals"][ii] in [
                            'NULL', u'NULL', None, 'None', u'None',
                            filter.sosConfig.aggregate_nodata
                    ]:
                        # TODO: add an else statement to add the
                        #  aggregate_nodata value OR delete the event time if
                        #  not filter.data[par]["vals"][ii] in ['NULL',u'NULL',
                        #  None]:
                        pqi = int(
                            float(filter.data[par +
                                              ":qualityIndex"]["vals"][ii]))
                        # Constraint quality is done only if the quality index
                        #  is equal to the default qi (RAW DATA)
                        if int(filter.sosConfig.default_qi) == pqi:
                            # quality check level I (gross error)
                            val = float(filter.data[par]["vals"][ii])
                            if filter.sosConfig.correct_qi is not None and (
                                    pco is not None):
                                if 'max' in pco:
                                    if val <= (float(pco['max'])):
                                        pqi = int(filter.sosConfig.correct_qi)

                                elif 'min' in pco:
                                    if val >= (float(pco['min'])):
                                        pqi = int(filter.sosConfig.correct_qi)

                                elif 'interval' in pco:
                                    if (float(pco['interval'][0]) <= val <=
                                            float(pco['interval'][1])):
                                        pqi = int(filter.sosConfig.correct_qi)

                                elif 'valueList' in pco:
                                    if val in [
                                            float(p)
                                            for p in (pco['valueList'])
                                    ]:
                                        pqi = int(filter.sosConfig.correct_qi)

                            # quality check level II (statistical range)
                            if filter.sosConfig.stat_qi is not None and (
                                    pcp is not None):
                                if 'max' in pcp:
                                    if val <= float(pcp['max']):
                                        pqi = int(filter.sosConfig.stat_qi)

                                elif 'min' in pcp:
                                    if val >= float(pcp['min']):
                                        pqi = int(filter.sosConfig.stat_qi)

                                elif 'interval' in pcp:
                                    if (float(pcp['interval'][0]) <= val <=
                                            float(pcp['interval'][1])):
                                        pqi = int(filter.sosConfig.stat_qi)

                                elif 'valueList' in pcp:
                                    if val in [
                                            float(p) for p in pcp['valueList']
                                    ]:
                                        pqi = int(filter.sosConfig.stat_qi)

                        params = (int(parsId[i]), int(id_et), pqi,
                                  float(filter.data[par]["vals"][ii]))
                        try:
                            nid_msr = pgdb.executeInTransaction(sql, params)
                            ids_msr.append(str(nid_msr[0]['id_msr']))
                        except Exception as e:
                            com = False
                            raise e

            #--insert position values if required
            if prc["name_oty"] == "insitu-mobile-point":
                xparspl = xobs.split(":")
                epsg = xparspl[xparspl.index("EPSG") + 1]
                params = []
                sql = """
                    INSERT INTO %s.positions (
                        id_qi_fk,
                        id_eti_fk,
                        geom_pos)""" % (filter.sosConfig.schema)
                sql += """
                    VALUES (
                        %s, %s,
                        ST_Transform(
                            ST_SetSRID(ST_MakePoint(%s, %s, %s), %s), %s))"""

                for i, id_et in enumerate(ids_eti):
                    params = (filter.sosConfig.default_qi, id_et,
                              filter.data[xobs]["vals"][i],
                              filter.data[yobs]["vals"][i],
                              filter.data[zobs]["vals"][i], epsg,
                              filter.sosConfig.istsosepsg)
                    try:
                        ids_pos = pgdb.executeInTransaction(sql, params)
                        com = True
                    except Exception as a:
                        com = False
                        raise Exception("%s\nSQL: %s" %
                                        (a, pgdb.mogrify(sql, params)))

            # register assigned IDs of measures
            self.assignedId = "@".join([str(p) for p in ids_eti])
            # commit executed operations

        #Register the transactional operation in Log table
        if filter.sosConfig.transactional_log in ['True', 'true', 1]:
            sqlLog = """
                INSERT INTO %s.tran_log (
                    operation_trl,
                    procedure_trl,
                    begin_trl,
                    end_trl,
                    count,
                    stime_prc,
                    etime_prc)""" % (filter.sosConfig.schema)
            sqlLog += """
                VALUES (
                    'InsertObservation',
                    %s,
                    %s::TIMESTAMPTZ,
                    %s::TIMESTAMPTZ,
                    %s,
                    %s::TIMESTAMPTZ,
                    %s::TIMESTAMPTZ)"""
            params = (str(filter.procedure), start, end, len(ids_eti),
                      prc["stime_prc"], prc["etime_prc"])
            try:
                pgdb.executeInTransaction(sqlLog, params)
                com = True
            except:
                raise Exception("SQL: %s" % (pgdb.mogrify(sqlLog, params)))

        if com is True:
            pgdb.commitTransaction()
            # broadcasting to mqtt broker if configured
            if filter.sosConfig.mqtt["broker_url"] != '' and (
                    filter.sosConfig.mqtt["broker_port"] != ''):
                from istmqttlib import PahoPublisher
                PahoPublisher({
                    "broker_url":
                    filter.sosConfig.mqtt["broker_url"],
                    "broker_port":
                    filter.sosConfig.mqtt["broker_port"],
                    "broker_topic":
                    "%s%s" %
                    (filter.sosConfig.mqtt["broker_topic"], filter.procedure),
                    "data":
                    filter.dataArray
                }).start()
Esempio n. 19
0
     op = go1['observedProperty']['component'][1]
 
 print "Gathering info for %s from destination" % procedure
 go2 = sos2.getSOSProcedure(procedure)
 samp2 = sos2.extractSamplingFromGOJson(go2)
 
 """interval = timedelta(days=int(30))
 begin = yearsago(1, samp1[1]) #samp1[0]
 if begin < samp1[0]:
     begin = samp1[0]"""
 
 begin = samp1[0]
 end = samp1[1]
 
 # Comment this if you want a full migration
 begin = isodate.parse_datetime('2015-04-29T07:30:00+00:00')
 end   = isodate.parse_datetime('2015-04-29T08:10:00+00:00')
 
 istsos2csv.execute({
     'begin': begin.isoformat(),
     'end': end.isoformat(),
     'procedure': procedure,
     'op': op,
     'url': '%s/%s' % (url1, srv1),
     'd': dirpath,
     'user': auth[0],
     'password': auth[1],
     'noqi': True
 })
 
 csv2istsos.execute({
Esempio n. 20
0
def execute(args, logger=None):
    def log(message):
        if debug:
            if logger:
                logger.log(message)
            else:
                print message

    # SCRIPT CONFIGURATION
    # =========================================================================

    # Activate and print verbose information
    debug = args['v'] if args.has_key('v') else False

    # Procedure name
    procedure = args['procedure']
    # Begin date
    begin = args['begin'] if args.has_key('begin') else "*"
    # End date
    end = args['end'] if args.has_key('end') else "*"
    # Global User and password valid for all connections
    suser = duser = auser = args['user'] if args.has_key('user') else None
    spwd = dpwd = apwd = args['pwd'] if args.has_key('pwd') else None

    # Activate this will copy also the quality index from source to destination
    cpqi = args['cpqi'] if args.has_key('cpqi') else False

    # Aggregating function configuration
    resolution = args['resolution'] if 'resolution' in args else None
    function = args['function'] if 'function' in args else None
    nodataValue = args['nodataValue'] if 'nodataValue' in args else None
    nodataQI = args['nodataQI'] if 'nodataQI' in args else None

    # Retroactive aggregation
    retro = args['retro'] if 'retro' in args else 0

    # Force using last position as end position during insert sensor operation
    lm = args['lm'] if 'lm' in args else False

    # SOURCE istSOS CONFIG ==================================
    # Location
    surl = args['surl']
    # Service instance name
    ssrv = args['ssrv']
    # User and password if given this will be used for source istSOS
    if args.has_key('suser'):
        suser = args['suser']
    if args.has_key('spwd'):
        spwd = args['spwd']

    # DESTINATION istSOS CONFIG =============================
    # Location (if not given, same as source will be used)
    durl = args['durl'] if (args.has_key('durl')
                            and args['durl'] is not None) else surl
    # Service instance name
    dsrv = args['dsrv']
    # User and password if given this will be used for destination istSOS
    if args.has_key('duser'):
        duser = args['duser']
    if args.has_key('dpwd'):
        dpwd = args['dpwd']

    # ALTERNATIVE istSOS SERVICE FOR QI EXTRAPOLATION =======
    # Location (if not given, same as source will be used)
    aurl = args['aurl'] if (args.has_key('aurl')
                            and args['aurl'] is not None) else None
    # Service instance name
    asrv = args['asrv'] if (args.has_key('asrv')
                            and args['asrv'] is not None) else None
    # User and password if given this will be used for extrapolation QI istSOS
    if args.has_key('auser'):
        auser = args['auser']
    if args.has_key('apwd'):
        apwd = args['apwd']

    # PROCESSING STARTS HERE ==================================================

    log("\nistSOS > 2 > istSOS STARTED:")
    log("==============================\n")

    #req = requests.session()
    req = requests

    # Load procedure description
    log("1. Loading procedure description: %s" % procedure)

    # Loading describe sensor from source =====================================
    res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                  (surl, ssrv, procedure),
                  auth=(suser, spwd),
                  verify=False)
    sdata = res.json()
    if sdata['success'] == False:
        raise Exception(
            "Description of procedure %s can not be loaded from source service: %s"
            % (procedure, sdata['message']))
    else:
        log("   > DS Source Ok.")

    # Loading describe sensor from destination ================================
    res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                  (durl, dsrv, procedure),
                  auth=(duser, dpwd),
                  verify=False)
    ddata = res.json()
    if ddata['success'] == False:
        raise Exception(
            "Description of procedure %s can not be loaded from destination service: %s"
            % (procedure, ddata['message']))
    else:
        log("   > DS Destination Ok.")

    # Load of a getobservation template from destination =======================================
    res = req.get(
        "%s/wa/istsos/services/%s/operations/getobservation/offerings/%s/procedures/%s/observedproperties/:/eventtime/last?qualityIndex=False"
        % (durl, dsrv, 'temporary', procedure),
        params={"qualityIndex": cpqi},
        auth=(duser, dpwd),
        verify=False)
    dtemplate = res.json()
    if dtemplate['success'] == False:
        raise Exception(
            "Observation template of procedure %s can not be loaded: %s" %
            (procedure, dtemplate['message']))
    else:
        dtemplate = dtemplate['data'][0]
        dtemplate['AssignedSensorId'] = ddata['data']['assignedSensorId']
        dtemplate['result']['DataArray']['values'] = []
        log("     > GO Template Ok.")

    # Loading describe sensor from QI EXTRAPOLATION service ===================
    if aurl and asrv:
        res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                      (aurl, asrv, procedure),
                      auth=(auser, apwd),
                      verify=False)
        adata = res.json()
        if adata['success'] == False:
            raise Exception(
                "Description of procedure %s can not be loaded from destination service: %s"
                % (procedure, adata['message']))
        else:
            log("   > DS QI Extrapolation Ok.")

    log("\n2. Identifying processing interval:")

    # Check if mesaures are present in source procedure, by identifying the sampling time constraint
    #  located always in the first position of the outputs, if it is empty an exception is thrown
    if (not 'constraint' in sdata['data']['outputs'][0]
            or not 'interval' in sdata['data']['outputs'][0]['constraint']):
        raise Exception(
            "There is no data in the source procedure to be copied to the destination procedure."
        )
    else:
        # Check if the contraint interval contains a valid ISO date begin position
        try:
            iso.parse_datetime(
                sdata['data']['outputs'][0]['constraint']['interval'][0])
        except Exception:
            raise Exception(
                "The date in the source procedure constraint interval (%s) is not valid."
                % sdata['data']['outputs'][0]['constraint']['interval'][0])

        # Check if the contraint interval contains a valid ISO date end position
        try:
            iso.parse_datetime(
                sdata['data']['outputs'][0]['constraint']['interval'][1])
        except Exception:
            raise Exception(
                "The date in the source procedure constraint interval (%s) is not valid."
                % sdata['data']['outputs'][0]['constraint']['interval'][1])

    log("   > Source interval is valid")

    # Looking for start (IO beginPOsition) instant processing
    #   If the default value (*) is used, then the endPosition of
    #   the "destination" service procedure will be used. But if the destination
    # procedure is empty , then the begin position of the source will be used
    start = None
    stop = None
    if begin == "*":
        if ('constraint' in ddata['data']['outputs'][0]
                and 'interval' in ddata['data']['outputs'][0]['constraint']):
            try:
                if function and resolution:
                    # getting last inserted observations of "destination" service
                    log("Aggregation requested: getting last inserted observations of \"destination\" service"
                        )
                    params = {
                        "request": "GetObservation",
                        "service": "SOS",
                        "version": "1.0.0",
                        "observedProperty": ':',
                        "procedure": procedure,
                        "responseFormat": "application/json",
                        "offering": 'temporary'
                    }
                    res = req.get("%s/%s" % (durl, dsrv),
                                  params=params,
                                  auth=(duser, dpwd),
                                  verify=False)
                    obs = res.json()
                    start = iso.parse_datetime(
                        obs['ObservationCollection']['member'][0]['result']
                        ['DataArray']['values'][0][0])
                else:
                    # The endPosition of the destination will be used as Start/IO BeginPosition
                    start = iso.parse_datetime(ddata['data']['outputs'][0]
                                               ['constraint']['interval'][1])
                if retro > 0:  # Retroactive aggregation
                    log("Retroactive aggregation active.")
                    if start - timedelta(minutes=retro) > iso.parse_datetime(
                            ddata['data']['outputs'][0]['constraint']
                        ['interval'][0]):
                        start = start - timedelta(minutes=retro)
                    else:
                        start = iso.parse_datetime(
                            ddata['data']['outputs'][0]['constraint']
                            ['interval'][0])

                log("Start: %s" % start)

            except Exception as ee:
                print "Error setting start date for proc %s: %s" % (procedure,
                                                                    ee)
                raise Exception(
                    "The date in the destination procedure %s constraint interval (%s) is not valid."
                    %
                    (procedure,
                     ddata['data']['outputs'][0]['constraint']['interval'][0]))
        else:
            # The beginPosition of the source will be used as Start/IO BeginPosition
            start = iso.parse_datetime(
                sdata['data']['outputs'][0]['constraint']['interval'][0])
    else:
        start = iso.parse_datetime(begin)

    if end == "*":
        # The endPosition of the source will be used as Stop/IO EndPosition
        stop = iso.parse_datetime(
            sdata['data']['outputs'][0]['constraint']['interval'][1])
    else:
        stop = iso.parse_datetime(end)

    log("   > Destination interval is valid")
    log("   > Start processing: %s" % start)
    log("   > Stop processing: %s" % stop)
    if retro > 0:
        log("   > Retro aggregation: %s minutes" % retro)

    # Insertion loop step timedelta
    interval = timedelta(days=15)
    if start < stop and start + interval > stop:
        interval = stop - start

    log("   > Insertion loop step: %s" % interval)

    if function and resolution:
        try:
            iso.duration_isoformat(resolution)
        except:
            raise Exception(
                "The resolution (%s) to apply in the aggregating function is not valid."
                % resolution)
        log("   > Function(Resolution) : %s(%s)" % (function, resolution))

    while start + interval <= stop:

        nextStart = start + interval

        params = {
            "request": "GetObservation",
            "service": "SOS",
            "version": "1.0.0",
            "observedProperty": ':',
            "procedure": procedure,
            "qualityIndex": str(cpqi),
            "responseFormat": "application/json",
            "offering": 'temporary',
            "eventTime": "%s/%s" % (start.isoformat(), nextStart.isoformat())
        }

        if function and resolution:

            params['aggregateFunction'] = function
            params['aggregateInterval'] = resolution

            if nodataValue != None:
                params['aggregateNodata'] = nodataValue
            if nodataQI != None:
                params['aggregateNodataQi'] = nodataQI

        res = req.get("%s/%s" % (surl, ssrv),
                      params=params,
                      auth=(suser, spwd),
                      verify=False)

        # Check if an Exception occured
        if 'ExceptionReport' in res.content:
            raise Exception(res.content)

        smeasures = res.json()['ObservationCollection']['member'][0]
        #pp.pprint(smeasures)

        log("   > %s measures from: %s to: %s" %
            (len(smeasures['result']['DataArray']['values']),
             start.isoformat(), nextStart.isoformat()))

        dtemplate["samplingTime"] = {}
        if lm and len(smeasures['result']['DataArray']['values']) > 0:
            dtemplate["samplingTime"]["beginPosition"] = smeasures['result'][
                'DataArray']['values'][0][0]
            dtemplate["samplingTime"]["endPosition"] = smeasures['result'][
                'DataArray']['values'][-1][0]
        else:
            dtemplate["samplingTime"]["beginPosition"] = start.isoformat()
            dtemplate["samplingTime"]["endPosition"] = nextStart.isoformat()

        dtemplate['result']['DataArray']['values'] = smeasures['result'][
            'DataArray']['values']
        dtemplate['result']['DataArray']['field'] = smeasures['result'][
            'DataArray']['field']

        # POST data to WA
        res = req.post(
            "%s/wa/istsos/services/%s/operations/insertobservation" %
            (durl, dsrv),
            auth=(duser, dpwd),
            verify=False,
            data=json.dumps({
                "ForceInsert":
                "true",
                "AssignedSensorId":
                ddata['data']['assignedSensorId'],
                "Observation":
                dtemplate
            }))

        # read response
        log("     > Insert observation success: %s" % res.json()['success'])

        #print res.json()

        if not res.json()['success']:
            raise Exception('Error inserting observation: %s' %
                            res.json()['message'])

        start = nextStart
        if start < stop and start + interval > stop:
            interval = stop - start
Esempio n. 21
0
    def __init__(self,filter,pgdb):

        #--get procedure information
        #============================================
        sql  = "SELECT id_prc, name_prc, name_oty, name_foi, stime_prc, etime_prc from %s.procedures,%s.obs_type,%s.foi" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        sql += " WHERE id_oty=id_oty_fk AND id_foi=id_foi_fk AND assignedid_prc=%s"
        params = (filter.assignedSensorId,)
        try:
            prc = pgdb.select(sql,params)[0]
        except:
            raise sosException.SOSException("InvalidParameterValue","assignedSensorId","assignedSensorId '%s' is not valid!" %(filter.assignedSensorId))

        #--check requested procedure name exists
        #=============================================
        if not prc["name_prc"]==filter.procedure:
            raise sosException.SOSException("NoApplicableCode",None,"procedure '%s' not associated with provided assignedSensorId!" %(filter.procedure))

        #--check requested  foi name exists
        #=============================================
        if not filter.foiName == prc["name_foi"]:
            raise sosException.SOSException("NoApplicableCode",None,"featureOfInterest '%s' not associated with provided assignedSensorId" %(filter.foiName))

        #--check provided samplingTime and upadate
        #  begin/end time procedure if necessary
        # (samplingTime=period or istant of provided
        #  observations defined by samplingTime filter)
        #=============================================
        if filter.samplingTime:
            stime = filter.samplingTime.split("/")
            #
            if len(stime)==2: # is a TimePeriod
                start = iso.parse_datetime(stime[0])
                end  = iso.parse_datetime(stime[1])
            elif len(stime)==1: # is a TimeInstant
                start = end = iso.parse_datetime(stime[0])
            else:
                raise Exception(" filter samplingTime error! given '%s'" %(filter.samplingTime))

            if start>end:
                raise Exception(" endPosition (%s) must be after beginPosition (%s)" %(end,start))

            #-- check samplingTime
            #==========================================
            # verify procedure begin/end exist
            #-----------------------------------
            if not (prc["stime_prc"].__class__.__name__ == "NoneType" and prc["etime_prc"].__class__.__name__ == "NoneType"):

                # check eventTime interval and update begin/end position when force flas is active
                #----------------------------------------------------------------------------------
                if filter.forceInsert:
                    #-- verify interval limits
                    '''if not (end>=prc["stime_prc"] and start<=prc["etime_prc"]):
                        raise sosException.SOSException(3,"observation eventTime (%s-%s) must overlap procedure samplingTime (%s-%s)" %(start,end,prc["stime_prc"],prc["etime_prc"]))
                    else:'''
                    #-- update begin time of procedure
                    if start<prc["stime_prc"]:
                        sql  = "UPDATE %s.procedures" %(filter.sosConfig.schema)
                        sql += " SET stime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s"
                        params = (stime[0],prc["id_prc"])
                        try:
                            a = pgdb.executeInTransaction(sql,params)
                            com=True
                        except:
                            raise Exception("SQL: %s" %(pgdb.mogrify(sql,params)))

                    #-- update end time of procedure
                    if end>prc["etime_prc"]:
                        sql  = "UPDATE %s.procedures" %(filter.sosConfig.schema)
                        sql += " SET etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s"
                        params = (stime[1],prc["id_prc"])
                        try:
                            b = pgdb.executeInTransaction(sql,params)
                            com=True
                        except Exception as err:
                            raise Exception("SQL: %s - %s" %(pgdb.mogrify(sql,params), err.pgerror))

                # check eventTime interval and update begin/end position when force flag is off
                #----------------------------------------------------------------------------------
                else:
                    sql  = "SELECT max(time_eti) as max_time_eti from %s.event_time" %(filter.sosConfig.schema)
                    sql += " WHERE id_prc_fk = %s group by id_prc_fk"
                    params = (prc["id_prc"],)
                    try:
                        lastMsr = pgdb.select(sql,params)[0]["max_time_eti"]
                    except:
                        lastMsr = None

                    if lastMsr!=None:
                        #-- verify begin observation is minor/equal then end time procedure and later then last observation
                        if not (end>=prc["etime_prc"] and start<=prc["etime_prc"] and start>=lastMsr):
                            raise Exception("begin observation (%s) must be between last observation (%s) and end procedure (%s); end observation (%s) must be after end procedure (%s)" %(start,lastMsr,prc["etime_prc"],end,prc["etime_prc"]))
                    else:
                        #-- verify begin observation is minor/equal then end time procedure and later then first observation
                        if not (end>=prc["etime_prc"] and start<=prc["etime_prc"] and start>=prc["stime_prc"]) :
                            raise Exception("begin observation (%s) must be between start procedure (%s) and end procedure (%s); end observation (%s) must be after end procedure (%s)" %(start,prc["stime_prc"],prc["etime_prc"],end,prc["etime_prc"]))

                    #-- update end time of procedure
                    sql  = "UPDATE %s.procedures" %(filter.sosConfig.schema)
                    sql += " SET etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s"
                    params = (str(stime[1]),int(prc["id_prc"]))
                    try:
                        b = pgdb.executeInTransaction(sql,params)
                        com=True
                    except Exception as err:
                        raise Exception("SQL: %s - %s" %(pgdb.mogrify(sql,params), err.pgerror))

            else:
                sql  = "UPDATE %s.procedures" %(filter.sosConfig.schema)
                sql += " SET stime_prc=%s::TIMESTAMPTZ, etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s"
                params = (str(stime[0]),str(stime[1]),int(prc["id_prc"]))
                try:
                    b = pgdb.executeInTransaction(sql,params)
                    com=True
                except:
                    raise Exception("SQL: %s" %(pgdb.mogrify(sql,params)))

        #  check data definition and uom (compare registered
        #  observed properties with provided observations)
        #==================================================
        # get values for provided data: UOM, NAME, URN, ID
        #--------------------------------------------------
        sql  = "SELECT id_pro, id_opr, def_opr, name_uom, constr_opr, constr_pro FROM %s.observed_properties, %s.proc_obs, %s.uoms" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        sql += " WHERE id_uom_fk=id_uom AND id_opr_fk=id_opr AND id_prc_fk=%s"
        params = (prc["id_prc"],)
        try:
            opr = pgdb.select(sql,params)
        except Exception as err:
            raise Exception("SQL2: %s -%s" %(pgdb.mogrify(sql,params), err.pgerror))

        #---- get list of available ObservedProperty, unit of measure, property id for this procedure -----
        oprNames=[]
        oprUoms=[]
        oprIds=[] #to be removed ????
        proIds=[]
        obsPropConstr=[]
        procConstr=[]


        # Building a matrix
        '''
        oprNames=       ["urn:ogc:def:parameter:x-istsos:1.0:meteo:air:temperature" , ...]
        oprUoms=        ["mm" , ...]
        oprIds=         [id_opr , ...]
        proIds=         [id_pro , ...]
        obsPropConstr=  [{"interval": ["-40", "50"], "role": "urn:ogc:def:classifiers:x-istsos:1.0:qualityIndex:check:acceptable"} , ...]
        procConstr=     [{"max": "100", "role": "urn:ogc:def:classifiers:x-istsos:1.0:qualityIndex:check:reasonable"} , ...]
        '''

        for row in opr:

            oprNames.append(row["def_opr"])
            oprUoms.append(row["name_uom"])
            oprIds.append(row["id_opr"])
            proIds.append(row["id_pro"])

            if not row["constr_opr"] in [None,'']:
                obsPropConstr.append(json.loads(row["constr_opr"]))
            else:
                obsPropConstr.append(None)

            if not row["constr_pro"] in [None,'']:
                procConstr.append(json.loads(row["constr_pro"]))
            else:
                procConstr.append(None)

        #---- get ordered list of observed properties in data----
        dataKeys = [ key for key in filter.data.keys() ]

        #----- get ordered list of unit of measures provided with data-------
        dataUoms = []
        for key in filter.data.keys():
            if "uom" in filter.data[key].keys():
                dataUoms.append(filter.data[key]["uom"])
            else:
                dataUoms.append('None')

        #------------------------------------------------------------------
        # verify that all the properties observed by this procedure
        # are provided with the correct data definition and uom
        #------------------------------------------------------------------
        for i,opr in enumerate(oprNames):
            try:
                k = dataKeys.index(opr)
            except:
                raise sosException.SOSException("NoApplicableCode",None,"parameter '%s' not observed by RegisteredSensor %s - %s" %(opr,oprNames,dataKeys))
            #if not str(dataUoms[k])==str(oprUoms[i]):
            if not dataUoms[k]==oprUoms[i]:
                raise sosException.SOSException("NoApplicableCode",None,"parameter '%s' not observed with provided unit of measure" %(opr))

        #---------------------------------------------------------------
        # verify if time and coordinates are passed as data parameters
        # and create the parameters list and parameters ID
        #--------------------------------------------------------------
        xobs=None
        yobs=None
        zobs=None
        tpar=None
        pars=[] # Observed parameters
        parsId=[]
        parsConsObs=[]
        parsConsPro=[]

        # urn of different parameters
        for i, dn in enumerate(dataKeys):
            if dn.split(":")[-1] in filter.sosConfig.parGeom["x"]:
                xobs = dataKeys[i]
            elif dn.split(":")[-1] in filter.sosConfig.parGeom["y"]:
                yobs = dataKeys[i]
            elif dn.split(":")[-1] in filter.sosConfig.parGeom["z"]:
                zobs = dataKeys[i]
            elif dn.find("iso8601")>=0:
                tpar = dataKeys[i]
            else:
                if dn.split(":")[-1] != "qualityIndex":
                    pars.append(dn)
                    try:
                        parsId.append(proIds[oprNames.index(dn)])
                        parsConsObs.append(obsPropConstr[oprNames.index(dn)])
                        parsConsPro.append(procConstr[oprNames.index(dn)])
                    except:
                        raise Exception("parameter %s not observed by this sensor %s - %s" %(dn,pars,oprNames))

        #----------------------------------------------------------------------------------
        # set default quality index if not provided
        #----------------------------------------------------------------------------------
        for par in pars:
            try:
                dataKeys.index(par+":qualityIndex")
            except:
                filter.data[par+":qualityIndex"]={"vals":[filter.sosConfig.default_qi]*len(filter.data[par]["vals"])}

        #---------------------------------------------------------------
        # verify that mobile sensors provide coordinates as X,Y,Z
        #---------------------------------------------------------------
        if (xobs==False and yobs==False and zobs==False) and prc["name_oty"] == "insitu-mobile-point":
            raise Exception("Mobile sensors require x,y,z parameters")

        #---------------------------------------------------------------
        # verify that time parameter is provided
        #---------------------------------------------------------------
        if not tpar:
            raise Exception("parameter 'time:iso8601' is required for InsertObservation")

        #---------------------------------------------------------------
        # verify that eventime are in provided samplingTime
        #---------------------------------------------------------------
        if len(filter.data[tpar]["vals"])>0:
            maxDate = iso.parse_datetime(max(filter.data[tpar]["vals"]))
            minDate = iso.parse_datetime(min(filter.data[tpar]["vals"]))
            if not maxDate <= end and minDate >= start:
                raise Exception("provided data (min: %s, max:%s) are not included in provided <samplingTime> period (%s / %s) for procedure %s" % 
                    (minDate.isoformat(), maxDate.isoformat(), start.isoformat(), end.isoformat(), prc["name_prc"]))

        #======================
        #-- insert observation
        #======================
        # delete existing observations if force flag is active
        #------------------------------------------------------
        if filter.forceInsert:
            sql  = "DELETE FROM %s.event_time" %(filter.sosConfig.schema)
            sql += " WHERE id_prc_fk=%s AND time_eti>=%s::TIMESTAMPTZ AND time_eti<=%s::TIMESTAMPTZ"
            params = (prc["id_prc"],stime[0],stime[1])
            try:
                b = pgdb.executeInTransaction(sql,params)
                com=True
            except:
                raise Exception("SQL: %s" %(pgdb.mogrify(sql,params)))

        #----------------------------------------
        # CASE I: observations list is void
        #----------------------------------------
        if len(filter.data[tpar]["vals"])==0:
            self.assignedId = ""
            ids_eti = []
        #----------------------------------------
        # CASE I: observations list contains data
        #----------------------------------------
        elif len(filter.data[tpar]["vals"])>0:
            #--------------------
            # insert event times
            #--------------------
            ids_eti = []
            params = []
            sql  = "INSERT INTO %s.event_time (id_prc_fk,time_eti)" %(filter.sosConfig.schema)
            sql += " VALUES (%s,%s::TIMESTAMPTZ) RETURNING id_eti"
            for val in filter.data[tpar]["vals"]:
                try:
                    ids_eti.append(pgdb.executeInTransaction(sql,(prc["id_prc"],val))[0]['id_eti'])
                    com = True
                except Exception as e:
                    raise Exception("Error inserting event times for %s: %s" % (prc["name_prc"], str(e)) )

            for i, par in enumerate(pars):
                params = []
                ids_msr = []
                sql = "INSERT INTO %s.measures (id_pro_fk, id_eti_fk,id_qi_fk,val_msr) VALUES" % (filter.sosConfig.schema)
                sql += " (%s,%s,%s,%s) RETURNING id_msr"

                #hasvalues = False
                for ii,id_et in enumerate(ids_eti):


                    if not filter.data[par]["vals"][ii] in ['NULL',u'NULL',None,-999,"-999",u"-999",filter.sosConfig.aggregate_nodata]:
                    #TODO: add a else statement to add the aggregate_nodata value OR delete the event time
                    #if not filter.data[par]["vals"][ii] in ['NULL',u'NULL',None]:
                        pqi = int(filter.data[par+":qualityIndex"]["vals"][ii])

                        # Constraint quality is done only if the quality index is equal to the default qi (RAW DATA)
                        if int(filter.sosConfig.default_qi) == pqi:

                            # quality check level I (gross error)
                            #------------------------------------
                            if filter.sosConfig.correct_qi != None and parsConsObs[i] != None:

                                if 'max' in parsConsObs[i]:
                                    if float(filter.data[par]["vals"][ii]) <= float(parsConsObs[i]['max']):
                                        pqi = int(filter.sosConfig.correct_qi)
                                elif 'min' in parsConsObs[i]:
                                    if float(filter.data[par]["vals"][ii]) >= float(parsConsObs[i]['min']):
                                        pqi = int(filter.sosConfig.correct_qi)
                                elif 'interval' in parsConsObs[i]:
                                    if float(parsConsObs[i]['interval'][0]) <= float(filter.data[par]["vals"][ii]) <= float(parsConsObs[i]['interval'][1]):
                                        pqi = int(filter.sosConfig.correct_qi)
                                elif 'valueList' in parsConsObs[i]:
                                    if float(filter.data[par]["vals"][ii]) in [float(p) for p in parsConsObs[i]['valueList']]:
                                        pqi = int(filter.sosConfig.correct_qi)

                            # quality check level II (statistical range)
                            #-------------------------------------------
                            if filter.sosConfig.stat_qi != None and parsConsPro[i] != None:

                                if 'max' in parsConsPro[i]:
                                    if float(filter.data[par]["vals"][ii]) <= float(parsConsPro[i]['max']):
                                        pqi = int(filter.sosConfig.stat_qi)
                                elif 'min' in parsConsPro[i]:
                                    if float(filter.data[par]["vals"][ii]) >= float(parsConsPro[i]['min']):
                                        pqi = int(filter.sosConfig.stat_qi)
                                elif 'interval' in parsConsPro[i]:
                                    if float(parsConsPro[i]['interval'][0]) <= float(filter.data[par]["vals"][ii]) <= float(parsConsPro[i]['interval'][1]):
                                        pqi = int(filter.sosConfig.stat_qi)
                                elif 'valueList' in parsConsPro[i]:
                                    if float(filter.data[par]["vals"][ii]) in [float(p) for p in parsConsPro[i]['valueList']]:
                                        pqi = int(filter.sosConfig.stat_qi)

                        params = (int(parsId[i]),int(id_et),pqi,float(filter.data[par]["vals"][ii]))
                        try:
                            nid_msr = pgdb.executeInTransaction(sql,params)
                            ids_msr.append(str(nid_msr[0]['id_msr']))
                        except Exception as e:
                            com=False
                            raise e
                            raise Exception("L: %s - %s - %s - %s") %(int(parsId[i]),int(id_et),pqi,float(filter.data[par]["vals"][ii]))

            #-------------------------------------
            #--insert position values if required
            #-------------------------------------
            if prc["name_oty"] == "insitu-mobile-point":
                xparspl = xobs.split(":")
                epsg = xparspl[xparspl.index("EPSG")+1]
                params = []
                sql = "INSERT INTO %s.positions (id_qi_fk, id_eti_fk,geom_pos) VALUES" %(filter.sosConfig.schema)
                sql += "(%s,%s,ST_Transform(ST_SetSRID(ST_MakePoint(%s, %s, %s), %s), %s))"

                for i,id_et in enumerate(ids_eti):
                    params = (filter.sosConfig.default_qi,id_et,filter.data[xobs]["vals"][i],filter.data[yobs]["vals"][i],filter.data[zobs]["vals"][i],epsg,filter.sosConfig.istsosepsg)
                    try:
                        ids_pos = pgdb.executeInTransaction(sql,params)
                        com=True
                    except Exception as a:
                        com=False
                        raise Exception("%s\nSQL: %s" %(a,pgdb.mogrify(sql,params)))

            # register assigned IDs of measures
            self.assignedId = "@".join([str(p) for p in ids_eti])
            # commit executed operations

        #Register the transactional operation in Log table
        if filter.sosConfig.transactional_log in ['True','true',1]:
            sqlLog  = "INSERT INTO %s.tran_log" %(filter.sosConfig.schema)
            sqlLog  += " (operation_trl,procedure_trl,begin_trl,end_trl,count,stime_prc,etime_prc)"
            sqlLog  += " VALUES ('InsertObservation', %s, %s::TIMESTAMPTZ, %s::TIMESTAMPTZ, %s, %s::TIMESTAMPTZ , %s::TIMESTAMPTZ)"
            params = (str(filter.procedure),start,end,len(ids_eti),prc["stime_prc"],prc["etime_prc"])
            try:
                pgdb.executeInTransaction(sqlLog,params)
                com=True
            except:
                raise Exception("SQL: %s" %(pgdb.mogrify(sqlLog,params)))

        if com==True:
            pgdb.commitTransaction()
Esempio n. 22
0
 go1 = sos1.getSOSProcedure(procedure)
 
 if ' component' in go1['observedProperty']:
     op = go1['observedProperty'][' component'][1]
 else:
     op = go1['observedProperty']['component'][1]
 
 '''print "Gathering info for %s from destination" % procedure
 go2 = sos2.getSOSProcedure(procedure)'''
 
 for line in lines:
 
     if procedure in ['Q_BOL_PTC', 'Q_CFER_ARO', 'Q_LAV_MEN', 'Q_MAG_VIS', 'Q_MOR_GIU', 'Q_RMUL_COM', 'Q_ROV_CVM2', 'Q_SBIB_PON', 'Q_TICTO_CHIR', 'Q_TRA_ARB_FFS1', 'Q_VED_ISO', 'Q_VVED_AGN', 'P_PON', 'Q_BON_QUA', 'Q_CUC_POR', 'Q_LAV_RSV', 'Q_MAR_MAR', 'Q_MUZ_MUZ', 'Q_RMUL_MAR', 'Q_ROV_CVM3', 'Q_SCA_LUG', 'Q_TRA_ARB', 'Q_TRA_ARB_FFS2', 'Q_VED_MUZ', 'V_TEST', 'Q_AETCAN_AIR', 'Q_CAL_AIR', 'Q_GNO_GNO', 'Q_MAG_LOD', 'Q_MAR_MAR2', 'Q_RMUL_ARB', 'Q_ROV_CVM1', 'Q_SAL_MAG', 'Q_TIC_BED', 'Q_TRA_ARB_FFS', 'Q_TRA_ARB_FFS3', 'Q_VER_BGU']:
         continue
     
     begin = isodate.parse_datetime(line.replace("\"","").split(",")[0])
     end   = isodate.parse_datetime(line.replace("\"","").split(",")[0])
     
     # Creating temporary directory where CSV files will be stored
     dirpath = tempfile.mkdtemp()
     
     try:
         
         istsos2csv.execute({
             'begin': begin.isoformat(),
             'end': end.isoformat(),
             'procedure': procedure,
             'op': op,
             'url': '%s/%s' % (url1, srv1),
             'd': dirpath
         })
Esempio n. 23
0
def execute (args, logger=None):
    
    print "istsos2csv start.."
    
    try:
        
        url = args['url']
        
        procedure = args['procedure']
        observedProperty = args['op']
        
        begin = iso.parse_datetime(args['begin'])
        end = iso.parse_datetime(args['end'])
        
        d = args['d']
        
        auth = None
        if 'user' in args:
            user = args['user']
        password = None
        if 'password' in args:
            password = args['password']
        if auth and password:
            auth = HTTPBasicAuth(user, password)
            
        qi = 'True'
        if 'noqi' in args:
            if args['noqi'] == True: 
                qi = 'False'
                
            
        params = {
            "request": "GetObservation",
            "offering": "temporary",
            "procedure": procedure,
            "eventTime": None,
            "observedProperty": observedProperty,
            "responseFormat": "text/plain",
            "service": "SOS", 
            "version":"1.0.0",
            "qualityIndex": qi
        }
        
        tmpBegin = begin
        tmpEnd = end
        if (end-begin)>step:
            tmpEnd = tmpBegin + step
        
        print params

        while tmpEnd <= end:
        
            print ("%s - %s") % (tmpBegin,tmpEnd)

            if tmpBegin == tmpEnd:
                params["eventTime"] = iso.datetime_isoformat(tmpBegin)
            else:
                params["eventTime"] = "%s/%s" % (iso.datetime_isoformat(tmpBegin), iso.datetime_isoformat(tmpEnd))
                
            res = req.get("%s?%s" % (url, urllib.urlencode(params)), auth=auth)
            
            makeFile(res, procedure, observedProperty, d)
            tmpBegin = tmpEnd
            tmpEnd = tmpBegin + step
            
            print " %s ************************** " % iso.datetime_isoformat(tmpEnd)
            
        if tmpBegin < end:
        
            tmpEnd = end
            if tmpBegin == tmpEnd:
                params["eventTime"] = iso.datetime_isoformat(tmpBegin)
            else:
                params["eventTime"] = "%s/%s" % (iso.datetime_isoformat(tmpBegin), iso.datetime_isoformat(tmpEnd))
            
            res = req.get("%s?%s" % (url, urllib.urlencode(params)), auth=auth)
            makeFile(res, procedure, observedProperty, d)
            
            print " %s ************************** " % iso.datetime_isoformat(end)
        
        print "Finish."
        
    except Exception as e:    
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
Esempio n. 24
0
    def __init__(self,filter,pgdb):
        self.offInfo = offInfo(filter.offering,pgdb,filter.sosConfig)
        self.refsys = filter.sosConfig.urn["refsystem"] + filter.srsName
        self.filter = filter
        
        #CHECK FILTER VALIDITY
        #=========================================
        """        
        off_list = BuildOfferingList(pgdb,filter.sosConfig)
        if not filter.offering in off_list:
            raise sosException.SOSException("InvalidParameterValue","offering","Parameter \"offering\" sent with invalid value: %s -  available options for offering are %s" %(filter.offering,off_list))
        """    
        if filter.procedure:
            pl = BuildProcedureList(pgdb,filter.offering,filter.sosConfig)
            for p in filter.procedure:
                if not p in pl:
                    raise sosException.SOSException("InvalidParameterValue","procedure","Parameter \"procedure\" sent with invalid value: %s -  available options for offering \"%s\": %s"%(p,filter.offering,pl))
        
        if filter.featureOfInterest:
            fl = BuildfeatureOfInterestList(pgdb,filter.offering,filter.sosConfig)
            if not filter.featureOfInterest in fl:
                raise sosException.SOSException("InvalidParameterValue","featureOfInterest","Parameter \"featureOfInterest\" sent with invalid value: %s - available options: %s"%(filter.featureOfInterest,fl))
        
        if filter.observedProperty:
            opl = BuildobservedPropertyList(pgdb, filter.offering,filter.sosConfig)
            opr_sel = "SELECT def_opr FROM %s.observed_properties WHERE " %(filter.sosConfig.schema,)
            opr_sel_w = []
            for op in filter.observedProperty:
                opr_sel_w += ["def_opr SIMILAR TO '%%(:|)%s(:|)%%'" %(op)]
            opr_sel = opr_sel + " OR ".join(opr_sel_w)
            try:
                opr_filtered = pgdb.select(opr_sel)
            except:
                raise Exception("SQL: %s"%(opr_sel))
            if not len(opr_filtered)>0:
                raise sosException.SOSException("InvalidParameterValue","observedProperty","Parameter \"observedProperty\" sent with invalid value: %s - available options: %s"%(filter.observedProperty,opl))
        
        #SET TIME PERIOD
        #=========================================
        tp=[]
        if filter.eventTime == None:
            tp = [None,None]
        else:
            for t in filter.eventTime:
                if len(t) == 2:
                    tp.append(iso.parse_datetime(t[0]))
                    tp.append(iso.parse_datetime(t[1]))
                if len(t)==1:
                    tp.append(iso.parse_datetime(t[0]))
                #else: rise error ???
        self.period = [min(tp),max(tp)]
        
        self.obs=[]
        
        # SET REQUEST TIMEZONE
        #===================================
        if filter.eventTime:
            if iso.parse_datetime(filter.eventTime[0][0]).tzinfo:
                self.reqTZ = iso.parse_datetime(filter.eventTime[0][0]).tzinfo
                pgdb.setTimeTZ(iso.parse_datetime(filter.eventTime[0][0]))
            else:
                self.reqTZ = pytz.utc
                pgdb.setTimeTZ("UTC")
        else:
            self.reqTZ = pytz.utc
            pgdb.setTimeTZ("UTC")
            
        
        
        #BUILD PROCEDURES LIST
        #=========================================
        #---select part of query
        sqlSel = "SELECT DISTINCT"
        sqlSel += " id_prc, name_prc, name_oty, stime_prc, etime_prc, time_res_prc"
        #---from part of query
#################################
# Rimosso codice di time_res_unit
#################################
        sqlFrom = "FROM %s.procedures, %s.proc_obs p, %s.observed_properties, %s.uoms," %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        sqlFrom += " %s.off_proc o, %s.offerings, %s.obs_type" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
        if filter.featureOfInterest or filter.featureOfInterestSpatial:
            sqlFrom += " ,%s.foi, %s.feature_type" %(filter.sosConfig.schema,filter.sosConfig.schema)
        
        sqlWhere = "WHERE id_prc=p.id_prc_fk AND id_opr_fk=id_opr AND o.id_prc_fk=id_prc AND id_off_fk=id_off AND id_uom=id_uom_fk AND id_oty=id_oty_fk"
        sqlWhere += " AND name_off='%s'" %(filter.offering) 
        
        #---where condition based on featureOfInterest
        if filter.featureOfInterest:
            #sqlWhere += " AND id_foi=id_foi_fk AND id_fty=id_fty_fk AND (name_foi='%s')" %(filter.featureOfInterest)
            sqlWhere += " AND id_foi=id_foi_fk AND id_fty=id_fty_fk AND (name_foi IN (%s))" %(",".join( [ "'"+f+"'" for f in filter.featureOfInterest.split(",")]))
        if filter.featureOfInterestSpatial:
            sqlWhere += " AND id_foi_fk=id_foi AND %s" %(filter.featureOfInterestSpatial)
        
        #---where condition based on procedures
        if filter.procedure:
            sqlWhere += " AND ("
            procWhere = []
            for proc in filter.procedure:
                procWhere.append("name_prc='%s'" %(proc))
            sqlWhere += " OR ".join(procWhere)
            sqlWhere += ")"
        
        #---where condition based on observed properties
        sqlWhere += " AND ("
        obsprWhere = []
        for obs in opr_filtered:
            obsprWhere.append("def_opr='%s'" %(obs["def_opr"])) 
        sqlWhere += " OR ".join(obsprWhere)
        sqlWhere += ")"
        
        try:
            res = pgdb.select(sqlSel + " " + sqlFrom + " " + sqlWhere)
        except:
            raise Exception("SQL: %s"%(sqlSel + " " + sqlFrom + " " + sqlWhere))
        
        #FOR EACH PROCEDURE
        #=========================================
        for o in res:
            #id_prc, name_prc, name_oty, stime_prc, etime_prc, time_res_prc, name_tru
            
            #CRETE OBSERVATION OBJECT
            #=================================================
            ob = Observation()
            
            #BUILD BASE INFOS FOR EACH PROCEDURE (Pi)
            #=================================================
            ob.baseInfo(pgdb,o,filter.sosConfig)
            
            #GET DATA FROM PROCEDURE ACCORDING TO THE FILTERS
            #=================================================
            ob.setData(pgdb,o,filter)
            
            #ADD OBSERVATIONS
            #=================================================
            self.obs.append(ob)
Esempio n. 25
0
def execute (args, logger=None):
    
    def log(message):
        if logger:
            logger.log(message)
        else:
            print message
    
    pp = pprint.PrettyPrinter(indent=2)
    
    try:
    
        # Initializing URLs
        url = args['u']
        
        # Service instance name
        service = args['s']
        
        # Quality index
        quality = '100'
        if 'q' in args:
            quality = args['q']
        
        # Procedures
        procs = args['p']
        
        # Working directory, where the CSV files are located
        wd = args['wd']
        
        # File extension
        ext = '.dat'
        if 'e' in args:
            ext = args['e']
        
        debug = False
        if 'v' in args:
            debug = args['v']
        
        test = False
        if 't' in args:
            test = args['t']
        
        user = None
        if 'user' in args:
            user = args['user']
        passw = None
        if 'password' in args:
            password = args['password']
            
        auth = None
        if user and password:
            auth = HTTPBasicAuth(user, password)
        
        noqi = False # False meas that quality index is also migrated
        if 'noqi' in args:
            if args['noqi'] == True: 
                noqi = True
            
        #print "noqi: %s" % noqi
            
        maxobs = 5000
        if 'm' in args:
            maxobs = int(args['m'])
            
        #req = requests.session()
        req = requests
        
        for proc in procs:
            
            log("\nProcedure: %s" % proc)
            
            # Load procedure description                
            res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (
                url,
                service,
                proc
                ), auth=auth, verify=False)
                
            data = res.json()
                
            if data['success']==False:
                raise Exception ("Description of procedure %s can not be loaded: %s" % (proc, data['message']))
            else:
                print " > %s" % data['message']
            
            data = data['data']
            
            aid = data['assignedSensorId']
            
            # Getting observed properties from describeSensor response
            op = []
            for out in data['outputs']:
                if not noqi or not ':qualityIndex' in out['definition']:
                    op.append(out['definition'])
                    
            # Load of a getobservation request
            res = req.get("%s/wa/istsos/services/%s/operations/getobservation/offerings/%s/procedures/%s/observedproperties/%s/eventtime/last" % (
                url,
                service,
                'temporary',
                proc,
                ','.join(op)
                ), auth=auth, verify=False)
            
            data = res.json()
            
            if data['success']==False:
                raise Exception ("Last observation of procedure %s can not be loaded: %s" % (proc, data['message']))
            else:
                print " > %s" % data['message']
                    
            data = data['data'][0]
            data['AssignedSensorId'] = aid
            
            # Set values array empty (can contain 1 value if procedure not empty)
            lastMeasure = data['result']['DataArray']['values'][0] if len(data['result']['DataArray']['values'])==1 else None
            data['result']['DataArray']['values'] = []
            
            # discover json observed property disposition
            jsonindex = {}
            for pos in range(0, len(data['result']['DataArray']['field'])):
                field = data['result']['DataArray']['field'][pos]
                if not noqi:
                    jsonindex[field['definition']] = pos
                elif not ':qualityIndex' in field['definition']:
                    jsonindex[field['definition']] = pos
                elif ':qualityIndex' in field['definition'] and noqi:
                    data['result']['DataArray']['field'].pop(pos)
            
            log ("Searching: %s" % os.path.join(wd, "%s_[0-9]*%s" % (proc,ext)))
                
            files = glob.glob(os.path.join(wd, "%s_*%s" % (proc,ext)))
            files.sort()
            
            if debug:
                print " > %s %s found" % (len(files), "Files" if len(files)>1 else "File")
                
            if len(files)>0:
                for f in files:
                
                    # open file
                    file = open(f, 'rU')
                    
                    # loop lines
                    lines = file.readlines()
                    
                    obsindex = lines[0].strip(' \t\n\r').split(",")
                    
                    # Check if all the observedProperties of the procedure are included in the CSV file (quality index is optional)
                    for k, v in jsonindex.iteritems():
                        if k in obsindex:
                            continue
                        elif ':qualityIndex' in k:
                            continue
                        else:
                            raise Exception ("Mandatory observed property %s is not present in the CSV." % k)
                    
                    # loop lines (skipping header)
                    for i in range(1, len(lines)):
                        try:
                            line = lines[i]
                            lineArray = line.strip(' \t\n\r').split(",")
                            
                            # Creating an empty array where the values will be inserted
                            observation =  ['']*len(jsonindex)
                            
                            for k, v in jsonindex.iteritems():
                                val = None
                                if k in obsindex:
                                    val = lineArray[obsindex.index(k)]
                                elif ':qualityIndex' in k: # Quality index is not present in the CSV so the default value will be set
                                    val = quality
                                    
                                observation[v] = val
                                
                            # attach to object
                            data['result']['DataArray']['values'].append(observation)
                            
                        except Exception as e:
                            print "Errore alla riga: %s - %s)" % (i, lines[i])
                            traceback.print_exc()
                            raise e
                            
                log ("Before insert ST:")
                if 'beginPosition' in data["samplingTime"]:
                    log (" > Begin: %s" % data["samplingTime"]["beginPosition"])
                if 'endPosition' in data["samplingTime"]:
                    log ("   + End: %s" % data["samplingTime"]["endPosition"])
                
                ep = datetime.strptime(
                    os.path.split(f)[1].replace("%s_" % proc, "").replace(ext, ""),"%Y%m%d%H%M%S%f"
                ).replace(tzinfo=timezone('UTC'))
                
                # Kick in the brain code snippet
                
                # If there is at least one measure:
                if len(data['result']['DataArray']['values'])>0:
                
                    # taking first observation as begin position
                    bp = iso.parse_datetime(
                        data['result']['DataArray']['values'][0][jsonindex['urn:ogc:def:parameter:x-istsos:1.0:time:iso8601']]
                    )
                        
                else: # otherwise this can be an irrebular procedure where just the end position is moved forward
                
                    if ep > iso.parse_datetime(data["samplingTime"]["endPosition"]):
                        bp = ep
                    else:
                        raise Exception("Something is wrong with begin position..")
                        
                data["samplingTime"] = {
                    "beginPosition": bp.isoformat(),
		            "endPosition":  ep.isoformat()
                }
                
                #data["result"]["DataArray"]["elementCount"] = str(len(data['result']['DataArray']['values']))
                
                log ("Insert ST:")
                log (" > Begin: %s" % bp.isoformat())
                log ("   + End: %s" % ep.isoformat())
                log (" > Values: %s" % len( data['result']['DataArray']['values']))
                    
                if not test and len(files)>0: # send to wa
                
                    if len(data['result']['DataArray']['values']) > maxobs:
                    
                        import copy
                        
                        total = len(data['result']['DataArray']['values'])
                        inserted = last = maxobs
                        
                        while len(data['result']['DataArray']['values'])>0:
                            
                            tmpData = copy.deepcopy(data)                            
                            tmpData['result']['DataArray']['values'] = data['result']['DataArray']['values'][:last]                            
                            data['result']['DataArray']['values']    = data['result']['DataArray']['values'][last:]
                            
                            if len(data['result']['DataArray']['values'])>0:
                                tmpData["samplingTime"] = {
                                    "beginPosition": tmpData['result']['DataArray']['values'][0][jsonindex['urn:ogc:def:parameter:x-istsos:1.0:time:iso8601']],
			                        "endPosition":      data['result']['DataArray']['values'][0][jsonindex['urn:ogc:def:parameter:x-istsos:1.0:time:iso8601']]
                                }
                            else:
                                tmpData["samplingTime"] = {
                                    "beginPosition": tmpData['result']['DataArray']['values'][0][jsonindex['urn:ogc:def:parameter:x-istsos:1.0:time:iso8601']],
			                        "endPosition":   ep.isoformat()
                                }
                                
                            res = req.post("%s/wa/istsos/services/%s/operations/insertobservation" % (
                                url,
                                service), 
                                auth=auth,
                                verify=False,
                                data=json.dumps({
                                    "ForceInsert": "true",
                                    "AssignedSensorId": aid,
                                    "Observation": tmpData
                                })
                            )
                            
                            # read response
                            res.raise_for_status()
                            log (" > Insert observation success of %s/%s (%s / %s) observations: %s" % (inserted,total,tmpData["samplingTime"]["beginPosition"],tmpData["samplingTime"]["endPosition"],res.json()['success']))
                            if not res.json()['success']:
                                log (res.json()['message'])
                                
                            if len(data['result']['DataArray']['values'])<maxobs:
                                last = len(data['result']['DataArray']['values'])
                            inserted += last
                            
                    
                    else:
                
                        res = req.post("%s/wa/istsos/services/%s/operations/insertobservation" % (
                            url,
                            service), 
                            auth=auth,
                            verify=False,
                            data=json.dumps({
                                "ForceInsert": "true",
                                "AssignedSensorId": aid,
                                "Observation": data
                            })
                        )
                        # read response
                        res.raise_for_status()
                        log (" > Insert observation success: %s" % res.json()['success'])
                        if not res.json()['success']:
                            log (res.json()['message'])
                        
                    
                    print "~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~"
        pass
    
    except requests.exceptions.HTTPError as eh:
        print "ERROR: %s\n\n" % eh
        traceback.print_exc()
        pass
    except Exception as e:    
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
        pass
        
    pass
Esempio n. 26
0
    def insert_observation(self, broker_url, port, topic, data):
        print("url: %s:%s, topic: %s, data: %s" %
              (broker_url, port, topic, data))
        with self.lock:
            broker = "%s:%s" % (broker_url, port)
            if (broker in self.broker) and (topic in self.broker[broker]):
                id_prc = self.broker[broker][topic]['id']
                name_prc = self.broker[broker][topic]['name']
                instance = self.broker[broker][topic]['instance']
                conn = self.services[instance]['conn']
                try:
                    sql = """
                        SELECT
                            procedures.id_prc,
                            proc_obs.id_pro,
                            proc_obs.constr_pro,
                            procedures.stime_prc,
                            procedures.etime_prc
                        FROM
                            %s.procedures,
                            %s.proc_obs
                        WHERE
                            proc_obs.id_prc_fk = procedures.id_prc
                    """ % (instance, instance)
                    sql += """
                      AND
                        id_prc = %s
                      ORDER BY
                        proc_obs.id_pro ASC;
                    """
                    rows = conn.select(sql, (int(id_prc), ))

                    if not isinstance(data, list):
                        #print (type(data))
                        if isinstance(data, str):
                            data = [data.split(",")]
                        else:
                            data = [data.decode('utf-8').split(",")]

                    # check if procedure observations length is ok
                    if len(rows) != (len(data[0]) - 1):
                        raise Exception(
                            "Array length missmatch with procedures "
                            "observation number")

                    else:
                        insertEventTime = """
                            INSERT INTO %s.event_time (id_prc_fk, time_eti)
                        """ % (instance)
                        insertEventTime += """
                            VALUES (%s, %s::TIMESTAMPTZ) RETURNING id_eti;
                        """

                        deleteEventTime = """
                            DELETE FROM %s.event_time
                        """ % (instance)
                        deleteEventTime += """
                            WHERE id_prc_fk = %s
                            AND time_eti = %s::TIMESTAMPTZ
                        """

                        insertMeasure = """
                            INSERT INTO %s.measures(id_eti_fk, id_qi_fk,
                                id_pro_fk,val_msr)
                        """ % (instance)
                        insertMeasure += """
                            VALUES (%s, 100, %s, %s);
                        """

                        updateBeginPosition = """
                            UPDATE %s.procedures""" % (instance)
                        updateBeginPosition += """
                            SET stime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
                        """
                        updateEndPosition = """
                            UPDATE %s.procedures""" % (instance)
                        updateEndPosition += """
                            SET etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
                        """

                        bp = rows[0][3]
                        bpu = False
                        ep = rows[0][4]
                        epu = False
                        for observation in data:
                            try:
                                id_eti = conn.executeInTransaction(
                                    insertEventTime,
                                    (rows[0][0], observation[0]))

                            except psycopg2.IntegrityError as ie:
                                conn.rollbackTransaction()
                                conn.executeInTransaction(
                                    deleteEventTime,
                                    (rows[0][0], observation[0]))
                                id_eti = conn.executeInTransaction(
                                    insertEventTime,
                                    (rows[0][0], observation[0]))

                            for idx in range(0, len(rows)):
                                conn.executeInTransaction(
                                    insertMeasure,
                                    (int(id_eti[0][0]), int(rows[idx][1]),
                                     float(observation[(idx + 1)])))

                            if (bp is None) or (bp == '') or (
                                    iso.parse_datetime(observation[0]) < bp):
                                bp = iso.parse_datetime(observation[0])
                                bpu = True

                            if (ep is None) or (ep == '') or (
                                    iso.parse_datetime(observation[0]) > ep):
                                ep = iso.parse_datetime(observation[0])
                                epu = True

                            conn.commitTransaction()

                        if bpu:
                            conn.executeInTransaction(updateBeginPosition,
                                                      (bp.isoformat(), id_prc))

                        if epu:
                            conn.executeInTransaction(updateEndPosition,
                                                      (ep.isoformat(), id_prc))

                        conn.commitTransaction()

                        # Publish / broadcast new data
                        mqttConf = self.services[instance]['config'].mqtt
                        print("mqttConf: ")
                        print(mqttConf)
                        if mqttConf["broker_url"] != '' and (
                                mqttConf["broker_port"] != ''):
                            print("Broadcasting new data!!")
                            istmqttlib.PahoPublisher({
                                "broker_url":
                                mqttConf["broker_url"],
                                "broker_port":
                                mqttConf["broker_port"],
                                "broker_topic":
                                "%s%s" % (mqttConf["broker_topic"], name_prc),
                                "data":
                                data
                            }).start()

                except Exception as e:
                    traceback.print_exc(file=sys.stderr)
                    conn.rollbackTransaction()
                    raise Exception("Error in fast insert (%s): %s" %
                                    (type(e), e))

            else:
                #print("Sensor unknown")
                pass
Esempio n. 27
0
    
    errors = False
    
    #print "**********************************************"
    #print "Stop: %s, %s" % (stop1,stop2)
    #print "**********************************************"
    
    for i in range(0,stop):

        row = []
        iso1 = None
        iso2 = None
        
        #toprint = i, ": "
        if (cnt1)<stop1:
            iso1 = isodate.parse_datetime(observations1[cnt1][0])
            #toprint = toprint , cnt1, observations1[cnt1], ' '
            
        if (cnt2)<stop2:
            iso2 = isodate.parse_datetime(observations2[cnt2][0])
            #toprint = toprint , cnt2, observations2[cnt2]
        
        #print toprint
        
        if iso1 == None:
            row = [None,None,None,iso2.isoformat(),observations2[cnt2][1],observations2[cnt2][2]]
            #errors = True
            #mergedErrors.append([None,None,None,iso2.isoformat(),observations2[cnt2][1],observations2[cnt2][2]])
            cnt1 += 1     
            if cnt2<stop2 and cnt1>stop1:
                cnt2 += 1
Esempio n. 28
0
def execute(args):
    pp = pprint.PrettyPrinter(indent=2)
    try:

        istsos_version = args['istsos'] if 'istsos' in args else None

        debug = args['v']
        test = args['t']

        procs = args['p']
        omit = args['o']
        procedures = {}

        src = args['s']
        dst = args['d']
        srv = args['n']

        duser = None
        if 'du' in args:
            duser = args['du']
        dpassw = None
        if 'dp' in args:
            dpassw = args['dp']

        #print "%s:%s" % (duser,'*'*len(dpassw))

        auth = None
        if duser and dpassw:
            print "User and password!"
            auth = HTTPBasicAuth(duser, dpassw)

        appendData = False
        if 'a' in args and args['a']:
            print "Append data: enabled."
            appendData = True

        dfrom = None
        dto = None
        if 'from' in args and type('') == type(args['from']):
            print "From: %s" % args['from']
            dfrom = iso.parse_datetime(args['from'])
            appendData = None
        if 'to' in args and type('') == type(args['to']):
            print "To: %s" % args['to']
            dto = iso.parse_datetime(args['to'])
            appendData = None

        registerOnly = args['r']

        virtual = False
        hq = False

        # Executing request
        res = req.get("%s" % (src),
                      params={
                          'service': 'SOS',
                          'version': '1.0.0',
                          'request': 'GetCapabilities',
                          'section': 'contents'
                      },
                      verify=False)

        # Parsing response
        gc, gcNs = parse_and_get_ns(StringIO(res.content))

        # Extract all offerings
        elOfferings = gc.findall(
            "{%s}Contents/{%s}ObservationOfferingList/{%s}ObservationOffering"
            % (gcNs['sos'], gcNs['sos'], gcNs['sos']))

        for offering in elOfferings:
            offeringName = offering.find("{%s}name" %
                                         (gcNs['gml'])).text.split(":")[-1]

            if offeringName != 'temporary':
                continue

            # For each offering get the procedures
            elProcs = offering.findall("{%s}procedure" % (gcNs['sos']))
            for p in elProcs:
                pname = p.get('{%s}href' % gcNs['xlink'])

                if (type(procs) == type([])
                        and pname not in procs) or (type(omit) == type([])
                                                    and pname in omit):
                    continue

                print "\n%s" % pname
                print "================================"

                procedures[pname] = Procedure(pname, offeringName, dst, srv,
                                              auth)

                if virtual:
                    procedures[pname].setSystemType('virtual')

                res = req.get("%s" % (src),
                              params={
                                  'service': 'SOS',
                                  'version': '1.0.0',
                                  'request': 'DescribeSensor',
                                  'outputFormat':
                                  'text/xml;subtype=\'sensorML/1.0.0\'',
                                  'procedure': pname
                              },
                              verify=False)

                #print res.content

                ds, dsNs = parse_and_get_ns(StringIO(res.content))

                #print res.content

                #print "Root: %s" % ds.getroot().tag
                if ds.getroot().tag == 'ExceptionReport':
                    print "Error on DS for %s" % pname
                    continue

                #print "Outputs found: %s" % len(elDescribe)

                observedProperties = []
                print "istsos_version: ", istsos_version
                uniqidurn = 'urn:ogc:def:parameter:x-ist::'
                if istsos_version != None and istsos_version == '2':
                    uniqidurn = 'urn:ogc:def:parameter:x-ist:1.0:'
                    elFields = ds.findall(
                        "{%s}member/{%s}System/{%s}outputs/{%s}OutputList/{%s}output/{%s}DataRecord/{%s}field"
                        % (dsNs['sml'], dsNs['sml'], dsNs['sml'], dsNs['sml'],
                           dsNs['sml'], dsNs['swe'], dsNs['swe']))
                    print "Observed properties (v2): %s " % len(elFields)
                    for fs in elFields:
                        print fs.get('name')
                        if fs.get('name') != 'Time':
                            observedProperties.append(
                                fs.find(
                                    "{%s}Quantity" %
                                    (dsNs['swe'])).get('definition').replace(
                                        uniqidurn, ''))

                else:
                    elDescribe = ds.findall(
                        "member/{%s}System/{%s}outputs/{%s}OutputList/{%s}output"
                        % (dsNs['sml'], dsNs['sml'], dsNs['sml'], dsNs['sml']))
                    print "Observed properties: %s " % len(elDescribe)
                    for ds in elDescribe:
                        definition = ds.find(
                            "{%s}ObservableProperty" %
                            (dsNs['swe'])).get('definition').replace(
                                uniqidurn, '')
                        #print definition
                        if definition.find('time:iso8601') < 0:
                            observedProperties.append(definition)

                #print {
                #    'service': 'SOS',
                #    'version': '1.0.0',
                #    'request': 'GetObservation',
                #    'offering': offeringName,
                #    'responseFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                #    'procedure': pname,
                #    'observedProperty': ",".join(observedProperties)
                #}

                res = req.get("%s" % (src),
                              params={
                                  'service': 'SOS',
                                  'version': '1.0.0',
                                  'request': 'GetObservation',
                                  'offering': offeringName,
                                  'responseFormat':
                                  'text/xml;subtype=\'sensorML/1.0.0\'',
                                  'procedure': pname,
                                  'observedProperty':
                                  ",".join(observedProperties)
                              },
                              verify=False)

                go, goNs = parse_and_get_ns(StringIO(res.content))

                if go.getroot().tag == 'ExceptionReport':
                    print "Error on GO for %s:\nparams:%s\n%s" % (
                        pname, {
                            'service': 'SOS',
                            'version': '1.0.0',
                            'request': 'GetObservation',
                            'offering': offeringName,
                            'responseFormat':
                            'text/xml;subtype=\'sensorML/1.0.0\'',
                            'procedure': pname,
                            'observedProperty': ",".join(observedProperties)
                        }, res.content)
                    continue

                # Extracting begin and end position
                begin = go.find(
                    "{%s}member/{%s}Observation/{%s}samplingTime/{%s}TimePeriod/{%s}beginPosition"
                    % (goNs['om'], goNs['om'], goNs['om'], goNs['gml'],
                       goNs['gml']))
                end = go.find(
                    "{%s}member/{%s}Observation/{%s}samplingTime/{%s}TimePeriod/{%s}endPosition"
                    % (goNs['om'], goNs['om'], goNs['om'], goNs['gml'],
                       goNs['gml']))
                procedures[pname].begin = begin.text
                procedures[pname].end = end.text

                # Extracting Feature of Interest and coordinates
                foi = go.find(
                    "{%s}member/{%s}Observation/{%s}featureOfInterest" %
                    (goNs['om'], goNs['om'], goNs['om']))
                point = foi.find("{%s}Point" % (goNs['gml']))

                if point == None:
                    point = foi.find(
                        "{%s}FeatureCollection/{%s}location/{%s}Point" %
                        (goNs['gml'], goNs['gml'], goNs['gml']))

                coord = point.find("{%s}coordinates" %
                                   (goNs['gml'])).text.split(",")

                if len(coord) == 2:
                    coord.append('0')

                procedures[pname].setFoi(
                    foi.get('{%s}href' % gcNs['xlink']).split(":")[-1],
                    point.get('srsName'), coord)

                # Extracting UOM
                fields = go.findall(
                    "{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}elementType/{%s}DataRecord/{%s}field"
                    % (goNs['om'], goNs['om'], goNs['om'], goNs['swe'],
                       goNs['swe'], goNs['swe'], goNs['swe']))

                for field in fields:
                    if field.get('name') != 'Time':
                        qty = field.find("{%s}Quantity" % (goNs['swe']))
                        uom = field.find("{%s}Quantity/{%s}uom" %
                                         (goNs['swe'], goNs['swe']))
                        procedures[pname].addObservedProperty(
                            field.get('name'),
                            qty.get('definition').replace(uniqidurn, ''),
                            uom.get('code'))

                if dfrom:
                    begin = dfrom
                    _begin = dfrom
                else:
                    begin = iso.parse_datetime(procedures[pname].begin)
                    _begin = iso.parse_datetime(procedures[pname].begin)

                if dto:
                    end = dto
                else:
                    end = iso.parse_datetime(procedures[pname].end)

                # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                # REGISTRATION PROCESS
                # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~

                # Check if procedure already exist
                res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                              (dst, srv, pname),
                              auth=auth,
                              verify=False)
                if not res.json()["success"]:
                    #print procedures[pname].data
                    # Registering procedure to istSOS
                    res = req.post("%s/wa/istsos/services/%s/procedures" %
                                   (dst, srv),
                                   data=json.dumps(procedures[pname].data),
                                   auth=auth)
                    if not res.json()["success"]:
                        #print json.dumps(procedures[pname].data)
                        raise Exception(
                            "Registering procedure %s failed: \n%s" %
                            (pname, res.json()["message"]))

                    # Getting details (describe sensor) to get the assignedSensorId
                    res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                                  (dst, srv, pname),
                                  auth=auth)

                    # Getting an InsertObservation template
                    template = procedures[pname].getIoTemplate()

                else:
                    # Getting an InsertObservation template
                    template = procedures[pname].getIoTemplate()
                    try:
                        if appendData and ('endPosition'
                                           in template['samplingTime']):
                            procedures[pname].begin = template['samplingTime'][
                                'endPosition']
                            begin = iso.parse_datetime(
                                template['samplingTime']['endPosition'])
                    except Exception as exproc:
                        print res.text
                        raise exproc

                procedures[pname].oid = res.json()["data"]["assignedSensorId"]
                days = int(args['i'])
                interval = timedelta(days=int(days))

                if not registerOnly:

                    if virtual and hq:
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        # VIRTUAL PROCEDURE CODE INITIALIZATION
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        pass

                    else:
                        print "Starting migration.."

                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        # PROCEDURE OBSERVATIONS MIGRATION
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~

                        oOrder = []

                        passedLoops = 0

                        lastPrint = ""

                        startTime = time.time()

                        print "%s: %s - %s" % (pname, procedures[pname].begin,
                                               procedures[pname].end)

                        if begin < end and begin + interval > end:
                            interval = end - begin

                        while (begin + interval) <= end:
                            loopTotalTime = time.time()

                            nextPosition = begin + interval

                            passedLoops = passedLoops + 1

                            t = float(
                                calendar.timegm(end.utctimetuple()) -
                                calendar.timegm(_begin.utctimetuple()))
                            t1 = float(
                                calendar.timegm(nextPosition.utctimetuple()) -
                                calendar.timegm(_begin.utctimetuple()))
                            try:
                                percentage = round((t1 / t) * 100, 2)
                            except:
                                percentage = 0
                            if percentage > 100:
                                percentage = 100
                            lastPrint = "%s > %s%% (%s / %s %s days)" % (
                                "\b" * len(lastPrint), percentage,
                                begin.strftime(fmtshort),
                                nextPosition.strftime(fmtshort), days)

                            looptime = time.time()
                            # GetObservation from source SOS
                            params = {
                                'service':
                                'SOS',
                                'version':
                                '1.0.0',
                                'request':
                                'GetObservation',
                                'eventTime':
                                '%s/%s' % (begin.strftime(fmt),
                                           nextPosition.strftime(fmt)),
                                'qualityIndex':
                                'True',
                                'offering':
                                offeringName,
                                'responseFormat':
                                'text/xml;subtype=\'sensorML/1.0.0\'',
                                'procedure':
                                pname,
                                'observedProperty':
                                ",".join(observedProperties)
                            }
                            try:
                                res = req.get("%s" % (src),
                                              params=params,
                                              verify=False)
                            except Exception:
                                res = req.get("%s" % (src),
                                              params=params,
                                              verify=False)

                            gotime = timedelta(seconds=int(time.time() -
                                                           looptime))

                            if gotime > timedelta(seconds=int(10)):
                                if days > 1:
                                    days = int(days / 2)
                                    if days <= 1:
                                        days = 1
                                interval = timedelta(days=days)
                            elif gotime < timedelta(seconds=int(5)):
                                days = days + 1
                                interval = timedelta(days=days)

                            lastPrint = "%s - GO: '%s'" % (lastPrint, gotime)

                            go, goNs = parse_and_get_ns(StringIO(res.content))

                            if len(oOrder) == 0:
                                fields = go.findall(
                                    "{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}elementType/{%s}DataRecord/{%s}field"
                                    % (goNs['om'], goNs['om'], goNs['om'],
                                       goNs['swe'], goNs['swe'], goNs['swe'],
                                       goNs['swe']))
                                for field in fields:
                                    oOrder.append(
                                        qty.get('definition').replace(
                                            'urn:ogc:def:parameter:x-ist::',
                                            ''))

                            values = go.find(
                                "{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}values"
                                % (goNs['om'], goNs['om'], goNs['om'],
                                   goNs['swe'], goNs['swe']))

                            if values.text:

                                rows = values.text.strip().split("@")

                                lastPrint = "%s " % (lastPrint)

                                copy = []

                                for row in rows:
                                    copy.append(row.split(","))

                                # InsertObservation to istSOS
                                template['result']['DataArray'][
                                    'values'] = copy
                                template['samplingTime'] = {
                                    "beginPosition": copy[0][0],
                                    "endPosition": nextPosition.strftime(fmt)
                                }
                                '''template['samplingTime'] = {
                                    "beginPosition": begin.strftime(fmt),
                                    "endPosition": nextPosition.strftime(fmt)
                                }'''

                                template[u"AssignedSensorId"] = procedures[
                                    pname].oid

                                looptime = time.time()
                                res = req.post(
                                    "%s/wa/istsos/services/%s/operations/insertobservation"
                                    % (dst, srv),
                                    auth=auth,
                                    data=json.dumps({
                                        u"AssignedSensorId":
                                        procedures[pname].oid,
                                        u"ForceInsert":
                                        u"true",
                                        u"Observation":
                                        template
                                    }))

                                iotime = timedelta(seconds=int(time.time() -
                                                               looptime))
                                lastPrint = "%s - IO: '%s'" % (lastPrint,
                                                               iotime)

                            begin = nextPosition
                            if begin < end and begin + interval > end:
                                interval = end - begin

                            if percentage < 100:
                                lastPrint = "%s - Step time: '%s' - Elapsed: %s  " % (
                                    lastPrint,
                                    timedelta(seconds=int(time.time() -
                                                          loopTotalTime)),
                                    timedelta(seconds=int(time.time() -
                                                          startTime)))
                            else:
                                lastPrint = "%s - Step time: '%s'  " % (
                                    lastPrint,
                                    timedelta(seconds=int(time.time() -
                                                          loopTotalTime)))

                            sys.stdout.write(lastPrint)
                            sys.stdout.flush()

                        print " > Completed in %s" % timedelta(
                            seconds=int(time.time() - startTime))
            break
    except Exception as e:
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
Esempio n. 29
0
 def getDSEndPosition(self):
     if u'constraint' in self.describe['outputs'][0]:
         return iso.parse_datetime(self.describe['outputs'][0]['constraint']['interval'][1])
     return None
Esempio n. 30
0
def execute(args, logger=None):

    print "istsos2csv start.."

    try:
        url = args['url']

        procedure = args['procedure']

        if 'filename' in args and args['filename'] is not None:
            filename = args['filename']
        else:
            filename = procedure

        observedProperty = args['op']

        begin = iso.parse_datetime(args['begin'])
        end = iso.parse_datetime(args['end'])

        d = args['d']

        auth = None
        if 'user' in args:
            user = args['user']
        password = None
        if 'password' in args:
            password = args['password']
        if user and password:
            auth = HTTPBasicAuth(user, password)

        qi = 'True'
        if 'noqi' in args:
            if args['noqi'] is True:
                qi = 'False'

        params = {
            "request": "GetObservation",
            "offering": "temporary",
            "procedure": procedure,
            "eventTime": None,
            "observedProperty": observedProperty,
            "responseFormat": "text/plain",
            "service": "SOS",
            "version": "1.0.0",
            "qualityIndex": qi
        }

        tmpBegin = begin
        tmpEnd = end
        if (end - begin) > step:
            tmpEnd = tmpBegin + step

        while tmpEnd <= end:
            print("%s - %s") % (tmpBegin, tmpEnd)

            if tmpBegin == tmpEnd:
                params["eventTime"] = iso.datetime_isoformat(tmpBegin)
            else:
                params["eventTime"] = "%s/%s" % (iso.datetime_isoformat(
                    tmpBegin), iso.datetime_isoformat(tmpEnd))

            res = req.get("%s?%s" % (url, urllib.urlencode(params)), auth=auth)

            makeFile(res, procedure, observedProperty, d, qi, filename)
            tmpBegin = tmpEnd
            tmpEnd = tmpBegin + step

            print " %s ************************** " % iso.datetime_isoformat(
                tmpEnd)

        if tmpBegin < end:
            tmpEnd = end
            if tmpBegin == tmpEnd:
                params["eventTime"] = iso.datetime_isoformat(tmpBegin)
            else:
                params["eventTime"] = "%s/%s" % (iso.datetime_isoformat(
                    tmpBegin), iso.datetime_isoformat(tmpEnd))

            res = req.get("%s?%s" % (url, urllib.urlencode(params)), auth=auth)
            makeFile(res, procedure, observedProperty, d, qi, filename)

            print " %s ************************** " % iso.datetime_isoformat(
                end)

        print "Finish."

    except Exception as e:
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
Esempio n. 31
0
    def __init__(self, sosRequest, method, requestObject, sosConfig):
        f.sosFilter.__init__(self, sosRequest, method, requestObject,
                             sosConfig)
        # @TODO Declare attribute first!
        # self.offering = None
        # etc..

        #**************************
        if method == "GET":
            #---------- THE OFFERING
            if requestObject.has_key("offering"):
                self.offering = get_name_from_urn(requestObject["offering"],
                                                  "offering", sosConfig)
            else:
                raise sosException.SOSException(
                    1,
                    "Parameter \"offering\" is mandatory with multiplicity 1")

            #---------- THE OBSERVED PROPERTY
            if requestObject.has_key("observedProperty"):
                self.observedProperty = []
                oprs = requestObject["observedProperty"].split(",")
                for opr in oprs:
                    # get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
                    # eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
                    #oprName = get_name_from_urn(opr,"property")
                    oprName = opr
                    self.observedProperty.append(oprName)  # one-many ID
            else:
                raise sosException.SOSException(
                    1,
                    "Parameter \"observedProperty\" is mandatory with multiplicity N"
                )

            #---------- RESPONSE FORMAT
            if requestObject.has_key("responseFormat"):
                if not requestObject["responseFormat"] in sosConfig.parameters[
                        "GO_responseFormat"]:
                    raise sosException.SOSException(
                        2,
                        "Parameter \"responseFormat\" sent with invalid value : use one of %s"
                        % "; ".join(sosConfig.parameters["GO_responseFormat"]))
                else:
                    self.responseFormat = requestObject["responseFormat"]
            else:
                raise sosException.SOSException(
                    1,
                    "Parameter \"responseFormat\" is mandatory with multiplicity 1"
                )  #one

            #OPTIONAL request parameters
            #---------- SRS FILTER
            if requestObject.has_key("srsName"):
                self.srsName = get_name_from_urn(requestObject["srsName"],
                                                 "refsystem", sosConfig)
                if not self.srsName in sosConfig.parameters["GO_srs"]:
                    raise sosException.SOSException(
                        2, "srsName \"%s\" not supported, use one of: %s" %
                        (self.srsName, ",".join(
                            sosConfig.parameters["GO_srs"])))
            else:
                self.srsName = sosConfig.parameters["GO_srs"][0]

            #---------- TIME FILTER
            if requestObject.has_key('eventTime'):
                self.eventTime = []
                for i in requestObject["eventTime"].replace(" ",
                                                            "+").split(","):
                    if len(i.split("/")) < 3:
                        self.eventTime.append(i.split("/"))
                    else:
                        raise sosException.SOSException(
                            2, "Parameter \"eventTime\" bad formatted")

                tp = []
                for t in self.eventTime:
                    if len(t) == 2:
                        tp.append(iso.parse_datetime(t[0]))
                        tp.append(iso.parse_datetime(t[1]))
                    if len(t) == 1:
                        tp.append(iso.parse_datetime(t[0]))

                # Checking if some event limitation is reached
                #if sosConfig["maxGoPeriod"]:
                if int(sosConfig.maxGoPeriod) > 0:
                    from datetime import timedelta
                    d = timedelta(hours=int(sosConfig.maxGoPeriod))
                    userPeriod = max(tp) - min(tp)
                    if d < userPeriod:
                        raise sosException.SOSException(
                            2,
                            "You are requesting data for a period of [%s hours], but you are not permitted to ask for a period longer than: %s hours"
                            % (userPeriod, d))

            else:
                self.eventTime = None

            #---------- PROCEDURES FILTER
            if requestObject.has_key("procedure"):
                self.procedure = []
                prcs = requestObject["procedure"].split(",")
                for prc in prcs:
                    prcName = get_name_from_urn(prc, "procedure", sosConfig)
                    self.procedure.append(prcName)
            else:
                self.procedure = None

            #---------- FEATURES OF INTEREST FILTER
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None
            if requestObject.has_key("featureOfInterest"):
                foi = requestObject["featureOfInterest"]
                if foi.find("<ogc:") >= 0 and foi.find("<gml:") >= 0:
                    #raise sosException.SOSException(3,"FOI SPATIAL: %s" %(foi))
                    self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(
                        foi, 'geom_foi', sosConfig.istsosepsg)
                else:
                    self.featureOfInterest = get_name_from_urn(
                        foi, "feature", sosConfig)

                #fois = requestObject["featureOfInterest"].split(",")
                #for foi in fois:
                #    foiName = get_name_from_urn(foi,"feature")
                #    self.featureOfInterest.append(foiName)

            #---------- FILTERS FOR QUERY NOT SUPPORTED YET
            if requestObject.has_key("result"):
                #raise sosException.SOSException(3,"Parameter \"result\" not yet supported")
                self.result = sosUtils.ogcCompCons2PostgisSql(
                    requestObject["result"])
            else:
                self.result = None  #zero-one optional

            #---------- RESULT MODEL
            if requestObject.has_key("resultModel"):
                if requestObject["resultModel"] in sosConfig.parameters[
                        "GO_resultModel"]:
                    self.resultModel = requestObject["resultModel"]
                else:
                    raise sosException.SOSException(
                        2,
                        "Parameter \"resultModel\" sent with invalid value: supported values are: %s"
                        % ",".join(sosConfig.parameters["GO_resultModel"]))
            else:
                self.resultModel = sosConfig.parameters["GO_resultModel"][0]

            #---------- RESPONSE MODE
            if requestObject.has_key("responseMode"):
                if requestObject["responseMode"] in sosConfig.parameters[
                        "GO_responseMode"]:
                    self.responseMode = requestObject["responseMode"]
                else:
                    raise sosException.SOSException(
                        2,
                        "Parameter \"responseMode\" sent with invalid value, supported values are: %s"
                        % (",".join(sosConfig.parameters["GO_responseMode"])))

            else:
                self.responseMode = sosConfig.parameters["GO_responseMode"][0]

            ###########################
            # NON STANDARD PARAMETERS #
            ###########################
            #---------- AGGREGATE INTERVAL
            # In ISO 8601 duration format
            if requestObject.has_key("aggregateInterval"):
                # Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime) != 1 or len(
                        self.eventTime[0]) != 2:
                    raise sosException.SOSException(2, exeMsg)
                self.aggregate_interval = requestObject["aggregateInterval"]
                try:
                    iso.parse_duration(self.aggregate_interval)
                except Exception as ex:
                    raise sosException.SOSException(
                        2,
                        "Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s"
                        % ex)
            else:
                self.aggregate_interval = None

            #---------- AGGREGATE FUNCTION
            # sum,avg,max,min
            if requestObject.has_key("aggregateFunction"):
                if self.aggregate_interval == None:
                    raise sosException.SOSException(
                        2,
                        "Using aggregate functions parameters \"aggregateInterval\" and \"aggregateFunction\" are both mandatory"
                    )
                self.aggregate_function = requestObject["aggregateFunction"]
                if not (self.aggregate_function.upper()
                        in ["AVG", "COUNT", "MAX", "MIN", "SUM"]):
                    raise sosException.SOSException(
                        2,
                        "Available aggregation functions: avg, count, max, min, sum."
                    )
            else:
                self.aggregate_function = None

            #---------- AGGREGATE NODATA
            if requestObject.has_key("aggregateNodata"):
                if self.aggregate_interval == None or self.aggregate_function == None:
                    raise sosException.SOSException(
                        2,
                        "Using aggregateNodata parameter requires both \"aggregateInterval\" and \"aggregateFunction\""
                    )
                self.aggregate_nodata = requestObject["aggregateNodata"]
            else:
                self.aggregate_nodata = sosConfig.aggregate_nodata

            #---------- AGGREGATE NODATA QUALITY INDEX
            if requestObject.has_key("aggregateNodataQi"):
                if self.aggregate_interval == None or self.aggregate_function == None:
                    raise sosException.SOSException(
                        2,
                        "Using aggregateNodataQi parameter requires both \"aggregateInterval\" and \"aggregateFunction\""
                    )
                self.aggregate_nodata_qi = requestObject["aggregateNodataQi"]
            else:
                self.aggregate_nodata_qi = sosConfig.aggregate_nodata_qi

            #------------ QUALITY INDEX
            self.qualityIndex = False
            if requestObject.has_key("qualityIndex"):
                if requestObject["qualityIndex"].upper() == "TRUE":
                    self.qualityIndex = True
                elif requestObject["qualityIndex"].upper() == "FALSE":
                    self.qualityIndex = False
                else:
                    raise sosException.SOSException(
                        2, "qualityIndex can only be True or False!")
                #    self.qualityIndex = sosUtils.CQLvalueFilter2PostgisSql("id_qi_fk",requestObject["qualityIndex"])

        #**********************
        if method == "POST":
            from xml.dom import minidom
            #---------- THE OFFERING
            offs = requestObject.getElementsByTagName('offering')
            if len(offs) == 1:
                val = offs[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.offering = get_name_from_urn(str(val.data),
                                                      "offering", sosConfig)
                else:
                    err_txt = "XML parsing error (get value: offering)"
                    raise sosException.SOSException(1, err_txt)
            else:
                err_txt = "Parameter \"offering\" is mandatory with multiplicity 1"
                raise sosException.SOSException(1, err_txt)

            #---------- THE OBSERVED PROPERTY
            obsProps = requestObject.getElementsByTagName('observedProperty')
            self.observedProperty = []
            if len(obsProps) > 0:
                for obsProp in obsProps:
                    val = obsProp.firstChild
                    if val.nodeType == val.TEXT_NODE:
                        # get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
                        # eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
                        #self.observedProperty.append(get_name_from_urn(str(val.data),"property"))
                        self.observedProperty.append(str(val.data))
                    else:
                        err_txt = "XML parsing error (get value: observedProperty)"
                        raise sosException.SOSException(1, err_txt)
            else:
                err_txt = "Parameter \"observedProperty\" is mandatory with multiplicity N"
                raise sosException.SOSException(1, err_txt)

            #---------- RESPONSE FORMAT
            respF = requestObject.getElementsByTagName('responseFormat')
            if len(respF) == 1:
                val = respF[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.responseFormat = str(val.data)
                    if self.responseFormat not in sosConfig.parameters[
                            "GO_responseFormat"]:
                        raise sosException.SOSException(
                            2,
                            "Parameter \"responseFormat\" sent with invalid value: use one of %s"
                            % "; ".join(
                                sosConfig.parameters["GO_responseFormat"]))
                else:
                    err_txt = "XML parsing error (get value: responseFormat)"
                    raise sosException.SOSException(1, err_txt)
            else:
                err_txt = "Parameter \"responseFormat\" is mandatory with multiplicity 1"
                raise sosException.SOSException(1, err_txt)

            #OPTIONAL request parameters
            #---------- SRS OF RETURNED GML FEATURES
            srss = requestObject.getElementsByTagName('srsName')
            if len(srss) > 0:
                if len(srss) < 2:
                    val = srss[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.srsName = get_name_from_urn(
                            str(val.data), "refsystem", sosConfig)
                    else:
                        err_txt = "XML parsing error (get value: srsName)"
                        raise sosException.SOSException(1, err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"srsName\""
                    raise sosException.SOSException(1, err_txt)
            else:
                self.srsName = sosConfig.parameters["GO_srs"][0]

            #---------- TIME FILTER
            evtms = requestObject.getElementsByTagName('eventTime')
            self.eventTime = []
            if len(evtms) > 0:
                for evtm in evtms:
                    tps = evtm.getElementsByTagName('gml:TimePeriod')
                    for tp in tps:
                        begin = tp.getElementsByTagName('gml:beginPosition')
                        end = tp.getElementsByTagName('gml:endPosition')
                        if len(begin) == 1 and len(end) == 1:
                            Bval = begin[0].firstChild
                            Eval = end[0].firstChild
                            #raise sosException.SOSException(1,end[0].toprettyxml())
                            if Bval.nodeType == Bval.TEXT_NODE and Eval.nodeType == Eval.TEXT_NODE:
                                self.eventTime.append([
                                    str(Bval.data).replace(" ", "+"),
                                    str(Eval.data).replace(" ", "+")
                                ])
                                #raise sosException.SOSException(1,str(self.eventTime))
                            else:
                                err_txt = "XML parsing error (get value: TimePeriod)"
                                raise sosException.SOSException(1, err_txt)

                    tis = evtm.getElementsByTagName('gml:TimeInstant')
                    for ti in tis:
                        instant = ti.getElementsByTagName('gml:timePosition')
                        if len(instant) > 0 and len(instant) < 2:
                            Ival = instant[0].firstChild
                            if Ival.nodeType == Ival.TEXT_NODE:
                                self.eventTime.append(
                                    [str(Ival.data).replace(" ", "+")])
                            else:
                                err_txt = "XML parsing error (get value: Timeinstant)"
                                raise sosException.SOSException(1, err_txt)
            else:
                self.eventTime = None

            #---------- PROCEDURES FILTER
            procs = requestObject.getElementsByTagName('procedure')
            if len(procs) > 0:
                self.procedure = []
                for proc in procs:
                    if "xlink:href" in proc.attributes.keys():
                        self.procedure.append(
                            str(proc.getAttribute("xlink:href")))
                    elif proc.hasChildNodes():
                        val = proc.firstChild
                        if val.nodeType == val.TEXT_NODE:
                            self.procedure.append(
                                get_name_from_urn(str(val.data), "procedure",
                                                  sosConfig))
                    else:
                        err_txt = "XML parsing error (get value: procedure)"
                        raise sosException.SOSException(1, err_txt)
            else:
                self.procedure = None

            #---------- FEATURES OF INTEREST FILTER
            fets = requestObject.getElementsByTagName('featureOfInterest')
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None
            if len(fets) > 0:
                if len(fets) < 2:
                    elements = [
                        e for e in fets[0].childNodes
                        if e.nodeType == e.ELEMENT_NODE
                    ]
                    if len(elements) == 1:
                        self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(
                            elements[0], 'geom_foi', sosConfig.istsosepsg)
                    else:
                        if "xlink:href" in fets[0].attributes.keys():
                            self.featureOfInterest = str(
                                fets[0].getAttribute("xlink:href"))
                        elif fets[0].hasChildNodes():
                            val = fets[0].firstChild
                            if val.nodeType == val.TEXT_NODE:
                                self.featureOfInterest = get_name_from_urn(
                                    str(val.data), "feature", sosConfig)
                        else:
                            err_txt = "XML parsing error (get value: featureOfInterest)"
                            raise sosException.SOSException(1, err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"featureOfInterest\""
                    raise sosException.SOSException(1, err_txt)

            #---------- FILTERS FOR QUERY NOT SUPPORTED YET
            ress = requestObject.getElementsByTagName('result')
            if len(ress) > 0:
                raise sosException.SOSException(
                    3, "Parameter \"result\" not yet supported")
            else:
                self.result = None  #zero-one optional

            #---------- RESULT MODEL
            mods = requestObject.getElementsByTagName('resultModel')
            if len(mods) > 0:
                if len(mods) < 2:
                    val = mods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.resultModel = str(val.data)
                        if self.resultModel not in sosConfig.parameters[
                                "GO_resultModel"]:
                            raise sosException.SOSException(
                                2,
                                "Parameter \"resultModel\" sent with invalid value"
                            )
                    else:
                        err_txt = "XML parsing error (get value: resultModel)"
                        raise sosException.SOSException(1, err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"resultModel\""
                    raise sosException.SOSException(1, err_txt)
            else:
                self.resultModel = None

            #---------- RESPONSE MODE
            rsmods = requestObject.getElementsByTagName('responseMode')
            if len(rsmods) > 0:
                if len(rsmods) < 2:
                    val = rsmods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.responseMode = str(val.data)
                        if self.responseMode not in sosConfig.parameters[
                                "GO_responseMode"]:
                            raise sosException.SOSException(
                                2,
                                "Parameter \"responseMode\" sent with invalid value"
                            )
                    else:
                        err_txt = "XML parsing error (get value: responseMode)"
                        raise sosException.SOSException(1, err_txt)
                else:
                    err_txt = "Allowed only ONE parameter \"responseMode\""
                    raise sosException.SOSException(1, err_txt)
            else:
                self.responseMode = sosConfig.parameters["GO_responseMode"][0]

            #-------------- AGGREGATE INTERVAL & FUNCTION
            self.aggregate_interval = None
            self.aggregate_function = None
            aggint = requestObject.getElementsByTagName('aggregateInterval')
            aggfun = requestObject.getElementsByTagName('aggregateFunction')
            aggnodata = requestObject.getElementsByTagName('aggregateNodata')

            if len(aggint) == 1 and len(aggfun) == 1:
                #-----------------------
                # -- aggregate_interval
                #-----------------------
                # Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime) != 1 or len(
                        self.eventTime[0]) != 2:
                    raise sosException.SOSException(2, exeMsg)
                val = aggint[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_interval = str(val.data)
                    try:
                        iso.parse_duration(self.aggregate_interval)
                    except Exception as ex:
                        raise sosException.SOSException(
                            2,
                            "Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s"
                            % ex)
                else:
                    err_txt = "cannot get ISO8601 duration value in \"aggregateInterval\""
                    raise sosException.SOSException(1, err_txt)
                #-----------------------
                # -- aggregate_function
                #-----------------------
                val = aggfun[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_function = str(val.data)
                    if not (self.aggregate_function.upper()
                            in ["AVG", "COUNT", "MAX", "MIN", "SUM"]):
                        raise sosException.SOSException(
                            2,
                            "Available aggregation functions: avg, count, max, min, sum."
                        )

                #-----------------------------------
                # -- aggregate_no_data default value
                #-----------------------------------
                if len(aggnodata) == 1:
                    val = aggnodata[0].firstChild
                    self.aggregate_nodata = str(val.data)
                else:
                    self.aggregate_nodata = sosConfig.aggregate_nodata

        #================================
        #MISSING AGGREGATE QUALITY INDEX
        #================================

            elif len(aggint) == 0 and len(aggfun) == 0:
                pass
            else:
                err_txt = "\"aggregateInterval\" and \"aggregate_function\" are both required with multiplicity 1"
                raise sosException.SOSException(1, err_txt)

            #------------ QUALITY INDEX
            self.qualityIndex = False
            qidx = requestObject.getElementsByTagName('qualityIndex')
            if len(qidx) > 0:
                if len(qidx) < 2:
                    val = qidx[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.qualityIndex = str(val.data)
                        if self.qualityIndex.upper() == "TRUE":
                            self.qualityIndex = True
                        elif self.qualityIndex.upper() == "FALSE":
                            pass
                        else:
                            raise sosException.SOSException(
                                2,
                                "qualityIndex can only be \'True\' or \'False\'"
                            )
            elif len(qidx) == 0:
                pass
            else:
                err_txt = "\"qualityIndex\" is allowed with multiplicity 1 only"
                raise sosException.SOSException(1, err_txt)
Esempio n. 32
0
    def __init__(self, sosRequest, method, requestObject, sosConfig):
        f.sosFilter.__init__(self, sosRequest, method, requestObject,
                             sosConfig)
        if method == "GET":

            self.eventTime = None
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None
            self.result = None  #zero-one optional
            self.observedProperty = [':']

            # OBSERVED PROPERTY
            #   get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
            #   eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
            if requestObject.has_key("observedproperty"):
                self.observedProperty = []
                oprs = requestObject["observedproperty"].split(",")

                for opr in oprs:
                    if opr == '':
                        raise sosException.SOSException(
                            "MissingParameterValue", "observedProperty",
                            "Missing 'observedProperty' parameter")

                    oprName = opr
                    self.observedProperty.append(oprName)  # one-many ID

            # PROCEDURES FILTER
            if requestObject.has_key("procedure"):
                self.procedure = []
                prcs = requestObject["procedure"].split(",")

                for prc in prcs:
                    if prc == '':
                        raise sosException.SOSException(
                            "MissingParameterValue", "procedure",
                            "Missing 'procedure' parameter")

                    try:
                        prcName = get_name_from_urn(prc, "procedure",
                                                    sosConfig)

                    except Exception as e:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "procedure", str(e))

                    self.procedure.append(prcName)

            else:
                self.procedure = None

            if self.version == '2.0.0':

                # THE OFFERING
                #  > in istSOS offerings are equals to procedures
                #  > so offerings are inserted into the procedure array filter

                if requestObject.has_key("offering"):
                    prcs = requestObject["offering"].split(",")
                    if self.procedure == None:
                        self.procedure = []

                    for prc in prcs:
                        if prc == '':
                            raise sosException.SOSException(
                                "MissingParameterValue", "offering",
                                "Missing 'offering' parameter")
                        try:
                            prcName = get_name_from_urn(
                                prc, "offering", sosConfig)

                        except Exception as e:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "offering", str(e))

                        # Check for name redundancy
                        if prcName not in self.procedure:
                            self.procedure.append(prcName)

                # RESPONSE FORMAT
                self.responseFormat = 'text/xml;subtype="om/2.0"'
                if requestObject.has_key("responseformat"):
                    if requestObject["responseformat"] == '':
                        raise sosException.SOSException(
                            "MissingParameterValue", "responseformat",
                            "Missing 'responseformat' parameter")
                    if not requestObject[
                            "responseformat"] in sosConfig.parameters[
                                "GO_responseFormat_2_0_0"]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "responseFormat",
                            "Parameter \"responseFormat\" sent with invalid value : use one of %s"
                            % "; ".join(sosConfig.
                                        parameters["GO_responseFormat_2_0_0"]))
                    elif requestObject[
                            "responseformat"] == sosConfig.parameters[
                                "GO_responseFormat_2_0_0"][0]:
                        self.responseFormat = 'text/xml;subtype="om/2.0"'
                    else:
                        self.responseFormat = requestObject["responseformat"]

                # OPTIONAL SRS FILTER
                if requestObject.has_key("crs"):
                    try:
                        self.srsName = requestObject["crs"].split(':')[-1]

                    except Exception as e:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "crs", "%s" % e)

                    if not self.srsName in sosConfig.parameters["GO_srs"]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "crs",
                            "crs \"%s\" not supported, use one of: %s" %
                            (self.srsName, ",".join(
                                sosConfig.parameters["GO_srs"])))

                else:
                    self.srsName = sosConfig.parameters["GO_srs"][0]

                # TIME FILTER
                # istSOS supports
                #     kvp examples:
                #     - during:       temporalFilter=om:phenomenonTime,2012-11-19T14:00:00+01:00/2012-11-19T14:15:00+01:00
                #     - equals:       temporalFilter=om:phenomenonTime,2012-11-19T14:00:00.000+01:00
                #     - combination:  temporalFilter=om:phenomenonTime,2012-11-19T14:00:00+01:00/2012-11-19T14:15:00+01:00,2012-11-19T14:00:00.000+01:00
                if 'temporalfilter' in requestObject:
                    self.eventTime = []
                    temporalfilter = requestObject["temporalfilter"].replace(
                        " ", "+").split(",")

                    #  > in istSOS om:phenomenonTime is equals to om:resultTime
                    if temporalfilter.pop(0) not in [
                            'om:phenomenonTime', 'phenomenonTime',
                            'om:resultTime', 'resultTime'
                    ]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "temporalfilter",
                            "Parameter \"temporalFilter\" bad formatted")

                    for i in temporalfilter:
                        if '/' in i:
                            interval = i.split("/")
                            if len(interval) != 2:
                                raise sosException.SOSException(
                                    "InvalidParameterValue", "temporalfilter",
                                    "Parameter \"temporalfilter\" bad formatted"
                                )
                            try:
                                iso.parse_date(interval[0])
                                iso.parse_date(interval[1])

                            except iso.ISO8601Error as isoerr:
                                raise sosException.SOSException(
                                    "InvalidParameterValue", "temporalfilter",
                                    "Parameter \"temporalfilter\" bad formatted, %s"
                                    % isoerr)

                            self.eventTime.append(interval)

                        else:
                            try:
                                iso.parse_date(i)

                            except iso.ISO8601Error as isoerr:
                                raise sosException.SOSException(
                                    "InvalidParameterValue", "temporalfilter",
                                    "Parameter \"temporalfilter\" bad formatted, %s"
                                    % isoerr)

                            self.eventTime.append([i])

                # FEATURES OF INTEREST FILTER
                if requestObject.has_key("featureofinterest"):
                    if requestObject["featureofinterest"] == '':
                        raise sosException.SOSException(
                            "MissingParameterValue", "featureOfInterest",
                            "Missing 'featureOfInterest' parameter")
                    if sosConfig.urn["feature"] in requestObject[
                            "featureofinterest"]:
                        self.featureOfInterest = get_name_from_urn(
                            requestObject["featureofinterest"], "feature",
                            sosConfig)
                    else:
                        self.featureOfInterest = requestObject[
                            "featureofinterest"]

                # SPATIAL FILTER
                # example1: spatialFilter=om:featureOfInterest/*/sams:shape,0.0,0.0,60.0,60.0,http://www.opengis.net/def/crs/EPSG/0/4326
                # example2: spatialFilter=om:featureOfInterest/*/sams:shape,0.0,0.0,60.0,60.0,urn:ogc:def:crs:EPSG::4326
                if requestObject.has_key("spatialfilter"):

                    sfs = requestObject["spatialfilter"].split(",")

                    if len(sfs) != 6:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "spatialfilter",
                            "Invalid spatial filter '%s'" %
                            requestObject["spatialfilter"])

                    if sfs[0] != 'om:featureOfInterest/*/sams:shape':
                        raise sosException.SOSException(
                            "InvalidParameterValue", "spatialfilter",
                            "Invalid spatial filter '%s'" %
                            requestObject["spatialfilter"])

                    srsName = None

                    if sfs[5].index(':') > -1:
                        srsName = sfs[5].split(':')[-1]

                    if sfs[5].index('/') > -1:
                        srsName = sfs[5].split('/')[-1]

                    ogcfilter = (
                        "<ogc:BBOX>" +
                        "<ogc:PropertyName>the_geom</ogc:PropertyName>" +
                        ("<gml:Box srsName='EPSG:%s'>" % (srsName)) +
                        ("<gml:coordinates>%s,%s %s,%s</gml:coordinates>" %
                         (sfs[1], sfs[2], sfs[3], sfs[4])) + "</gml:Box>" +
                        "</ogc:BBOX>")

                    self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(
                        ogcfilter, 'geom_foi', sosConfig.istsosepsg)

            else:

                # THE OFFERING
                if requestObject.has_key("offering"):
                    try:
                        self.offering = get_name_from_urn(
                            requestObject["offering"], "offering", sosConfig)

                    except Exception as e:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "offering", str(e))

                # RESPONSE FORMAT
                if requestObject.has_key("responseformat"):
                    if requestObject["responseformat"] == '':
                        raise sosException.SOSException(
                            "MissingParameterValue", "responseFormat",
                            "Missing 'responseFormat' parameter")

                    if not requestObject[
                            "responseformat"] in sosConfig.parameters[
                                "GO_responseFormat"]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "responseFormat",
                            "Parameter \"responseFormat\" sent with invalid value : use one of %s"
                            % "; ".join(
                                sosConfig.parameters["GO_responseFormat"]))

                    else:
                        self.responseFormat = requestObject["responseformat"]

                if not requestObject.has_key("offering"):
                    raise sosException.SOSException(
                        "MissingParameterValue", "offering",
                        "Parameter \"offering\" is mandatory with multiplicity 1"
                    )

                if not requestObject.has_key("observedproperty"):
                    raise sosException.SOSException(
                        "MissingParameterValue", "observedProperty",
                        "Parameter \"observedProperty\" is mandatory with multiplicity N"
                    )

                if not requestObject.has_key("responseformat"):
                    raise sosException.SOSException(
                        "MissingParameterValue", "responseFormat",
                        "Parameter \"responseFormat\" is mandatory with multiplicity 1"
                    )  #one

                # OPTIONAL SRS FILTER
                if requestObject.has_key("srsname"):
                    try:
                        self.srsName = get_name_from_urn(
                            requestObject["srsname"], "refsystem", sosConfig)
                    except Exception as e:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "srsname", "%s" % e)

                    if not self.srsName in sosConfig.parameters["GO_srs"]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "srsName",
                            "srsName \"%s\" not supported, use one of: %s" %
                            (self.srsName, ",".join(
                                sosConfig.parameters["GO_srs"])))
                else:
                    self.srsName = sosConfig.parameters["GO_srs"][0]

                # TIME FILTER
                if requestObject.has_key('eventtime'):
                    self.eventTime = []
                    for i in requestObject["eventtime"].replace(
                            " ", "+").split(","):
                        if len(i.split("/")) < 3:
                            self.eventTime.append(i.split("/"))

                        else:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "eventTime",
                                "Parameter \"eventTime\" bad formatted")

                # FEATURES OF INTEREST FILTER
                if requestObject.has_key("featureofinterest"):
                    foi = requestObject["featureofinterest"]
                    if foi.find("<ogc:") >= 0 and foi.find("<gml:") >= 0:
                        self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(
                            foi, 'geom_foi', sosConfig.istsosepsg)

                    else:
                        try:
                            self.featureOfInterest = get_name_from_urn(
                                foi, "feature", sosConfig)

                        except Exception as e:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "featureofinterest",
                                str(e))

                # FILTERS FOR QUERY NOT SUPPORTED YET
                if requestObject.has_key("result"):
                    self.result = sosUtils.ogcCompCons2PostgisSql(
                        requestObject["result"])

                # RESULT MODEL
                if requestObject.has_key("resultmodel"):
                    if requestObject["resultmodel"] in sosConfig.parameters[
                            "GO_resultModel"]:
                        self.resultModel = requestObject["resultmodel"]

                    else:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "resultModel",
                            "Parameter \"resultModel\" sent with invalid value: supported values are: %s"
                            % ",".join(sosConfig.parameters["GO_resultModel"]))

                else:
                    self.resultModel = sosConfig.parameters["GO_resultModel"][
                        0]

                # RESPONSE MODE
                if requestObject.has_key("responsemode"):
                    if requestObject["responsemode"] in sosConfig.parameters[
                            "GO_responseMode"]:
                        self.responseMode = requestObject["responsemode"]

                    else:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "responseMode",
                            "Parameter \"responseMode\" sent with invalid value, supported values are: %s"
                            % (",".join(
                                sosConfig.parameters["GO_responseMode"])))

                else:
                    self.responseMode = sosConfig.parameters[
                        "GO_responseMode"][0]

            # Checking if some event limitation is reached
            if self.eventTime != None:
                tp = []
                for t in self.eventTime:
                    if len(t) == 2:
                        tp.append(iso.parse_datetime(t[0]))
                        tp.append(iso.parse_datetime(t[1]))

                    if len(t) == 1:
                        tp.append(iso.parse_datetime(t[0]))

                if int(sosConfig.maxGoPeriod) > 0:
                    maxhours = timedelta(hours=int(sosConfig.maxGoPeriod))
                    userPeriod = max(tp) - min(tp)

                    if maxhours < userPeriod:
                        if self.version == '2.0.0':
                            # REQ39 - http://www.opengis.net/spec/SOS/2.0/req/core/go-too-many-obs-exception
                            #     The service determined that the requested result set exceeds the response
                            #     size limit of the service and thus cannot be delivered.
                            raise sosException.SOSException(
                                "ResponseExceedsSizeLimit", "",
                                "You are requesting data for a period of [%s hours], but you are not permitted to ask for a period longer than: %s hours"
                                % (userPeriod, maxhours))

                        else:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "eventTime",
                                "You are requesting data for a period of [%s hours], but you are not permitted to ask for a period longer than: %s hours"
                                % (userPeriod, maxhours))

            elif (sosConfig.strictogc in ['True', 'true', 1]
                  and self.version == '2.0.0' and self.eventTime == None
                  and self.featureOfInterest == None
                  and self.featureOfInterestSpatial == None
                  and self.procedure == None):
                # ResponseExceedsSizeLimit fake exception
                raise sosException.SOSException(
                    "ResponseExceedsSizeLimit", "",
                    "Sorry but, You are requesting too many data")

            #####################################
            # NON STANDARD PARAMETERS by istSOS #
            #####################################

            # AGGREGATE INTERVAL
            #  In ISO 8601 duration format
            if requestObject.has_key("aggregateinterval"):
                # Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime) != 1 or len(
                        self.eventTime[0]) != 2:
                    raise sosException.SOSException("InvalidParameterValue",
                                                    "aggregateInterval",
                                                    exeMsg)

                self.aggregate_interval = requestObject["aggregateinterval"]
                try:
                    iso.parse_duration(self.aggregate_interval)

                except Exception as ex:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "aggregateInterval",
                        "Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s"
                        % ex)
            else:
                self.aggregate_interval = None

            # AGGREGATE FUNCTION
            #  sum,avg,max,min
            if requestObject.has_key("aggregatefunction"):
                if self.aggregate_interval == None:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "aggregateFunction",
                        "Using aggregate functions parameters \"aggregateInterval\" and \"aggregateFunction\" are both mandatory"
                    )

                self.aggregate_function = requestObject["aggregatefunction"]
                if not (self.aggregate_function.upper()
                        in ["AVG", "COUNT", "MAX", "MIN", "SUM"]):
                    raise sosException.SOSException(
                        "InvalidParameterValue", "aggregateFunction",
                        "Available aggregation functions: avg, count, max, min, sum."
                    )

            else:
                self.aggregate_function = None

            # AGGREGATE NODATA
            if requestObject.has_key("aggregatenodata"):
                if self.aggregate_interval == None or self.aggregate_function == None:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "aggregateNodata",
                        "Using aggregateNodata parameter requires both \"aggregateInterval\" and \"aggregateFunction\""
                    )

                self.aggregate_nodata = requestObject["aggregatenodata"]

            else:
                self.aggregate_nodata = sosConfig.aggregate_nodata

            # AGGREGATE NODATA QUALITY INDEX
            if requestObject.has_key("aggregatenodataqi"):
                if self.aggregate_interval == None or self.aggregate_function == None:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "aggregateNodataQi",
                        "Using aggregateNodataQi parameter requires both \"aggregateInterval\" and \"aggregateFunction\""
                    )
                self.aggregate_nodata_qi = requestObject["aggregatenodataqi"]

            else:
                self.aggregate_nodata_qi = sosConfig.aggregate_nodata_qi

            # QUALITY INDEX
            self.qualityIndex = False
            if requestObject.has_key("qualityindex"):
                if requestObject["qualityindex"].upper() == "TRUE":
                    self.qualityIndex = True

                elif requestObject["qualityindex"].upper() == "FALSE":
                    self.qualityIndex = False

                else:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "qualityIndex",
                        "qualityIndex can only be True or False!")

            # QUALITY INDEX FILTERING
            self.qualityFilter = False
            if requestObject.has_key("qualityfilter"):
                if len(requestObject["qualityfilter"]) >= 2:
                    try:
                        if requestObject["qualityfilter"][
                                0:2] == '<=' or requestObject["qualityfilter"][
                                    0:2] == '>=':
                            self.qualityFilter = (
                                requestObject["qualityfilter"][0:2],
                                float(requestObject["qualityfilter"][2:]))

                        elif (requestObject["qualityfilter"][0] == '>'
                              or requestObject["qualityfilter"][0] == '='
                              or requestObject["qualityfilter"][0] == '<'):
                            self.qualityFilter = (
                                requestObject["qualityfilter"][0],
                                float(requestObject["qualityfilter"][1:]))

                        # If qualityFilter is defined qualityIndex are automatically returned
                        self.qualityIndex = True

                    except ValueError as ve:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "qualityFilter",
                            "invalid quality index value in qualityFilter")

                else:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "qualityFilter",
                        "qualityFilter operator can only be in ['<','>','<=','>=','=']"
                    )

        if method == "POST":
            from xml.dom import minidom
            # THE OFFERING
            offs = requestObject.getElementsByTagName('offering')
            if len(offs) == 1:
                val = offs[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    try:
                        self.offering = get_name_from_urn(
                            str(val.data), "offering", sosConfig)

                    except Exception as e:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "offering", str(e))

                else:
                    err_txt = "XML parsing error (get value: offering)"
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    err_txt)

            else:
                err_txt = "Parameter \"offering\" is mandatory with multiplicity 1"
                raise sosException.SOSException("MissingParameterValue",
                                                "offering", err_txt)

            # THE OBSERVED PROPERTY
            obsProps = requestObject.getElementsByTagName('observedProperty')
            self.observedProperty = []
            if len(obsProps) > 0:
                for obsProp in obsProps:
                    val = obsProp.firstChild
                    if val.nodeType == val.TEXT_NODE:
                        # get_name_from_urn limit the ability to ask for an observedProperty with LIKE:
                        # eg: ask "water" to get all the water related data, "water:discharge", "water:temperature" ...
                        #self.observedProperty.append(get_name_from_urn(str(val.data),"property"))
                        self.observedProperty.append(str(val.data))

                    else:
                        err_txt = "XML parsing error (get value: observedProperty)"
                        raise sosException.SOSException(
                            "NoApplicableCode", None, err_txt)

            else:
                err_txt = "Parameter \"observedProperty\" is mandatory with multiplicity N"
                raise sosException.SOSException("MissingParameterValue",
                                                "observedProperty", err_txt)

            # RESPONSE FORMAT
            respF = requestObject.getElementsByTagName('responseFormat')
            if len(respF) == 1:
                val = respF[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.responseFormat = str(val.data)
                    if self.responseFormat not in sosConfig.parameters[
                            "GO_responseFormat"]:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "responseFormat",
                            "Parameter \"responseFormat\" sent with invalid value: use one of %s"
                            % "; ".join(
                                sosConfig.parameters["GO_responseFormat"]))

                else:
                    err_txt = "XML parsing error (get value: responseFormat)"
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    err_txt)

            else:
                err_txt = "Parameter \"responseFormat\" is mandatory with multiplicity 1"
                raise sosException.SOSException("MissingParameterValue",
                                                "responseFormat", err_txt)

            # OPTIONAL request parameters
            #  SRS OF RETURNED GML FEATURES
            srs = requestObject.getAttributeNode('srsName')
            if srs:
                self.srsName = srs.nodeValue
                if not self.srsName in sosConfig.parameters["GO_srs"]:
                    raise sosException.SOSException(
                        "InvalidParameterValue", "srsName",
                        "srsName \"%s\" not supported, use one of: %s" %
                        (self.srsName, ",".join(
                            sosConfig.parameters["GO_srs"])))

            else:
                self.srsName = sosConfig.parameters["GO_srs"][0]

            # TIME FILTER
            evtms = requestObject.getElementsByTagName('eventTime')
            self.eventTime = []
            if len(evtms) > 0:
                for evtm in evtms:
                    tps = evtm.getElementsByTagName('gml:TimePeriod')
                    for tp in tps:
                        begin = tp.getElementsByTagName('gml:beginPosition')
                        end = tp.getElementsByTagName('gml:endPosition')
                        if len(begin) == 1 and len(end) == 1:
                            Bval = begin[0].firstChild
                            Eval = end[0].firstChild
                            if Bval.nodeType == Bval.TEXT_NODE and Eval.nodeType == Eval.TEXT_NODE:
                                self.eventTime.append([
                                    str(Bval.data).replace(" ", "+"),
                                    str(Eval.data).replace(" ", "+")
                                ])

                            else:
                                err_txt = "XML parsing error (get value: TimePeriod)"
                                raise sosException.SOSException(
                                    "NoApplicableCode", None, err_txt)

                    tis = evtm.getElementsByTagName('gml:TimeInstant')
                    for ti in tis:
                        instant = ti.getElementsByTagName('gml:timePosition')
                        if len(instant) > 0 and len(instant) < 2:
                            Ival = instant[0].firstChild
                            if Ival.nodeType == Ival.TEXT_NODE:
                                self.eventTime.append(
                                    [str(Ival.data).replace(" ", "+")])
                            else:
                                err_txt = "XML parsing error (get value: Timeinstant)"
                                raise sosException.SOSException(
                                    "NoApplicableCode", None, err_txt)
            else:
                self.eventTime = None

            # PROCEDURES FILTER
            procs = requestObject.getElementsByTagName('procedure')
            if len(procs) > 0:
                self.procedure = []
                for proc in procs:
                    if "xlink:href" in proc.attributes.keys():
                        self.procedure.append(
                            str(proc.getAttribute("xlink:href")))

                    elif proc.hasChildNodes():
                        val = proc.firstChild
                        if val.nodeType == val.TEXT_NODE:
                            try:
                                self.procedure.append(
                                    get_name_from_urn(str(val.data),
                                                      "procedure", sosConfig))

                            except Exception as e:
                                raise sosException.SOSException(
                                    "InvalidParameterValue", "procedure",
                                    str(e))
                    else:
                        err_txt = "XML parsing error (get value: procedure)"
                        raise sosException.SOSException(
                            "NoApplicableCode", None, err_txt)

            else:
                self.procedure = None

            # FEATURES OF INTEREST FILTER
            fets = requestObject.getElementsByTagName('featureOfInterest')
            self.featureOfInterest = None
            self.featureOfInterestSpatial = None

            # get sub-elements of FOI
            if fets:
                elements = [
                    e for e in fets[0].childNodes
                    if e.nodeType == e.ELEMENT_NODE
                ]
                if len(elements) == 0:
                    err_txt = "ObjectID or ogc:spatialOps elements in parameter \"featureOfInterest\" are mandatory"
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    err_txt)

                # only one sub element
                elif len(elements) == 1 and elements[0].tagName != "ObjectID":
                    self.featureOfInterestSpatial = sosUtils.ogcSpatCons2PostgisSql(
                        elements[0], 'geom_foi', sosConfig.istsosepsg)

                else:
                    tempfois = []
                    for e in elements:
                        if not e.tagName == "ObjectID":
                            err_txt = "Allowed only ObjectID or ogc:spatialOps elements in parameter \"featureOfInterest\""
                            raise sosException.SOSException(
                                "NoApplicableCode", None, err_txt)

                        try:
                            val = e.firstChild
                            if val.nodeType == val.TEXT_NODE:
                                try:
                                    tempfois.append(
                                        get_name_from_urn(
                                            str(val.data), "feature",
                                            sosConfig))

                                except Exception as e:
                                    raise sosException.SOSException(
                                        "InvalidParameterValue",
                                        "featureOfInterest", str(e))

                        except Exception as e:
                            raise e

                    self.featureOfInterest = ",".join(tempfois)

            # FILTERS FOR QUERY NOT SUPPORTED YET
            ress = requestObject.getElementsByTagName('result')
            if len(ress) > 0:
                raise sosException.SOSException(
                    "NoApplicableCode", None,
                    "Parameter \"result\" not yet supported")
            else:
                self.result = None  #zero-one optional

            # RESULT MODEL
            mods = requestObject.getElementsByTagName('resultModel')
            if len(mods) > 0:
                if len(mods) < 2:
                    val = mods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.resultModel = str(val.data)
                        if self.resultModel not in sosConfig.parameters[
                                "GO_resultModel"]:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "resultModel",
                                "Parameter \"resultModel\" sent with invalid value"
                            )

                    else:
                        err_txt = "XML parsing error (get value: resultModel)"
                        raise sosException.SOSException(
                            "NoApplicableCode", None, err_txt)

                else:
                    err_txt = "Allowed only ONE parameter \"resultModel\""
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    err_txt)

            else:
                self.resultModel = None

            # RESPONSE MODE
            rsmods = requestObject.getElementsByTagName('responseMode')
            if len(rsmods) > 0:
                if len(rsmods) < 2:
                    val = rsmods[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.responseMode = str(val.data)
                        if self.responseMode not in sosConfig.parameters[
                                "GO_responseMode"]:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "responseMode",
                                "Parameter \"responseMode\" sent with invalid value"
                            )

                    else:
                        err_txt = "XML parsing error (get value: responseMode)"
                        raise sosException.SOSException(
                            "NoApplicableCode", None, err_txt)

                else:
                    err_txt = "Allowed only ONE parameter \"responseMode\""
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    err_txt)

            else:
                self.responseMode = sosConfig.parameters["GO_responseMode"][0]

            # AGGREGATE INTERVAL & FUNCTION
            self.aggregate_interval = None
            self.aggregate_function = None
            self.aggregate_nodata = None
            self.aggregate_nodata_qi = None

            aggint = requestObject.getElementsByTagName('aggregateInterval')
            aggfun = requestObject.getElementsByTagName('aggregateFunction')
            aggnodata = requestObject.getElementsByTagName('aggregateNodata')

            if len(aggint) == 1 and len(aggfun) == 1:
                # aggregate_interval
                #  Check on the eventTime parameter: it must be only one interval: 2010-01-01T00:00:00+00/2011-01-01T00:00:01+00
                exeMsg = "Using aggregate functions, the event time must exist with an interval composed by a begin and an end date (ISO8601)"
                if self.eventTime == None or len(self.eventTime) != 1 or len(
                        self.eventTime[0]) != 2:
                    raise sosException.SOSException("NoApplicableCode", None,
                                                    exeMsg)

                val = aggint[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_interval = str(val.data)
                    try:
                        iso.parse_duration(self.aggregate_interval)

                    except Exception as ex:
                        raise sosException.SOSException(
                            "InvalidParameterValue", "aggregateInterval",
                            "Parameter \"aggregate_interval\" sent with invalid format (check ISO8601 duration spec): %s"
                            % ex)

                else:
                    err_txt = "cannot get ISO8601 duration value in \"aggregateInterval\""
                    raise sosException.SOSException("InvalidParameterValue",
                                                    "aggregateInterval",
                                                    err_txt)

                # aggregate_function
                val = aggfun[0].firstChild
                if val.nodeType == val.TEXT_NODE:
                    self.aggregate_function = str(val.data)
                    if not (self.aggregate_function.upper()
                            in ["AVG", "COUNT", "MAX", "MIN", "SUM"]):
                        raise sosException.SOSException(
                            "InvalidParameterValue", "aggregateFunction",
                            "Available aggregation functions: avg, count, max, min, sum."
                        )

                # aggregate_no_data default value
                if len(aggnodata) == 1:
                    val = aggnodata[0].firstChild
                    self.aggregate_nodata = str(val.data)

                else:
                    self.aggregate_nodata = sosConfig.aggregate_nodata

        #================================
        # MISSING AGGREGATE QUALITY INDEX
        #================================

            elif len(aggint) == 0 and len(aggfun) == 0:
                pass
            else:
                err_txt = "\"aggregateInterval\" and \"aggregate_function\" are both required with multiplicity 1"
                raise sosException.SOSException("NoApplicableCode", None,
                                                err_txt)

            # QUALITY INDEX
            self.qualityIndex = False
            qidx = requestObject.getElementsByTagName('qualityIndex')

            if len(qidx) > 0:
                if len(qidx) < 2:
                    val = qidx[0].firstChild
                    if val.nodeType == val.TEXT_NODE:
                        self.qualityIndex = str(val.data)
                        if self.qualityIndex.upper() == "TRUE":
                            self.qualityIndex = True

                        elif self.qualityIndex.upper() == "FALSE":
                            pass

                        else:
                            raise sosException.SOSException(
                                "InvalidParameterValue", "qualityIndex",
                                "qualityIndex can only be \'True\' or \'False\'"
                            )

            elif len(qidx) == 0:
                pass

            else:
                err_txt = "\"qualityIndex\" is allowed with multiplicity 1 only"
                raise sosException.SOSException("NoApplicableCode", None,
                                                err_txt)

            self.qualityFilter = False
Esempio n. 33
0
    def executePost(self, db=True):
        if self.procedurename is None:
            raise Exception("POST action without procedure name not allowed")

        now = datetime.now(iso.UTC)
        non_blocking_exceptions = []

        # Create data array
        data = self.waEnviron['wsgi_input'].split(";")

        # Assigned id always in the first position
        assignedid = data[0]

        if len(data) == 4:  # regular time series
            mode = self.MODE_REGULAR

        elif len(data) == 2:  # irregular time series
            mode = self.MODE_IRREGULAR

        else:
            raise Exception(
                "Body content wrongly formatted. Please read the docs.")

        try:
            conn = databaseManager.PgDB(
                self.serviceconf.connection['user'],
                self.serviceconf.connection['password'],
                self.serviceconf.connection['dbname'],
                self.serviceconf.connection['host'],
                self.serviceconf.connection['port'])

            rows = conn.select(("""
                    SELECT
                        procedures.id_prc,
                        proc_obs.id_pro,
                        proc_obs.constr_pro,
                        procedures.stime_prc,
                        procedures.etime_prc,
                        procedures.name_prc
                    FROM
                        %s.procedures,
                        %s.proc_obs
                    WHERE
                        proc_obs.id_prc_fk = procedures.id_prc
                """ % (self.servicename, self.servicename)) + """
                  AND
                    assignedid_prc = %s
                  ORDER BY
                    proc_obs.id_pro ASC;
                """, (assignedid, ))

            if len(rows) == 0:
                raise Exception("Procedure with aid %s not found." %
                                assignedid)

            id_prc = rows[0][0]
            name_prc = rows[0][5]
            bp = rows[0][3]
            bpu = False
            ep = rows[0][4]
            epu = False

            def check_sampling(sampling):

                # If the end position exists the new measures must be after
                if ep is not None and sampling_time < ep:
                    non_blocking_exceptions.append(
                        "Procedure %s, Sampling time (%s) "
                        "is before the end position (%s)" %
                        (name_prc, sampling_time.isoformat(), ep.isoformat()))
                    return False

                # Check that the sampling time is before now
                if sampling_time > now:
                    non_blocking_exceptions.append(
                        "Procedure %s, Sampling time (%s) "
                        "is in the future (%s)" %
                        (name_prc, sampling_time.isoformat(), now.isoformat()))
                    return False

                return True

            tmp_data = []
            if mode == self.MODE_REGULAR:

                try:
                    start = iso.parse_datetime(data[1])
                except Exception:
                    raise Exception(
                        "Procedure %s, Sampling time (%s) "
                        "wrong format" % name_prc, data[1])

                try:
                    step = iso.parse_duration(data[2])
                except Exception:
                    raise Exception("Procedure %s, duration (%s) "
                                    "wrong format" % (name_prc, data[2]))

                data = data[3].split("@")
                for idx in range(0, len(data)):

                    sampling_time = start + (step * idx)

                    if not check_sampling(sampling_time):
                        continue

                    tmp_data.append([sampling_time.isoformat()] +
                                    data[idx].split(","))

            elif mode == self.MODE_IRREGULAR:
                data = data[1].split("@")
                for i in range(0, len(data)):
                    data[i] = data[i].split(",")

                    try:
                        try:
                            sampling_time = iso.parse_datetime(data[i][0])
                            if not check_sampling(sampling_time):
                                continue
                        except Exception:
                            raise Exception("Procedure %s, Sampling time (%s) "
                                            "wrong format" %
                                            (name_prc, data[i][0]))

                        tmp_data.append(data[i])

                    except Exception:
                        non_blocking_exceptions.append(
                            "Procedure %s, Sampling time (%s) "
                            "wrong format" % (name_prc, data[1]))
                        continue

            data = tmp_data

            op_cnt = len(rows)

            for observation in data:

                id_eti = conn.executeInTransaction(("""
                        INSERT INTO %s.event_time (id_prc_fk, time_eti)
                    """ % self.servicename) + """
                        VALUES (%s, %s::TIMESTAMPTZ) RETURNING id_eti;
                    """, (id_prc, observation[0]))

                if (bp is None) or (bp == '') or (iso.parse_datetime(
                        observation[0]) < bp):
                    bp = iso.parse_datetime(observation[0])
                    bpu = True

                if (ep is None) or (ep == '') or (iso.parse_datetime(
                        observation[0]) > ep):
                    ep = iso.parse_datetime(observation[0])
                    epu = True

                # check if procedure observations length is ok
                #   (-1 remove datetime from lenght of observations array)
                if op_cnt != (len(observation) - 1):
                    non_blocking_exceptions.append(
                        "Procedure %s, Array length missmatch with procedures "
                        "observation number: %s" % (name_prc, observation))
                    continue

                for idx in range(0, op_cnt):
                    try:
                        conn.executeInTransaction(
                            ("""
                                INSERT INTO %s.measures(
                                    id_eti_fk,
                                    id_qi_fk,
                                    id_pro_fk,
                                    val_msr
                                )
                            """ % self.servicename) + """
                                VALUES (%s, 100, %s, %s);
                            """,
                            (
                                int(id_eti[0][0]),  # id_eti
                                int(rows[idx][1]),  # id_pro
                                float(observation[(idx + 1)])))
                    except Exception as ie:
                        non_blocking_exceptions.append("Procedure %s, %s" %
                                                       (name_prc, ie))

            if bpu:
                conn.executeInTransaction(("""
                        UPDATE %s.procedures
                    """ % self.servicename) + """
                        SET stime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
                    """, (bp.isoformat(), id_prc))

            if epu:
                conn.executeInTransaction(("""
                        UPDATE %s.procedures
                    """ % self.servicename) + """
                        SET etime_prc=%s::TIMESTAMPTZ WHERE id_prc=%s
                    """, (ep.isoformat(), id_prc))

            conn.commitTransaction()

            # self.setData(ret)
            self.setMessage("Thanks for data")

            if len(non_blocking_exceptions) > 0:
                print >> sys.stderr, str(non_blocking_exceptions)

        except Exception as e:
            print >> sys.stderr, traceback.print_exc()
            #traceback.print_exc(file=sys.stderr)
            conn.rollbackTransaction()
            raise Exception("Error in fast insert (%s): %s" % (type(e), e))
Esempio n. 34
0
def execute(args, conf=None):
    def log(message):
        if conf is not None and 'logger' in conf:
            conf['logger'].log(message)
        else:
            print message

    def addMessage(self, message):
        if 'logger' in conf:
            conf['logger'].addMessage(message)
        else:
            print message

    def addWarning(self, message):
        if 'logger' in conf:
            conf['logger'].addWarning(message)
        else:
            print message

    def addException(self, message):
        if 'logger' in conf:
            conf['logger'].addException(message)
        else:
            print message

    pp = pprint.PrettyPrinter(indent=2)

    try:

        # Initializing URLs
        url = args['u']

        # Service instance name
        service = args['s']

        # Quality index
        quality = '100'
        if 'q' in args:
            quality = args['q']

        # Procedures
        procs = args['p']

        # Working directory, where the CSV files are located
        wd = args['wd']

        # File extension
        ext = '.dat'
        if 'e' in args:
            ext = args['e']

        debug = False
        if 'v' in args:
            debug = args['v']

        test = False
        if 't' in args:
            test = args['t']

        user = None
        if 'user' in args and args['user'] is not None:
            user = args['user']

        password = None
        if 'password' in args and args['password'] is not None:
            password = args['password']

        auth = None
        if user and password:
            auth = HTTPBasicAuth(user, password)

        noqi = False  # False meas that quality index is also migrated
        if 'noqi' in args:
            if args['noqi'] is True:
                noqi = True

        maxobs = 5000
        if 'm' in args:
            maxobs = int(args['m'])

        #req = requests.session()
        req = requests

        for proc in procs:

            log("\nProcedure: %s" % proc)

            if conf is not None and 'description' in conf:
                data = conf['description']
            else:
                # Load procedure description
                res = req.get("%s/wa/istsos/services/%s/procedures/%s" %
                              (url, service, proc),
                              auth=auth,
                              verify=False)

                data = res.json()

                if data['success'] is False:
                    raise Exception(
                        "Description of procedure %s can not be loaded: %s" %
                        (proc, data['message']))
                else:
                    log("%s > %s" % (proc, data['message']))

                data = data['data']

            aid = data['assignedSensorId']

            # Getting observed properties from describeSensor response
            op = []
            for out in data['outputs']:
                if not noqi or not ':qualityIndex' in out['definition']:
                    op.append(out['definition'])

            # Load of a getobservation request
            res = req.get("%s/wa/istsos/services/%s/operations/getobservation/"
                          "offerings/%s/procedures/%s/observedproperties/%s/ev"
                          "enttime/last" %
                          (url, service, 'temporary', proc, ','.join(op)),
                          auth=auth,
                          verify=False)

            data = res.json()

            if data['success'] is False:
                raise Exception("Last observation of procedure %s can not be "
                                "loaded: %s" % (proc, data['message']))
            else:
                log("%s > %s" % (proc, data['message']))

            data = data['data'][0]
            data['AssignedSensorId'] = aid

            # Set values array empty, can contain 1 value if
            # procedure not empty
            if len(data['result']['DataArray']['values']) == 1:
                lastMeasure = data['result']['DataArray']['values'][0]
            else:
                lastMeasure = None

            data['result']['DataArray']['values'] = []

            # discover json observed property disposition
            jsonindex = {}
            for pos in range(0, len(data['result']['DataArray']['field'])):
                field = data['result']['DataArray']['field'][pos]
                if not noqi:
                    jsonindex[field['definition']] = pos
                elif not ':qualityIndex' in field['definition']:
                    jsonindex[field['definition']] = pos
                elif ':qualityIndex' in field['definition'] and noqi:
                    data['result']['DataArray']['field'].pop(pos)

            log("Searching: %s" % os.path.join(wd, "%s_[0-9]*%s" %
                                               (proc, ext)))

            files = glob.glob(os.path.join(wd, "%s_*%s" % (proc, ext)))
            files.sort()

            if debug:
                log("%s > %s %s found" %
                    (proc, len(files), "Files" if len(files) > 1 else "File"))

            if len(files) > 0:
                for f in files:

                    # open file
                    file = open(f, 'rU')

                    # loop lines
                    lines = file.readlines()

                    obsindex = lines[0].strip(' \t\n\r').split(",")

                    # Check if all the observedProperties of the procedure are
                    # included in the CSV file (quality index is optional)
                    for k, v in jsonindex.iteritems():
                        if k in obsindex:
                            continue
                        elif ':qualityIndex' in k:
                            continue
                        else:
                            raise Exception(
                                "Mandatory observed property %s is not present"
                                " in the CSV." % k)

                    # loop lines (skipping header)
                    for i in range(1, len(lines)):
                        try:
                            line = lines[i]
                            lineArray = line.strip(' \t\n\r').split(",")

                            # Creating an empty array where the values will
                            # be inserted
                            observation = [''] * len(jsonindex)

                            for k, v in jsonindex.iteritems():
                                val = None
                                if k in obsindex:
                                    val = lineArray[obsindex.index(k)]
                                elif ':qualityIndex' in k:
                                    # Quality index is not present in the CSV
                                    # so the default value will be set
                                    val = quality

                                observation[v] = val

                            # attach to object
                            data['result']['DataArray']['values'].append(
                                observation)

                        except Exception as e:
                            raise Exception("Errore alla riga: %s - %s\n%s" %
                                            (i, lines[i], str(e)))

                log("Before insert ST: %s" % proc)
                if 'beginPosition' in data["samplingTime"]:
                    log(" > Begin: %s" % data["samplingTime"]["beginPosition"])
                if 'endPosition' in data["samplingTime"]:
                    log("   + End: %s" % data["samplingTime"]["endPosition"])

                dtstr = os.path.split(f)[1].replace("%s_" % proc,
                                                    "").replace(ext, "")

                offset = False

                if '+' in dtstr:
                    offset = dtstr[dtstr.index('+'):]
                    offset = [offset[0:3], offset[3:5]]
                    dtstr = dtstr[:dtstr.index('+')]

                elif '-' in dtstr:
                    offset = dtstr[dtstr.index('-'):]
                    offset = [offset[0:3], offset[3:5]]
                    dtstr = dtstr[:dtstr.index('-')]

                ep = datetime.strptime(
                    dtstr, "%Y%m%d%H%M%S%f").replace(tzinfo=timezone('UTC'))

                if offset:
                    ep = ep - timedelta(hours=int(offset[0]),
                                        minutes=int(offset[1]))

                # A kick in the brain code snippet
                # If there is at least one measure:
                if len(data['result']['DataArray']['values']) > 0:
                    # taking first observation as begin position
                    bp = iso.parse_datetime(data['result']['DataArray']
                                            ['values'][0][jsonindex[isourn]])

                else:
                    # otherwise this can be an irrebular procedure where just
                    # the end position is moved forward
                    if ep > iso.parse_datetime(
                            data["samplingTime"]["endPosition"]):
                        bp = ep
                    else:
                        raise Exception(
                            "Something is wrong with begin position..")

                data["samplingTime"] = {
                    "beginPosition": bp.isoformat(),
                    "endPosition": ep.isoformat()
                }

                log("Insert ST: %s" % proc)
                log(" > Begin: %s" % bp.isoformat())
                log("   + End: %s" % ep.isoformat())
                log(" > Values: %s" %
                    len(data['result']['DataArray']['values']))

                if not test and len(files) > 0:  # send to wa

                    if len(data['result']['DataArray']['values']) > maxobs:

                        total = len(data['result']['DataArray']['values'])
                        inserted = last = maxobs

                        while len(data['result']['DataArray']['values']) > 0:

                            tmpData = copy.deepcopy(data)
                            tmpData['result']['DataArray']['values'] = (
                                data['result']['DataArray']['values'][:last])
                            data['result']['DataArray']['values'] = (
                                data['result']['DataArray']['values'][last:])

                            if len(data['result']['DataArray']['values']) > 0:
                                tmpData["samplingTime"] = {
                                    "beginPosition":
                                    tmpData['result']['DataArray']['values'][0]
                                    [jsonindex[isourn]],
                                    "endPosition":
                                    data['result']['DataArray']['values'][0][
                                        jsonindex[isourn]]
                                }
                            else:
                                tmpData["samplingTime"] = {
                                    "beginPosition":
                                    tmpData['result']['DataArray']['values'][0]
                                    [jsonindex[isourn]],
                                    "endPosition":
                                    ep.isoformat()
                                }

                            res = req.post("%s/wa/istsos/services/%s/"
                                           "operations/insertobservation" %
                                           (url, service),
                                           auth=auth,
                                           verify=False,
                                           data=json.dumps({
                                               "ForceInsert":
                                               "true",
                                               "AssignedSensorId":
                                               aid,
                                               "Observation":
                                               tmpData
                                           }))

                            # read response
                            res.raise_for_status()
                            log(" > Insert observation success of %s/%s (%s / "
                                "%s) observations: %s" %
                                (inserted, total,
                                 tmpData["samplingTime"]["beginPosition"],
                                 tmpData["samplingTime"]["endPosition"],
                                 res.json()['success']))

                            if not res.json()['success']:
                                log(res.json()['message'])

                            if len(data['result']['DataArray']
                                   ['values']) < maxobs:
                                last = len(
                                    data['result']['DataArray']['values'])
                            inserted += last

                    else:

                        res = req.post("%s/wa/istsos/services/%s/operations/"
                                       "insertobservation" % (url, service),
                                       auth=auth,
                                       verify=False,
                                       data=json.dumps({
                                           "ForceInsert": "true",
                                           "AssignedSensorId": aid,
                                           "Observation": data
                                       }))

                        # read response
                        res.raise_for_status()
                        log(" > Insert observation success: %s" %
                            (res.json()['success']))

                        if not res.json()['success']:
                            log(res.json()['message'])

        pass

    except requests.exceptions.HTTPError as eh:
        #addException(str(eh))
        traceback.print_exc()

    except Exception as e:
        #addException(str(e))
        traceback.print_exc()
Esempio n. 35
0
def execute (args):
    pp = pprint.PrettyPrinter(indent=2)
    try:
    
        istsos_version = args['istsos'] if 'istsos' in args else None
        
        debug = args['v']
        test = args['t']
        
        procs = args['p']
        omit = args['o']
        procedures = {}
        
        src = args['s']
        dst = args['d']
        srv = args['n']
        
        appendData = False
        if 'a' in args:
            appendData = True
        
        
        dfrom = None
        dto = None
        if 'from' in args and type('') == type(args['from']):
            print "From: %s" % args['from']
            dfrom = iso.parse_datetime(args['from'])
            appendData = None
        if 'to' in args and type('') == type(args['to']):
            print "To: %s" % args['to']
            dto = iso.parse_datetime(args['to'])
            appendData = None
        
        registerOnly = args['r']
        
        virtual = False
        hq = False
        
        
        # Executing request
        res = req.get("%s" % (src), params={
            'service': 'SOS', 
            'version': '1.0.0',
            'request': 'GetCapabilities',
            'section': 'contents'
        }, verify=False)
        
        # Parsing response
        gc, gcNs = parse_and_get_ns(StringIO(res.content))
        
        # Extract all offerings
        elOfferings = gc.findall("{%s}Contents/{%s}ObservationOfferingList/{%s}ObservationOffering" % (gcNs['sos'],gcNs['sos'],gcNs['sos']))
        
        for offering in elOfferings:
            offeringName = offering.find("{%s}name" % (gcNs['gml']) ).text.split(":")[-1]
            
            # For each offering get the procedures
            elProcs = offering.findall("{%s}procedure" % (gcNs['sos']) )
            for p in elProcs:
                pname = p.get('{%s}href' % gcNs['xlink'])
                
                if (
                        type(procs) == type([]) and pname not in procs
                    ) or (
                        type(omit) == type([]) and pname in omit
                    ):
                    continue
                    
                #print pname
                procedures[pname] = Procedure(pname, offeringName, dst, srv)
                
                if virtual:
                    procedures[pname].setSystemType('virtual')
                               
                res = req.get("%s" % (src), params={
                    'service': 'SOS', 
                    'version': '1.0.0',
                    'request': 'DescribeSensor',
                    'outputFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                    'procedure': pname
                }, verify=False)
                
                ds, dsNs = parse_and_get_ns(StringIO(res.content))
                
                #print res.content
                
                
                #print "Root: %s" % ds.getroot().tag
                if ds.getroot().tag == 'ExceptionReport':
                    print "Error on DS for %s" % pname
                    continue
                    
                
                
                #print "Outputs found: %s" % len(elDescribe)
                
                observedProperties = []
                print "istsos_version: ", istsos_version
                uniqidurn = 'urn:ogc:def:parameter:x-ist::'
                if istsos_version != None and istsos_version == '2':
                    uniqidurn = 'urn:ogc:def:parameter:x-ist:1.0:'
                    elFields = ds.findall("{%s}member/{%s}System/{%s}outputs/{%s}OutputList/{%s}output/{%s}DataRecord/{%s}field" % (
                                    dsNs['sml'],dsNs['sml'],dsNs['sml'],dsNs['sml'],dsNs['sml'],dsNs['swe'],dsNs['swe']) )
                    print "Observed properties (v2): %s " % len(elFields)
                    for fs in elFields:
                        print fs.get('name')
                        if fs.get('name') != 'Time':
                            observedProperties.append(fs.find("{%s}Quantity" % (dsNs['swe'])).get('definition').replace(uniqidurn,''))
                        
                else:
                    elDescribe = ds.findall("member/{%s}System/{%s}outputs/{%s}OutputList/{%s}output" % (dsNs['sml'],dsNs['sml'],dsNs['sml'],dsNs['sml']) )
                    print "Observed properties: %s " % len(elDescribe)
                    for ds in elDescribe:
                        definition = ds.find("{%s}ObservableProperty" % (dsNs['swe'])).get('definition').replace(uniqidurn,'')
                        #print definition
                        if definition.find('time:iso8601')<0:
                            observedProperties.append(definition)
                            
                
                print {
                    'service': 'SOS', 
                    'version': '1.0.0',
                    'request': 'GetObservation',
                    'offering': offeringName,
                    'responseFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                    'procedure': pname,
                    'observedProperty': ",".join(observedProperties)
                }
                
                res = req.get("%s" % (src), params={
                    'service': 'SOS', 
                    'version': '1.0.0',
                    'request': 'GetObservation',
                    'offering': offeringName,
                    'responseFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                    'procedure': pname,
                    'observedProperty': ",".join(observedProperties)
                }, verify=False)
                              
                go, goNs = parse_and_get_ns(StringIO(res.content))
                
                
                if go.getroot().tag == 'ExceptionReport':
                    print "Error on GO for %s:\nparams:%s\n%s" % (pname,{
                        'service': 'SOS', 
                        'version': '1.0.0',
                        'request': 'GetObservation',
                        'offering': offeringName,
                        'responseFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                        'procedure': pname,
                        'observedProperty': ",".join(observedProperties)
                    },res.content)
                    continue
                
                # Extracting begin and end position
                begin = go.find("{%s}member/{%s}Observation/{%s}samplingTime/{%s}TimePeriod/{%s}beginPosition" % (
                    goNs['om'], goNs['om'], goNs['om'], goNs['gml'], goNs['gml'])
                )
                end = go.find("{%s}member/{%s}Observation/{%s}samplingTime/{%s}TimePeriod/{%s}endPosition" % (
                    goNs['om'], goNs['om'], goNs['om'], goNs['gml'], goNs['gml'])
                )
                procedures[pname].begin = begin.text
                procedures[pname].end  = end.text
                
                # Extracting Feature of Interest and coordinates
                foi = go.find("{%s}member/{%s}Observation/{%s}featureOfInterest" % (
                    goNs['om'], goNs['om'], goNs['om'])
                )
                point = foi.find("{%s}Point" % (
                    goNs['gml'])
                )
                
                if point == None:
                    point = foi.find("{%s}FeatureCollection/{%s}location/{%s}Point" % (
                        goNs['gml'],goNs['gml'],goNs['gml'])
                    )
                
                coord = point.find("{%s}coordinates" % (
                    goNs['gml'])
                ).text.split(",")
                
                if len(coord) == 2:
                    coord.append('0')
                
                procedures[pname].setFoi(
                    foi.get('{%s}href' % gcNs['xlink']).split(":")[-1],
                    point.get('srsName'),
                    coord
                )
                
                # Extracting UOM
                fields = go.findall("{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}elementType/{%s}DataRecord/{%s}field" % (
                    goNs['om'], goNs['om'], goNs['om'], goNs['swe'], goNs['swe'], goNs['swe'], goNs['swe'])
                )
                
                for field in fields:
                    if field.get('name')!='Time':
                        qty = field.find("{%s}Quantity" % (
                            goNs['swe'])
                        )
                        uom = field.find("{%s}Quantity/{%s}uom" % (
                            goNs['swe'],goNs['swe'])
                        )                        
                        procedures[pname].addObservedProperty(
                            field.get('name'),
                            qty.get('definition').replace(uniqidurn,''), 
                            uom.get('code')
                        )
                
                if dfrom:
                    begin = dfrom
                    _begin = dfrom
                else:
                    begin = iso.parse_datetime(procedures[pname].begin)
                    _begin = iso.parse_datetime(procedures[pname].begin)
                    
                if dto:
                    end = dto
                else:
                    end = iso.parse_datetime(procedures[pname].end)
                
                # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                # REGISTRATION PROCESS
                # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                
                # Check if procedure already exist
                res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (dst,srv,pname), verify=False)  
                if not res.json()["success"]:
                    # Registering procedure to istSOS   
                    res = req.post("%s/wa/istsos/services/%s/procedures" % (dst,srv), 
                            data=json.dumps(procedures[pname].data)
                    ) 
                    if not res.json()["success"]:
                        #print json.dumps(procedures[pname].data)
                        raise Exception("Registering procedure %s failed: \n%s" % (pname, res.json()["message"]))
                    
                    # Getting details (describe sensor) to get the assignedSensorId
                    res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (dst,srv,pname))  
                    
                    # Getting an InsertObservation template
                    template = procedures[pname].getIoTemplate()  
                    
                else:
                    # Getting an InsertObservation template
                    template = procedures[pname].getIoTemplate()  
                    try:
                        if appendData and ('endPosition' in template['samplingTime']):
                            procedures[pname].begin = template['samplingTime']['endPosition']
                            begin = iso.parse_datetime(template['samplingTime']['endPosition'])     
                    except Exception as exproc:
                        print res.text
                        raise exproc
                
                procedures[pname].oid = res.json()["data"]["assignedSensorId"]
                days = int(args['i'])
                interval = timedelta(days=int(days))
                
                
                if not registerOnly:
                
                    if virtual and hq:
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        # VIRTUAL PROCEDURE CODE INITIALIZATION
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        pass
                        
                    else:
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        # PROCEDURE OBSERVATIONS MIGRATION
                        # ~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
                        
                        oOrder = []
                        
                        passedLoops = 0
                        
                        lastPrint = ""
                        
                        startTime = time.time()
                        
                        print "%s: %s - %s" % (pname, procedures[pname].begin, procedures[pname].end)
                        
                        if begin<end and begin+interval>end:
                            interval = end-begin
                        
                        while (begin+interval)<=end:
                            loopTotalTime = time.time()
                                
                            nextPosition = begin + interval
                            
                            passedLoops = passedLoops+1
                            
                            t = float(calendar.timegm(end.utctimetuple())-calendar.timegm(_begin.utctimetuple()))
                            t1 = float(calendar.timegm(nextPosition.utctimetuple())-calendar.timegm(_begin.utctimetuple()))
                            try:
                                percentage = round((t1/t)*100,2)
                            except:
                                percentage = 0
                            if percentage > 100:
                                percentage = 100
                            lastPrint = "%s > %s%% (%s / %s %s days)" % ("\b"*len(lastPrint),percentage, begin.strftime(fmtshort), nextPosition.strftime(fmtshort), days)
                            
                            looptime = time.time()
                            # GetObservation from source SOS
                            params={
                                'service': 'SOS', 
                                'version': '1.0.0',
                                'request': 'GetObservation',
                                'eventTime': '%s/%s' % (
                                        begin.strftime(fmt),
                                        nextPosition.strftime(fmt)),
                                'qualityIndex': 'True',
                                'offering': offeringName,
                                'responseFormat': 'text/xml;subtype=\'sensorML/1.0.0\'',
                                'procedure': pname,
                                'observedProperty': ",".join(observedProperties)
                            }
                            try:
                                res = req.get("%s" % (src), params=params, verify=False)
                            except Exception:
                                res = req.get("%s" % (src), params=params, verify=False)
                            
                            gotime = timedelta(seconds=int(time.time() - looptime))
                            
                            if gotime > timedelta(seconds=int(10)):
                                if days > 1:
                                    days = int(days/2)
                                    if days <= 1:
                                        days = 1 
                                interval = timedelta(days=days)
                            elif gotime < timedelta(seconds=int(5)):
                                days = days + 1
                                interval = timedelta(days=days)
                            
                            lastPrint = "%s - GO: '%s'" % (lastPrint, gotime)
                            
                            go, goNs = parse_and_get_ns(StringIO(res.content))
                                                        
                            if len(oOrder)==0:
                                fields = go.findall("{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}elementType/{%s}DataRecord/{%s}field" % (
                                    goNs['om'], goNs['om'], goNs['om'], goNs['swe'], goNs['swe'], goNs['swe'], goNs['swe'])
                                )
                                for field in fields:
                                    oOrder.append(qty.get('definition').replace('urn:ogc:def:parameter:x-ist::',''))
                            
                            values = go.find("{%s}member/{%s}Observation/{%s}result/{%s}DataArray/{%s}values" % (
                                goNs['om'], goNs['om'], goNs['om'], goNs['swe'], goNs['swe'])
                            )
                            
                            if values.text:
                            
                                rows = values.text.strip().split("@")
                                
                                lastPrint = "%s " % (lastPrint)
                                
                                copy = []
                                
                                for row in rows:
                                    copy.append(row.split(","))
                                
                                # InsertObservation to istSOS
                                template['result']['DataArray']['values'] = copy
                                template['samplingTime'] = {
                                    "beginPosition": copy[0][0],
                                    "endPosition": nextPosition.strftime(fmt)
                                }
                                '''template['samplingTime'] = {
                                    "beginPosition": begin.strftime(fmt),
                                    "endPosition": nextPosition.strftime(fmt)
                                }'''
                                
                                template[u"AssignedSensorId"] = procedures[pname].oid
                                
                                looptime = time.time() 
                                res = req.post("%s/wa/istsos/services/%s/operations/insertobservation" % (
                                    dst,
                                    srv
                                ),data = json.dumps({
                                    u"AssignedSensorId": procedures[pname].oid,
                                    u"ForceInsert": u"true",
                                    u"Observation": template
                                }))
                                iotime = timedelta(seconds=int(time.time() - looptime))
                                lastPrint = "%s - IO: '%s'" % (lastPrint, iotime)
                                                    
                            begin = nextPosition
                            if begin<end and begin+interval>end:
                                interval = end-begin
                                
                            if percentage < 100:
                                lastPrint = "%s - Step time: '%s' - Elapsed: %s  " % (
                                    lastPrint, 
                                    timedelta(seconds=int(time.time() - loopTotalTime)), 
                                    timedelta(seconds=int(time.time() - startTime))
                                )
                            else:
                                lastPrint = "%s - Step time: '%s'  " % (
                                    lastPrint, 
                                    timedelta(seconds=int(time.time() - loopTotalTime))
                                )
                            
                            sys.stdout.write(lastPrint)
                            sys.stdout.flush()
                        
                        
                        print " > Completed in %s" % timedelta(seconds=int(time.time() - startTime))
            break
    except Exception as e:    
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
Esempio n. 36
0
def execute (args, logger=None):
    
    # SCRIPT CONFIGURATION
    # =========================================================================
    
    # Activate and print verbose information
    debug = args['v'] if args.has_key('v') else False
        
    # Procedure name
    procedure = args['procedure']
    # Begin date
    begin = args['begin'] if args.has_key('begin') else "*"
    # End date
    end = args['end'] if args.has_key('end') else "*"    
    # Global User and password valid for all connections
    suser = duser = auser = args['user'] if args.has_key('user') else None
    spwd = dpwd = apwd = args['pwd'] if args.has_key('pwd') else None
    
    # Activate this will copy also the quality index from source to destination
    cpqi = args['cpqi'] if args.has_key('cpqi') else False
    
    # Aggregating function configuration
    resolution = args['resolution'] if 'resolution' in args else None
    function = args['function'] if 'function' in args else None
    nodataValue = args['nodataValue'] if 'nodataValue' in args else None
    nodataQI = args['nodataQI'] if 'nodataQI' in args else None
    
    # Retroactive aggregation
    retro = args['retro'] if 'retro' in args else 0
    
    # Force using last position as end position during insert sensor operation
    lm = args['lm'] if 'lm' in args else False
    
    # SOURCE istSOS CONFIG ==================================
    # Location
    surl = args['surl']
    # Service instance name
    ssrv = args['ssrv']
    # User and password if given this will be used for source istSOS
    if args.has_key('suser'):
        suser = args['suser']
    if args.has_key('spwd'):
        spwd = args['spwd']
    
    # DESTINATION istSOS CONFIG =============================
    # Location (if not given, same as source will be used)
    durl = args['durl'] if (args.has_key('durl') and args['durl'] is not None) else surl
    # Service instance name
    dsrv = args['dsrv']
    # User and password if given this will be used for destination istSOS
    if args.has_key('duser'):
        duser = args['duser']
    if args.has_key('dpwd'):
        dpwd = args['dpwd']
        
    # ALTERNATIVE istSOS SERVICE FOR QI EXTRAPOLATION =======
    # Location (if not given, same as source will be used)
    aurl = args['aurl'] if (args.has_key('aurl') and args['aurl'] is not None) else None
    # Service instance name
    asrv = args['asrv'] if (args.has_key('asrv') and args['asrv'] is not None) else None
    # User and password if given this will be used for extrapolation QI istSOS
    if args.has_key('auser'):
        auser = args['auser']
    if args.has_key('apwd'):
        apwd = args['apwd']
    
    def log(message):
        if debug:
            if logger:
                logger.log(message)
            else:
                print message
            
    # PROCESSING STARTS HERE ==================================================
    
    log("\nistSOS > 2 > istSOS STARTED:")
    log("==============================\n")
    
    #req = requests.session()
    req = requests
    
    # Load procedure description        
    log("1. Loading procedure description: %s" % procedure)
        
    # Loading describe sensor from source =====================================
    res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (
        surl,
        ssrv,
        procedure
        ), auth=(suser, spwd), verify=False)
    sdata = res.json()
    if sdata['success']==False:
        raise Exception ("Description of procedure %s can not be loaded from source service: %s" % (procedure, sdata['message']))
    else:
        log("   > DS Source Ok.")
            
    # Loading describe sensor from destination ================================
    res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (
            durl, dsrv, procedure
        ), auth=(duser, dpwd), verify=False)
    ddata = res.json()
    if ddata['success']==False:
        raise Exception ("Description of procedure %s can not be loaded from destination service: %s" % (procedure, ddata['message']))
    else:
        log("   > DS Destination Ok.")
        
    # Load of a getobservation template from destination =======================================
    res = req.get("%s/wa/istsos/services/%s/operations/getobservation/offerings/%s/procedures/%s/observedproperties/:/eventtime/last?qualityIndex=False" % (
            durl, dsrv, 'temporary', procedure
        ),  params={
            "qualityIndex": cpqi
        }, auth=(duser, dpwd), verify=False)
    dtemplate = res.json()
    if dtemplate['success']==False:
        raise Exception ("Observation template of procedure %s can not be loaded: %s" % (procedure, dtemplate['message']))
    else:
        dtemplate = dtemplate['data'][0]
        dtemplate['AssignedSensorId'] = ddata['data']['assignedSensorId']
        dtemplate['result']['DataArray']['values'] = []
        log("     > GO Template Ok.")

    # Loading describe sensor from QI EXTRAPOLATION service ===================
    if aurl and asrv:
        res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (
                aurl, asrv, procedure
            ), auth=(auser, apwd), verify=False)
        adata = res.json()
        if adata['success']==False:
            raise Exception ("Description of procedure %s can not be loaded from destination service: %s" % (procedure, adata['message']))
        else:
            log("   > DS QI Extrapolation Ok.")

    log("\n2. Identifying processing interval:")
    
    # Check if mesaures are present in source procedure, by identifying the sampling time constraint 
    #  located always in the first position of the outputs, if it is empty an exception is thrown
    if (not 'constraint' in sdata['data']['outputs'][0] 
            or not 'interval' in sdata['data']['outputs'][0]['constraint'] ):
        raise Exception ("There is no data in the source procedure to be copied to the destination procedure.")
    else:
        # Check if the contraint interval contains a valid ISO date begin position
        try:
            iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][0])
        except Exception:
            raise Exception ("The date in the source procedure constraint interval (%s) is not valid." % 
                sdata['data']['outputs'][0]['constraint']['interval'][0])
        
        # Check if the contraint interval contains a valid ISO date end position
        try:
            iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][1])
        except Exception:
            raise Exception ("The date in the source procedure constraint interval (%s) is not valid." % 
                sdata['data']['outputs'][0]['constraint']['interval'][1])
    
    log("   > Source interval is valid")
    
    # Looking for start (IO beginPOsition) instant processing 
    #   If the default value (*) is used, then the endPosition of 
    #   the "destination" service procedure will be used. But if the destination
    # procedure is empty , then the begin position of the source will be used
    start = None
    stop = None
    if begin == "*":
        if ('constraint' in ddata['data']['outputs'][0] 
            and 'interval' in ddata['data']['outputs'][0]['constraint']):
                try:
                    if function and resolution:
                        # getting last inserted observations of "destination" service
                        log("Aggregation requested: getting last inserted observations of \"destination\" service")
                        params = {
                            "request": "GetObservation",
                            "service": "SOS",
                            "version": "1.0.0",
                            "observedProperty": ':',
                            "procedure": procedure,
                            "responseFormat": "application/json",
                            "offering": 'temporary'
                        }
                        res = req.get("%s/%s" % (durl,dsrv), params=params, auth=(duser, dpwd), verify=False)
                        obs = res.json()
                        start = iso.parse_datetime(obs['ObservationCollection']['member'][0]['result']['DataArray']['values'][0][0])
                    else:
                        # The endPosition of the destination will be used as Start/IO BeginPosition
                        start = iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][1])
                    if retro > 0: # Retroactive aggregation
                        log("Retroactive aggregation active.")
                        if start-timedelta(minutes=retro) > iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][0]):
                            start = start-timedelta(minutes=retro)
                        else:
                            start = iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][0])
                            
                    log("Start: %s" % start)
                    
                except Exception as ee:
                    print "Error setting start date for proc %s: %s" % (procedure,ee)
                    raise Exception ("The date in the destination procedure %s constraint interval (%s) is not valid." % 
                        (procedure,ddata['data']['outputs'][0]['constraint']['interval'][0]))
        else:
            # The beginPosition of the source will be used as Start/IO BeginPosition
            start = iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][0])           
    else:
        start = iso.parse_datetime(begin)
    
    if end == "*":
        # The endPosition of the source will be used as Stop/IO EndPosition
        stop = iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][1])   
    else:
        stop = iso.parse_datetime(end)
    
    log("   > Destination interval is valid")
    log("   > Start processing: %s" % start)
    log("   > Stop processing: %s" % stop)
    if retro > 0:
        log("   > Retro aggregation: %s minutes" % retro)
    
    # Insertion loop step timedelta
    interval = timedelta(days=15)
    if start<stop and start+interval>stop:
        interval = stop-start
    
    log("   > Insertion loop step: %s" % interval)
        
    if function and resolution:
        try:
            iso.duration_isoformat(resolution)
        except:
            raise Exception ("The resolution (%s) to apply in the aggregating function is not valid." % resolution)
        log("   > Function(Resolution) : %s(%s)" % (function,resolution))
        
    while start+interval<=stop:        
        
        nextStart = start + interval
        
        params = {
            "request": "GetObservation",
            "service": "SOS",
            "version": "1.0.0",
            "observedProperty": ':',
            "procedure": procedure,
            "qualityIndex": str(cpqi),
            "responseFormat": "application/json",
            "offering": 'temporary',
            "eventTime": "%s/%s" % (start.isoformat(), nextStart.isoformat())
        }
        
        if function and resolution:
            
            params['aggregateFunction'] = function
            params['aggregateInterval'] = resolution
            
            if nodataValue != None:
                params['aggregateNodata'] = nodataValue
            if nodataQI != None:
                params['aggregateNodataQi'] = nodataQI
        
        res = req.get("%s/%s" % (surl,ssrv),  params=params, auth=(suser, spwd), verify=False)
        
        # Check if an Exception occured
        if 'ExceptionReport' in res.content:
            raise Exception (res.content)
        
        smeasures = res.json()['ObservationCollection']['member'][0]
        #pp.pprint(smeasures)
        
        log("   > %s measures from: %s to: %s" % (len(smeasures['result']['DataArray']['values']), start.isoformat(), nextStart.isoformat()))
        
        dtemplate["samplingTime"] = {}
        if lm and len(smeasures['result']['DataArray']['values'])>0:
            dtemplate["samplingTime"]["beginPosition"] = smeasures['result']['DataArray']['values'][0][0]
            dtemplate["samplingTime"]["endPosition"] = smeasures['result']['DataArray']['values'][-1][0]
        else:
            dtemplate["samplingTime"]["beginPosition"] = start.isoformat()
            dtemplate["samplingTime"]["endPosition"] = nextStart.isoformat()
            
        dtemplate['result']['DataArray']['values'] = smeasures['result']['DataArray']['values']
        dtemplate['result']['DataArray']['field'] = smeasures['result']['DataArray']['field']
        
        # POST data to WA
        res = req.post("%s/wa/istsos/services/%s/operations/insertobservation" % (
            durl,
            dsrv), 
            auth=(duser, dpwd),
            verify=False,
            data=json.dumps({
                "ForceInsert": "true",
                "AssignedSensorId": ddata['data']['assignedSensorId'],
                "Observation": dtemplate
            })
        )
        
        # read response
        log("     > Insert observation success: %s" % res.json()['success'])
        
        #print res.json()
        
        if not res.json()['success']:
            raise Exception ('Error inserting observation: %s' % res.json()['message'])
        
        start = nextStart
        if start<stop and start+interval>stop:
            interval = stop-start