def render(GF,sosConfig): r = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" if GF.type.lower()=="station" or GF.type.lower()=="point": r += "<sa:SamplingPoint \n" elif GF.type=="surface": r += "<sa:SamplingSurface \n" r += "gml:id=\"" + GF.name + "\" \n" r += "xmlns:sa=\"http://www.opengis.net/sampling/1.0\" \n" r += "xmlns:swe=\"http://www.opengis.net/swe/1.0.1\" \n" r += "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n" r += "xmlns:xlink=\"http://www.w3.org/1999/xlink\" \n" r += "xmlns:gml=\"http://www.opengis.net/gml\" \n" r += "xmlns:om=\"http://www.opengis.net/om/1.0\" \n" r += "xsi:schemaLocation=\"http://www.opengis.net/sampling/1.0 http://schemas.opengis.net/sampling/1.0.0/sampling.xsd\">\n" r += " <gml:description>" + GF.desc + "</gml:description>\n" r += " <gml:name>" + GF.name + "</gml:name> \n" r += " <sa:sampledFeature/>\n" for i in range(len(GF.procedures)): r += " <sa:relatedObservation>\n" r += " <om:Observation>\n" # Sampling time for a in range(len(GF.samplingTime[i])): if len(GF.samplingTime[i][a])==2: r += " <om:samplingTime>\n" r += " <gml:TimePeriod>\n" r += " <gml:beginPosition>" + iso.datetime_isoformat(GF.samplingTime[i][a][0]) + "</gml:beginPosition>\n" r += " <gml:endPosition>" + iso.datetime_isoformat(GF.samplingTime[i][a][1]) + "</gml:endPosition>\n" r += " <gml:duration>" + iso.duration_isoformat(GF.samplingTime[i][a][1]-GF.samplingTime[i][a][0]) + "</gml:duration>\n" ''' r += " <gml:TimeLength>\n" r += " <gml:duration>" + iso.duration_isoformat(a[1]-a[0]) + "</gml:duration>\n" r += " <gml:timeInterval unit=\"" + str(ob.timeResUnit) + "\">" + str(ob.timeResVal) + "</gml:timeInterval>\n" r += " </gml:TimeLength>\n" ''' r += " </gml:TimePeriod>\n" r += " </om:samplingTime>\n" # Procedure r += " <om:procedure xlink:href=\"" + GF.procedures[i] + "\"/>\n" # ObservationProperty r += " <om:observedProperty>\n" if GF.obsType[i] == "insitu-fixed-point": ii=1 elif GF.obsType[i] == "insitu-mobile-point": ii=4 r += " <swe:CompositePhenomenon gml:id=\"comp_" + str(GF.idPrc[i]) + "\" dimension=\"" + str(len(GF.properties[i])+ii) + "\">\n" r += " <gml:name/>\n" r += " <swe:component xlink:href=\"" + sosConfig.urn["parameter"] + "time:iso8601" + " \" />\n" #if ob.procedureType == "insitu-fixed-point": if GF.obsType[i]=="insitu-mobile-point": r += " <swe:component xlink:href=\"" + sosConfig.urn["refsystem"] + ":x-position\" />\n" r += " <swe:component xlink:href=\"" + sosConfig.urn["refsystem"] + ":y-position\" />\n" r += " <swe:component xlink:href=\"" + sosConfig.urn["refsystem"] + ":z-position\" />\n" for c in range(len(GF.properties[i])): r += " <swe:component xlink:href=\"" + sosConfig.urn["parameter"] + GF.properties[i][c] + "\"/>\n" r += " </swe:CompositePhenomenon>\n" r += " </om:observedProperty>\n" #FEATURE OF INTEREST r += " <om:featureOfInterest xlink:href=\"" + sosConfig.urn["feature"] + GF.type + ":" + GF.name + "\"/>\n" #RESULT EMPTY (?) r += " <om:result/>\n" r += " </om:Observation>\n" r += " </sa:relatedObservation>\n" r += " <sa:position> \n" r += " " + GF.geom + "\n" r += " </sa:position>\n" if GF.type.lower()=="station" or GF.type.lower()=="point": r += "</sa:SamplingPoint> \n" elif GF.type=="surface": r += "</sa:SamplingSurface> \n" return r
def JSONformat(GO): import json oc = { "ObservationCollection": { "description": GO.offInfo.desc, "name": GO.offInfo.name, "member": [] } } for iob, ob in enumerate(GO.obs): member = { "name": ob.name, "samplingTime": {}, "procedure": ob.procedure } if ob.samplingTime != None: member["samplingTime"]["beginPosition"] = ob.samplingTime[ 0].astimezone(GO.reqTZ).isoformat() if ob.samplingTime[1]: member["samplingTime"]["endPosition"] = ob.samplingTime[ 1].astimezone(GO.reqTZ).isoformat() member["samplingTime"]["duration"] = iso.duration_isoformat( ob.samplingTime[1] - ob.samplingTime[0]) else: member["samplingTime"]["endPosition"] = ob.samplingTime[ 0].astimezone(GO.reqTZ).isoformat() if ob.procedureType == "insitu-fixed-point": ii = 1 elif ob.procedureType == "insitu-mobile-point": ii = 4 elif ob.procedureType == "virtual": ii = 1 member['observedProperty'] = { "CompositePhenomenon": { "id": "comp_%s" % str(ob.id_prc), "dimension": str(len(ob.opr_urn) + ii), "name": "timeSeriesOfObservations" } } member['observedProperty']["component"] = [ob.timedef] if ii == 4: member['observedProperty']["component"] += [ ("%s:x-position" % GO.refsys), ("%s:y-position" % GO.refsys), ("%s:z-position" % GO.refsys) ] member['observedProperty']["component"] += ob.opr_urn member['featureOfInterest'] = { "name": ob.foi_urn, "geom": ob.foiGml.replace("\"", "'") } member['result'] = { "DataArray": { "elementCount": str(len(ob.observedProperty) + ii), "field": [{ "name": "Time", "definition": ob.timedef }] } } if ii == 4: member['result']['DataArray']['field'] += [{ "name": "x-position", "definition": "%s:x-position" % GO.refsys }, { "name": "y-position", "definition": "%s:y-position" % GO.refsys }, { "name": "z-position", "definition": "%s:z-position" % GO.refsys }] for idx in range(len(ob.observedProperty)): member['result']['DataArray']['field'] += [{ "name": ob.observedPropertyName[idx], "definition": ob.observedProperty[idx], "uom": ob.uom[idx] }] member['result']['DataArray']['values'] = [] for row in range(len(ob.data)): data = [ob.data[row][0].isoformat()] for i in range(1, len(ob.data[0])): data.append(str(ob.data[row][i])) member['result']['DataArray']['values'].append(data) # append member to collection oc["ObservationCollection"]["member"].append(member) return json.dumps(oc)
def XMLformat(GO): r = """<om:ObservationCollection xmlns:sos="http://www.opengis.net/sos/1.0" xmlns:om="http://www.opengis.net/om/1.0" xmlns:swe="http://www.opengis.net/swe/1.0.1" xmlns:gml="http://www.opengis.net/gml" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/om/1.0 http://schemas.opengis.net/om/1.0.0/om.xsd"> """ r += "<gml:description>" + GO.offInfo.desc + "</gml:description>\n" r += "<gml:name>" + GO.offInfo.name + "</gml:name>\n" if len(GO.obs) == 0: r += "<om:member/>\n" #raise sosException.SOSException("NoApplicableCode",None,"No matching observation was found according the request parameters!") for ob in GO.obs: #OBSERVATION OBJ r += "<om:member>\n" r += " <om:Observation>\n" r += " <gml:name>" + ob.name + "</gml:name>\n" #PERIODO DI CAMPIONAMENTO DEI DATI ESTRATTI if ob.samplingTime != None: r += " <om:samplingTime>\n" r += " <gml:TimePeriod>\n" # r += " <gml:beginPosition>" + ob.samplingTime[0].astimezone(GO.reqTZ).strftime("%Y-%m-%dT%H:%M:%S.%f%z") + "</gml:beginPosition>\n" r += " <gml:beginPosition>" + ob.samplingTime[0].astimezone( GO.reqTZ).isoformat() + "</gml:beginPosition>\n" if ob.samplingTime[1]: r += " <gml:endPosition>" + ob.samplingTime[ 1].astimezone( GO.reqTZ).isoformat() + "</gml:endPosition>\n" else: r += " <gml:endPosition>" + ob.samplingTime[ 0].astimezone( GO.reqTZ).isoformat() + "</gml:endPosition>\n" if ob.samplingTime[1]: r += " <gml:duration>" + iso.duration_isoformat( ob.samplingTime[1] - ob.samplingTime[0]) + "</gml:duration>\n" r += " </gml:TimePeriod>\n" r += " </om:samplingTime>\n" else: r += " <om:samplingTime/>\n" #PROCEDURE r += " <om:procedure xlink:href=\"" + ob.procedure + "\"/>\n" #PROPRIETA OSSERVATA if ob.procedureType == "insitu-fixed-point": ii = 1 elif ob.procedureType == "insitu-mobile-point": ii = 4 elif ob.procedureType == "virtual": ii = 1 #OBSERVED PROPERTIES r += " <om:observedProperty>\n" r += " <swe:CompositePhenomenon gml:id=\"comp_" + str( ob.id_prc) + "\" dimension=\"" + str(len(ob.opr_urn) + ii) + "\">\n" r += " <gml:name>timeSeriesOfObservations</gml:name>\n" r += " <swe:component xlink:href=\"" + ob.timedef + "\"/>\n" if ob.procedureType == "insitu-mobile-point": r += " <swe:component xlink:href=\"" + GO.refsys + ":x-position\"/>\n" r += " <swe:component xlink:href=\"" + GO.refsys + ":y-position\"/>\n" r += " <swe:component xlink:href=\"" + GO.refsys + ":z-position\"/>\n" for urn in ob.opr_urn: r += " <swe:component xlink:href=\"" + urn + "\"/>\n" r += " </swe:CompositePhenomenon>\n" r += " </om:observedProperty>\n" #FEATURE OF INTEREST r += " <om:featureOfInterest xlink:href=\"" + ob.foi_urn + "\">\n" r += " <gml:FeatureCollection>\n" r += " <gml:location>\n" r += " " + ob.foiGml + "\n" r += " </gml:location>\n" r += " </gml:FeatureCollection>\n" r += " </om:featureOfInterest>" #SERIE TEMPORALE r += " <om:result>\n" #ii = 1 #if ob.procedureType=="insitu-mobile-point": # ii = 4 #DESCRIZIONE DEI DATI ESTRATTI: VARIA A SECONDA DEL TIPO DI PROCEDURA #-- CASO GENERALE r += " <swe:DataArray>\n" r += " <swe:elementCount>\n" r += " <swe:Count>\n" r += " <swe:value>" + str(len(ob.observedProperty) + ii) + "</swe:value>\n" r += " </swe:Count>\n" r += " </swe:elementCount>\n" r += " <swe:elementType name=\"SimpleDataArray\">\n" r += " <swe:DataRecord>\n" r += " <swe:field name=\"Time\">\n" r += " <swe:Time definition=\"" + ob.timedef + "\"/>\n" r += " </swe:field>\n" if ob.procedureType == "insitu-mobile-point": r += " <swe:field name=\"x-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":x-position\"/>\n" r += " </swe:field>\n" r += " <swe:field name=\"y-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":y-position\"/>\n" r += " </swe:field>\n" r += " <swe:field name=\"z-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":z-position\"/>\n" r += " </swe:field>\n" if ob.qualityIndex: r += " <swe:field name=\"position-qualityIndex\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":position:qualityIndex\"/>\n" r += " </swe:field>\n" for idx in range(len(ob.observedProperty)): if ob.aggregate_function: if ob.observedProperty[idx].split(":")[-1] == "qualityIndex": r += " <swe:field name=\"%s\">\n" % ( ob.observedPropertyName[idx]) r += " <swe:Quantity definition=\"%s\">\n" % ( ob.observedProperty[idx]) else: r += " <swe:field name=\"%s:%s\">\n" % ( ob.observedPropertyName[idx], ob.aggregate_function) r += " <swe:Quantity definition=\"%s:%s\">\n" % ( ob.observedProperty[idx], ob.aggregate_function) if ob.aggregate_function.upper() == "COUNT": r += " <swe:uom code=\"None\"/>\n" else: r += " <swe:uom code=\"" + ob.uom[ idx] + "\"/>\n" else: r += " <swe:field name=\"%s\">\n" % ( ob.observedPropertyName[idx]) r += " <swe:Quantity definition=\"" + ob.observedProperty[ idx] + "\">\n" r += " <swe:uom code=\"" + ob.uom[idx] + "\"/>\n" r += " </swe:Quantity>\n" r += " </swe:field>\n" r += " </swe:DataRecord>\n" r += " </swe:elementType>\n" r += " <swe:encoding>\n" r += " <swe:TextBlock tokenSeparator=\",\" blockSeparator=\"@\" decimalSeparator=\".\"/>\n" r += " </swe:encoding>\n" if len(ob.data) > 0: r += " <swe:values>" data = [] for row in range(len(ob.data)): str_data = [ob.data[row][0].isoformat()] for i in range(1, len(ob.data[0])): str_data.append(str(ob.data[row][i])) data.append(",".join(str_data)) r += "@".join(data) r += "</swe:values>\n" else: r += " <swe:values/>" r += " </swe:DataArray>\n" r += " </om:result>\n" r += " </om:Observation>\n" r += "</om:member>\n" r += "</om:ObservationCollection>" return r
def execute(args, logger=None): def log(message): if debug: if logger: logger.log(message) else: print message # SCRIPT CONFIGURATION # ========================================================================= # Activate and print verbose information debug = args['v'] if args.has_key('v') else False # Procedure name procedure = args['procedure'] # Begin date begin = args['begin'] if args.has_key('begin') else "*" # End date end = args['end'] if args.has_key('end') else "*" # Global User and password valid for all connections suser = duser = auser = args['user'] if args.has_key('user') else None spwd = dpwd = apwd = args['pwd'] if args.has_key('pwd') else None # Activate this will copy also the quality index from source to destination cpqi = args['cpqi'] if args.has_key('cpqi') else False # Aggregating function configuration resolution = args['resolution'] if 'resolution' in args else None function = args['function'] if 'function' in args else None nodataValue = args['nodataValue'] if 'nodataValue' in args else None nodataQI = args['nodataQI'] if 'nodataQI' in args else None # Retroactive aggregation retro = args['retro'] if 'retro' in args else 0 # Force using last position as end position during insert sensor operation lm = args['lm'] if 'lm' in args else False # SOURCE istSOS CONFIG ================================== # Location surl = args['surl'] # Service instance name ssrv = args['ssrv'] # User and password if given this will be used for source istSOS if args.has_key('suser'): suser = args['suser'] if args.has_key('spwd'): spwd = args['spwd'] # DESTINATION istSOS CONFIG ============================= # Location (if not given, same as source will be used) durl = args['durl'] if (args.has_key('durl') and args['durl'] is not None) else surl # Service instance name dsrv = args['dsrv'] # User and password if given this will be used for destination istSOS if args.has_key('duser'): duser = args['duser'] if args.has_key('dpwd'): dpwd = args['dpwd'] # ALTERNATIVE istSOS SERVICE FOR QI EXTRAPOLATION ======= # Location (if not given, same as source will be used) aurl = args['aurl'] if (args.has_key('aurl') and args['aurl'] is not None) else None # Service instance name asrv = args['asrv'] if (args.has_key('asrv') and args['asrv'] is not None) else None # User and password if given this will be used for extrapolation QI istSOS if args.has_key('auser'): auser = args['auser'] if args.has_key('apwd'): apwd = args['apwd'] # PROCESSING STARTS HERE ================================================== log("\nistSOS > 2 > istSOS STARTED:") log("==============================\n") #req = requests.session() req = requests # Load procedure description log("1. Loading procedure description: %s" % procedure) # Loading describe sensor from source ===================================== res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (surl, ssrv, procedure), auth=(suser, spwd), verify=False) sdata = res.json() if sdata['success'] == False: raise Exception( "Description of procedure %s can not be loaded from source service: %s" % (procedure, sdata['message'])) else: log(" > DS Source Ok.") # Loading describe sensor from destination ================================ res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (durl, dsrv, procedure), auth=(duser, dpwd), verify=False) ddata = res.json() if ddata['success'] == False: raise Exception( "Description of procedure %s can not be loaded from destination service: %s" % (procedure, ddata['message'])) else: log(" > DS Destination Ok.") # Load of a getobservation template from destination ======================================= res = req.get( "%s/wa/istsos/services/%s/operations/getobservation/offerings/%s/procedures/%s/observedproperties/:/eventtime/last?qualityIndex=False" % (durl, dsrv, 'temporary', procedure), params={"qualityIndex": cpqi}, auth=(duser, dpwd), verify=False) dtemplate = res.json() if dtemplate['success'] == False: raise Exception( "Observation template of procedure %s can not be loaded: %s" % (procedure, dtemplate['message'])) else: dtemplate = dtemplate['data'][0] dtemplate['AssignedSensorId'] = ddata['data']['assignedSensorId'] dtemplate['result']['DataArray']['values'] = [] log(" > GO Template Ok.") # Loading describe sensor from QI EXTRAPOLATION service =================== if aurl and asrv: res = req.get("%s/wa/istsos/services/%s/procedures/%s" % (aurl, asrv, procedure), auth=(auser, apwd), verify=False) adata = res.json() if adata['success'] == False: raise Exception( "Description of procedure %s can not be loaded from destination service: %s" % (procedure, adata['message'])) else: log(" > DS QI Extrapolation Ok.") log("\n2. Identifying processing interval:") # Check if mesaures are present in source procedure, by identifying the sampling time constraint # located always in the first position of the outputs, if it is empty an exception is thrown if (not 'constraint' in sdata['data']['outputs'][0] or not 'interval' in sdata['data']['outputs'][0]['constraint']): raise Exception( "There is no data in the source procedure to be copied to the destination procedure." ) else: # Check if the contraint interval contains a valid ISO date begin position try: iso.parse_datetime( sdata['data']['outputs'][0]['constraint']['interval'][0]) except Exception: raise Exception( "The date in the source procedure constraint interval (%s) is not valid." % sdata['data']['outputs'][0]['constraint']['interval'][0]) # Check if the contraint interval contains a valid ISO date end position try: iso.parse_datetime( sdata['data']['outputs'][0]['constraint']['interval'][1]) except Exception: raise Exception( "The date in the source procedure constraint interval (%s) is not valid." % sdata['data']['outputs'][0]['constraint']['interval'][1]) log(" > Source interval is valid") # Looking for start (IO beginPOsition) instant processing # If the default value (*) is used, then the endPosition of # the "destination" service procedure will be used. But if the destination # procedure is empty , then the begin position of the source will be used start = None stop = None if begin == "*": if ('constraint' in ddata['data']['outputs'][0] and 'interval' in ddata['data']['outputs'][0]['constraint']): try: if function and resolution: # getting last inserted observations of "destination" service log("Aggregation requested: getting last inserted observations of \"destination\" service" ) params = { "request": "GetObservation", "service": "SOS", "version": "1.0.0", "observedProperty": ':', "procedure": procedure, "responseFormat": "application/json", "offering": 'temporary' } res = req.get("%s/%s" % (durl, dsrv), params=params, auth=(duser, dpwd), verify=False) obs = res.json() start = iso.parse_datetime( obs['ObservationCollection']['member'][0]['result'] ['DataArray']['values'][0][0]) else: # The endPosition of the destination will be used as Start/IO BeginPosition start = iso.parse_datetime(ddata['data']['outputs'][0] ['constraint']['interval'][1]) if retro > 0: # Retroactive aggregation log("Retroactive aggregation active.") if start - timedelta(minutes=retro) > iso.parse_datetime( ddata['data']['outputs'][0]['constraint'] ['interval'][0]): start = start - timedelta(minutes=retro) else: start = iso.parse_datetime( ddata['data']['outputs'][0]['constraint'] ['interval'][0]) log("Start: %s" % start) except Exception as ee: print "Error setting start date for proc %s: %s" % (procedure, ee) raise Exception( "The date in the destination procedure %s constraint interval (%s) is not valid." % (procedure, ddata['data']['outputs'][0]['constraint']['interval'][0])) else: # The beginPosition of the source will be used as Start/IO BeginPosition start = iso.parse_datetime( sdata['data']['outputs'][0]['constraint']['interval'][0]) else: start = iso.parse_datetime(begin) if end == "*": # The endPosition of the source will be used as Stop/IO EndPosition stop = iso.parse_datetime( sdata['data']['outputs'][0]['constraint']['interval'][1]) else: stop = iso.parse_datetime(end) log(" > Destination interval is valid") log(" > Start processing: %s" % start) log(" > Stop processing: %s" % stop) if retro > 0: log(" > Retro aggregation: %s minutes" % retro) # Insertion loop step timedelta interval = timedelta(days=15) if start < stop and start + interval > stop: interval = stop - start log(" > Insertion loop step: %s" % interval) if function and resolution: try: iso.duration_isoformat(resolution) except: raise Exception( "The resolution (%s) to apply in the aggregating function is not valid." % resolution) log(" > Function(Resolution) : %s(%s)" % (function, resolution)) while start + interval <= stop: nextStart = start + interval params = { "request": "GetObservation", "service": "SOS", "version": "1.0.0", "observedProperty": ':', "procedure": procedure, "qualityIndex": str(cpqi), "responseFormat": "application/json", "offering": 'temporary', "eventTime": "%s/%s" % (start.isoformat(), nextStart.isoformat()) } if function and resolution: params['aggregateFunction'] = function params['aggregateInterval'] = resolution if nodataValue != None: params['aggregateNodata'] = nodataValue if nodataQI != None: params['aggregateNodataQi'] = nodataQI res = req.get("%s/%s" % (surl, ssrv), params=params, auth=(suser, spwd), verify=False) # Check if an Exception occured if 'ExceptionReport' in res.content: raise Exception(res.content) smeasures = res.json()['ObservationCollection']['member'][0] #pp.pprint(smeasures) log(" > %s measures from: %s to: %s" % (len(smeasures['result']['DataArray']['values']), start.isoformat(), nextStart.isoformat())) dtemplate["samplingTime"] = {} if lm and len(smeasures['result']['DataArray']['values']) > 0: dtemplate["samplingTime"]["beginPosition"] = smeasures['result'][ 'DataArray']['values'][0][0] dtemplate["samplingTime"]["endPosition"] = smeasures['result'][ 'DataArray']['values'][-1][0] else: dtemplate["samplingTime"]["beginPosition"] = start.isoformat() dtemplate["samplingTime"]["endPosition"] = nextStart.isoformat() dtemplate['result']['DataArray']['values'] = smeasures['result'][ 'DataArray']['values'] dtemplate['result']['DataArray']['field'] = smeasures['result'][ 'DataArray']['field'] # POST data to WA res = req.post( "%s/wa/istsos/services/%s/operations/insertobservation" % (durl, dsrv), auth=(duser, dpwd), verify=False, data=json.dumps({ "ForceInsert": "true", "AssignedSensorId": ddata['data']['assignedSensorId'], "Observation": dtemplate })) # read response log(" > Insert observation success: %s" % res.json()['success']) #print res.json() if not res.json()['success']: raise Exception('Error inserting observation: %s' % res.json()['message']) start = nextStart if start < stop and start + interval > stop: interval = stop - start
def render(GF, sosConfig): r = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" if GF.type.lower() == "station" or GF.type.lower() == "point": r += "<sa:SamplingPoint \n" elif GF.type == "surface": r += "<sa:SamplingSurface \n" r += "gml:id=\"" + GF.name + "\" \n" r += "xmlns:sa=\"http://www.opengis.net/sampling/1.0\" \n" r += "xmlns:swe=\"http://www.opengis.net/swe/1.0.1\" \n" r += "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n" r += "xmlns:xlink=\"http://www.w3.org/1999/xlink\" \n" r += "xmlns:gml=\"http://www.opengis.net/gml\" \n" r += "xmlns:om=\"http://www.opengis.net/om/1.0\" \n" r += "xsi:schemaLocation=\"http://www.opengis.net/sampling/1.0 http://schemas.opengis.net/sampling/1.0.0/sampling.xsd\">\n" r += " <gml:description>" + GF.desc + "</gml:description>\n" r += " <gml:name>" + GF.name + "</gml:name> \n" r += " <sa:sampledFeature/>\n" for i in range(len(GF.procedures)): r += " <sa:relatedObservation>\n" r += " <om:Observation>\n" # Sampling time for a in range(len(GF.samplingTime[i])): if len(GF.samplingTime[i][a]) == 2: r += " <om:samplingTime>\n" r += " <gml:TimePeriod>\n" r += " <gml:beginPosition>" + iso.datetime_isoformat( GF.samplingTime[i][a][0]) + "</gml:beginPosition>\n" r += " <gml:endPosition>" + iso.datetime_isoformat( GF.samplingTime[i][a][1]) + "</gml:endPosition>\n" r += " <gml:duration>" + iso.duration_isoformat( GF.samplingTime[i][a][1] - GF.samplingTime[i][a][0]) + "</gml:duration>\n" ''' r += " <gml:TimeLength>\n" r += " <gml:duration>" + iso.duration_isoformat(a[1]-a[0]) + "</gml:duration>\n" r += " <gml:timeInterval unit=\"" + str(ob.timeResUnit) + "\">" + str(ob.timeResVal) + "</gml:timeInterval>\n" r += " </gml:TimeLength>\n" ''' r += " </gml:TimePeriod>\n" r += " </om:samplingTime>\n" # Procedure r += " <om:procedure xlink:href=\"" + GF.procedures[i] + "\"/>\n" # ObservationProperty r += " <om:observedProperty>\n" if GF.obsType[i] == "insitu-fixed-point": ii = 1 elif GF.obsType[i] == "insitu-mobile-point": ii = 4 r += " <swe:CompositePhenomenon gml:id=\"comp_" + str( GF.idPrc[i]) + "\" dimension=\"" + str(len(GF.properties[i]) + ii) + "\">\n" r += " <gml:name/>\n" r += " <swe:component xlink:href=\"" + sosConfig.urn[ "parameter"] + "time:iso8601" + " \" />\n" #if ob.procedureType == "insitu-fixed-point": if GF.obsType[i] == "insitu-mobile-point": r += " <swe:component xlink:href=\"" + sosConfig.urn[ "refsystem"] + ":x-position\" />\n" r += " <swe:component xlink:href=\"" + sosConfig.urn[ "refsystem"] + ":y-position\" />\n" r += " <swe:component xlink:href=\"" + sosConfig.urn[ "refsystem"] + ":z-position\" />\n" for c in range(len(GF.properties[i])): r += " <swe:component xlink:href=\"" + sosConfig.urn[ "parameter"] + GF.properties[i][c] + "\"/>\n" r += " </swe:CompositePhenomenon>\n" r += " </om:observedProperty>\n" #FEATURE OF INTEREST r += " <om:featureOfInterest xlink:href=\"" + sosConfig.urn[ "feature"] + GF.type + ":" + GF.name + "\"/>\n" #RESULT EMPTY (?) r += " <om:result/>\n" r += " </om:Observation>\n" r += " </sa:relatedObservation>\n" r += " <sa:position> \n" r += " " + GF.geom + "\n" r += " </sa:position>\n" if GF.type.lower() == "station" or GF.type.lower() == "point": r += "</sa:SamplingPoint> \n" elif GF.type == "surface": r += "</sa:SamplingSurface> \n" return r
def XMLformat(GO): r = """<om:ObservationCollection xmlns:sos="http://www.opengis.net/sos/1.0" xmlns:om="http://www.opengis.net/om/1.0" xmlns:swe="http://www.opengis.net/swe/1.0.1" xmlns:gml="http://www.opengis.net/gml" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/om/1.0 http://schemas.opengis.net/om/1.0.0/om.xsd"> """ r += "<gml:description>" + GO.offInfo.desc + "</gml:description>\n" r += "<gml:name>" + GO.offInfo.name + "</gml:name>\n" if len(GO.obs)==0: raise sosException.SOSException("NoApplicableCode",None,"No matching observation was found according the request parameters!") r += "<om:member/>\n" for ob in GO.obs: #OBSERVATION OBJ r += "<om:member>\n" r += " <om:Observation>\n" r += " <gml:name>" + ob.name + "</gml:name>\n" #PERIODO DI CAMPIONAMENTO DEI DATI ESTRATTI if ob.samplingTime != None: r += " <om:samplingTime>\n" r += " <gml:TimePeriod>\n" # r += " <gml:beginPosition>" + ob.samplingTime[0].astimezone(GO.reqTZ).strftime("%Y-%m-%dT%H:%M:%S.%f%z") + "</gml:beginPosition>\n" r += " <gml:beginPosition>" + ob.samplingTime[0].astimezone(GO.reqTZ).isoformat() + "</gml:beginPosition>\n" if ob.samplingTime[1]: r += " <gml:endPosition>" + ob.samplingTime[1].astimezone(GO.reqTZ).isoformat() + "</gml:endPosition>\n" else: r += " <gml:endPosition>" + ob.samplingTime[0].astimezone(GO.reqTZ).isoformat() + "</gml:endPosition>\n" if ob.samplingTime[1]: r += " <gml:duration>" + iso.duration_isoformat(ob.samplingTime[1]-ob.samplingTime[0]) + "</gml:duration>\n" r += " </gml:TimePeriod>\n" r += " </om:samplingTime>\n" else: r += " <om:samplingTime/>\n" #PROCEDURE r += " <om:procedure xlink:href=\"" + ob.procedure + "\"/>\n" #PROPRIETA OSSERVATA if ob.procedureType == "insitu-fixed-point": ii=1 elif ob.procedureType == "insitu-mobile-point": ii=4 elif ob.procedureType == "virtual": ii=1 #OBSERVED PROPERTIES r += " <om:observedProperty>\n" r += " <swe:CompositePhenomenon gml:id=\"comp_" + str(ob.id_prc) + "\" dimension=\"" + str(len(ob.opr_urn)+ii) + "\">\n" r += " <gml:name>timeSeriesOfObservations</gml:name>\n" r += " <swe:component xlink:href=\"" + ob.timedef + "\"/>\n" if ob.procedureType=="insitu-mobile-point": r += " <swe:component xlink:href=\"" + GO.refsys + ":x-position\"/>\n" r += " <swe:component xlink:href=\"" + GO.refsys + ":y-position\"/>\n" r += " <swe:component xlink:href=\"" + GO.refsys + ":z-position\"/>\n" for urn in ob.opr_urn: r += " <swe:component xlink:href=\"" + urn + "\"/>\n" r += " </swe:CompositePhenomenon>\n" r += " </om:observedProperty>\n" #FEATURE OF INTEREST r += " <om:featureOfInterest xlink:href=\"" + ob.foi_urn + "\">\n" r += " <gml:FeatureCollection>\n" r += " <gml:location>\n" r += " " + ob.foiGml + "\n" r += " </gml:location>\n" r += " </gml:FeatureCollection>\n" r += " </om:featureOfInterest>" #SERIE TEMPORALE r += " <om:result>\n" #ii = 1 #if ob.procedureType=="insitu-mobile-point": # ii = 4 #DESCRIZIONE DEI DATI ESTRATTI: VARIA A SECONDA DEL TIPO DI PROCEDURA #-- CASO GENERALE r += " <swe:DataArray>\n" r += " <swe:elementCount>\n" r += " <swe:Count>\n" r += " <swe:value>" + str(len(ob.observedProperty)+ii) + "</swe:value>\n" r += " </swe:Count>\n" r += " </swe:elementCount>\n" r += " <swe:elementType name=\"SimpleDataArray\">\n" r += " <swe:DataRecord>\n" r += " <swe:field name=\"Time\">\n" r += " <swe:Time definition=\"" + ob.timedef + "\"/>\n" r += " </swe:field>\n" if ob.procedureType=="insitu-mobile-point": r += " <swe:field name=\"x-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":x-position\"/>\n" r += " </swe:field>\n" r += " <swe:field name=\"y-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":y-position\"/>\n" r += " </swe:field>\n" r += " <swe:field name=\"z-position\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":z-position\"/>\n" r += " </swe:field>\n" if ob.qualityIndex: r += " <swe:field name=\"position-qualityIndex\">\n" r += " <swe:Quantity definition=\"" + GO.refsys + ":position:qualityIndex\"/>\n" r += " </swe:field>\n" for idx in range(len(ob.observedProperty)): if ob.aggregate_function: if ob.observedProperty[idx].split(":")[-1] == "qualityIndex": r += " <swe:field name=\"%s\">\n" % (ob.observedPropertyName[idx]) r += " <swe:Quantity definition=\"%s\">\n" % (ob.observedProperty[idx]) else: r += " <swe:field name=\"%s:%s\">\n" % (ob.observedPropertyName[idx],ob.aggregate_function) r += " <swe:Quantity definition=\"%s:%s\">\n" % (ob.observedProperty[idx],ob.aggregate_function) if ob.aggregate_function.upper()=="COUNT": r += " <swe:uom code=\"None\"/>\n" else: r += " <swe:uom code=\"" + ob.uom[idx] + "\"/>\n" else: r += " <swe:field name=\"%s\">\n" % (ob.observedPropertyName[idx]) r += " <swe:Quantity definition=\"" + ob.observedProperty[idx] + "\">\n" r += " <swe:uom code=\"" + ob.uom[idx] + "\"/>\n" r += " </swe:Quantity>\n" r += " </swe:field>\n" r += " </swe:DataRecord>\n" r += " </swe:elementType>\n" r += " <swe:encoding>\n" r += " <swe:TextBlock tokenSeparator=\",\" blockSeparator=\"@\" decimalSeparator=\".\"/>\n" r += " </swe:encoding>\n" if len(ob.data)>0: r += " <swe:values>" data=[] for row in range(len(ob.data)): str_data=[ob.data[row][0].isoformat()] for i in range(1,len(ob.data[0])): str_data.append(str(ob.data[row][i])) data.append(",".join(str_data)) r += "@".join(data) r += "</swe:values>\n" else: r += " <swe:values/>" r += " </swe:DataArray>\n" r += " </om:result>\n" r += " </om:Observation>\n" r += "</om:member>\n" r += "</om:ObservationCollection>" return r
def JSONformat(GO): import json oc = { "ObservationCollection": { "description": GO.offInfo.desc, "name": GO.offInfo.name, "member": [] } } for iob, ob in enumerate(GO.obs): member = { "name": ob.name, "samplingTime": {}, "procedure": ob.procedure } if ob.samplingTime != None: member["samplingTime"]["beginPosition"] = ob.samplingTime[0].astimezone(GO.reqTZ).isoformat() if ob.samplingTime[1]: member["samplingTime"]["endPosition"] = ob.samplingTime[1].astimezone(GO.reqTZ).isoformat() member["samplingTime"]["duration"] = iso.duration_isoformat(ob.samplingTime[1]-ob.samplingTime[0]) else: member["samplingTime"]["endPosition"] = ob.samplingTime[0].astimezone(GO.reqTZ).isoformat() if ob.procedureType == "insitu-fixed-point": ii=1 elif ob.procedureType == "insitu-mobile-point": ii=4 elif ob.procedureType == "virtual": ii=1 member['observedProperty'] = { "CompositePhenomenon": { "id": "comp_%s" % str(ob.id_prc), "dimension": str(len(ob.opr_urn)+ii), "name": "timeSeriesOfObservations" } } member['observedProperty']["component"] = [ob.timedef] if ii==4: member['observedProperty']["component"] += [ ("%s:x-position" % GO.refsys), ("%s:y-position" % GO.refsys), ("%s:z-position" % GO.refsys) ] member['observedProperty']["component"] += ob.opr_urn member['featureOfInterest'] = { "name": ob.foi_urn, "geom": ob.foiGml.replace("\"","'") } member['result'] = { "DataArray": { "elementCount": str(len(ob.observedProperty)+ii), "field": [ { "name": "Time", "definition": ob.timedef } ] } } if ii==4: member['result']['DataArray']['field'] += [ { "name": "x-position", "definition": "%s:x-position" % GO.refsys }, { "name": "y-position", "definition": "%s:y-position" % GO.refsys }, { "name": "z-position", "definition": "%s:z-position" % GO.refsys } ] for idx in range(len(ob.observedProperty)): member['result']['DataArray']['field'] += [ { "name": ob.observedPropertyName[idx], "definition": ob.observedProperty[idx], "uom": ob.uom[idx] } ] member['result']['DataArray']['values'] = [] for row in range(len(ob.data)): data = [ob.data[row][0].isoformat()] for i in range(1,len(ob.data[0])): data.append(str(ob.data[row][i])) member['result']['DataArray']['values'].append(data) # append member to collection oc["ObservationCollection"]["member"].append(member) return json.dumps(oc)
def execute (args, logger=None): # SCRIPT CONFIGURATION # ========================================================================= # Activate and print verbose information debug = args['v'] if args.has_key('v') else False # Procedure name procedure = args['procedure'] # Begin date begin = args['begin'] if args.has_key('begin') else "*" # End date end = args['end'] if args.has_key('end') else "*" # Global User and password valid for all connections suser = duser = auser = args['user'] if args.has_key('user') else None spwd = dpwd = apwd = args['pwd'] if args.has_key('pwd') else None # Activate this will copy also the quality index from source to destination cpqi = args['cpqi'] if args.has_key('cpqi') else False # Aggregating function configuration resolution = args['resolution'] if 'resolution' in args else None function = args['function'] if 'function' in args else None nodataValue = args['nodataValue'] if 'nodataValue' in args else None nodataQI = args['nodataQI'] if 'nodataQI' in args else None # Retroactive aggregation retro = args['retro'] if 'retro' in args else 0 # Force using last position as end position during insert sensor operation lm = args['lm'] if 'lm' in args else False # SOURCE istSOS CONFIG ================================== # Location surl = args['surl'] # Service instance name ssrv = args['ssrv'] # User and password if given this will be used for source istSOS if args.has_key('suser'): suser = args['suser'] if args.has_key('spwd'): spwd = args['spwd'] # DESTINATION istSOS CONFIG ============================= # Location (if not given, same as source will be used) durl = args['durl'] if (args.has_key('durl') and args['durl'] is not None) else surl # Service instance name dsrv = args['dsrv'] # User and password if given this will be used for destination istSOS if args.has_key('duser'): duser = args['duser'] if args.has_key('dpwd'): dpwd = args['dpwd'] # ALTERNATIVE istSOS SERVICE FOR QI EXTRAPOLATION ======= # Location (if not given, same as source will be used) aurl = args['aurl'] if (args.has_key('aurl') and args['aurl'] is not None) else None # Service instance name asrv = args['asrv'] if (args.has_key('asrv') and args['asrv'] is not None) else None # User and password if given this will be used for extrapolation QI istSOS if args.has_key('auser'): auser = args['auser'] if args.has_key('apwd'): apwd = args['apwd'] def log(message): if debug: if logger: logger.log(message) else: print message # PROCESSING STARTS HERE ================================================== log("\nistSOS > 2 > istSOS STARTED:") log("==============================\n") #req = requests.session() req = requests # Load procedure description log("1. Loading procedure description: %s" % procedure) # Loading describe sensor from source ===================================== res = req.get("%s/wa/istsos/services/%s/procedures/%s" % ( surl, ssrv, procedure ), auth=(suser, spwd), verify=False) sdata = res.json() if sdata['success']==False: raise Exception ("Description of procedure %s can not be loaded from source service: %s" % (procedure, sdata['message'])) else: log(" > DS Source Ok.") # Loading describe sensor from destination ================================ res = req.get("%s/wa/istsos/services/%s/procedures/%s" % ( durl, dsrv, procedure ), auth=(duser, dpwd), verify=False) ddata = res.json() if ddata['success']==False: raise Exception ("Description of procedure %s can not be loaded from destination service: %s" % (procedure, ddata['message'])) else: log(" > DS Destination Ok.") # Load of a getobservation template from destination ======================================= res = req.get("%s/wa/istsos/services/%s/operations/getobservation/offerings/%s/procedures/%s/observedproperties/:/eventtime/last?qualityIndex=False" % ( durl, dsrv, 'temporary', procedure ), params={ "qualityIndex": cpqi }, auth=(duser, dpwd), verify=False) dtemplate = res.json() if dtemplate['success']==False: raise Exception ("Observation template of procedure %s can not be loaded: %s" % (procedure, dtemplate['message'])) else: dtemplate = dtemplate['data'][0] dtemplate['AssignedSensorId'] = ddata['data']['assignedSensorId'] dtemplate['result']['DataArray']['values'] = [] log(" > GO Template Ok.") # Loading describe sensor from QI EXTRAPOLATION service =================== if aurl and asrv: res = req.get("%s/wa/istsos/services/%s/procedures/%s" % ( aurl, asrv, procedure ), auth=(auser, apwd), verify=False) adata = res.json() if adata['success']==False: raise Exception ("Description of procedure %s can not be loaded from destination service: %s" % (procedure, adata['message'])) else: log(" > DS QI Extrapolation Ok.") log("\n2. Identifying processing interval:") # Check if mesaures are present in source procedure, by identifying the sampling time constraint # located always in the first position of the outputs, if it is empty an exception is thrown if (not 'constraint' in sdata['data']['outputs'][0] or not 'interval' in sdata['data']['outputs'][0]['constraint'] ): raise Exception ("There is no data in the source procedure to be copied to the destination procedure.") else: # Check if the contraint interval contains a valid ISO date begin position try: iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][0]) except Exception: raise Exception ("The date in the source procedure constraint interval (%s) is not valid." % sdata['data']['outputs'][0]['constraint']['interval'][0]) # Check if the contraint interval contains a valid ISO date end position try: iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][1]) except Exception: raise Exception ("The date in the source procedure constraint interval (%s) is not valid." % sdata['data']['outputs'][0]['constraint']['interval'][1]) log(" > Source interval is valid") # Looking for start (IO beginPOsition) instant processing # If the default value (*) is used, then the endPosition of # the "destination" service procedure will be used. But if the destination # procedure is empty , then the begin position of the source will be used start = None stop = None if begin == "*": if ('constraint' in ddata['data']['outputs'][0] and 'interval' in ddata['data']['outputs'][0]['constraint']): try: if function and resolution: # getting last inserted observations of "destination" service log("Aggregation requested: getting last inserted observations of \"destination\" service") params = { "request": "GetObservation", "service": "SOS", "version": "1.0.0", "observedProperty": ':', "procedure": procedure, "responseFormat": "application/json", "offering": 'temporary' } res = req.get("%s/%s" % (durl,dsrv), params=params, auth=(duser, dpwd), verify=False) obs = res.json() start = iso.parse_datetime(obs['ObservationCollection']['member'][0]['result']['DataArray']['values'][0][0]) else: # The endPosition of the destination will be used as Start/IO BeginPosition start = iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][1]) if retro > 0: # Retroactive aggregation log("Retroactive aggregation active.") if start-timedelta(minutes=retro) > iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][0]): start = start-timedelta(minutes=retro) else: start = iso.parse_datetime(ddata['data']['outputs'][0]['constraint']['interval'][0]) log("Start: %s" % start) except Exception as ee: print "Error setting start date for proc %s: %s" % (procedure,ee) raise Exception ("The date in the destination procedure %s constraint interval (%s) is not valid." % (procedure,ddata['data']['outputs'][0]['constraint']['interval'][0])) else: # The beginPosition of the source will be used as Start/IO BeginPosition start = iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][0]) else: start = iso.parse_datetime(begin) if end == "*": # The endPosition of the source will be used as Stop/IO EndPosition stop = iso.parse_datetime(sdata['data']['outputs'][0]['constraint']['interval'][1]) else: stop = iso.parse_datetime(end) log(" > Destination interval is valid") log(" > Start processing: %s" % start) log(" > Stop processing: %s" % stop) if retro > 0: log(" > Retro aggregation: %s minutes" % retro) # Insertion loop step timedelta interval = timedelta(days=15) if start<stop and start+interval>stop: interval = stop-start log(" > Insertion loop step: %s" % interval) if function and resolution: try: iso.duration_isoformat(resolution) except: raise Exception ("The resolution (%s) to apply in the aggregating function is not valid." % resolution) log(" > Function(Resolution) : %s(%s)" % (function,resolution)) while start+interval<=stop: nextStart = start + interval params = { "request": "GetObservation", "service": "SOS", "version": "1.0.0", "observedProperty": ':', "procedure": procedure, "qualityIndex": str(cpqi), "responseFormat": "application/json", "offering": 'temporary', "eventTime": "%s/%s" % (start.isoformat(), nextStart.isoformat()) } if function and resolution: params['aggregateFunction'] = function params['aggregateInterval'] = resolution if nodataValue != None: params['aggregateNodata'] = nodataValue if nodataQI != None: params['aggregateNodataQi'] = nodataQI res = req.get("%s/%s" % (surl,ssrv), params=params, auth=(suser, spwd), verify=False) # Check if an Exception occured if 'ExceptionReport' in res.content: raise Exception (res.content) smeasures = res.json()['ObservationCollection']['member'][0] #pp.pprint(smeasures) log(" > %s measures from: %s to: %s" % (len(smeasures['result']['DataArray']['values']), start.isoformat(), nextStart.isoformat())) dtemplate["samplingTime"] = {} if lm and len(smeasures['result']['DataArray']['values'])>0: dtemplate["samplingTime"]["beginPosition"] = smeasures['result']['DataArray']['values'][0][0] dtemplate["samplingTime"]["endPosition"] = smeasures['result']['DataArray']['values'][-1][0] else: dtemplate["samplingTime"]["beginPosition"] = start.isoformat() dtemplate["samplingTime"]["endPosition"] = nextStart.isoformat() dtemplate['result']['DataArray']['values'] = smeasures['result']['DataArray']['values'] dtemplate['result']['DataArray']['field'] = smeasures['result']['DataArray']['field'] # POST data to WA res = req.post("%s/wa/istsos/services/%s/operations/insertobservation" % ( durl, dsrv), auth=(duser, dpwd), verify=False, data=json.dumps({ "ForceInsert": "true", "AssignedSensorId": ddata['data']['assignedSensorId'], "Observation": dtemplate }) ) # read response log(" > Insert observation success: %s" % res.json()['success']) #print res.json() if not res.json()['success']: raise Exception ('Error inserting observation: %s' % res.json()['message']) start = nextStart if start<stop and start+interval>stop: interval = stop-start