def read_ts(file): fh = open(file) line = fh.readline() while not line.startswith("DATE"): line = fh.readline() line = fh.readline() line = fh.readline() tf = TimeFactory.getInstance() timeinterval = "IR-DAY" vals = [] tvals = [] ncount = 0 while line != None or line.strip() != "": fields = line.split() if len(fields) < 4: print "No data in line #%d: %s" % (ncount, line) break else: date, time = fields[0:2] vtime = tf.createTime(date + " " + time, "MM/dd/yyyy HH:mm:ss") tvals.append(vtime.create(vtime)) vals.append(float(fields[3])) ncount = ncount + 1 line = fh.readline() fh.close() attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, "", "UMHOS/CM", "TIME", "INST-VAL") return IrregularTimeSeries("TIME SERIES", tvals, vals, None, attr)
def grow_window(tw,left,right=None): if right == None: right = left st = tw.getStartTime() - timeinterval(left) et = tw.getEndTime() + timeinterval(right) return TimeFactory.getInstance().createTimeWindow(st,et)
def spline(ref,outint,offset=0): ''' Usage example: interpolate(ref,outint = timeinterval("15min"),offset = 48) Interpolating spline Eli Ateljevich 9/27/99 This functions is designed to map a coarser time series into a smaller one covering the same time window. The spline is monotonicity-preserving and fourth order accurate (except near boundaries) offset shifts the output as appropriate. Typically, offset will be zero for inst-val input. For per-ave input, offset will often be half of the output frequency. In the example above, NDO input is treated as "daily averaged". Output is in units of 15minutes. Since there are are 96 15min samples per 24 hours offset = 0.5*96 = 48. Output is a regular time series (rts). Reference: Huynh, HT "Accurate Monotone Cubic Interpolation", SIAM J. Numer. Analysis V30 No. 1 pp 57-100 All equation numbers refer to this paper. The variable names are also almost the same. Double letters like "ee" to indicate that the subscript should have "+1/2" added to it. ''' got_ref = 0 if isinstance(ref, DataReference): data = ref.getData() got_ref = 1 else: data = ref got_ref = 0 # check for regular time series if not isinstance(data,RegularTimeSeries): print ref, " is not a regular time-series data set" return None yt = data.getIterator() div = TimeFactory.getInstance().createTimeInterval(outint) nsub = data.getTimeInterval()/div from jarray import zeros vals = zeros(1+nsub*(data.size()-1),'d') vals=map(lambda x: -901.0, vals) vallength = len(vals) dflags = zeros(vallength,'l') lastone = vallength + 4*nsub -1 firstone = 4*nsub y4,y3,y2,y1,y0,ss3,ss2,ss1,ss0,s1,s0 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0. dd1,dd0,d3,d2,d1,d1,d0,e1,e0,ee2,ee1,ee0,eem1,df0,df1 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0. count = 0 subcount = 0 nextbad = 5 atstart=1 atend =0 while not yt.atEnd() or nextbad>1: if not (yt.atEnd() or atend==1): el = yt.getElement() #Read new value, good or bad count = count + 1 if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): y4 = el.getY(); # ss3 = y4-y3 d3 = ss3 - ss2 ee2 = d3-d2 atstart = 0 else: if atstart == 0: atend = 1 nextbad = nextbad - 1 #NoMissing = "Internal missing values not allowed." #raise NoMissing else: nextbad = nextbad - 1; # # minmod-based estimates s2 = minmod(ss1,ss2) dd2 = minmod(d2,d3) e2 = minmod(ee1,ee2) #polynomial slopes dpp1adj1 = ss1 - dd1 dpp0adj1 = ss0 + dd0 t1 = minmod(dpp0adj1,dpp1adj1) dqq1adj1 = ss1 - minmod(d1+e1,d2-2*e2) #Eq4.7a dqq0adj1 = ss0 + minmod(d0+2*e0,d1-e1) #Eq4.7b ttilde1 = minmod(dqq0adj1,dqq1adj1) df1 = 0.5*(dqq0adj1 + dqq1adj1) #First cut, Eq. 4.16 tmin = min(0.,3*s1,1.5*t1,ttilde1) tmax = max(0.,3*s1,1.5*t1,ttilde1) # #If count == 3: # have enough to make up boundary quantities # gamma = (ee1 - ee0)/4 #5.8, # eex0 = ee0 - 4*gamma #5.9 x is the boundary value # qqx = median3(ssm1,qqx, df1 = df1 + minmod(tmin-df1, tmax-df1) #Revise, Eq. 4.16 for j in range(nsub): jfrac = (float(j)+offset)/ float(nsub) c0 = y0 # from eq. 2 in the paper c1 = df0 c2 = 3*ss0 - 2*df0 - df1 c3 = df0 + df1 - 2*ss0 if count > 4: if subcount <= lastone and subcount >= firstone: vals[subcount-4*nsub] = c0 + c1*jfrac +c2*jfrac**2 + c3*jfrac**3; subcount = subcount + 1 # Now lag all data and estimates to make room for next time step y3,y2,y1,y0 = y4,y3,y2,y1 ss2,ss1,ss0,ssm1 = ss3,ss2,ss1,ss0 s1,s0 = s2,s1 dd1,dd0 = dd2,dd1 d2,d1,d0 = d3,d2,d1 e1,e0 = e2,e1 ee1,ee0,eem1 = ee2,ee1,ee0 df0 = df1 if not yt.atEnd(): yt.advance() # # #refpath=ref.getPathname() #refpath.setPart(Pathname.E_PART,outint) rts = RegularTimeSeries(data.getName(), data.getStartTime().toString(), outint,vals) return rts
def read_dss_txt(file,dssts=True,flag=False): """ read_dss_txt(file, dssts=True,flag=False): reads from a ascii file in dssts or dssits format and writes out the data to the appropriate dss file and pathnames. If dssts == True then dssts format is assumed in the file else dssits format is assumed For more info look up doc on read_dssts(file) and read_dssits(file) If flag == True then it expects a flag value in the ascii file as well The flag values in the ascii file are represented as flag_type|user_name where flag_type is one of UNSCREENED,QUESTIONABLE,MISSING,REJECT,OK & user_name is one of the authorized users/agency. e.g. MISSING|nsandhu or REJECT|kate """ import string tf = TimeFactory.getInstance() f = open(file) line = f.readline()[:-1] dssfile = line while line : try : line = string.upper(f.readline()[:-1]) if line == "FINISH": break; path = Pathname.createPathname(string.upper(line)) except : print 'Incorrect format for path: ', line break try : line = f.readline()[:-1] units = string.upper(line) except : print 'Incorrect format for units: ', line break try : line = f.readline()[:-1] type = string.upper(line) except : print 'Incorrect format for type: ', line break if dssts: try : line = f.readline()[:-1] stime = tf.createTime(line) except : print 'Incorrect format for time: ', line break # line = string.upper(f.readline()[:-1]) xvals = [] yvals = [] if flag: flags = [] else: flags = None while line != "END" : if dssts: try: if flag: vals = string.split(line) if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line) yvals.append(float(vals[0])) flags.append(make_flag_value(vals[1])) else: # no flags yvals.append(float(line)) except: yvals.append(Constants.MISSING_VALUE); if flag: flags.append(make_flag_value('MISSING|null')) else : try : line = string.strip(line) tmstr = line[0:14] tm = tf.createTime(tmstr) try : if flag: vals = string.split(line[14:]) if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line) val = float(vals[0]) flag_val = vals[1] else: # no flag val = float(line[14:]) except : val = Constants.MISSING_VALUE xvals.append(tm) yvals.append(val) flags.append(make_flag_value(flag_val)) except Exception, exc: print exc print "Error reading line: ", line line = string.upper(f.readline()[:-1]) # create appropriate time series object if len(yvals) == 0 : continue #print yvals, flags if ( dssts ): attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,"TIME",units,"",type) ts = RegularTimeSeries("",repr(stime),path.getPart(Pathname.E_PART), yvals, flags, attr) else : attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,"TIME",units,"",type) ts = IrregularTimeSeries("",xvals, yvals, flags, attr)
from vdss import * from vista.set import * #from vista.set import Group from vista.set import PathnamePredicate from vista.db.dss import * from vutils import * from vista.time import TimeFactory from tideUtils import * if __name__ == '__main__': if len(sys.argv) <= 1: PFilter = None else: PFilter = sys.argv[1] TF = TimeFactory.getInstance() # Trim perturbation test output DSS files and consolidate into a single file # Then calculate the 4 desired averaged metrics: # stage amplitude # flow amplitude # flow net tidal # EC # reRun = False # for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch','Len_Ch','Depth_Res',\ # 'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']: for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch',\ 'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']: print 'Trimming', Param Prefix = 'HIST-CLB2K-'
def retrieve_ts(station,sensor,\ start_date, end_date='now',_verbose=1): """ retrieve(station,sensor, start_date, end_date='now',_verbose=1) Retrieves data from cdec station and sensor and constructs a regular time series with given pathname, units. The start_date is a starting time value in format mm/dd/yyyy e.g. 15-Jun-2000 would be 06/15/2000 The end_date can either be the keyword "now" or the same format as the start_date verbose=1 sets it to more verbose and 0 for silence """ _debug = 0 station_name = station.id sensor_number = sensor.sensor_number c_part = "%s_%s" % (sensor.type, sensor.subType) f_part = "SENSOR %s" % (sensor.id) pathname = '/%s/%s/%s//%s/%s/' % ('CDEC-RAW', station_name, c_part, DSS_INTERVAL[sensor.duration], "SENSOR " + sensor.id) units = sensor.units dur_code = sensor.getDurationCode() c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?' + 'station_id=' + str(station_name) + '&dur_code=' + str(dur_code) + '&sensor_num=' + str(sensor_number) + '&start_date=' + start_date + '&end_date=' + end_date) if _verbose: print "station name:%s & sensor number:%s " % (station_name, sensor_number) print "dur_code:%s " % (dur_code) print "pathname:%s & units:%s" % (pathname, units) print 'URL: %s' % c_url lr = LineNumberReader( InputStreamReader(c_url.openConnection().getInputStream())) # jump all the way to data lr.readLine() lr.readLine() lr.readLine() # create starting date and time and of the format the # data is at cdec line = lr.readLine() tf = TimeFactory.getInstance() starray = string.split(line, ',') dtm = starray[0] + ' ' + starray[1] tp = tf.createTime(dtm, 'yyyyMMdd HHmm') tp = tf.createTime(tp.getTimeInMinutes()) ti_str = '1HOUR' irreg = 0 # indicates the data as irregular time series if dur_code == 'E': irreg = 1 ti_str = 'IR-DAY' elif dur_code == 'D': ti_str = '1DAY' elif dur_code == 'H': ti_str = '1HOUR' elif dur_code == 'M': ti_str = '1MON' if not irreg: ti = tf.createTimeInterval(ti_str) ltime = tp if _debug: print line, tp, dtm yvals = [] if irreg: tvals = [] if _verbose: print 'Data starting at ', tp # read in all the data and append missing values if no data # is available while line != None: st = StringTokenizer(line, ',') if (st.countTokens() != 3): raise "Invalid CDEC format, need 3 tokens on line: " + line # get time ctime = tf.createTime(st.nextToken() + ' ' + st.nextToken(), 'yyyyMMdd HHmm') # if time is not in increasing order -> quit! if ctime.compare(ltime) < 0: raise "Invalid time sequence: %s followed by %s"+\ " ? -> should be always increasing"%(str(ctime),str(ltime)) # check if current time is only one time interval from last time if not irreg: nskip = ltime.getNumberOfIntervalsTo(ctime, ti) # if skip is greater than one then fill with -901's while nskip > 1: yvals.append(Constants.MISSING_VALUE) nskip = nskip - 1 ltime = ctime # now get current value val_str = st.nextToken() try: if (val_str == "m"): # if missing val = Constants.MISSING_VALUE else: # else just save the value val = float(val_str) if irreg: tvals.append(ctime) yvals.append(val) except: tvals.append(ctime) yvals.append(Constants.MISSING_VALUE) print "Exception! for string ", val_str line = lr.readLine() # create a time series data set from the array of values, # the start time and the time interval if irreg: attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, '', units, 'TIME', 'INST-VAL') its = IrregularTimeSeries(pathname, tvals, yvals, None, attr) return its else: attr = DataSetAttr(DataType.REGULAR_TIME_SERIES, '', units, 'TIME', 'INST-VAL') rts = RegularTimeSeries(pathname, tp.toString(), ti_str, yvals, None, attr) return rts
def retrieve_ts(station,sensor,\ start_date, end_date='now',_verbose=1): """ retrieve(station,sensor, start_date, end_date='now',_verbose=1) Retrieves data from cdec station and sensor and constructs a regular time series with given pathname, units. The start_date is a starting time value in format mm/dd/yyyy e.g. 15-Jun-2000 would be 06/15/2000 The end_date can either be the keyword "now" or the same format as the start_date verbose=1 sets it to more verbose and 0 for silence """ _debug=0 station_name=station.id sensor_number=sensor.sensor_number c_part="%s_%s"%(sensor.type,sensor.subType) f_part="SENSOR %s"%(sensor.id) pathname='/%s/%s/%s//%s/%s/'%('CDEC-RAW',station_name,c_part,DSS_INTERVAL[sensor.duration],"SENSOR "+sensor.id) units=sensor.units dur_code = sensor.getDurationCode() c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?' +'station_id='+str(station_name) +'&dur_code='+str(dur_code)+'&sensor_num='+str(sensor_number) +'&start_date='+start_date +'&end_date='+end_date) if _verbose: print "station name:%s & sensor number:%s "%(station_name,sensor_number) print "dur_code:%s "%(dur_code) print "pathname:%s & units:%s"%(pathname,units) print 'URL: %s'%c_url lr = LineNumberReader(InputStreamReader(c_url.openConnection().getInputStream())) # jump all the way to data lr.readLine() lr.readLine() lr.readLine() # create starting date and time and of the format the # data is at cdec line = lr.readLine() tf = TimeFactory.getInstance() starray = string.split(line,',') dtm = starray[0] + ' ' + starray[1] tp = tf.createTime(dtm,'yyyyMMdd HHmm') tp = tf.createTime(tp.getTimeInMinutes()) ti_str='1HOUR' irreg=0 # indicates the data as irregular time series if dur_code=='E': irreg=1 ti_str='IR-DAY' elif dur_code=='D': ti_str='1DAY' elif dur_code=='H': ti_str='1HOUR' elif dur_code=='M': ti_str='1MON' if not irreg: ti = tf.createTimeInterval(ti_str) ltime = tp if _debug: print line, tp,dtm yvals = [] if irreg: tvals=[] if _verbose: print 'Data starting at ', tp # read in all the data and append missing values if no data # is available while line != None: st = StringTokenizer(line,',') if ( st.countTokens() != 3 ) : raise "Invalid CDEC format, need 3 tokens on line: " + line # get time ctime = tf.createTime(st.nextToken()+' '+st.nextToken(),'yyyyMMdd HHmm') # if time is not in increasing order -> quit! if ctime.compare(ltime) < 0: raise "Invalid time sequence: %s followed by %s"+\ " ? -> should be always increasing"%(str(ctime),str(ltime)) # check if current time is only one time interval from last time if not irreg: nskip = ltime.getNumberOfIntervalsTo(ctime,ti) # if skip is greater than one then fill with -901's while nskip > 1: yvals.append(Constants.MISSING_VALUE) nskip=nskip-1 ltime = ctime # now get current value val_str = st.nextToken() try : if ( val_str == "m" ): # if missing val=Constants.MISSING_VALUE else: # else just save the value val=float(val_str) if irreg: tvals.append(ctime) yvals.append(val) except: tvals.append(ctime) yvals.append(Constants.MISSING_VALUE) print "Exception! for string " ,val_str line = lr.readLine(); # create a time series data set from the array of values, # the start time and the time interval if irreg: attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,'',units,'TIME','INST-VAL') its = IrregularTimeSeries(pathname,tvals,yvals,None,attr) return its else: attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,'',units,'TIME','INST-VAL') rts = RegularTimeSeries(pathname,tp.toString(),ti_str,yvals,None,attr) return rts
def grow_window(tw, left, right=None): if right == None: right = left st = tw.getStartTime() - timeinterval(left) et = tw.getEndTime() + timeinterval(right) return TimeFactory.getInstance().createTimeWindow(st, et)