Esempio n. 1
0
def read_ts(file):
    fh = open(file)
    line = fh.readline()
    while not line.startswith("DATE"):
        line = fh.readline()
    line = fh.readline()
    line = fh.readline()
    tf = TimeFactory.getInstance()
    timeinterval = "IR-DAY"
    vals = []
    tvals = []
    ncount = 0
    while line != None or line.strip() != "":
        fields = line.split()
        if len(fields) < 4:
            print "No data in line #%d: %s" % (ncount, line)
            break
        else:
            date, time = fields[0:2]
            vtime = tf.createTime(date + " " + time, "MM/dd/yyyy HH:mm:ss")
            tvals.append(vtime.create(vtime))
            vals.append(float(fields[3]))
        ncount = ncount + 1
        line = fh.readline()
    fh.close()
    attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, "", "UMHOS/CM", "TIME", "INST-VAL")
    return IrregularTimeSeries("TIME SERIES", tvals, vals, None, attr)
def grow_window(tw,left,right=None):
    if right == None: right = left
    st = tw.getStartTime() - timeinterval(left)
    et = tw.getEndTime() + timeinterval(right)
    return TimeFactory.getInstance().createTimeWindow(st,et)
                
                
Esempio n. 3
0
def spline(ref,outint,offset=0):
    '''
    Usage example:  interpolate(ref,outint = timeinterval("15min"),offset = 48)

    Interpolating spline
    Eli Ateljevich 9/27/99

    This functions is designed to map a coarser time series into a smaller one
    covering the same time window. The spline is monotonicity-preserving
    and fourth order accurate (except near boundaries)
    
    offset shifts the output as appropriate. Typically, offset will be
    zero for inst-val input. For per-ave input, offset will often be
    half of the output frequency. In the example above, NDO 
    input is treated as  "daily averaged". Output is in units of
    15minutes. Since there are are 96 15min samples per 24 hours
    offset = 0.5*96 = 48.
    
    Output is a regular time series (rts).
    
    Reference: Huynh, HT "Accurate Monotone Cubic Interpolation",
    SIAM J. Numer. Analysis V30 No. 1 pp 57-100 
    All equation numbers refer to this paper. The variable names are 
    also almost the same. Double letters like "ee" to indicate 
    that the subscript should have "+1/2" added to it.
    '''
    got_ref = 0
    if isinstance(ref, DataReference):
        data = ref.getData()
        got_ref = 1
    else:
        data = ref
        got_ref = 0
    # check for regular time series
    if not isinstance(data,RegularTimeSeries):
        print ref, " is not a regular time-series data set"
        return None
    
    yt = data.getIterator()
    div =  TimeFactory.getInstance().createTimeInterval(outint)
    nsub = data.getTimeInterval()/div
    from jarray import zeros
    vals = zeros(1+nsub*(data.size()-1),'d')
    vals=map(lambda x: -901.0, vals)
    vallength = len(vals)
    dflags = zeros(vallength,'l')
    lastone = vallength + 4*nsub  -1
    firstone = 4*nsub
 
    y4,y3,y2,y1,y0,ss3,ss2,ss1,ss0,s1,s0 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.
    dd1,dd0,d3,d2,d1,d1,d0,e1,e0,ee2,ee1,ee0,eem1,df0,df1 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.
    count = 0
    subcount = 0
    nextbad = 5
    atstart=1
    atend =0
    while not yt.atEnd() or nextbad>1: 
	if not (yt.atEnd() or atend==1):
            el = yt.getElement()     #Read new value, good or bad
            count = count + 1
            if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):           
                y4 = el.getY();
             
                #
                ss3 = y4-y3
                d3 = ss3 - ss2
                ee2 = d3-d2
                atstart = 0
            else:
                if atstart == 0:
                    atend = 1
                    nextbad = nextbad - 1
                #NoMissing = "Internal missing values not allowed."
                #raise NoMissing
        else:
            nextbad = nextbad - 1;
        #
        # minmod-based estimates
        s2 = minmod(ss1,ss2)
        dd2 = minmod(d2,d3)
        e2 = minmod(ee1,ee2)

        #polynomial slopes
        dpp1adj1 = ss1 - dd1
        dpp0adj1 = ss0 + dd0

        t1 = minmod(dpp0adj1,dpp1adj1)
        dqq1adj1 = ss1 - minmod(d1+e1,d2-2*e2)   #Eq4.7a
        dqq0adj1 = ss0 + minmod(d0+2*e0,d1-e1)   #Eq4.7b
        ttilde1 = minmod(dqq0adj1,dqq1adj1)

        df1 = 0.5*(dqq0adj1 + dqq1adj1)          #First cut, Eq. 4.16
        tmin = min(0.,3*s1,1.5*t1,ttilde1)
        tmax = max(0.,3*s1,1.5*t1,ttilde1)

#	#If count == 3:         # have enough to make up boundary quantities
#	gamma = (ee1 - ee0)/4   #5.8, 
#        eex0 = ee0 - 4*gamma  #5.9 x is the boundary value
#        qqx = median3(ssm1,qqx,
        
        df1 = df1 + minmod(tmin-df1, tmax-df1)   #Revise,  Eq. 4.16
        
        for j in range(nsub):
            jfrac = (float(j)+offset)/ float(nsub)
            c0 = y0                  # from eq. 2 in the paper
            c1 = df0
            c2 = 3*ss0 - 2*df0 - df1
            c3 = df0 + df1 - 2*ss0
            if count > 4:
                if subcount <= lastone and subcount >= firstone:  
                    vals[subcount-4*nsub] = c0 + c1*jfrac +c2*jfrac**2 + c3*jfrac**3;
   
            subcount = subcount + 1

        # Now lag all data and estimates to make room for next time step
        y3,y2,y1,y0 = y4,y3,y2,y1
        ss2,ss1,ss0,ssm1 = ss3,ss2,ss1,ss0
        s1,s0 = s2,s1
        dd1,dd0 = dd2,dd1
        d2,d1,d0 = d3,d2,d1
        e1,e0 = e2,e1
        ee1,ee0,eem1 = ee2,ee1,ee0
        df0 = df1

        if not  yt.atEnd():
            yt.advance()
        #
    #
    #refpath=ref.getPathname()
    #refpath.setPart(Pathname.E_PART,outint)
    rts = RegularTimeSeries(data.getName(),
			    data.getStartTime().toString(),
			    outint,vals)
 
    return rts
Esempio n. 4
0
def read_dss_txt(file,dssts=True,flag=False):
    """
    read_dss_txt(file, dssts=True,flag=False): reads from a ascii file in dssts or dssits format
    and writes out the data to the appropriate dss file and pathnames.
    If dssts == True then dssts format is assumed in the file else dssits format is assumed
    For more info  look up doc on read_dssts(file) and read_dssits(file)
    If flag == True then it expects a flag value in the ascii file as well
    The flag values in the ascii file are represented as
    flag_type|user_name
    where flag_type is one of UNSCREENED,QUESTIONABLE,MISSING,REJECT,OK
    & user_name is one of the authorized users/agency.
    e.g. MISSING|nsandhu or REJECT|kate
    """
    import string
    tf = TimeFactory.getInstance()
    f = open(file)
    line = f.readline()[:-1]
    dssfile = line
    while line :
        try :
            line = string.upper(f.readline()[:-1])
            if line == "FINISH": break;
            path = Pathname.createPathname(string.upper(line))
        except :
            print 'Incorrect format for path: ', line
            break
        try :
            line = f.readline()[:-1]
            units = string.upper(line)
        except :
            print 'Incorrect format for units: ', line
            break
        try :
            line = f.readline()[:-1]
            type = string.upper(line)
        except :
            print 'Incorrect format for type: ', line
            break
        if dssts:
            try :
                line = f.readline()[:-1]
                stime = tf.createTime(line)
            except :
                print 'Incorrect format for time: ', line
                break
        #
        line = string.upper(f.readline()[:-1])
        xvals = []
        yvals = []
    if flag:
        flags = []
    else:
        flags = None
        while line != "END" :
            if dssts:
                try:
                    if flag:
                        vals = string.split(line)
                        if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line)
                        yvals.append(float(vals[0]))
                        flags.append(make_flag_value(vals[1]))
                    else:   # no flags
                        yvals.append(float(line))
                except:
                    yvals.append(Constants.MISSING_VALUE);
                    if flag: flags.append(make_flag_value('MISSING|null'))
            else :
                try :
                    line = string.strip(line)
                    tmstr = line[0:14]
                    tm = tf.createTime(tmstr)
                    try :
                        if flag:
                            vals = string.split(line[14:])
                            if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line)
                            val = float(vals[0])
                            flag_val = vals[1]
                        else:   # no flag
                            val = float(line[14:])
                    except :
                        val = Constants.MISSING_VALUE
                    xvals.append(tm)
                    yvals.append(val)
                    flags.append(make_flag_value(flag_val))
                except Exception, exc:
                    print exc
                    print "Error reading line: ", line
            line = string.upper(f.readline()[:-1])
            # create appropriate time series object
            if len(yvals) == 0 : continue
            #print yvals, flags
        if ( dssts ):
            attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,"TIME",units,"",type)
            ts = RegularTimeSeries("",repr(stime),path.getPart(Pathname.E_PART),
                                   yvals, flags, attr)
        else :
            attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,"TIME",units,"",type)
            ts = IrregularTimeSeries("",xvals, yvals, flags, attr)
from vdss import *
from vista.set import *
#from vista.set import Group
from vista.set import PathnamePredicate
from vista.db.dss import *
from vutils import *
from vista.time import TimeFactory
from tideUtils import *

if __name__ == '__main__':
    if len(sys.argv) <= 1:
        PFilter = None
    else:
        PFilter = sys.argv[1]
 
    TF = TimeFactory.getInstance()
    
    # Trim perturbation test output DSS files and consolidate into a single file
    # Then calculate the 4 desired averaged metrics:
    # stage amplitude
    # flow amplitude
    # flow net tidal
    # EC
    #
    reRun = False
#    for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch','Len_Ch','Depth_Res',\
#                  'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']:
    for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch',\
                  'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']:
        print 'Trimming', Param
        Prefix = 'HIST-CLB2K-'
Esempio n. 6
0
from vdss import *
from vista.set import *
#from vista.set import Group
from vista.set import PathnamePredicate
from vista.db.dss import *
from vutils import *
from vista.time import TimeFactory
from tideUtils import *

if __name__ == '__main__':
    if len(sys.argv) <= 1:
        PFilter = None
    else:
        PFilter = sys.argv[1]

    TF = TimeFactory.getInstance()

    # Trim perturbation test output DSS files and consolidate into a single file
    # Then calculate the 4 desired averaged metrics:
    # stage amplitude
    # flow amplitude
    # flow net tidal
    # EC
    #
    reRun = False
    #    for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch','Len_Ch','Depth_Res',\
    #                  'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']:
    for Param in ['ManN_Ch','Disp_Ch','XTopW_Ch','XElev_Ch',\
                  'DICU-QDIV_Nd','DICU-QRET_Nd','DICU-ECRET_Nd']:
        print 'Trimming', Param
        Prefix = 'HIST-CLB2K-'
Esempio n. 7
0
def retrieve_ts(station,sensor,\
   start_date, end_date='now',_verbose=1):
    """
	retrieve(station,sensor, start_date, end_date='now',_verbose=1)

	Retrieves data from cdec station and sensor and
	constructs a regular time series with given pathname, units.

	The start_date is a starting time value in format mm/dd/yyyy
	e.g. 15-Jun-2000 would be 06/15/2000

	The end_date can either be the keyword "now" or the same format as
	the start_date

	verbose=1 sets it to more verbose and 0 for silence
	"""
    _debug = 0
    station_name = station.id
    sensor_number = sensor.sensor_number
    c_part = "%s_%s" % (sensor.type, sensor.subType)
    f_part = "SENSOR %s" % (sensor.id)
    pathname = '/%s/%s/%s//%s/%s/' % ('CDEC-RAW', station_name, c_part,
                                      DSS_INTERVAL[sensor.duration],
                                      "SENSOR " + sensor.id)
    units = sensor.units
    dur_code = sensor.getDurationCode()
    c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?' +
                'station_id=' + str(station_name) + '&dur_code=' +
                str(dur_code) + '&sensor_num=' + str(sensor_number) +
                '&start_date=' + start_date + '&end_date=' + end_date)
    if _verbose:
        print "station name:%s & sensor number:%s " % (station_name,
                                                       sensor_number)
        print "dur_code:%s " % (dur_code)
        print "pathname:%s & units:%s" % (pathname, units)
        print 'URL: %s' % c_url
    lr = LineNumberReader(
        InputStreamReader(c_url.openConnection().getInputStream()))
    # jump all the way to data
    lr.readLine()
    lr.readLine()
    lr.readLine()
    # create starting date and time and of the format the
    # data is at cdec
    line = lr.readLine()
    tf = TimeFactory.getInstance()
    starray = string.split(line, ',')
    dtm = starray[0] + ' ' + starray[1]
    tp = tf.createTime(dtm, 'yyyyMMdd HHmm')
    tp = tf.createTime(tp.getTimeInMinutes())
    ti_str = '1HOUR'
    irreg = 0  # indicates the data as irregular time series
    if dur_code == 'E':
        irreg = 1
        ti_str = 'IR-DAY'
    elif dur_code == 'D':
        ti_str = '1DAY'
    elif dur_code == 'H':
        ti_str = '1HOUR'
    elif dur_code == 'M':
        ti_str = '1MON'
    if not irreg:
        ti = tf.createTimeInterval(ti_str)
    ltime = tp
    if _debug: print line, tp, dtm
    yvals = []
    if irreg:
        tvals = []
    if _verbose: print 'Data starting at ', tp
    # read in all the data and append missing values if no data
    # is available
    while line != None:
        st = StringTokenizer(line, ',')
        if (st.countTokens() != 3):
            raise "Invalid CDEC format, need 3 tokens on line: " + line
        # get time
        ctime = tf.createTime(st.nextToken() + ' ' + st.nextToken(),
                              'yyyyMMdd HHmm')
        # if time is not in increasing order -> quit!
        if ctime.compare(ltime) < 0:
            raise "Invalid time sequence: %s followed by %s"+\
              " ? -> should be always increasing"%(str(ctime),str(ltime))
            # check if current time is only one time interval from last time
            if not irreg:
                nskip = ltime.getNumberOfIntervalsTo(ctime, ti)
                # if skip is greater than one then fill with -901's
                while nskip > 1:
                    yvals.append(Constants.MISSING_VALUE)
                    nskip = nskip - 1
                ltime = ctime
        # now get current value
        val_str = st.nextToken()
        try:
            if (val_str == "m"):  # if missing
                val = Constants.MISSING_VALUE
            else:  # else just save the value
                val = float(val_str)
                if irreg:
                    tvals.append(ctime)
            yvals.append(val)
        except:
            tvals.append(ctime)
            yvals.append(Constants.MISSING_VALUE)
            print "Exception! for string ", val_str
        line = lr.readLine()
    # create a time series data set from the array of values,
    # the start time and the time interval
    if irreg:
        attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, '', units, 'TIME',
                           'INST-VAL')
        its = IrregularTimeSeries(pathname, tvals, yvals, None, attr)
        return its
    else:
        attr = DataSetAttr(DataType.REGULAR_TIME_SERIES, '', units, 'TIME',
                           'INST-VAL')
        rts = RegularTimeSeries(pathname, tp.toString(), ti_str, yvals, None,
                                attr)
        return rts
Esempio n. 8
0
def retrieve_ts(station,sensor,\
		 start_date, end_date='now',_verbose=1):
	"""
	retrieve(station,sensor, start_date, end_date='now',_verbose=1)

	Retrieves data from cdec station and sensor and
	constructs a regular time series with given pathname, units.

	The start_date is a starting time value in format mm/dd/yyyy
	e.g. 15-Jun-2000 would be 06/15/2000

	The end_date can either be the keyword "now" or the same format as
	the start_date

	verbose=1 sets it to more verbose and 0 for silence
	"""
	_debug=0
	station_name=station.id
	sensor_number=sensor.sensor_number
	c_part="%s_%s"%(sensor.type,sensor.subType)
	f_part="SENSOR %s"%(sensor.id)
	pathname='/%s/%s/%s//%s/%s/'%('CDEC-RAW',station_name,c_part,DSS_INTERVAL[sensor.duration],"SENSOR "+sensor.id)
	units=sensor.units
	dur_code = sensor.getDurationCode()
	c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?'
		+'station_id='+str(station_name)
		+'&dur_code='+str(dur_code)+'&sensor_num='+str(sensor_number)
		+'&start_date='+start_date
		+'&end_date='+end_date)
	if _verbose:
		print "station name:%s & sensor number:%s "%(station_name,sensor_number)
	   	print "dur_code:%s "%(dur_code)
	   	print "pathname:%s & units:%s"%(pathname,units)
	   	print 'URL: %s'%c_url
	lr = LineNumberReader(InputStreamReader(c_url.openConnection().getInputStream()))
	# jump all the way to data
	lr.readLine()
	lr.readLine()
	lr.readLine()
	# create starting date and time and of the format the
	# data is at cdec
	line = lr.readLine()
	tf = TimeFactory.getInstance()
	starray = string.split(line,',')
	dtm = starray[0] + ' ' + starray[1]
	tp = tf.createTime(dtm,'yyyyMMdd HHmm')
	tp = tf.createTime(tp.getTimeInMinutes())
	ti_str='1HOUR'
	irreg=0 # indicates the data as irregular time series
	if dur_code=='E':
		irreg=1
		ti_str='IR-DAY'
	elif dur_code=='D':
		ti_str='1DAY'
	elif dur_code=='H':
		ti_str='1HOUR'
	elif dur_code=='M':
		ti_str='1MON'
	if not irreg:
		ti = tf.createTimeInterval(ti_str)
	ltime = tp
	if _debug: print line, tp,dtm
	yvals = []
	if irreg:
		tvals=[]
	if _verbose: print 'Data starting at ', tp
	# read in all the data and append missing values if no data 
	# is available
	while line != None:
		st = StringTokenizer(line,',')
		if ( st.countTokens() != 3 ) :
			raise "Invalid CDEC format, need 3 tokens on line: " + line
		# get time 
		ctime = tf.createTime(st.nextToken()+' '+st.nextToken(),'yyyyMMdd HHmm')
		# if time is not in increasing order -> quit!
		if ctime.compare(ltime) < 0:
			raise "Invalid time sequence: %s followed by %s"+\
			  " ? -> should be always increasing"%(str(ctime),str(ltime))
		# check if current time is only one time interval from last time
			if not irreg:
				nskip = ltime.getNumberOfIntervalsTo(ctime,ti)
				# if skip is greater than one then fill with -901's
				while nskip > 1:
					yvals.append(Constants.MISSING_VALUE)
					nskip=nskip-1
				ltime = ctime
		# now get current value
		val_str = st.nextToken()
		try :
			if ( val_str == "m" ): # if missing
				val=Constants.MISSING_VALUE
			else: # else just save the value
				val=float(val_str)
				if irreg:
					tvals.append(ctime)
			yvals.append(val)
		except:
			tvals.append(ctime)
			yvals.append(Constants.MISSING_VALUE)
			print "Exception! for string " ,val_str
		line = lr.readLine();
	# create a time series data set from the array of values,
	# the start time and the time interval
	if irreg:
		attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,'',units,'TIME','INST-VAL')
		its = IrregularTimeSeries(pathname,tvals,yvals,None,attr)
		return its
	else:
		attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,'',units,'TIME','INST-VAL')
		rts = RegularTimeSeries(pathname,tp.toString(),ti_str,yvals,None,attr)
		return rts
Esempio n. 9
0
def grow_window(tw, left, right=None):
    if right == None: right = left
    st = tw.getStartTime() - timeinterval(left)
    et = tw.getEndTime() + timeinterval(right)
    return TimeFactory.getInstance().createTimeWindow(st, et)