def month_numbers(series): n = len(series) values = zeros(n, 'd') for el, i in zip(series, range(n)): t = el.getXString() values[i] = monthlist.index(t[2:5]) out = RegularTimeSeries("/MONTH_NUMBER//////", series.getStartTime().toString(), series.getTimeInterval().toString(), values) out.getAttributes().setYType(series.getAttributes().getYType()) return out
def month_numbers(series): n=len(series) values=zeros(n,'d') for el,i in zip(series,range(n)): t=el.getXString() values[i]=monthlist.index(t[2:5]) out=RegularTimeSeries("/MONTH_NUMBER//////", series.getStartTime().toString(), series.getTimeInterval().toString(), values) out.getAttributes().setYType(series.getAttributes().getYType()) return out
def days_in_month(series): n = len(series) values = zeros(n, 'd') for el, i in zip(series, range(n)): t = el.getXString() m = monthlist.index(t[2:5]) y = int(t[5:9]) values[i] = monthrange(y, m)[1] out = RegularTimeSeries("/DAY_IN_MONTH//////", series.getStartTime().toString(), series.getTimeInterval().toString(), values) out.getAttributes().setYType(series.getAttributes().getYType()) return out
def setUp(self): self.ts_1 = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[6000,8000,4000,2000]) self.ts_2 = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[10000,10000,10000,10000]) self.ts_3 = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[1000,1000,1000,1000]) self.list_1 = [6000 for i in range(31)] self.list_2 = [8000 for i in range(31)] self.list_3 = [10000 for i in range(31)] self.list_4 = [4000 for i in range(31)] self.list_5 = [2000 for i in range(31)] self.list_6 = [1000 for i in range(31)]
def days_in_month(series): n=len(series) values=zeros(n,'d') for el,i in zip(series,range(n)): t=el.getXString() m=monthlist.index(t[2:5]) y=int(t[5:9]) values[i]=monthrange(y,m)[1] out=RegularTimeSeries("/DAY_IN_MONTH//////", series.getStartTime().toString(), series.getTimeInterval().toString(), values) out.getAttributes().setYType(series.getAttributes().getYType()) return out
def extend_flow(nodes_to_extend): """ Copying WY1922 data to WY1921 for allowing to preprocessing and running DSM2 from 01Jan1921. """ calsimfile = getAttr("CALSIMFILE") f = opendss(calsimfile) # open CALSIM file outfile = getAttr("BOUNDARYFILE") if not outfile or outfile == "": raise "Config variable BOUNDARYFILE not set and needed for prepro output" tw = timewindow("01OCT1921 0000 - 01OCT1922 0000") for calsimname in nodes_to_extend: print calsimname dsspath = calsim_path(calsimname) paths = findpath(f, dsspath) if not paths or len(paths) > 1: print "File: %s" % calsimfile raise "Path %s not found or not unique" % dsspath ref = DataReference.create(paths[0], tw) monthly = ref.getData() itr = monthly.getIterator() d = zeros(len(monthly), 'd') count = 0 while not itr.atEnd(): el = itr.getElement() d[count] = el.getY() count = count + 1 itr.advance() stime = "01OCT1920 0000" rts = RegularTimeSeries(monthly.getName(),stime, \ timeinterval("1MON").toString(), d, None, monthly.getAttributes()) writedss(calsimfile, ref.getPathname().toString(), rts)
def replace_vamp(non_pulse,pulse,include_shoulder=0): """Creates a new series based on non_pulse values, replacing them with pulse values during the VAMP season (April 15-May 15 inclusive) or April1 - May 31 if include_shoulder is set to true (1) Arguments: non_pulse: monthly or daily series of values to use for non-pulse pulse: pulse values, must have same start, length, interval as non-pulse Output: New series with non-pulse values replaced by pulse during vamp """ if not isinstance(non_pulse,RegularTimeSeries): raise TypeError("Non-Pulse flow must be regular time series") if not isinstance(pulse,RegularTimeSeries): raise TypeError("Pulse flow must be regular time series") if include_shoulder: first_april_day=1 last_may_day=31 else: first_april_day=15 last_may_day=15 #if non_pulse.getTimeInterval() == timeinterval("1MON"): non_pulse=interpolate(non_pulse,"1DAY") #if pulse.getTimeInterval() == timeinterval("1MON"): pulse=interpolate(pulse,"1DAY") if non_pulse.getStartTime() != pulse.getStartTime() or \ len(non_pulse) != len(pulse): raise ValueError("Pulse and Non-pulse must have the same start time," " interval and length") values=zeros(len(non_pulse),'d') for np,p,i in zip(non_pulse,pulse,range(len(values))): values[i]=np.getY() xstr=np.getXString() day,month=int(xstr[0:2]),xstr[2:5] if month=="APR" and day >= first_april_day: values[i]=p.getY() elif month=="MAY" and day <= last_may_day: values[i]=p.getY() out=RegularTimeSeries("/vamp//////", non_pulse.getStartTime().toString(), non_pulse.getTimeInterval().toString(),values) out.getAttributes().setYUnits("CFS") out.getAttributes().setYType("PER-AVER") return out
def conserveSpline(avedata, interval): if not isinstance(interval, TimeInterval): interval = timeinterval(interval) start = avedata.getStartTime() try: nout = start.getExactNumberOfIntervalsTo(avedata.getEndTime(), interval) + 1 except: nout = start.getNumberOfIntervalsTo(avedata.getEndTime(), interval) n = avedata.size() from jarray import zeros x = zeros(n + 1, 'd') y = zeros(n + 1, 'd') i = 0 start2 = time(start.toString()) # Pain in the neck way to clone rev = timeinterval("-" + interval.toString()) start2.incrementBy(avedata.getTimeInterval(), -1) x[0] = start2.getTimeInMinutes() for el in avedata: x[i + 1] = el.getX() y[i] = el.getY() i = i + 1 if not Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): raise "Missing or bad data not allowed in conservative spline" p = map(lambda x: splineparam, y) spline = interpolator.ConservativeSpline(x, y, p, y[0], y[n - 1]) ynew = zeros(nout, 'd') newseries = RegularTimeSeries( '/Smooth/Data///' + interval.toString() + '//', start.toString(), interval.toString(), ynew) dsi = newseries.getIterator() while not dsi.atEnd(): el = dsi.getElement() if spline.rh2val(el.getX()) < 0: # useY=y(el.getX()) useY = spline.rh2val(el.getX()) else: useY = spline.rh2val(el.getX()) # el.setY(useY) dsi.putElement(el) dsi.advance() #tabulate(newseries,avedata) return newseries
def gCalc(ndo,beta,g0=None): """ Calculates antecedent outflow from a stream of ndo Arguments: ndo: a regular time series. Must be 15MIN, 1HOUR. Thus, NDO has been interpolated first. g0: initial condition. If g0 is not given it is equal to ndo at the first time step. beta: g-model parameter. Output: g: a regular time series, same sampling rate as input with the same start time as ndo """ ti = ndo.getTimeInterval() if not ((ti.toString() == "15MIN") | (ti.toString() == "1HOUR")): raise "NDO time step must be 15MIN or 1HOUR." dt=1 nstep=ndo.size() g=zeros(nstep,'d') if ti.toString() == "15MIN": dt=0.25 g=zeros(nstep,'d') g=map(lambda x: -901.0, g) beta=beta*24*365 div2dt=2*beta/dt dsi=ndo.getIterator() q0 = dsi.getElement().getY() # Set initial condition if g0==None: g[0]= q0 # ??? else: g[0]=g0 # Loop through and integrate gmodel using trapazoidal. atend=0 i=1 dsi.advance() while atend == 0: el=dsi.getElement() if el and Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): q=el.getY() qterm=(q-div2dt) g[i]= 0.5*( qterm + sqrt(qterm*qterm + 4*g[i-1]*(q0-g[i-1]+div2dt)) ) if dsi.atEnd(): atend=1 else: dsi.advance() q0=q i=i+1 else: atend=1 rts = RegularTimeSeries("/gcalc//////",ndo.getStartTime().toString(), ndo.getTimeInterval().toString(),g) return rts
def ts_where(criteria,trueval,falseval): if not( len(criteria) == len(trueval) and \ len(criteria) == len(falseval) ): raise ValueError("Criteria (len %s), trueval (len %s) and falseval (len %s)" " must be the same length" % (len(criteria),len(trueval),len(falseval))) n=len(criteria) values=zeros(n,'d') for c,t,f,i in zip(criteria,trueval,falseval,range(n)): if filter.isAcceptable(c): if c.getY(): values[i]=t.getY() else: values[i]=f.getY() else: values[i]=NA_VAL out=RegularTimeSeries("/WHERE//////",criteria.getStartTime().toString(), criteria.getTimeInterval().toString(),values) out.getAttributes().setYType(trueval.getAttributes().getYType()) return out
def conserveSpline(avedata, interval): if not isinstance(interval,TimeInterval): interval=timeinterval(interval) start=avedata.getStartTime() try: nout=start.getExactNumberOfIntervalsTo(avedata.getEndTime(),interval)+1 except: nout=start.getNumberOfIntervalsTo(avedata.getEndTime(), interval) n=avedata.size() from jarray import zeros x=zeros(n+1,'d') y=zeros(n+1,'d') i=0 start2=time(start.toString()) # Pain in the neck way to clone rev=timeinterval("-"+interval.toString()) start2.incrementBy(avedata.getTimeInterval(),-1) x[0]=start2.getTimeInMinutes() for el in avedata: x[i+1]=el.getX() y[i]=el.getY() i=i+1 if not Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): raise "Missing or bad data not allowed in conservative spline" p=map(lambda x: splineparam, y) spline=interpolator.ConservativeSpline(x,y,p,y[0],y[n-1]) ynew=zeros(nout,'d') newseries=RegularTimeSeries('/Smooth/Data///'+interval.toString()+'//',start.toString(),interval.toString(),ynew) dsi=newseries.getIterator() while not dsi.atEnd(): el=dsi.getElement() if spline.rh2val(el.getX()) < 0: useY=y(el.getX()) else: useY=spline.rh2val(el.getX()) # el.setY(useY) dsi.putElement(el) dsi.advance() #tabulate(newseries,avedata) return newseries
def test_project_export_only_apr_may(self): print "" print "" print "test_project_export_only_apr_may..." pulse_limit = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[0,2500,2000,0]) delta_inflow = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[20000,10000,8000,8000]) ei_ratio = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[0.35,0.35,0.35,0.35]) #delta_inflow*ei_ratio = [7000, 3500, 2800, 2800] cvp_out = [3500, 1250, 1000, 1400] swp_out = [3500, 1250, 1000, 1400] swp_out_from_function, cvp_out_from_function = prep_vamp.project_export_limits(pulse_limit, ei_ratio, delta_inflow) swp_array = swp_out_from_function.getYArray() cvp_array = cvp_out_from_function.getYArray() for i in range(0, len(cvp_array)): self.assertEqual(cvp_out[i], round(cvp_array[i], 0)), 'project_export function returned wrong cvp_limit value' self.assertEqual(swp_out[i], round(swp_array[i], 0)), 'project_export function returned wrong swp_limit value'
def ts_where(criteria, trueval, falseval): if not( len(criteria) == len(trueval) and \ len(criteria) == len(falseval) ): raise ValueError( "Criteria (len %s), trueval (len %s) and falseval (len %s)" " must be the same length" % (len(criteria), len(trueval), len(falseval))) n = len(criteria) values = zeros(n, 'd') for c, t, f, i in zip(criteria, trueval, falseval, range(n)): if filter.isAcceptable(c): if c.getY(): values[i] = t.getY() else: values[i] = f.getY() else: values[i] = NA_VAL out = RegularTimeSeries("/WHERE//////", criteria.getStartTime().toString(), criteria.getTimeInterval().toString(), values) out.getAttributes().setYType(trueval.getAttributes().getYType()) return out
def plot_vol_calcs(tidefile,chan,tw,run_name): volume = get_volume_data(tidefile, chan, tw) upflow = get_upstream_flow_data(tidefile, chan, tw) downflow = get_downstream_flow_data(tidefile, chan, tw) #import vdisplay #vdisplay.plot(volume) initial_volume = volume[0].y final_volume = volume[len(volume) - 1].y ndata = len(upflow) vol = [initial_volume] for i in range(0, ndata): vol.append(vol[i] + (upflow[i].y - downflow[i].y) * time_step) print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, final_volume) print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, vol[ndata - 1]) print 'Difference in Volume %f' % (final_volume-vol[ndata-1]) from vista.set import RegularTimeSeries, DataSetAttr attr = volume.getAttributes().createClone() attr.setTypeName("VOLCALC") volcalc = RegularTimeSeries("/VOL/%s/VOLCALC//1HOUR/%s/"%(chan,run_name),"01OCT1974 0000","1HOUR",vol[1:]) volcalc.setAttributes(attr) import vdisplay vdisplay.plot(volcalc,volume) vdisplay.plot(volcalc-volume)
def test_calculate_exports_average_below_limit(self): print "" print "" print "test_calculate_exports_average_below_limit..." pulse_limit = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[6000,4000,2000,8000]) average_export = RegularTimeSeries("///////","31MAR1980 2400", "1MON",[3000,3000,3000,3000]) export_limit = [3000 for i in range(0,31)] export_limit.extend([3000 for i in range(14)]) export_limit.extend([3000 for i in range(16)]) export_limit.extend([2000 for i in range(15)]) export_limit.extend([3938 for i in range(16)]) export_limit.extend([3000 for i in range(30)]) export_limit_from_function = prep_vamp.calculate_exports(pulse_limit, average_export) export_limit_array = export_limit_from_function.getYArray() for i in range(0, len(export_limit_array)): self.assertEqual(export_limit[i], round(export_limit_array[i], 0)), 'calculate_exports function returned wrong value'
def expand_seasonal(seasonal, tw): """ Given an input time series with seasonal (monthly) values, create a larger monthly time series recycling the seasonal values over and over. Arguments: seasonal: input time series with seasonal tw: time window of output """ from vtimeseries import time, timewindow, timeinterval import string from vista.set import Constants, RegularTimeSeries ti = timeinterval("1MON") seasonalVal = {} filter = Constants.DEFAULT_FLAG_FILTER for el in seasonal: if el and filter.isAcceptable(el): xval = el.getX() xdate = time(long(xval)) xstr = xdate.toString() yval = el.getY() month = string.lower(xstr[2:5]) if seasonalVal.has_key(month): raise "Ambiguous seasonal values for month of: %s " % month seasonalVal[month] = yval # # if len(seasonalVal) != 12: raise "Not all seasonal values found" t = tw.getStartTime() t = t.create(t.floor(ti)) # need copy start = t.create(tw.getStartTime().floor(ti)) end = tw.getEndTime().ceiling(ti) n = start.getNumberOfIntervalsTo(end, ti) from jarray import zeros y = zeros(n + 1, 'd') i = 0 while (t.getTimeInMinutes() <= end.getTimeInMinutes()): xstr = t.toString() month = string.lower(xstr[2:5]) y[i] = seasonalVal[month] t.incrementBy(ti) i = i + 1 out = RegularTimeSeries(seasonal.getName(), start.toString(), timeinterval("1MON").toString(), y, None, seasonal.getAttributes()) return out
def calculate_vamp_times(series): """Calculate vamp pulse period lengths for each month in series Arguments: series: series that provides the months Output: (total, pulse, nonpulse) total: total days in month pulse: days in month in pulse period nonpulse: days in month not in pulse period """ total = days_in_month(series) n = len(series) first_month = monthlist.index(series.getStartTime().toString()[2:5]) # Now switch to entirely zero-based arrays for remainder of routine first_month_ndx_yr1 = first_month - 1 #zero based index of first month # This array has the number of pulse days per month for a single # year starting in January. We are going to repeat it over and over # with a subarray at the beginning and end because the input doesn't # necessarily start in January and end in December. single_year_pulse_days = [0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0] if first_month_ndx_yr1 + n > 12: months_year1 = 12 - first_month_ndx_yr1 last_month_ndx_yr1 = 11 repeat_months = (n - months_year1) / 12 extra_months = (n - months_year1) % 12 else: months_year1 = n last_month_ndx_yr1 = first_month_ndx_yr1 + n - 1 repeat_months = 0 extra_months = 0 pulse_days = array( single_year_pulse_days[first_month_ndx_yr1:(last_month_ndx_yr1 + 1)] + repeat_months * single_year_pulse_days + single_year_pulse_days[0:extra_months], 'd') pulse = RegularTimeSeries("/pulse//////", series.getStartTime().toString(), series.getTimeInterval().toString(), pulse_days) nonpulse = total - pulse return total, pulse, nonpulse
def replace_vamp(non_pulse, pulse, include_shoulder=0): """Creates a new series based on non_pulse values, replacing them with pulse values during the VAMP season (April 15-May 15 inclusive) or April1 - May 31 if include_shoulder is set to true (1) Arguments: non_pulse: monthly or daily series of values to use for non-pulse pulse: pulse values, must have same start, length, interval as non-pulse Output: New series with non-pulse values replaced by pulse during vamp """ if not isinstance(non_pulse, RegularTimeSeries): raise TypeError("Non-Pulse flow must be regular time series") if not isinstance(pulse, RegularTimeSeries): raise TypeError("Pulse flow must be regular time series") if include_shoulder: first_april_day = 1 last_may_day = 31 else: first_april_day = 15 last_may_day = 15 #if non_pulse.getTimeInterval() == timeinterval("1MON"): non_pulse = interpolate(non_pulse, "1DAY") #if pulse.getTimeInterval() == timeinterval("1MON"): pulse = interpolate(pulse, "1DAY") if non_pulse.getStartTime() != pulse.getStartTime() or \ len(non_pulse) != len(pulse): raise ValueError("Pulse and Non-pulse must have the same start time," " interval and length") values = zeros(len(non_pulse), 'd') for np, p, i in zip(non_pulse, pulse, range(len(values))): values[i] = np.getY() xstr = np.getXString() day, month = int(xstr[0:2]), xstr[2:5] if month == "APR" and day >= first_april_day: values[i] = p.getY() elif month == "MAY" and day <= last_may_day: values[i] = p.getY() out = RegularTimeSeries("/vamp//////", non_pulse.getStartTime().toString(), non_pulse.getTimeInterval().toString(), values) out.getAttributes().setYUnits("CFS") out.getAttributes().setYType("PER-AVER") return out
def dsAddFlags(dataset): """ dsAddFlags(dataset) Add UNSCREENED_FLAG to dataset that does not have any flags """ if dataset.isFlagged(): return dataset # create copy of incoming dataset but with flags import jarray fa = jarray.zeros(len(dataset), 'i') if dataset.getAttributes().getType() == DataType.REGULAR_TIME_SERIES: #RTS datasetFlagged = RegularTimeSeries(dataset.getName(), str(dataset.getStartTime()), \ str(dataset.getTimeInterval()), dataset.getYArray(), \ fa, dataset.getAttributes()) else: # ITS xa = jarray.zeros(len(dataset), 'd') ya = jarray.zeros(len(dataset), 'd') for i in range(len(dataset)): xa[i] = dataset[i].getX() ya[i] = dataset[i].getY() datasetFlagged = IrregularTimeSeries( dataset.getName(), xa, ya, fa, dataset.getAttributes()) return datasetFlagged
def ECEst(stage, ndo, so, sb, beta, npow1, npow2, g0, zrms, c): """ Estimate 15min EC at the boundary. Inputs: stage astronomical tide estimate. Only 15min data are acceptable ndo ndo estimate -- e.g., from CALSIM """ import interpolate from vista.set import Units if not isinstance(stage,RegularTimeSeries) or \ not isinstance(ndo,RegularTimeSeries): raise "stage and ndo must be RegularTimeSeries" if ndo.getTimeInterval().toString() == "1DAY": ndo = interpolate.spline(ndo, "15MIN", 0.5) << 95 elif ndo.getTimeInterval().toString() != "15MIN": raise "ndo must be a one day or 15 minute series" if not stage.getTimeInterval().toString() == "15MIN": raise "stage must be an hourly or 15 minute series" # if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString(): raise "stage and ndo must have the same window" # #if not len(c) ==9: # raise "Wrong number (%s) of coefficients in the array c" % len(c) if (first_missing(ndo)) >= 0: raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing( ndo) if (first_missing(stage)) >= 0: raise "missing data not allowed in stage. First missing data at index: %s" % first_missing( stage) newstart = ndo.getStartTime() - "21HOURS" newend = ndo.getEndTime() - "3HOURS" if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() > 0): print "Stage record starts %s and NDO starts %s" % ( stage.getStartTime().toString(), ndo.getStartTime().toString()) raise "stage record must begin at least 21 hours before ndo" if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0): raise "stage record must end no more than 3 hours before end of ndo" ztw = timewindow(newstart.toString() + ' - ' + newend.toString()) z = stage.createSlice(ztw) g = gCalc(ndo, beta, g0) # writedss("planning_ec_input.dss","/CALC/RSAC054/G//15MIN/CALC",g) # for debug zarr = z.getYArray() giter = g.getIterator() ec = zeros(g.size(), 'd') ec = map(lambda x: -901.0, ec) zrmsiter = zrms.getIterator() i = 0 while (not giter.atEnd()): gval = giter.getElement().getY() zrmsval = zrmsiter.getElement().getY() ecfrac = gval * c[0] + 1.1 * gval**npow1 * ( c[1] * zarr[i + 72] + c[2] * zarr[i + 60] + c[3] * zarr[i + 48] + c[4] * zarr[i + 36] + c[5] * zarr[i + 24] + c[6] * zarr[i + 12] + c[7] * zarr[i]) ec[i] = max(200, exp(ecfrac) * (so - sb) + sb) zrmsiter.advance() giter.advance() i = i + 1 # ec needs to be set umhos/cm rts = RegularTimeSeries("/ECest//////", g.getStartTime().toString(), g.getTimeInterval().toString(), ec) rts.getAttributes().setYUnits(Units.UMHOS_CM) return [rts, gval]
def gCalcFlatQ(ndo, beta, g0, out="inst"): """ Calculates antecedent outflow from ndo based on the flat ndo assumption in the g documentation. In this case, the integration of g is exact rather than numeric, but the approximation to ndo is a series of flat lines. In the case of daily data this is probably acceptable. In the case of monthly data it leads to large errors, though it is common Arguments: ndo: a regular time series. Must be 1DAY, 1MONTH g0: initial condition. If g0 is not given it is equal to ndo at the first time step. beta: g-model parameter. out: must be "inst" to calculate instantaneous values of g or "ave" to calculate averages over the time step. Output: g: a regular time series, same sampling rate as input with the same start time as ndo, ending at the end of ndo or the first piece of bad data in ndo. """ if ndo.getTimeInterval().toString( ) != "1DAY" | ndo.getTimeInterval().toString() != "1MONTH": raise "Time step for input must be 1DAY or 1MONTH" dsi = ndo.getIterator() nstep = ndo.size() g = zeros(nstep, 'd') g = map(lambda x: -901.0, g) bdt = beta / dt if g0 == None: g[0] = q[0] else: g[0] = g0 atend = 0 i = 1 if out[:4] == "inst": while atend == 0: el = dsi.getElement() if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): q = el.getY() g[i] = q / (1 + (q / g[i - 1] - 1) * exp(-q / bdt)) i = i + 1 if not dsi.atEnd(): dsi.advance() else: atend = 1 else: atend = 1 elif out[:3] == "ave": while atend == 0: el = dsi.getElement() if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): q = el.getY() g[i] = q / (1 + (q / g[i - 1] - 1) * exp(-q / bdt)) i = i + 1 if not dsi.atEnd(): dsi.advance() else: atend = 1 else: atend = 1 else: raise "Argument out must be either \"inst\" or \"ave\")" rts = RegularTimeSeries("gcalc", ndo.getStartTime().toString(), ndo.getTimeInterval(), g) if out[:4] == "inst": raise "dummy exception" if out[:3] == "ave": rts.getAttributes().setXType("PER-VAL") return rts
def spline(ref,outint,offset=0): ''' Usage example: interpolate(ref,outint = timeinterval("15min"),offset = 48) Interpolating spline Eli Ateljevich 9/27/99 This functions is designed to map a coarser time series into a smaller one covering the same time window. The spline is monotonicity-preserving and fourth order accurate (except near boundaries) offset shifts the output as appropriate. Typically, offset will be zero for inst-val input. For per-ave input, offset will often be half of the output frequency. In the example above, NDO input is treated as "daily averaged". Output is in units of 15minutes. Since there are are 96 15min samples per 24 hours offset = 0.5*96 = 48. Output is a regular time series (rts). Reference: Huynh, HT "Accurate Monotone Cubic Interpolation", SIAM J. Numer. Analysis V30 No. 1 pp 57-100 All equation numbers refer to this paper. The variable names are also almost the same. Double letters like "ee" to indicate that the subscript should have "+1/2" added to it. ''' got_ref = 0 if isinstance(ref, DataReference): data = ref.getData() got_ref = 1 else: data = ref got_ref = 0 # check for regular time series if not isinstance(data,RegularTimeSeries): print ref, " is not a regular time-series data set" return None yt = data.getIterator() div = TimeFactory.getInstance().createTimeInterval(outint) nsub = data.getTimeInterval()/div from jarray import zeros vals = zeros(1+nsub*(data.size()-1),'d') vals=map(lambda x: -901.0, vals) vallength = len(vals) dflags = zeros(vallength,'l') lastone = vallength + 4*nsub -1 firstone = 4*nsub y4,y3,y2,y1,y0,ss3,ss2,ss1,ss0,s1,s0 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0. dd1,dd0,d3,d2,d1,d1,d0,e1,e0,ee2,ee1,ee0,eem1,df0,df1 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0. count = 0 subcount = 0 nextbad = 5 atstart=1 atend =0 while not yt.atEnd() or nextbad>1: if not (yt.atEnd() or atend==1): el = yt.getElement() #Read new value, good or bad count = count + 1 if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): y4 = el.getY(); # ss3 = y4-y3 d3 = ss3 - ss2 ee2 = d3-d2 atstart = 0 else: if atstart == 0: atend = 1 nextbad = nextbad - 1 #NoMissing = "Internal missing values not allowed." #raise NoMissing else: nextbad = nextbad - 1; # # minmod-based estimates s2 = minmod(ss1,ss2) dd2 = minmod(d2,d3) e2 = minmod(ee1,ee2) #polynomial slopes dpp1adj1 = ss1 - dd1 dpp0adj1 = ss0 + dd0 t1 = minmod(dpp0adj1,dpp1adj1) dqq1adj1 = ss1 - minmod(d1+e1,d2-2*e2) #Eq4.7a dqq0adj1 = ss0 + minmod(d0+2*e0,d1-e1) #Eq4.7b ttilde1 = minmod(dqq0adj1,dqq1adj1) df1 = 0.5*(dqq0adj1 + dqq1adj1) #First cut, Eq. 4.16 tmin = min(0.,3*s1,1.5*t1,ttilde1) tmax = max(0.,3*s1,1.5*t1,ttilde1) # #If count == 3: # have enough to make up boundary quantities # gamma = (ee1 - ee0)/4 #5.8, # eex0 = ee0 - 4*gamma #5.9 x is the boundary value # qqx = median3(ssm1,qqx, df1 = df1 + minmod(tmin-df1, tmax-df1) #Revise, Eq. 4.16 for j in range(nsub): jfrac = (float(j)+offset)/ float(nsub) c0 = y0 # from eq. 2 in the paper c1 = df0 c2 = 3*ss0 - 2*df0 - df1 c3 = df0 + df1 - 2*ss0 if count > 4: if subcount <= lastone and subcount >= firstone: vals[subcount-4*nsub] = c0 + c1*jfrac +c2*jfrac**2 + c3*jfrac**3; subcount = subcount + 1 # Now lag all data and estimates to make room for next time step y3,y2,y1,y0 = y4,y3,y2,y1 ss2,ss1,ss0,ssm1 = ss3,ss2,ss1,ss0 s1,s0 = s2,s1 dd1,dd0 = dd2,dd1 d2,d1,d0 = d3,d2,d1 e1,e0 = e2,e1 ee1,ee0,eem1 = ee2,ee1,ee0 df0 = df1 if not yt.atEnd(): yt.advance() # # #refpath=ref.getPathname() #refpath.setPart(Pathname.E_PART,outint) rts = RegularTimeSeries(data.getName(), data.getStartTime().toString(), outint,vals) return rts
def linear(ref,myfilter=Constants.DEFAULT_FLAG_FILTER): ''' Linearly interpolate missing data in a time series Eli Ateljevich 9/27/99 ''' got_ref = 0 if isinstance(ref, DataReference): data = ref.getData() got_ref = 1 else: data = ref got_ref = 0 # check for regular time series if not isinstance(data,RegularTimeSeries): print ref, " is not a regular time-series data set" return None yt = data.getIterator() st = data.getStartTime() et = data.getEndTime() ti = data.getTimeInterval() dsi = data.getIterator() n = st.getExactNumberOfIntervalsTo(et,ti) + 1 from jarray import zeros vals = zeros(n,'d') vals=map(lambda x: -901.0, vals) i=0 if myfilter.isAcceptable(dsi.getElement()): firstmissing=0 else: firstmissing=1 while not dsi.atEnd(): el=dsi.getElement() if el: if myfilter.isAcceptable(el): vals[i]=el.getY() lasty=vals[i] lasti=i else: while not dsi.atEnd(): el=dsi.getElement() if myfilter.isAcceptable(el): nexty=el.getY() nexti=i if not firstmissing: # no interpolation at begin or end of record for ii in range(lasti,nexti): vals[ii]=lasty + (ii-lasti)*(nexty-lasty)/(nexti-lasti) vals[nexti]=nexty # this one gets filled even at beginning of record. firstmissing=0 break else: i=i+1 dsi.advance() # else: if not firstmissing: break # # dsi.advance() i=i+1 # rts = RegularTimeSeries(data.getName(), data.getStartTime().toString(), data.getTimeInterval().toString(),vals) return rts
import sys
def gCalcFlatQ(ndo, beta, g0,out = "inst"): """ Calculates antecedent outflow from ndo based on the flat ndo assumption in the g documentation. In this case, the integration of g is exact rather than numeric, but the approximation to ndo is a series of flat lines. In the case of daily data this is probably acceptable. In the case of monthly data it leads to large errors, though it is common Arguments: ndo: a regular time series. Must be 1DAY, 1MONTH g0: initial condition. If g0 is not given it is equal to ndo at the first time step. beta: g-model parameter. out: must be "inst" to calculate instantaneous values of g or "ave" to calculate averages over the time step. Output: g: a regular time series, same sampling rate as input with the same start time as ndo, ending at the end of ndo or the first piece of bad data in ndo. """ if ndo.getTimeInterval().toString() != "1DAY" | ndo.getTimeInterval().toString() != "1MONTH": raise "Time step for input must be 1DAY or 1MONTH" dsi=ndo.getIterator() nstep=ndo.size() g=zeros(nstep,'d') g=map(lambda x: -901.0, g) bdt=beta/dt if g0==None: g[0]=q[0] else: g[0]=g0 atend=0 i=1 if out[:4] == "inst": while atend == 0: el=dsi.getElement() if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): q=el.getY() g[i] = q/(1+(q/g[i-1] -1)*exp(-q/bdt)) i=i+1 if not dsi.atEnd(): dsi.advance() else: atend=1 else: atend=1 elif out[:3]=="ave": while atend == 0: el=dsi.getElement() if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el): q=el.getY() g[i] = q/(1+(q/g[i-1] -1)*exp(-q/bdt)) i=i+1 if not dsi.atEnd(): dsi.advance() else: atend=1 else: atend=1 else: raise "Argument out must be either \"inst\" or \"ave\")" rts = RegularTimeSeries("gcalc", ndo.getStartTime().toString(), ndo.getTimeInterval(),g) if out[:4] == "inst": raise "dummy exception" if out[:3]=="ave": rts.getAttributes().setXType("PER-VAL") return rts
def ECEst(stage, ndo, beta, npow1, npow2,g0=None, zrms=None, c=getMTZCoef()): """ Estimate 15min EC at the boundary. Inputs: stage astronomical tide estimate. Only 15min data are acceptable ndo ndo estimate -- e.g., from CALSIM """ import interpolate from vista.set import Units if not isinstance(stage,RegularTimeSeries) or \ not isinstance(ndo,RegularTimeSeries): raise "stage and ndo must be RegularTimeSeries" if ndo.getTimeInterval().toString() == "1DAY": ndo=interpolate.spline(ndo,"15MIN",0.5) << 95 elif ndo.getTimeInterval().toString()!= "15MIN": raise "ndo must be a one day or 15 minute series" if not stage.getTimeInterval().toString() == "15MIN": raise "stage must be an hourly or 15 minute series" # if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString(): raise "stage and ndo must have the same window" # #if not len(c) ==9: # raise "Wrong number (%s) of coefficients in the array c" % len(c) if (first_missing(ndo)) >= 0: raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing(ndo) if (first_missing(stage)) >= 0: raise "missing data not allowed in stage. First missing data at index: %s" % first_missing(stage) newstart=ndo.getStartTime() - "21HOURS" newend =ndo.getEndTime() - "3HOURS" if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() > 0): print "Stage record starts %s and NDO starts %s" % (stage.getStartTime().toString(),ndo.getStartTime().toString()) raise "stage record must begin at least 21 hours before ndo" if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0): raise "stage record must end no more than 3 hours before end of ndo" ztw=timewindow(newstart.toString()+' - '+newend.toString()) z=stage.createSlice(ztw) g=gCalc(ndo,beta,g0) zarr=z.getYArray() giter=g.getIterator() ec=zeros(g.size(),'d') ec=map(lambda x: -901.0, ec) so=32797 #ocean salinity parameter for gmodel sb=200 #river salinity parameter for gmodel zrmsiter=zrms.getIterator() i=0 while (not giter.atEnd()): gval=giter.getElement().getY() zrmsval=zrmsiter.getElement().getY() ecfrac = c[0] + gval*c[1] + 1.1*gval**npow1*(c[2]*zarr[i+72] + c[3]*zarr[i+60] + c[4]*zarr[i+48] + c[5]*zarr[i+36] + c[6]*+zarr[i+24] + c[7]*zarr[i+12] + c[8]*zarr[i])-0.1 #+ c[9]*zrmsval ec[i]=max(200,exp(ecfrac)*(so-sb) + sb) #ec[i]=max((1+ecfrac)*(so-sb)+sb,200) zrmsiter.advance() giter.advance() i=i+1 # ec needs to be set umhos/cm rts = RegularTimeSeries("/ECest//////",g.getStartTime().toString(),g.getTimeInterval().toString(),ec) rts.getAttributes().setYUnits(Units.UMHOS_CM) return [rts,gval]
def read_dss_txt(file,dssts=True,flag=False): """ read_dss_txt(file, dssts=True,flag=False): reads from a ascii file in dssts or dssits format and writes out the data to the appropriate dss file and pathnames. If dssts == True then dssts format is assumed in the file else dssits format is assumed For more info look up doc on read_dssts(file) and read_dssits(file) If flag == True then it expects a flag value in the ascii file as well The flag values in the ascii file are represented as flag_type|user_name where flag_type is one of UNSCREENED,QUESTIONABLE,MISSING,REJECT,OK & user_name is one of the authorized users/agency. e.g. MISSING|nsandhu or REJECT|kate """ import string tf = TimeFactory.getInstance() f = open(file) line = f.readline()[:-1] dssfile = line while line : try : line = string.upper(f.readline()[:-1]) if line == "FINISH": break; path = Pathname.createPathname(string.upper(line)) except : print 'Incorrect format for path: ', line break try : line = f.readline()[:-1] units = string.upper(line) except : print 'Incorrect format for units: ', line break try : line = f.readline()[:-1] type = string.upper(line) except : print 'Incorrect format for type: ', line break if dssts: try : line = f.readline()[:-1] stime = tf.createTime(line) except : print 'Incorrect format for time: ', line break # line = string.upper(f.readline()[:-1]) xvals = [] yvals = [] if flag: flags = [] else: flags = None while line != "END" : if dssts: try: if flag: vals = string.split(line) if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line) yvals.append(float(vals[0])) flags.append(make_flag_value(vals[1])) else: # no flags yvals.append(float(line)) except: yvals.append(Constants.MISSING_VALUE); if flag: flags.append(make_flag_value('MISSING|null')) else : try : line = string.strip(line) tmstr = line[0:14] tm = tf.createTime(tmstr) try : if flag: vals = string.split(line[14:]) if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line) val = float(vals[0]) flag_val = vals[1] else: # no flag val = float(line[14:]) except : val = Constants.MISSING_VALUE xvals.append(tm) yvals.append(val) flags.append(make_flag_value(flag_val)) except Exception, exc: print exc print "Error reading line: ", line line = string.upper(f.readline()[:-1]) # create appropriate time series object if len(yvals) == 0 : continue #print yvals, flags if ( dssts ): attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,"TIME",units,"",type) ts = RegularTimeSeries("",repr(stime),path.getPart(Pathname.E_PART), yvals, flags, attr) else : attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,"TIME",units,"",type) ts = IrregularTimeSeries("",xvals, yvals, flags, attr)
def retrieve_ts(station,sensor,\ start_date, end_date='now',_verbose=1): """ retrieve(station,sensor, start_date, end_date='now',_verbose=1) Retrieves data from cdec station and sensor and constructs a regular time series with given pathname, units. The start_date is a starting time value in format mm/dd/yyyy e.g. 15-Jun-2000 would be 06/15/2000 The end_date can either be the keyword "now" or the same format as the start_date verbose=1 sets it to more verbose and 0 for silence """ _debug = 0 station_name = station.id sensor_number = sensor.sensor_number c_part = "%s_%s" % (sensor.type, sensor.subType) f_part = "SENSOR %s" % (sensor.id) pathname = '/%s/%s/%s//%s/%s/' % ('CDEC-RAW', station_name, c_part, DSS_INTERVAL[sensor.duration], "SENSOR " + sensor.id) units = sensor.units dur_code = sensor.getDurationCode() c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?' + 'station_id=' + str(station_name) + '&dur_code=' + str(dur_code) + '&sensor_num=' + str(sensor_number) + '&start_date=' + start_date + '&end_date=' + end_date) if _verbose: print "station name:%s & sensor number:%s " % (station_name, sensor_number) print "dur_code:%s " % (dur_code) print "pathname:%s & units:%s" % (pathname, units) print 'URL: %s' % c_url lr = LineNumberReader( InputStreamReader(c_url.openConnection().getInputStream())) # jump all the way to data lr.readLine() lr.readLine() lr.readLine() # create starting date and time and of the format the # data is at cdec line = lr.readLine() tf = TimeFactory.getInstance() starray = string.split(line, ',') dtm = starray[0] + ' ' + starray[1] tp = tf.createTime(dtm, 'yyyyMMdd HHmm') tp = tf.createTime(tp.getTimeInMinutes()) ti_str = '1HOUR' irreg = 0 # indicates the data as irregular time series if dur_code == 'E': irreg = 1 ti_str = 'IR-DAY' elif dur_code == 'D': ti_str = '1DAY' elif dur_code == 'H': ti_str = '1HOUR' elif dur_code == 'M': ti_str = '1MON' if not irreg: ti = tf.createTimeInterval(ti_str) ltime = tp if _debug: print line, tp, dtm yvals = [] if irreg: tvals = [] if _verbose: print 'Data starting at ', tp # read in all the data and append missing values if no data # is available while line != None: st = StringTokenizer(line, ',') if (st.countTokens() != 3): raise "Invalid CDEC format, need 3 tokens on line: " + line # get time ctime = tf.createTime(st.nextToken() + ' ' + st.nextToken(), 'yyyyMMdd HHmm') # if time is not in increasing order -> quit! if ctime.compare(ltime) < 0: raise "Invalid time sequence: %s followed by %s"+\ " ? -> should be always increasing"%(str(ctime),str(ltime)) # check if current time is only one time interval from last time if not irreg: nskip = ltime.getNumberOfIntervalsTo(ctime, ti) # if skip is greater than one then fill with -901's while nskip > 1: yvals.append(Constants.MISSING_VALUE) nskip = nskip - 1 ltime = ctime # now get current value val_str = st.nextToken() try: if (val_str == "m"): # if missing val = Constants.MISSING_VALUE else: # else just save the value val = float(val_str) if irreg: tvals.append(ctime) yvals.append(val) except: tvals.append(ctime) yvals.append(Constants.MISSING_VALUE) print "Exception! for string ", val_str line = lr.readLine() # create a time series data set from the array of values, # the start time and the time interval if irreg: attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, '', units, 'TIME', 'INST-VAL') its = IrregularTimeSeries(pathname, tvals, yvals, None, attr) return its else: attr = DataSetAttr(DataType.REGULAR_TIME_SERIES, '', units, 'TIME', 'INST-VAL') rts = RegularTimeSeries(pathname, tp.toString(), ti_str, yvals, None, attr) return rts