def month_numbers(series):
n = len(series)
values = zeros(n, 'd')
for el, i in zip(series, range(n)):
t = el.getXString()
values[i] = monthlist.index(t[2:5])
out = RegularTimeSeries("/MONTH_NUMBER//////",
series.getStartTime().toString(),
series.getTimeInterval().toString(), values)
out.getAttributes().setYType(series.getAttributes().getYType())
return out
def month_numbers(series):
n=len(series)
values=zeros(n,'d')
for el,i in zip(series,range(n)):
t=el.getXString()
values[i]=monthlist.index(t[2:5])
out=RegularTimeSeries("/MONTH_NUMBER//////",
series.getStartTime().toString(),
series.getTimeInterval().toString(),
values)
out.getAttributes().setYType(series.getAttributes().getYType())
return out
def days_in_month(series):
n = len(series)
values = zeros(n, 'd')
for el, i in zip(series, range(n)):
t = el.getXString()
m = monthlist.index(t[2:5])
y = int(t[5:9])
values[i] = monthrange(y, m)[1]
out = RegularTimeSeries("/DAY_IN_MONTH//////",
series.getStartTime().toString(),
series.getTimeInterval().toString(), values)
out.getAttributes().setYType(series.getAttributes().getYType())
return out
def setUp(self):
self.ts_1 = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[6000,8000,4000,2000])
self.ts_2 = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[10000,10000,10000,10000])
self.ts_3 = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[1000,1000,1000,1000])
self.list_1 = [6000 for i in range(31)]
self.list_2 = [8000 for i in range(31)]
self.list_3 = [10000 for i in range(31)]
self.list_4 = [4000 for i in range(31)]
self.list_5 = [2000 for i in range(31)]
self.list_6 = [1000 for i in range(31)]
def days_in_month(series):
n=len(series)
values=zeros(n,'d')
for el,i in zip(series,range(n)):
t=el.getXString()
m=monthlist.index(t[2:5])
y=int(t[5:9])
values[i]=monthrange(y,m)[1]
out=RegularTimeSeries("/DAY_IN_MONTH//////",
series.getStartTime().toString(),
series.getTimeInterval().toString(),
values)
out.getAttributes().setYType(series.getAttributes().getYType())
return out
def extend_flow(nodes_to_extend):
""" Copying WY1922 data to WY1921 for allowing to preprocessing and running DSM2
from 01Jan1921.
"""
calsimfile = getAttr("CALSIMFILE")
f = opendss(calsimfile) # open CALSIM file
outfile = getAttr("BOUNDARYFILE")
if not outfile or outfile == "":
raise "Config variable BOUNDARYFILE not set and needed for prepro output"
tw = timewindow("01OCT1921 0000 - 01OCT1922 0000")
for calsimname in nodes_to_extend:
print calsimname
dsspath = calsim_path(calsimname)
paths = findpath(f, dsspath)
if not paths or len(paths) > 1:
print "File: %s" % calsimfile
raise "Path %s not found or not unique" % dsspath
ref = DataReference.create(paths[0], tw)
monthly = ref.getData()
itr = monthly.getIterator()
d = zeros(len(monthly), 'd')
count = 0
while not itr.atEnd():
el = itr.getElement()
d[count] = el.getY()
count = count + 1
itr.advance()
stime = "01OCT1920 0000"
rts = RegularTimeSeries(monthly.getName(),stime, \
timeinterval("1MON").toString(), d, None, monthly.getAttributes())
writedss(calsimfile, ref.getPathname().toString(), rts)
def replace_vamp(non_pulse,pulse,include_shoulder=0):
"""Creates a new series based on non_pulse values, replacing them
with pulse values during the VAMP season (April 15-May 15 inclusive)
or April1 - May 31 if include_shoulder is set to true (1)
Arguments:
non_pulse: monthly or daily series of values to use for non-pulse
pulse: pulse values, must have same start, length, interval as non-pulse
Output: New series with non-pulse values replaced by pulse during vamp
"""
if not isinstance(non_pulse,RegularTimeSeries):
raise TypeError("Non-Pulse flow must be regular time series")
if not isinstance(pulse,RegularTimeSeries):
raise TypeError("Pulse flow must be regular time series")
if include_shoulder:
first_april_day=1
last_may_day=31
else:
first_april_day=15
last_may_day=15
#if non_pulse.getTimeInterval() == timeinterval("1MON"):
non_pulse=interpolate(non_pulse,"1DAY")
#if pulse.getTimeInterval() == timeinterval("1MON"):
pulse=interpolate(pulse,"1DAY")
if non_pulse.getStartTime() != pulse.getStartTime() or \
len(non_pulse) != len(pulse):
raise ValueError("Pulse and Non-pulse must have the same start time,"
" interval and length")
values=zeros(len(non_pulse),'d')
for np,p,i in zip(non_pulse,pulse,range(len(values))):
values[i]=np.getY()
xstr=np.getXString()
day,month=int(xstr[0:2]),xstr[2:5]
if month=="APR" and day >= first_april_day:
values[i]=p.getY()
elif month=="MAY" and day <= last_may_day:
values[i]=p.getY()
out=RegularTimeSeries("/vamp//////", non_pulse.getStartTime().toString(),
non_pulse.getTimeInterval().toString(),values)
out.getAttributes().setYUnits("CFS")
out.getAttributes().setYType("PER-AVER")
return out
def conserveSpline(avedata, interval):
if not isinstance(interval, TimeInterval):
interval = timeinterval(interval)
start = avedata.getStartTime()
try:
nout = start.getExactNumberOfIntervalsTo(avedata.getEndTime(),
interval) + 1
except:
nout = start.getNumberOfIntervalsTo(avedata.getEndTime(), interval)
n = avedata.size()
from jarray import zeros
x = zeros(n + 1, 'd')
y = zeros(n + 1, 'd')
i = 0
start2 = time(start.toString()) # Pain in the neck way to clone
rev = timeinterval("-" + interval.toString())
start2.incrementBy(avedata.getTimeInterval(), -1)
x[0] = start2.getTimeInMinutes()
for el in avedata:
x[i + 1] = el.getX()
y[i] = el.getY()
i = i + 1
if not Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
raise "Missing or bad data not allowed in conservative spline"
p = map(lambda x: splineparam, y)
spline = interpolator.ConservativeSpline(x, y, p, y[0], y[n - 1])
ynew = zeros(nout, 'd')
newseries = RegularTimeSeries(
'/Smooth/Data///' + interval.toString() + '//', start.toString(),
interval.toString(), ynew)
dsi = newseries.getIterator()
while not dsi.atEnd():
el = dsi.getElement()
if spline.rh2val(el.getX()) < 0:
# useY=y(el.getX())
useY = spline.rh2val(el.getX())
else:
useY = spline.rh2val(el.getX())
#
el.setY(useY)
dsi.putElement(el)
dsi.advance()
#tabulate(newseries,avedata)
return newseries
def gCalc(ndo,beta,g0=None):
""" Calculates antecedent outflow from a stream of ndo
Arguments:
ndo: a regular time series. Must be 15MIN, 1HOUR. Thus, NDO has been interpolated first.
g0: initial condition. If g0 is not given it is equal to ndo at the first time step.
beta: g-model parameter.
Output:
g: a regular time series, same sampling rate as input
with the same start time as ndo
"""
ti = ndo.getTimeInterval()
if not ((ti.toString() == "15MIN") | (ti.toString() == "1HOUR")):
raise "NDO time step must be 15MIN or 1HOUR."
dt=1
nstep=ndo.size()
g=zeros(nstep,'d')
if ti.toString() == "15MIN":
dt=0.25
g=zeros(nstep,'d')
g=map(lambda x: -901.0, g)
beta=beta*24*365
div2dt=2*beta/dt
dsi=ndo.getIterator()
q0 = dsi.getElement().getY()
# Set initial condition
if g0==None:
g[0]= q0 # ???
else:
g[0]=g0
# Loop through and integrate gmodel using trapazoidal.
atend=0
i=1
dsi.advance()
while atend == 0:
el=dsi.getElement()
if el and Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
q=el.getY()
qterm=(q-div2dt)
g[i]= 0.5*( qterm + sqrt(qterm*qterm + 4*g[i-1]*(q0-g[i-1]+div2dt)) )
if dsi.atEnd():
atend=1
else:
dsi.advance()
q0=q
i=i+1
else:
atend=1
rts = RegularTimeSeries("/gcalc//////",ndo.getStartTime().toString(),
ndo.getTimeInterval().toString(),g)
return rts
def ts_where(criteria,trueval,falseval):
if not( len(criteria) == len(trueval) and \
len(criteria) == len(falseval) ):
raise ValueError("Criteria (len %s), trueval (len %s) and falseval (len %s)"
" must be the same length" % (len(criteria),len(trueval),len(falseval)))
n=len(criteria)
values=zeros(n,'d')
for c,t,f,i in zip(criteria,trueval,falseval,range(n)):
if filter.isAcceptable(c):
if c.getY():
values[i]=t.getY()
else:
values[i]=f.getY()
else:
values[i]=NA_VAL
out=RegularTimeSeries("/WHERE//////",criteria.getStartTime().toString(),
criteria.getTimeInterval().toString(),values)
out.getAttributes().setYType(trueval.getAttributes().getYType())
return out
def conserveSpline(avedata, interval):
if not isinstance(interval,TimeInterval):
interval=timeinterval(interval)
start=avedata.getStartTime()
try:
nout=start.getExactNumberOfIntervalsTo(avedata.getEndTime(),interval)+1
except:
nout=start.getNumberOfIntervalsTo(avedata.getEndTime(), interval)
n=avedata.size()
from jarray import zeros
x=zeros(n+1,'d')
y=zeros(n+1,'d')
i=0
start2=time(start.toString()) # Pain in the neck way to clone
rev=timeinterval("-"+interval.toString())
start2.incrementBy(avedata.getTimeInterval(),-1)
x[0]=start2.getTimeInMinutes()
for el in avedata:
x[i+1]=el.getX()
y[i]=el.getY()
i=i+1
if not Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
raise "Missing or bad data not allowed in conservative spline"
p=map(lambda x: splineparam, y)
spline=interpolator.ConservativeSpline(x,y,p,y[0],y[n-1])
ynew=zeros(nout,'d')
newseries=RegularTimeSeries('/Smooth/Data///'+interval.toString()+'//',start.toString(),interval.toString(),ynew)
dsi=newseries.getIterator()
while not dsi.atEnd():
el=dsi.getElement()
if spline.rh2val(el.getX()) < 0:
useY=y(el.getX())
else:
useY=spline.rh2val(el.getX())
#
el.setY(useY)
dsi.putElement(el)
dsi.advance()
#tabulate(newseries,avedata)
return newseries
def test_project_export_only_apr_may(self):
print ""
print ""
print "test_project_export_only_apr_may..."
pulse_limit = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[0,2500,2000,0])
delta_inflow = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[20000,10000,8000,8000])
ei_ratio = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[0.35,0.35,0.35,0.35])
#delta_inflow*ei_ratio = [7000, 3500, 2800, 2800]
cvp_out = [3500, 1250, 1000, 1400]
swp_out = [3500, 1250, 1000, 1400]
swp_out_from_function, cvp_out_from_function = prep_vamp.project_export_limits(pulse_limit, ei_ratio, delta_inflow)
swp_array = swp_out_from_function.getYArray()
cvp_array = cvp_out_from_function.getYArray()
for i in range(0, len(cvp_array)):
self.assertEqual(cvp_out[i], round(cvp_array[i], 0)), 'project_export function returned wrong cvp_limit value'
self.assertEqual(swp_out[i], round(swp_array[i], 0)), 'project_export function returned wrong swp_limit value'
def ts_where(criteria, trueval, falseval):
if not( len(criteria) == len(trueval) and \
len(criteria) == len(falseval) ):
raise ValueError(
"Criteria (len %s), trueval (len %s) and falseval (len %s)"
" must be the same length" %
(len(criteria), len(trueval), len(falseval)))
n = len(criteria)
values = zeros(n, 'd')
for c, t, f, i in zip(criteria, trueval, falseval, range(n)):
if filter.isAcceptable(c):
if c.getY():
values[i] = t.getY()
else:
values[i] = f.getY()
else:
values[i] = NA_VAL
out = RegularTimeSeries("/WHERE//////",
criteria.getStartTime().toString(),
criteria.getTimeInterval().toString(), values)
out.getAttributes().setYType(trueval.getAttributes().getYType())
return out
def plot_vol_calcs(tidefile,chan,tw,run_name):
volume = get_volume_data(tidefile, chan, tw)
upflow = get_upstream_flow_data(tidefile, chan, tw)
downflow = get_downstream_flow_data(tidefile, chan, tw)
#import vdisplay
#vdisplay.plot(volume)
initial_volume = volume[0].y
final_volume = volume[len(volume) - 1].y
ndata = len(upflow)
vol = [initial_volume]
for i in range(0, ndata):
vol.append(vol[i] + (upflow[i].y - downflow[i].y) * time_step)
print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, final_volume)
print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, vol[ndata - 1])
print 'Difference in Volume %f' % (final_volume-vol[ndata-1])
from vista.set import RegularTimeSeries, DataSetAttr
attr = volume.getAttributes().createClone()
attr.setTypeName("VOLCALC")
volcalc = RegularTimeSeries("/VOL/%s/VOLCALC//1HOUR/%s/"%(chan,run_name),"01OCT1974 0000","1HOUR",vol[1:])
volcalc.setAttributes(attr)
import vdisplay
vdisplay.plot(volcalc,volume)
vdisplay.plot(volcalc-volume)
def plot_vol_calcs(tidefile,chan,tw,run_name):
volume = get_volume_data(tidefile, chan, tw)
upflow = get_upstream_flow_data(tidefile, chan, tw)
downflow = get_downstream_flow_data(tidefile, chan, tw)
#import vdisplay
#vdisplay.plot(volume)
initial_volume = volume[0].y
final_volume = volume[len(volume) - 1].y
ndata = len(upflow)
vol = [initial_volume]
for i in range(0, ndata):
vol.append(vol[i] + (upflow[i].y - downflow[i].y) * time_step)
print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, final_volume)
print 'Initial Volume: %f -> Final Volume: %f' % (initial_volume, vol[ndata - 1])
print 'Difference in Volume %f' % (final_volume-vol[ndata-1])
from vista.set import RegularTimeSeries, DataSetAttr
attr = volume.getAttributes().createClone()
attr.setTypeName("VOLCALC")
volcalc = RegularTimeSeries("/VOL/%s/VOLCALC//1HOUR/%s/"%(chan,run_name),"01OCT1974 0000","1HOUR",vol[1:])
volcalc.setAttributes(attr)
import vdisplay
vdisplay.plot(volcalc,volume)
vdisplay.plot(volcalc-volume)
def test_calculate_exports_average_below_limit(self):
print ""
print ""
print "test_calculate_exports_average_below_limit..."
pulse_limit = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[6000,4000,2000,8000])
average_export = RegularTimeSeries("///////","31MAR1980 2400",
"1MON",[3000,3000,3000,3000])
export_limit = [3000 for i in range(0,31)]
export_limit.extend([3000 for i in range(14)])
export_limit.extend([3000 for i in range(16)])
export_limit.extend([2000 for i in range(15)])
export_limit.extend([3938 for i in range(16)])
export_limit.extend([3000 for i in range(30)])
export_limit_from_function = prep_vamp.calculate_exports(pulse_limit, average_export)
export_limit_array = export_limit_from_function.getYArray()
for i in range(0, len(export_limit_array)):
self.assertEqual(export_limit[i], round(export_limit_array[i], 0)), 'calculate_exports function returned wrong value'
def expand_seasonal(seasonal, tw):
"""
Given an input time series with seasonal (monthly) values,
create a larger monthly time series
recycling the seasonal values over and over.
Arguments:
seasonal: input time series with seasonal
tw: time window of output
"""
from vtimeseries import time, timewindow, timeinterval
import string
from vista.set import Constants, RegularTimeSeries
ti = timeinterval("1MON")
seasonalVal = {}
filter = Constants.DEFAULT_FLAG_FILTER
for el in seasonal:
if el and filter.isAcceptable(el):
xval = el.getX()
xdate = time(long(xval))
xstr = xdate.toString()
yval = el.getY()
month = string.lower(xstr[2:5])
if seasonalVal.has_key(month):
raise "Ambiguous seasonal values for month of: %s " % month
seasonalVal[month] = yval
#
#
if len(seasonalVal) != 12:
raise "Not all seasonal values found"
t = tw.getStartTime()
t = t.create(t.floor(ti)) # need copy
start = t.create(tw.getStartTime().floor(ti))
end = tw.getEndTime().ceiling(ti)
n = start.getNumberOfIntervalsTo(end, ti)
from jarray import zeros
y = zeros(n + 1, 'd')
i = 0
while (t.getTimeInMinutes() <= end.getTimeInMinutes()):
xstr = t.toString()
month = string.lower(xstr[2:5])
y[i] = seasonalVal[month]
t.incrementBy(ti)
i = i + 1
out = RegularTimeSeries(seasonal.getName(), start.toString(),
timeinterval("1MON").toString(), y, None,
seasonal.getAttributes())
return out
def calculate_vamp_times(series):
"""Calculate vamp pulse period lengths for each month in series
Arguments:
series: series that provides the months
Output: (total, pulse, nonpulse)
total: total days in month
pulse: days in month in pulse period
nonpulse: days in month not in pulse period
"""
total = days_in_month(series)
n = len(series)
first_month = monthlist.index(series.getStartTime().toString()[2:5])
# Now switch to entirely zero-based arrays for remainder of routine
first_month_ndx_yr1 = first_month - 1 #zero based index of first month
# This array has the number of pulse days per month for a single
# year starting in January. We are going to repeat it over and over
# with a subarray at the beginning and end because the input doesn't
# necessarily start in January and end in December.
single_year_pulse_days = [0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0]
if first_month_ndx_yr1 + n > 12:
months_year1 = 12 - first_month_ndx_yr1
last_month_ndx_yr1 = 11
repeat_months = (n - months_year1) / 12
extra_months = (n - months_year1) % 12
else:
months_year1 = n
last_month_ndx_yr1 = first_month_ndx_yr1 + n - 1
repeat_months = 0
extra_months = 0
pulse_days = array(
single_year_pulse_days[first_month_ndx_yr1:(last_month_ndx_yr1 + 1)] +
repeat_months * single_year_pulse_days +
single_year_pulse_days[0:extra_months], 'd')
pulse = RegularTimeSeries("/pulse//////",
series.getStartTime().toString(),
series.getTimeInterval().toString(), pulse_days)
nonpulse = total - pulse
return total, pulse, nonpulse
def replace_vamp(non_pulse, pulse, include_shoulder=0):
"""Creates a new series based on non_pulse values, replacing them
with pulse values during the VAMP season (April 15-May 15 inclusive)
or April1 - May 31 if include_shoulder is set to true (1)
Arguments:
non_pulse: monthly or daily series of values to use for non-pulse
pulse: pulse values, must have same start, length, interval as non-pulse
Output: New series with non-pulse values replaced by pulse during vamp
"""
if not isinstance(non_pulse, RegularTimeSeries):
raise TypeError("Non-Pulse flow must be regular time series")
if not isinstance(pulse, RegularTimeSeries):
raise TypeError("Pulse flow must be regular time series")
if include_shoulder:
first_april_day = 1
last_may_day = 31
else:
first_april_day = 15
last_may_day = 15
#if non_pulse.getTimeInterval() == timeinterval("1MON"):
non_pulse = interpolate(non_pulse, "1DAY")
#if pulse.getTimeInterval() == timeinterval("1MON"):
pulse = interpolate(pulse, "1DAY")
if non_pulse.getStartTime() != pulse.getStartTime() or \
len(non_pulse) != len(pulse):
raise ValueError("Pulse and Non-pulse must have the same start time,"
" interval and length")
values = zeros(len(non_pulse), 'd')
for np, p, i in zip(non_pulse, pulse, range(len(values))):
values[i] = np.getY()
xstr = np.getXString()
day, month = int(xstr[0:2]), xstr[2:5]
if month == "APR" and day >= first_april_day:
values[i] = p.getY()
elif month == "MAY" and day <= last_may_day:
values[i] = p.getY()
out = RegularTimeSeries("/vamp//////",
non_pulse.getStartTime().toString(),
non_pulse.getTimeInterval().toString(), values)
out.getAttributes().setYUnits("CFS")
out.getAttributes().setYType("PER-AVER")
return out
def dsAddFlags(dataset):
"""
dsAddFlags(dataset)
Add UNSCREENED_FLAG to dataset that does not have any flags
"""
if dataset.isFlagged(): return dataset
# create copy of incoming dataset but with flags
import jarray
fa = jarray.zeros(len(dataset), 'i')
if dataset.getAttributes().getType() == DataType.REGULAR_TIME_SERIES: #RTS
datasetFlagged = RegularTimeSeries(dataset.getName(), str(dataset.getStartTime()), \
str(dataset.getTimeInterval()), dataset.getYArray(), \
fa, dataset.getAttributes())
else: # ITS
xa = jarray.zeros(len(dataset), 'd')
ya = jarray.zeros(len(dataset), 'd')
for i in range(len(dataset)):
xa[i] = dataset[i].getX()
ya[i] = dataset[i].getY()
datasetFlagged = IrregularTimeSeries(
dataset.getName(), xa, ya, fa, dataset.getAttributes())
return datasetFlagged
def ECEst(stage, ndo, so, sb, beta, npow1, npow2, g0, zrms, c):
""" Estimate 15min EC at the boundary.
Inputs:
stage astronomical tide estimate. Only 15min data are acceptable
ndo ndo estimate -- e.g., from CALSIM
"""
import interpolate
from vista.set import Units
if not isinstance(stage,RegularTimeSeries) or \
not isinstance(ndo,RegularTimeSeries):
raise "stage and ndo must be RegularTimeSeries"
if ndo.getTimeInterval().toString() == "1DAY":
ndo = interpolate.spline(ndo, "15MIN", 0.5) << 95
elif ndo.getTimeInterval().toString() != "15MIN":
raise "ndo must be a one day or 15 minute series"
if not stage.getTimeInterval().toString() == "15MIN":
raise "stage must be an hourly or 15 minute series"
#
if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString():
raise "stage and ndo must have the same window"
#
#if not len(c) ==9:
# raise "Wrong number (%s) of coefficients in the array c" % len(c)
if (first_missing(ndo)) >= 0:
raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing(
ndo)
if (first_missing(stage)) >= 0:
raise "missing data not allowed in stage. First missing data at index: %s" % first_missing(
stage)
newstart = ndo.getStartTime() - "21HOURS"
newend = ndo.getEndTime() - "3HOURS"
if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() >
0):
print "Stage record starts %s and NDO starts %s" % (
stage.getStartTime().toString(), ndo.getStartTime().toString())
raise "stage record must begin at least 21 hours before ndo"
if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0):
raise "stage record must end no more than 3 hours before end of ndo"
ztw = timewindow(newstart.toString() + ' - ' + newend.toString())
z = stage.createSlice(ztw)
g = gCalc(ndo, beta, g0)
# writedss("planning_ec_input.dss","/CALC/RSAC054/G//15MIN/CALC",g) # for debug
zarr = z.getYArray()
giter = g.getIterator()
ec = zeros(g.size(), 'd')
ec = map(lambda x: -901.0, ec)
zrmsiter = zrms.getIterator()
i = 0
while (not giter.atEnd()):
gval = giter.getElement().getY()
zrmsval = zrmsiter.getElement().getY()
ecfrac = gval * c[0] + 1.1 * gval**npow1 * (
c[1] * zarr[i + 72] + c[2] * zarr[i + 60] + c[3] * zarr[i + 48] +
c[4] * zarr[i + 36] + c[5] * zarr[i + 24] + c[6] * zarr[i + 12] +
c[7] * zarr[i])
ec[i] = max(200, exp(ecfrac) * (so - sb) + sb)
zrmsiter.advance()
giter.advance()
i = i + 1
# ec needs to be set umhos/cm
rts = RegularTimeSeries("/ECest//////",
g.getStartTime().toString(),
g.getTimeInterval().toString(), ec)
rts.getAttributes().setYUnits(Units.UMHOS_CM)
return [rts, gval]
def gCalcFlatQ(ndo, beta, g0, out="inst"):
""" Calculates antecedent outflow from ndo based on the flat ndo
assumption in the g documentation. In this case, the integration of g is exact
rather than numeric, but the approximation to ndo is a series of flat lines. In the
case of daily data this is probably acceptable. In the case of monthly data it leads
to large errors, though it is common
Arguments:
ndo: a regular time series. Must be 1DAY, 1MONTH
g0: initial condition. If g0 is not given it is equal to ndo at the first time step.
beta: g-model parameter.
out: must be "inst" to calculate instantaneous values of g or "ave" to calculate averages over
the time step.
Output:
g: a regular time series, same sampling rate as input
with the same start time as ndo, ending at the end of ndo or the first piece of bad
data in ndo.
"""
if ndo.getTimeInterval().toString(
) != "1DAY" | ndo.getTimeInterval().toString() != "1MONTH":
raise "Time step for input must be 1DAY or 1MONTH"
dsi = ndo.getIterator()
nstep = ndo.size()
g = zeros(nstep, 'd')
g = map(lambda x: -901.0, g)
bdt = beta / dt
if g0 == None:
g[0] = q[0]
else:
g[0] = g0
atend = 0
i = 1
if out[:4] == "inst":
while atend == 0:
el = dsi.getElement()
if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
q = el.getY()
g[i] = q / (1 + (q / g[i - 1] - 1) * exp(-q / bdt))
i = i + 1
if not dsi.atEnd():
dsi.advance()
else:
atend = 1
else:
atend = 1
elif out[:3] == "ave":
while atend == 0:
el = dsi.getElement()
if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
q = el.getY()
g[i] = q / (1 + (q / g[i - 1] - 1) * exp(-q / bdt))
i = i + 1
if not dsi.atEnd():
dsi.advance()
else:
atend = 1
else:
atend = 1
else:
raise "Argument out must be either \"inst\" or \"ave\")"
rts = RegularTimeSeries("gcalc",
ndo.getStartTime().toString(),
ndo.getTimeInterval(), g)
if out[:4] == "inst":
raise "dummy exception"
if out[:3] == "ave":
rts.getAttributes().setXType("PER-VAL")
return rts
def spline(ref,outint,offset=0):
'''
Usage example: interpolate(ref,outint = timeinterval("15min"),offset = 48)
Interpolating spline
Eli Ateljevich 9/27/99
This functions is designed to map a coarser time series into a smaller one
covering the same time window. The spline is monotonicity-preserving
and fourth order accurate (except near boundaries)
offset shifts the output as appropriate. Typically, offset will be
zero for inst-val input. For per-ave input, offset will often be
half of the output frequency. In the example above, NDO
input is treated as "daily averaged". Output is in units of
15minutes. Since there are are 96 15min samples per 24 hours
offset = 0.5*96 = 48.
Output is a regular time series (rts).
Reference: Huynh, HT "Accurate Monotone Cubic Interpolation",
SIAM J. Numer. Analysis V30 No. 1 pp 57-100
All equation numbers refer to this paper. The variable names are
also almost the same. Double letters like "ee" to indicate
that the subscript should have "+1/2" added to it.
'''
got_ref = 0
if isinstance(ref, DataReference):
data = ref.getData()
got_ref = 1
else:
data = ref
got_ref = 0
# check for regular time series
if not isinstance(data,RegularTimeSeries):
print ref, " is not a regular time-series data set"
return None
yt = data.getIterator()
div = TimeFactory.getInstance().createTimeInterval(outint)
nsub = data.getTimeInterval()/div
from jarray import zeros
vals = zeros(1+nsub*(data.size()-1),'d')
vals=map(lambda x: -901.0, vals)
vallength = len(vals)
dflags = zeros(vallength,'l')
lastone = vallength + 4*nsub -1
firstone = 4*nsub
y4,y3,y2,y1,y0,ss3,ss2,ss1,ss0,s1,s0 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.
dd1,dd0,d3,d2,d1,d1,d0,e1,e0,ee2,ee1,ee0,eem1,df0,df1 = 0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.
count = 0
subcount = 0
nextbad = 5
atstart=1
atend =0
while not yt.atEnd() or nextbad>1:
if not (yt.atEnd() or atend==1):
el = yt.getElement() #Read new value, good or bad
count = count + 1
if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
y4 = el.getY();
#
ss3 = y4-y3
d3 = ss3 - ss2
ee2 = d3-d2
atstart = 0
else:
if atstart == 0:
atend = 1
nextbad = nextbad - 1
#NoMissing = "Internal missing values not allowed."
#raise NoMissing
else:
nextbad = nextbad - 1;
#
# minmod-based estimates
s2 = minmod(ss1,ss2)
dd2 = minmod(d2,d3)
e2 = minmod(ee1,ee2)
#polynomial slopes
dpp1adj1 = ss1 - dd1
dpp0adj1 = ss0 + dd0
t1 = minmod(dpp0adj1,dpp1adj1)
dqq1adj1 = ss1 - minmod(d1+e1,d2-2*e2) #Eq4.7a
dqq0adj1 = ss0 + minmod(d0+2*e0,d1-e1) #Eq4.7b
ttilde1 = minmod(dqq0adj1,dqq1adj1)
df1 = 0.5*(dqq0adj1 + dqq1adj1) #First cut, Eq. 4.16
tmin = min(0.,3*s1,1.5*t1,ttilde1)
tmax = max(0.,3*s1,1.5*t1,ttilde1)
# #If count == 3: # have enough to make up boundary quantities
# gamma = (ee1 - ee0)/4 #5.8,
# eex0 = ee0 - 4*gamma #5.9 x is the boundary value
# qqx = median3(ssm1,qqx,
df1 = df1 + minmod(tmin-df1, tmax-df1) #Revise, Eq. 4.16
for j in range(nsub):
jfrac = (float(j)+offset)/ float(nsub)
c0 = y0 # from eq. 2 in the paper
c1 = df0
c2 = 3*ss0 - 2*df0 - df1
c3 = df0 + df1 - 2*ss0
if count > 4:
if subcount <= lastone and subcount >= firstone:
vals[subcount-4*nsub] = c0 + c1*jfrac +c2*jfrac**2 + c3*jfrac**3;
subcount = subcount + 1
# Now lag all data and estimates to make room for next time step
y3,y2,y1,y0 = y4,y3,y2,y1
ss2,ss1,ss0,ssm1 = ss3,ss2,ss1,ss0
s1,s0 = s2,s1
dd1,dd0 = dd2,dd1
d2,d1,d0 = d3,d2,d1
e1,e0 = e2,e1
ee1,ee0,eem1 = ee2,ee1,ee0
df0 = df1
if not yt.atEnd():
yt.advance()
#
#
#refpath=ref.getPathname()
#refpath.setPart(Pathname.E_PART,outint)
rts = RegularTimeSeries(data.getName(),
data.getStartTime().toString(),
outint,vals)
return rts
def linear(ref,myfilter=Constants.DEFAULT_FLAG_FILTER):
'''
Linearly interpolate missing data in a time series
Eli Ateljevich 9/27/99
'''
got_ref = 0
if isinstance(ref, DataReference):
data = ref.getData()
got_ref = 1
else:
data = ref
got_ref = 0
# check for regular time series
if not isinstance(data,RegularTimeSeries):
print ref, " is not a regular time-series data set"
return None
yt = data.getIterator()
st = data.getStartTime()
et = data.getEndTime()
ti = data.getTimeInterval()
dsi = data.getIterator()
n = st.getExactNumberOfIntervalsTo(et,ti) + 1
from jarray import zeros
vals = zeros(n,'d')
vals=map(lambda x: -901.0, vals)
i=0
if myfilter.isAcceptable(dsi.getElement()):
firstmissing=0
else:
firstmissing=1
while not dsi.atEnd():
el=dsi.getElement()
if el:
if myfilter.isAcceptable(el):
vals[i]=el.getY()
lasty=vals[i]
lasti=i
else:
while not dsi.atEnd():
el=dsi.getElement()
if myfilter.isAcceptable(el):
nexty=el.getY()
nexti=i
if not firstmissing: # no interpolation at begin or end of record
for ii in range(lasti,nexti):
vals[ii]=lasty + (ii-lasti)*(nexty-lasty)/(nexti-lasti)
vals[nexti]=nexty # this one gets filled even at beginning of record.
firstmissing=0
break
else:
i=i+1
dsi.advance()
#
else:
if not firstmissing: break
#
#
dsi.advance()
i=i+1
#
rts = RegularTimeSeries(data.getName(),
data.getStartTime().toString(),
data.getTimeInterval().toString(),vals)
return rts
import sys
def gCalcFlatQ(ndo, beta, g0,out = "inst"):
""" Calculates antecedent outflow from ndo based on the flat ndo
assumption in the g documentation. In this case, the integration of g is exact
rather than numeric, but the approximation to ndo is a series of flat lines. In the
case of daily data this is probably acceptable. In the case of monthly data it leads
to large errors, though it is common
Arguments:
ndo: a regular time series. Must be 1DAY, 1MONTH
g0: initial condition. If g0 is not given it is equal to ndo at the first time step.
beta: g-model parameter.
out: must be "inst" to calculate instantaneous values of g or "ave" to calculate averages over
the time step.
Output:
g: a regular time series, same sampling rate as input
with the same start time as ndo, ending at the end of ndo or the first piece of bad
data in ndo.
"""
if ndo.getTimeInterval().toString() != "1DAY" | ndo.getTimeInterval().toString() != "1MONTH":
raise "Time step for input must be 1DAY or 1MONTH"
dsi=ndo.getIterator()
nstep=ndo.size()
g=zeros(nstep,'d')
g=map(lambda x: -901.0, g)
bdt=beta/dt
if g0==None:
g[0]=q[0]
else:
g[0]=g0
atend=0
i=1
if out[:4] == "inst":
while atend == 0:
el=dsi.getElement()
if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
q=el.getY()
g[i] = q/(1+(q/g[i-1] -1)*exp(-q/bdt))
i=i+1
if not dsi.atEnd():
dsi.advance()
else:
atend=1
else:
atend=1
elif out[:3]=="ave":
while atend == 0:
el=dsi.getElement()
if Constants.DEFAULT_FLAG_FILTER.isAcceptable(el):
q=el.getY()
g[i] = q/(1+(q/g[i-1] -1)*exp(-q/bdt))
i=i+1
if not dsi.atEnd():
dsi.advance()
else:
atend=1
else:
atend=1
else:
raise "Argument out must be either \"inst\" or \"ave\")"
rts = RegularTimeSeries("gcalc",
ndo.getStartTime().toString(),
ndo.getTimeInterval(),g)
if out[:4] == "inst":
raise "dummy exception"
if out[:3]=="ave":
rts.getAttributes().setXType("PER-VAL")
return rts
def ECEst(stage, ndo, beta, npow1, npow2,g0=None, zrms=None, c=getMTZCoef()):
""" Estimate 15min EC at the boundary.
Inputs:
stage astronomical tide estimate. Only 15min data are acceptable
ndo ndo estimate -- e.g., from CALSIM
"""
import interpolate
from vista.set import Units
if not isinstance(stage,RegularTimeSeries) or \
not isinstance(ndo,RegularTimeSeries):
raise "stage and ndo must be RegularTimeSeries"
if ndo.getTimeInterval().toString() == "1DAY":
ndo=interpolate.spline(ndo,"15MIN",0.5) << 95
elif ndo.getTimeInterval().toString()!= "15MIN":
raise "ndo must be a one day or 15 minute series"
if not stage.getTimeInterval().toString() == "15MIN":
raise "stage must be an hourly or 15 minute series"
#
if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString():
raise "stage and ndo must have the same window"
#
#if not len(c) ==9:
# raise "Wrong number (%s) of coefficients in the array c" % len(c)
if (first_missing(ndo)) >= 0:
raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing(ndo)
if (first_missing(stage)) >= 0:
raise "missing data not allowed in stage. First missing data at index: %s" % first_missing(stage)
newstart=ndo.getStartTime() - "21HOURS"
newend =ndo.getEndTime() - "3HOURS"
if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() > 0):
print "Stage record starts %s and NDO starts %s" % (stage.getStartTime().toString(),ndo.getStartTime().toString())
raise "stage record must begin at least 21 hours before ndo"
if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0):
raise "stage record must end no more than 3 hours before end of ndo"
ztw=timewindow(newstart.toString()+' - '+newend.toString())
z=stage.createSlice(ztw)
g=gCalc(ndo,beta,g0)
zarr=z.getYArray()
giter=g.getIterator()
ec=zeros(g.size(),'d')
ec=map(lambda x: -901.0, ec)
so=32797 #ocean salinity parameter for gmodel
sb=200 #river salinity parameter for gmodel
zrmsiter=zrms.getIterator()
i=0
while (not giter.atEnd()):
gval=giter.getElement().getY()
zrmsval=zrmsiter.getElement().getY()
ecfrac = c[0] + gval*c[1] + 1.1*gval**npow1*(c[2]*zarr[i+72] + c[3]*zarr[i+60]
+ c[4]*zarr[i+48] + c[5]*zarr[i+36]
+ c[6]*+zarr[i+24] + c[7]*zarr[i+12] + c[8]*zarr[i])-0.1
#+ c[9]*zrmsval
ec[i]=max(200,exp(ecfrac)*(so-sb) + sb)
#ec[i]=max((1+ecfrac)*(so-sb)+sb,200)
zrmsiter.advance()
giter.advance()
i=i+1
# ec needs to be set umhos/cm
rts = RegularTimeSeries("/ECest//////",g.getStartTime().toString(),g.getTimeInterval().toString(),ec)
rts.getAttributes().setYUnits(Units.UMHOS_CM)
return [rts,gval]
def read_dss_txt(file,dssts=True,flag=False):
"""
read_dss_txt(file, dssts=True,flag=False): reads from a ascii file in dssts or dssits format
and writes out the data to the appropriate dss file and pathnames.
If dssts == True then dssts format is assumed in the file else dssits format is assumed
For more info look up doc on read_dssts(file) and read_dssits(file)
If flag == True then it expects a flag value in the ascii file as well
The flag values in the ascii file are represented as
flag_type|user_name
where flag_type is one of UNSCREENED,QUESTIONABLE,MISSING,REJECT,OK
& user_name is one of the authorized users/agency.
e.g. MISSING|nsandhu or REJECT|kate
"""
import string
tf = TimeFactory.getInstance()
f = open(file)
line = f.readline()[:-1]
dssfile = line
while line :
try :
line = string.upper(f.readline()[:-1])
if line == "FINISH": break;
path = Pathname.createPathname(string.upper(line))
except :
print 'Incorrect format for path: ', line
break
try :
line = f.readline()[:-1]
units = string.upper(line)
except :
print 'Incorrect format for units: ', line
break
try :
line = f.readline()[:-1]
type = string.upper(line)
except :
print 'Incorrect format for type: ', line
break
if dssts:
try :
line = f.readline()[:-1]
stime = tf.createTime(line)
except :
print 'Incorrect format for time: ', line
break
#
line = string.upper(f.readline()[:-1])
xvals = []
yvals = []
if flag:
flags = []
else:
flags = None
while line != "END" :
if dssts:
try:
if flag:
vals = string.split(line)
if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line)
yvals.append(float(vals[0]))
flags.append(make_flag_value(vals[1]))
else: # no flags
yvals.append(float(line))
except:
yvals.append(Constants.MISSING_VALUE);
if flag: flags.append(make_flag_value('MISSING|null'))
else :
try :
line = string.strip(line)
tmstr = line[0:14]
tm = tf.createTime(tmstr)
try :
if flag:
vals = string.split(line[14:])
if len(vals) != 2: raise "No flags in file %s @ line: %s"%(file,line)
val = float(vals[0])
flag_val = vals[1]
else: # no flag
val = float(line[14:])
except :
val = Constants.MISSING_VALUE
xvals.append(tm)
yvals.append(val)
flags.append(make_flag_value(flag_val))
except Exception, exc:
print exc
print "Error reading line: ", line
line = string.upper(f.readline()[:-1])
# create appropriate time series object
if len(yvals) == 0 : continue
#print yvals, flags
if ( dssts ):
attr = DataSetAttr(DataType.REGULAR_TIME_SERIES,"TIME",units,"",type)
ts = RegularTimeSeries("",repr(stime),path.getPart(Pathname.E_PART),
yvals, flags, attr)
else :
attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES,"TIME",units,"",type)
ts = IrregularTimeSeries("",xvals, yvals, flags, attr)
def retrieve_ts(station,sensor,\
start_date, end_date='now',_verbose=1):
"""
retrieve(station,sensor, start_date, end_date='now',_verbose=1)
Retrieves data from cdec station and sensor and
constructs a regular time series with given pathname, units.
The start_date is a starting time value in format mm/dd/yyyy
e.g. 15-Jun-2000 would be 06/15/2000
The end_date can either be the keyword "now" or the same format as
the start_date
verbose=1 sets it to more verbose and 0 for silence
"""
_debug = 0
station_name = station.id
sensor_number = sensor.sensor_number
c_part = "%s_%s" % (sensor.type, sensor.subType)
f_part = "SENSOR %s" % (sensor.id)
pathname = '/%s/%s/%s//%s/%s/' % ('CDEC-RAW', station_name, c_part,
DSS_INTERVAL[sensor.duration],
"SENSOR " + sensor.id)
units = sensor.units
dur_code = sensor.getDurationCode()
c_url = URL('http://cdec.water.ca.gov/cgi-progs/queryCSV?' +
'station_id=' + str(station_name) + '&dur_code=' +
str(dur_code) + '&sensor_num=' + str(sensor_number) +
'&start_date=' + start_date + '&end_date=' + end_date)
if _verbose:
print "station name:%s & sensor number:%s " % (station_name,
sensor_number)
print "dur_code:%s " % (dur_code)
print "pathname:%s & units:%s" % (pathname, units)
print 'URL: %s' % c_url
lr = LineNumberReader(
InputStreamReader(c_url.openConnection().getInputStream()))
# jump all the way to data
lr.readLine()
lr.readLine()
lr.readLine()
# create starting date and time and of the format the
# data is at cdec
line = lr.readLine()
tf = TimeFactory.getInstance()
starray = string.split(line, ',')
dtm = starray[0] + ' ' + starray[1]
tp = tf.createTime(dtm, 'yyyyMMdd HHmm')
tp = tf.createTime(tp.getTimeInMinutes())
ti_str = '1HOUR'
irreg = 0 # indicates the data as irregular time series
if dur_code == 'E':
irreg = 1
ti_str = 'IR-DAY'
elif dur_code == 'D':
ti_str = '1DAY'
elif dur_code == 'H':
ti_str = '1HOUR'
elif dur_code == 'M':
ti_str = '1MON'
if not irreg:
ti = tf.createTimeInterval(ti_str)
ltime = tp
if _debug: print line, tp, dtm
yvals = []
if irreg:
tvals = []
if _verbose: print 'Data starting at ', tp
# read in all the data and append missing values if no data
# is available
while line != None:
st = StringTokenizer(line, ',')
if (st.countTokens() != 3):
raise "Invalid CDEC format, need 3 tokens on line: " + line
# get time
ctime = tf.createTime(st.nextToken() + ' ' + st.nextToken(),
'yyyyMMdd HHmm')
# if time is not in increasing order -> quit!
if ctime.compare(ltime) < 0:
raise "Invalid time sequence: %s followed by %s"+\
" ? -> should be always increasing"%(str(ctime),str(ltime))
# check if current time is only one time interval from last time
if not irreg:
nskip = ltime.getNumberOfIntervalsTo(ctime, ti)
# if skip is greater than one then fill with -901's
while nskip > 1:
yvals.append(Constants.MISSING_VALUE)
nskip = nskip - 1
ltime = ctime
# now get current value
val_str = st.nextToken()
try:
if (val_str == "m"): # if missing
val = Constants.MISSING_VALUE
else: # else just save the value
val = float(val_str)
if irreg:
tvals.append(ctime)
yvals.append(val)
except:
tvals.append(ctime)
yvals.append(Constants.MISSING_VALUE)
print "Exception! for string ", val_str
line = lr.readLine()
# create a time series data set from the array of values,
# the start time and the time interval
if irreg:
attr = DataSetAttr(DataType.IRREGULAR_TIME_SERIES, '', units, 'TIME',
'INST-VAL')
its = IrregularTimeSeries(pathname, tvals, yvals, None, attr)
return its
else:
attr = DataSetAttr(DataType.REGULAR_TIME_SERIES, '', units, 'TIME',
'INST-VAL')
rts = RegularTimeSeries(pathname, tp.toString(), ti_str, yvals, None,
attr)
return rts
