def _retrieve_ts(book, selection, time_column, **args): """ private helper funcs to retrieve mixed its from excel. """ rvar = _parse_range(selection) sheet = book.Worksheets(rvar[0]) if "header_labels" in args.keys(): header_labels = args["header_labels"] else: header_labels = None (header_lst,data_start_row)=_retrieve_headers_by_tryerror\ (sheet,rvar,header_labels) data_range = sheet.Range(rvar[1] + str(data_start_row) + ":" + rvar[3] + str(data_start_row)) ts_time_data = data_range.Value num_ts = len(ts_time_data[0]) / 2 tsl = [] t1 = sheet.Range(rvar[1] + str(data_start_row)) col = t1.Columns[0].Column t2 = sheet.Cells(rvar[4], col) d1 = sheet.Cells(data_start_row, col + 1) d2 = sheet.Cells(rvar[4], col + 1) for i in range(0, num_ts): ts_time = sheet.Range(t1, t2).Value ts_data = sheet.Range(d1, d2).Value ts_time = array(ts_time).flatten() ts_data = array(ts_data).flatten() ## removing possilbe trailing none ts_time = _strip_ending_none(ts_time) ts_data = ts_data[0:len(ts_time)] ts_time = map(_pytime2datetime, ts_time) if header_lst: ts = its(ts_time, ts_data, header_lst[i]) else: ts = its(ts_time, ts_data) tsl.append(ts) col = col + 2 t1 = sheet.Cells(data_start_row, col) t2 = sheet.Cells(rvar[4], col) d1 = sheet.Cells(data_start_row, col + 1) d2 = sheet.Cells(rvar[4], col + 1) if len(tsl) == 1: return tsl[0] else: return tsl
def interpolate_ts(ts, times, method=LINEAR, filter_nan=True, **dic): """ Interpolate a time series to a new time sequence. Parameters ----------- ts : :class:`~vtools.data.timeseries.TimeSeries` Series to interpolate. Must has data of one dimension, regular or irregular. times : :ref:`time_interval <time_intervals>` or :ref:`time_sequence <time_sequence>` The new times to which the series will be interpolated. Can also be a string that can be parsed into a time interval. method : string, optional See :func:`interpolate_ts_nan` for a list of methods filter_nan : boolean, optional True if nan should be omitted or not. If retained, nan values in the source series will be used in interpolation algorithm, which may cause new nan points in resulting ts. **dic : dictonary Extra parameters. Returns ------- result : :class:`~vtools.data.timeseries.TimeSeries` A regular or irregular series with times based on times and values interpolated from ts. See Also -------- interpolate_ts_nan : Fill internal nan values using interpolation """ from vtools.data.api import TimeSeries if type(times) == TimeSeries: if times.start < ts.start or times.end > ts.end: raise ValueError( "Time series used to provide requested interpolation times is outside the original series" ) interval = times.interval if times.is_regular() else None seq = times.times data=_interpolate_ts2array(ts,seq,method=method,\ filter_nan=filter_nan,**dic) if interval: return rts(data, seq[0], interval) else: return its(times, data) elif not (type(times) == str) and isSequenceType(times): data=_interpolate_ts2array(ts,times,method=method,\ filter_nan=filter_nan,**dic) ts = its(times, data, {}) else: # interval ts=_interpolate2rts(ts,times,method=method,\ filter_nan=filter_nan,**dic) return ts
def test_save_data(self): ## save some ts into dss file, ts may contain ## header. ## save rts first. data = range(1000) start = "12/21/2000 2:00" interval = "1hour" prop = {} prop[TIMESTAMP] = PERIOD_START prop[AGGREGATION] = MEAN prop["datum"] = "NGVD88" prop["manager"] = "John Doe" prop["model"] = "hydro 7.5" rt1 = rts(data, start, interval, prop) id = "vtools.datastore.dss.DssService" path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/" source = self.test_file_path data_ref = DataReference(id, source=source, selector=path) self.dss_service.add_data(data_ref, rt1) dssc = self.dss_service.get_catalog(source) path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/" data_ref = dssc.data_references(path).next() rtt = self.dss_service.get_data(data_ref) self.assertTrue(len(rtt) == len(data)) self.assertTrue(rtt.props[TIMESTAMP] == PERIOD_START) self.assertTrue(rtt.props[AGGREGATION] == MEAN) self.assertTrue(rtt.times[0], dtm.datetime(2000, 12, 21, 2)) extent = "time_window=(12/21/2000 02:00,01/31/2001 18:00)" data_ref = DataReference(id, source, None, path, extent) rtt2 = self.dss_service.get_data(data_ref) self.assertTrue(rtt.start == rtt2.start) self.assertTrue(rtt.end == rtt2.end) ## then its. path = "/HERE/IS/ITS//IR-YEAR/TEST/" data = range(20) data_ref = DataReference(id, source=source, selector=path) prop[AGGREGATION] = INDIVIDUAL times=["01/15/1997","02/17/1997","03/5/1997",\ "04/25/1997","05/1/1997","06/15/1997",\ "07/25/1997","08/14/1997","09/17/1997",\ "10/15/1997","11/21/1997","12/3/1997",\ "01/9/1998","02/15/1998","03/19/1998",\ "04/15/1998","05/19/1998","06/30/1998",\ "07/15/1998","08/24/1998"] times = map(parse_time, times) itt = its(times, data, prop) self.dss_service.add_data(data_ref, itt) extent = "time_window=(1/10/1997 02:00,09/30/1998 18:00)" data_ref = DataReference(id, source, None, path, extent) rtt3 = self.dss_service.get_data(data_ref) self.assertTrue(parse_time("01/15/1997") == rtt3.start) self.assertTrue(parse_time("08/24/1998") == rtt3.end)
def dss_its_to_ts(data,jbdate,itimes,prop,flags): """ Convert dss irregular time series into TimeSeries class. data: list of raw data returned by zrits or zritsx jbadte: integer dss base date returned by zrits or zritsx itimes: offset list in minutes from base date for each time point it is also generated by zritx or zritsx prop: dict save info get from dss func flags: list of data quality flag returned by zritx or zritsx """ if len(itimes)<len(data): raise ValueError("lenght of input itimes must not be less than data") (data,start_i)=_validate_dss_data_series(data,flags) itimes=itimes[start_i:start_i+len(data)] ts=None new_times=[] for itime in itimes: a_time=_dss_julian_datetime_to_python_datetime(jbdate,itime) new_times.append(a_time) new_times.sort() prop[TIMESTAMP]=INST prop[AGGREGATION]=INDIVIDUAL ts= its(new_times[start_i:start_i+len(data)],data,prop) return ts
def ts_split(ts, shared=True): """ Splits a 2D multivariate series into constituent univariate series. Parameters ---------- ts : :class:`~vtools.data.timeseries.TimeSeries` shared :: Boolean Return time sereis share or copy data array of input one Returns ------- out1,out2 : :class:`~vtools.data.timeseries.TimeSeries` Two comonent time series. """ if ts.data.ndim > 2: raise ValueError("Only 2D arrays can be split") if shared: dsource = ts.data else: dsource = ts.data.copy() ncol = dsource.shape[1] out = [] for jcol in range(ncol): if ts.is_regular(): colts = rts(dsource[:, jcol], ts.start, ts.interval) else: colts = its(ts.ticks, dsource[:, jcol]) out.append(colts) return tuple(out)
def make_depth_average(self, time_basis): """ Make depth averaged values from numpy array of outputs Parameters ---------- time_basis: datetime.datetime time base of the outputs Returns ------- lists of a set vtools.data.timeseries of depth and depth-averaged values For scalars, the list has only one set of time series. For vectors, the list has two sets of time series. Each time series has multiple columns of data, and each column is for stations. """ nvrt = self._nvrt n_casts = self._n_casts # collect time stamp first times = list() output = self._outputs[0] for cast_i in range(n_casts): i_begin = cast_i * nvrt time = time_basis + time_interval(days=output[i_begin, 0]) times.append(time) # collect data values = list() for output in self._outputs: values_at_point = list() depths_at_point = list() for xy_i in range(self._nxy): depths_at_cast = list() values_at_cast = list() for cast_i in range(n_casts): i_begin = cast_i * nvrt + xy_i * (n_casts * nvrt) depth = -output[i_begin + nvrt - 1, 2] x = -output[i_begin:i_begin + nvrt, 2] y = output[i_begin:i_begin + nvrt, 1] avg = scipy.integrate.simps(y, x) / depth depths_at_cast.append(depth) values_at_cast.append(avg) depths_at_point.append(depths_at_cast) values_at_point.append(values_at_cast) ts_depths = its(times, numpy.array(depths_at_point).transpose()) ts_values = its(times, numpy.array(values_at_point).transpose()) values.append((ts_depths, ts_values)) return values
def _bind(ts1, ts2): """ bind data from timeseries ts1 and ts2. Parameters ---------- ts1,ts2 : :class:`~vtools.data.timeseries.TimeSeries` Two timeseries Returns ------- merged : :class:`~vtools.data.timeseries.TimeSeries` A new binded time series if success. """ if (not ((ts1.data.ndim == 1) and (ts2.data.ndim == 1))): raise ValueError("bind only support time series of univariate") ts = None ts_is_regular = False new_ts_time_sequence = [] new_start = None new_interval = None if ((ts1.is_regular()) and (ts2.is_regular())): ts1_start = ts1.times[0] ts1_end = ts1.times[-1] ts2_start = ts2.times[0] ts2_end = ts2.times[-1] new_start = ts1_start if new_start > ts2_start: new_start = ts2_start new_end = ts1_end if new_end < ts2_end: new_end = ts2_end new_interval = ts1.interval ts2_interval = ts2.interval if new_interval > ts2_interval: new_interval = ts2_interval num_data = number_intervals(new_start, new_end, new_interval) + 1 new_ts_time_sequence = time_sequence(new_start, new_interval, num_data) ts_is_regular = True else: new_ts_time_sequence = np.union1d(ts1.ticks, ts2.ticks) new_ts_len = len(new_ts_time_sequence) new_data = np.array([[np.nan] * new_ts_len, [np.nan] * new_ts_len]) ts1_data_id = np.searchsorted(new_ts_time_sequence, ts1.ticks) ts2_data_id = np.searchsorted(new_ts_time_sequence, ts2.ticks) new_data[0, ts1_data_id] = ts1.data new_data[1, ts2_data_id] = ts2.data new_data = new_data.transpose() if ts_is_regular: ts = rts(new_data, new_start, new_interval) else: ts = its(new_ts_time_sequence, new_data) return ts
def test_shift_operation_its(self): ts_len = 1000 data = repeat(10.0, ts_len) ts_start = datetime(year=1990, month=2, day=3, hour=11, minute=15) times = [ts_start] * ts_len for i in range(1, ts_len): times[i] = times[i - 1] + time_interval(hours=i % 5) ts0 = its(times, data, {}) test_input = [hours(1), months(1), days(3), years(1)] for shift_interval in test_input: ts = its(times, data) ts = shift(ts, shift_interval) t = ts0.times + shift_interval for a1, b1 in zip(t, ts.times): self.assertEqual(a1, b1)
def test_bind_op_irregular(self): """ Test behaviour of bind operation on irregular TS.""" times=[12,15,32,38,43,52,84,138,161,172] #times=sciadd.accumulate(times) start_datetime = parse_time("1996-2-1") start_ticks = ticks(start_datetime) times=scimultiply(times,ticks_per_minute) times=sciadd(times,start_ticks) data=sciarray([1.0,1.0,1.0,1.0,1.0,1.0,2.0,3.0,3.0,3.0]) ts1=its(times,data,{}) ts2=its(times,data,{}) new_ts = ts_bind(ts1,ts2) self.assertEqual(len(new_ts),len(ts1)) self.assertEqual(new_ts.start,ts1.start) for (d1,d2),d in zip(new_ts.data,data): self.assertEqual(d1,d) self.assertEqual(d2,d)
def ts_accumulate(ts, ufunc): """Apply ufunc.accumulate to the data and produce a neatened time series The function will be applied cumulatively to the data. So... ts_apply(ts, add) will produce a time series, where each entry is the cumulative sum up to that index. """ if ts.is_regular(): return rts(ufunc.accumulate(ts.data), ts.start, ts.interval, ts.props) else: return its(ts.ticks, ufunc.accumulate(ts.data), ts.props)
def create_its(self): """ Only create a irregular time series for usage of testing. """ times = self.create_irregular_timesequence(2000, 1) self.its1_years = 2 num = len(times) data=[random.uniform(self.min_val,self.max_val) \ for k in range(num)] ts = its(times, data, {}) return ts
def _retrieve_irregularTS(self, data_ref, overlap=None): """ Retrieve irregular time sereis referenced by data_re. An instance of class TimeSereis is returned. """ path = data_ref.selector dss_file_path = data_ref.source dssf = open_dss(dss_file_path) juls, jule, istime, ietime = self._gen_its_jultime(data_ref) # Max number of vals can be retrived one time, this consant is defined in dss_constants.py kval = DSS_MAX_ITS_POINTS lflags = True kheadu = DSS_MAX_HEADER_ITEMS if (overlap == (0, 0)) or (overlap is None): inflag = int(0) elif (overlap == (1, 1)): inflag = int(3) elif (overlap == (1, 0)): inflag = int(1) else: inflag = int(2) (itimes,data,nval,jbdate,flags,lfread,cunits,ctype,headu,nheadu,istat)= \ dssf.zritsx(path,juls,istime,jule,ietime,kval,lflags,kheadu,inflag) del dssf ## add those props. prop = {} #prop[CTYPE]=ctype prop[UNIT] = cunits if istat == 5: raise DssAccessError("no record of %s is found" % path) if istat > 5: raise DssAccessError("error in access data of %s" % path) if istat == 4: ##no data found,return a 0 len its return its([], [], prop) if (nval == kval): message ="Input time window contains more data than equals or exceeds the size of the cache used to retrieve "\ "irregular time series at path %s. You may try to increase the size of irregular time series "\ "cache by changing the value of DSS_MAX_ITS_POINTS defined in the file dss_constants.py under vtools\dastastore\dss." % path raise ValueError(message) if nheadu > 0: hdic = self._unstuff_header(headu, nheadu, 2) for key in hdic.keys(): if not ((key == UNIT) or (key == CTYPE)): prop[key] = hdic[key] data = dss_its_to_ts(data, jbdate, itimes, prop, flags) return data
def ts_apply(ts, ufunc, other=None): """Apply a numpy ufunc to the data and produce a neatened time series The function will be applied pointwise to the data. The argument other must be a scalar and serves as the second argument to a binary operator or function. So... ts_apply(ts, add, 5) will add 5 to every data member """ if not (other is None): ndata = ufunc(ts.data, other) else: ndata = ufunc(ts.data) if ts.is_regular(): return rts(ndata, ts.start, ts.interval, ts.props) else: return its(ts.ticks, ndata, ts.props)
def test_flat(self): """ Test the behavior of flat interpolation. """ data = [sin(i * 3.14 / 3) for i in range(20)] times = [ 1, 3, 7, 14, 19, 29, 35, 41, 42, 44, 60, 61, 67, 69, 72, 76, 77, 79, 89, 90 ] data[4] = nan data[5] = nan ts = its(times, data) ## Interpolation time locations. st = [2, 9, 19, 20, 29, 34, 50, 72.4] for func in [nearest_neighbor, previous_neighbor, next_neighbor]: irt = func(ts, st) for vv in irt.data: self.assertTrue(vv in ts.data)
def test_split_op_irregular(self): """ Test behaviour of split operation on irregular TS.""" times=[12,15,32,38,43,52,84,138,161,172] #times=sciadd.accumulate(times) start_datetime = parse_time("1996-2-1") start_ticks = ticks(start_datetime) times=scimultiply(times,ticks_per_minute) times=sciadd(times,start_ticks) data1=sciarray([1.0,1.0,1.0,1.0,1.0,1.0,2.0,3.0,3.0,3.0]) data2=sciarray([7.0,1.2,10.5,3.0,1.0,1.0,9.0,3.0,0.0,0.2]) ts = its(times,sciarray([data1,data2]).transpose()) ts1,ts2 =ts_split(ts,False) for d1,d2 in zip(ts.data[:,0],ts1.data): self.assertEqual(d1,d2) for d1,d2 in zip(ts.data[:,1],ts2.data): self.assertEqual(d1,d2) ts1.data[5] = -9999.0 ts2.data[2] = -9999.0 self.assertNotEqual(ts1.data[5],ts.data[5,0]) self.assertNotEqual(ts2.data[2],ts.data[2,1]) for t1,t2 in zip(ts1.times,ts.times): self.assertEqual(t1,t2) for t1,t2 in zip(ts2.times,ts.times): self.assertEqual(t1,t2) ts1,ts2 =ts_split(ts,True) ts1.data[5] = -9999.0 ts2.data[2] = -9999.0 for d1,d2 in zip(ts.data[:,0],ts1.data): self.assertEqual(d1,d2) for d1,d2 in zip(ts.data[:,1],ts2.data): self.assertEqual(d1,d2) for t1,t2 in zip(ts1.times,ts.times): self.assertEqual(t1,t2) for t1,t2 in zip(ts2.times,ts.times): self.assertEqual(t1,t2)
def shift(ts, interval, copy_data=True): """ Shift entire time series by a given interval Parameters ---------- ts : :class:`~vtools.data.timeseries.TimeSeries` A regular timeseries to be shifted. interval : :ref:`time_interval <time_intervals>` Interval of the shifting. copy_data : boolean,optional If True, the result is an entirely new series with deep copy of all data and properties. Otherwise, it will share data and properties. Returns ------- shifted : :class:`~vtools.data.timeseries.TimeSeries` A new time series with shifted times. """ if copy_data: new_data = numpy.copy(ts.data) new_props = deepcopy(ts.props) else: new_data = ts.data new_props = ts.props if not is_interval(interval): interval = parse_interval(interval) if ts.is_regular(): return rts(new_data, increment(ts.start, interval, 1), ts.interval, new_props) else: if is_calendar_dependent(interval): tms = ts.times + interval else: tms = ts.ticks + ticks(interval) # ts._ticks=scipy.array(map(ticks,time_op(ts.times,interval))) return its(tms, new_data, new_props)
def test_save_its_data(self): """ test adding a its to source.""" ## save some data. dssfile_path=self.dss_file_path path="/HERE/IS/ITS//IR-YEAR/TEST/" data=[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0, \ 10.0,11.0,12.0,13.0,14.5,15.1,16.8,\ 17.2,14.2,19.2,20.0] times=["01/15/1997","02/17/1997","03/5/1997",\ "04/25/1997","05/1/1997","06/15/1997",\ "07/25/1997","08/14/1997","09/17/1997",\ "10/15/1997","11/21/1997","12/3/1997",\ "01/9/1998","02/15/1998","03/19/1998",\ "04/15/1998","05/19/1998","06/30/1998",\ "07/15/1998","08/24/1998"] times =map(parse,times) props={} ts = its(times,data,props=props) dss_store_ts(ts,dssfile_path,path)
def test_retrievesave_longits(self): ## save some ts into dss file, ts may contain ## header. ## create rts first, and use it ticks and ## data to create its total_len = 700000 its_len = 40000 ## 80000 will cause zsitx fail,for dss interal ## cache size is 10000. data size ## close to this number will cause dss lib abort data = range(total_len) start = "12/21/1940 2:00" interval = "1hour" prop = {} prop[TIMESTAMP] = INST prop[AGGREGATION] = INDIVIDUAL prop["datum"] = "NGVD88" prop["manager"] = "John Doe" prop["model"] = "hydro 7.5" rt1 = rts(data, start, interval, prop) id = "vtools.datastore.dss.DssService" path = "/TEST/DOWNSTREAM/EC//1HOUR/SRT/" source = self.test_file_path data_ref = DataReference(id, source=source, selector=path) self.dss_service.add_data(data_ref, rt1) path = "/TEST/DOWNSTREAM/EC//IR-DAY/STAGE/" data_ref = DataReference(id, source=source, selector=path) i = 0 start = i * its_len end = (i + 1) * its_len it1 = its(data[start:end], rt1.ticks[start:end], prop) self.dss_service.add_data(data_ref, it1)
def test_interpolate_rts_to_larger_dt(self): """ Test interpolating a fine ts to a coarser one """ import numpy as np import datetime as dt ts = rts(np.arange(100.), dt.datetime(2000, 1, 1, 0, 6), minutes(6)) ts1 = interpolate_ts(ts, hours(1), method=LINEAR) self.assertTrue(ts[9].value == ts1[0].value) ts2 = interpolate_ts(ts, ts1, method=LINEAR) self.assertTrue(ts1[3].value == ts2[3].value) its1 = its(ts1.times, ts1.data) ts3 = interpolate_ts(ts, its1, method=LINEAR) self.assertTrue(ts1[3].value == ts3[3].value) ts = rts(np.arange(98.), dt.datetime(2000, 1, 1, 0, 3), minutes(6)) ts2 = interpolate_ts(ts, hours(1), method=LINEAR) self.assertTrue(ts2[0].value == 9.5) #self.assertTrue(rt.is_regular()) #self.assertEqual(rt.interval,parse_interval(interval)) ts_too_wide = rts(np.arange(120.), dt.datetime(1999, 12, 31, 23), hours(1)) self.assertRaises(ValueError, interpolate_ts, ts, ts_too_wide)
def read(self, fpath, start=None, end=None, force_regular=True, selector=None): """ Read a text file with the given pattern and parsers. Parsers and a pattern must be defined and set in the child class. Parameters ---------- fpath: str file to read start: datetime.datetime, optional datetime to start reading in. If None, read from the start of the file end: datetime.datetime, optional datetime to finish reading in. If None, read till the end of the file force_regular: boolean, optional If it is true, it returns a regular time series Returns ------- vtools.data.timeseries.TimeSeries time series from the file """ # The selector (if it exists) can probably be precalculated or at least recorded. # Almost always this amounts to picking variables out of a list of column names # and recording indexes, but here we don't ask any questions about what "selector" is. n_headerlines, metadata = self.process_header(fpath, selector) metadata = dict() if not self._header_regexs is None: metadata = self.read_metadata_from_header(fpath) print "Here we are working on %s" % fpath with open(fpath, 'r') as f_in: times = list() values = list() # fast forward past header if n_headerlines > 0: for i in range(n_headerlines): f_in.readline() # process lines starting from current file pointer for i, line in enumerate(f_in): if self.is_comment(line): continue timestamp, vals = self.parse_record(line) if start and timestamp < start: continue if end and timestamp > end: break times.append(timestamp) values.append(vals) if len(times) < 1: return None arr = numpy.array(values) # Here I assume that it is more effective to retrieve too much # in the reading stage and then do this with numpy fancy indexing. # I But you can override this function arr = self.cull_using_selector(arr) ts = vts.its(times, numpy.array(values)) if force_regular: interval = vt_infer_interval(times[:11], fraction=0.5, standard=[ vtt.minutes(6), vtt.minutes(10), vtt.minutes(15), vtt.hours(1) ]) if not interval: for t in times[:10]: print t.strftime("%Y-%m-%d %H:%M:%S") raise ValueError( "Interval could not be inferred from first time steps in %s" % fpath) import warnings # todo: this really should be an option with warnings.catch_warnings(): warnings.simplefilter("ignore") ts = vts.its2rts(ts, interval) if start is not None: if start < ts.start: ts = vts.extrapolate_ts(ts, start=start) else: ts = ts.window(start=start) if end is not None: if end > ts.end: ts = vts.extrapolate_ts(ts, end=end) else: ts = ts.window(end=end) for k, v in metadata.iteritems(): ts.props[k] = v return ts
def test_period_op_irregular(self): """ Test behaviour of period operation on irregular TS.""" times = [12, 15, 32, 38, 43, 52, 84, 138, 161, 172] #times=sciadd.accumulate(times) start_datetime = parse_time("1996-2-1") start_ticks = ticks(start_datetime) times = scimultiply(times, ticks_per_minute) times = sciadd(times, start_ticks) data = sciarray([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0]) ts = its(times, data, {}) op = MEAN ts_op = period_op(ts, "1 hour", op) self.assertEqual(len(ts_op), 3) self.assertEqual(ts_op.data[0], 1.0) self.assertEqual(ts_op.data[1], 2.0) self.assertEqual(ts_op.data[2], 3.0) times = [0, 15, 32, 38, 43, 52, 60, 120, 138, 161, 180] data = sciarray( [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0, 4.0]) times = scimultiply(times, ticks_per_minute) times = sciadd(times, start_ticks) ts = its(times, data, {}) op = MEAN ts_op = period_op(ts, "1 hour", op) self.assertEqual(len(ts_op), 4) self.assertEqual(ts_op.data[0], 1.0) self.assertEqual(ts_op.data[1], 2.0) self.assertEqual(ts_op.data[2], 3.0) self.assertEqual(ts_op.data[3], 4.0) data = sciarray( [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 2.0, 3.0, 4.0, 5.0, 4.0]) ts = its(times, data, {}) op = MIN ts_op = period_op(ts, "1 hour", op) self.assertEqual(len(ts_op), 4) self.assertEqual(ts_op.data[0], 1.0) self.assertEqual(ts_op.data[1], 2.0) self.assertEqual(ts_op.data[2], 3.0) self.assertEqual(ts_op.data[3], 4.0) op = MAX ts_op = period_op(ts, "1 hour", op) self.assertEqual(len(ts_op), 4) self.assertEqual(ts_op.data[0], 6.0) self.assertEqual(ts_op.data[1], 2.0) self.assertEqual(ts_op.data[2], 5.0) self.assertEqual(ts_op.data[3], 4.0) times = [0, 15, 28, 30, 58, 64, 80, 90, 91] start_datetime = parse_time("1996-1-1") start_ticks = ticks(start_datetime) times = scimultiply(times, ticks_per_day) times = sciadd(times, start_ticks) data = sciarray([1.0, 1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0, 4.0]) ts = its(times, data, {}) op = MEAN ts_op = period_op(ts, "1 month", op) self.assertEqual(len(ts_op), 4) self.assertEqual(ts_op.data[0], 1.0) self.assertEqual(ts_op.data[1], 2.0) self.assertEqual(ts_op.data[2], 3.0) self.assertEqual(ts_op.data[3], 4.0)
def _flat(ts_data, ts_ticks, tticks, method=NEAREST): """ Estimate timeseries values at given times by flat methods use previous known value, next known vlaue, and nearest known value to fill missed data. Parameters ----------- ts : :class:`~vtools.data.timeseries.TimeSeries` Series to be interpolated times : :ref:`time_sequence <time_sequence>` The new times to which the series will be interpolated. method: int,optional Choice of PREVIOUS,NEXT,NEAREST filter_nan:boolean,optional if true, null data points of input timeseries are omitted. **dic : Dictionary Dictonary of extra parameters to be passed in. For instance , input 'extroplate' as integer (num of interval,whether or not do a number of extra extroplation at the end of new ts data). Returns ------- result: array interpolated values. """ tss = its(ts_ticks, ts_data) ## find out index of interpolating points that exists in ## in original ts, thus orginal value can be reused in ## interpolated ts. ts_len = len(tss) index1 = tss.ticks.searchsorted(tticks) index2 = where(index1 < ts_len, index1, ts_len - 1) tticks2 = take(tss.ticks, index2) same_points_index = where(tticks2 == tticks, index1, ts_len) indexes = tss.index_after(tticks) ## if not(alltrue(greater(indexes,0))) or not(alltrue(less(indexes,len(tss.data)))): ## raise ValueError("interpolation list has time points falling out of input "+ ## "timeseries range.") ## removed the exception above, that will by default make interpolating point ## locate on boundary points using boundary values (value before take first order). indexes1 = where(indexes < 1, 1, indexes) indexes2 = where(indexes > len(tss.data) - 1, len(tss.data) - 1, indexes1) indexes = indexes2 if method == PREVIOUS: indexes = indexes - 1 elif method == NEXT: indexes = indexes else: pre = indexes - 1 after = indexes interval_to_pre = abs(tticks - take(tss.ticks, pre)) interval_to_after = abs(tticks - take(tss.ticks, after)) indexes = where(interval_to_pre <= interval_to_after, pre, after) vals1 = take(tss.data, indexes) dum_index = where(same_points_index < ts_len, same_points_index, 0) dum_val = take(tss.data, dum_index) vals2 = where(same_points_index < ts_len, dum_val, vals1) ## if is required by user, do some extra extrapolation at the ## end of new ts using last value, thus new ts will be longer # for keyword in dic.keys(): # if not(keyword=="extrapolate"): # raise TypeError("unexpected keyword %s"%keyword) # # if "extrapolate" in dic.keys(): # new_ts_len=len(vals2) # extrapolate_num=dic["extrapolate"] # dum_val2=zeros(new_ts_len+extrapolate_num) # dum_val2[0:new_ts_len]=vals2[0:new_ts_len] # last_val=vals2[new_ts_len-1] # dum_val2[new_ts_len:new_ts_len+extrapolate_num]=last_val # vals2=dum_val2 return vals2