Exemple #1
0
 def test_merge_rts_2d_intersect(self): 
     
     """ Test merging of two rts of 2-d data and has intersection."""
     
     d1=[[1.0,2.0]]*int(spy.math.pow(2,10))
     d2=[[3.0,4.0]]*int(spy.math.pow(2,11))
     
     st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
     st2=datetime.datetime(year=1990,month=3,day=3,hour=11, minute=15)
     
     ts1=rts(d1,st1,time_interval(hours=1))
     ts2=rts(d2,st2,time_interval(hours=1))
     
     ## Intentionally put some nan into ts1 to see effect.
     ii=ts1.index_after(st2)
     num_nan=10
     ts1.data[ii+1:ii+num_nan+2,]=spy.nan  
     
     nt=merge(ts1,ts2)
     
     self.assertEqual(nt.start,ts1.start)
     self.assertEqual(nt.end,ts2.end)
     
     total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
     
     self.assertEqual(len(nt.data),total_n)
     s1=nt.index_after(ts1.end)
     s2=nt.index_after(ts2.start)
     s3=ts2.index_after(ts1.end)
     self.assert_(spy.allclose(nt.data[0:ii+1,],ts1.data[0:ii+1,]))
     self.assert_(spy.allclose(nt.data[ii+1:ii+num_nan+2,],ts2.data[0:num_nan+1,]))
     self.assert_(spy.allclose(nt.data[ii+num_nan+2:s1+1,],ts1.data[ii+num_nan+2:s1+1,]))
     self.assert_(spy.allclose(nt.data[s1+1:len(nt.data),],ts2.data[s3+1:len(ts2.data),]))
Exemple #2
0
 def test_merge_rts_no_intersect(self):
     """ Test merging two rts without intersection."""
     
     d1=[1]*int(spy.math.pow(2,10))
     d2=[2]*int(spy.math.pow(2,11))
     
     st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
     st2=datetime.datetime(year=1990,month=5,day=3,hour=11, minute=15)
     
     ts1=rts(d1,st1,time_interval(hours=1))
     ts2=rts(d2,st2,time_interval(hours=1))
     nt=merge(ts1,ts2)
     
     self.assertEqual(nt.start,ts1.start)
     self.assertEqual(nt.end,ts2.end)
     
     total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
     
     self.assertEqual(len(nt.data),total_n)
     s1=nt.index_after(ts1.end)
     s2=nt.index_after(ts2.start)
     
     self.assert_(spy.alltrue(spy.isnan(nt.data[s1+1:s2])))
     self.assert_(spy.allclose(nt.data[0:s1+1],ts1.data))
     self.assert_(spy.allclose(nt.data[s2:len(nt.data)],ts2.data))
Exemple #3
0
    def test_period_op_uncompatible_interval(self):
        """ Test behaviour of period operation on TS with interval uncompatible
            with operation time interval
        """
        test_input = [
            (datetime.datetime(year=1990, month=2, day=3,
                               hour=11, minute=15), 3005,
             time_interval(minutes=45), "1hour", time_interval(hours=1)),
            (datetime.datetime(year=1990, month=2,
                               day=3, hour=11, minute=15), 3301,
             time_interval(days=1), "1hour", time_interval(hours=1)),
            (datetime.datetime(year=1990, month=1,
                               day=1, hour=00, minute=00), 10957,
             time_interval(days=2), "1 month", time_interval(months=1)),
            (datetime.datetime(year=1990, month=1, day=1,
                               hour=00, minute=00), 10957,
             time_interval(minutes=35), "3 days", time_interval(days=3)),
            (datetime.datetime(year=1990, month=1,
                               day=1, hour=00, minute=00), 59,
             time_interval(months=5), "3 years", time_interval(years=3)),
        ]

        for (st, num, delta, interval, op_delta) in test_input:
            data=[random.uniform(self.min_val,self.max_val) \
                  for k in range(num)]
            ts = rts(data, st, delta, {})
            for op in [MIN, MAX, MEAN, SUM]:
                self.assertRaises(ValueError, period_op, ts, interval, op)
Exemple #4
0
    def test_period_op3(self):
        """ Test period operation on time series with 2-dimensional data."""

        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        num = 3005
        dimension2 = 3
        delta = time_interval(minutes=5)
        interval = '1 hour'
        op_delta = time_interval(hours=1)
        aligned_start = datetime.datetime(year=1990,
                                          month=2,
                                          day=3,
                                          hour=12,
                                          minute=0)

        data=[[random.uniform(self.min_val,self.max_val) for i in range(dimension2)] \
              for k in range(num)]

        data = sciarray(data)

        # Reformalize raw data, insert known mini_val and max_val
        # and calcuate hourly mean to use later.
        i0 = 9  # this is the first index with aligned calendar
        num_interval = (num - i0 + 1) // 12

        for k in range(num_interval):
            index = i0 + k * 12 + 1
            data[index, ] = self.min_val
            index = index + 1
            data[index, ] = self.max_val

        nt_data = data[i0:12 * num_interval + i0, ]
        nt_data = sciarray(nt_data)
        nt_data.shape = (num_interval, 12, -1)

        nt_mean = sciadd.reduce(
            nt_data,
            1,
        ) / 12
        nt_sum = sciadd.reduce(
            nt_data,
            1,
        )
        nt_min = sciminimum.reduce(
            nt_data,
            1,
        )
        nt_max = scimaximum.reduce(
            nt_data,
            1,
        )

        ts = rts(data, st, delta, {})

        for (op, op_data) in [(MIN, nt_min), (MAX, nt_max), (MEAN, nt_mean),
                              (SUM, nt_sum)]:
            nt = period_op(ts, interval, op)
            assert_array_equal(nt.data,op_data,\
            err_msg="two array not equal in average" \
            " by %s"%(op))
Exemple #5
0
 def test_resample_rts_aligned(self):
     # test resampling regular time series with aligned start
     data=range(100)
     st=datetime.datetime(year=2000,month=2,day=1,hour=2,minute=15)
     delta=time_interval(minutes=15)
     ts=rts(data,st,delta,{})
     resample_interval=time_interval(hours=1)
     nts=resample(ts,resample_interval,aligned=True)
     rstart=datetime.datetime(year=2000,month=2,day=1,hour=3)
     self.assertEqual(rstart,nts.start)
Exemple #6
0
    def __init__(self,methodName="runTest"):

        super(TestMerge,self).__init__(methodName)
                         
        # Number of data in a time series.
        self.num_ts=1000
        self.max_val=1000
        self.min_val=0.01
        self.large_data_size=100000
        self.test_interval=[time_interval(minutes=30),time_interval(hours=2),
                            time_interval(days=1)]
Exemple #7
0
 def test_resample_rts_in(self):
     # Test operations on ts of with incomplete end hour
     st=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=45)
     delta=time_interval(minutes=15)
     resample_interval=time_interval(hours=1)
     data=range(-1,10)
     ts=rts(data,st,delta,{})
     nts=resample(ts,resample_interval)
     self.assert_(nts.data[2]==ts.data[9])
     self.assert_(len(nts)==3)
      
     nt2=resample(ts,resample_interval,aligned=False)
     self.assert_(nt2.data[2]==ts.data[8])
     self.assert_(len(nt2)==3)
Exemple #8
0
    def test_butterworth(self):
        """ Butterworth filter on a series of 1hour interval with four
            frequencies.
        """
        # Test operations on ts of varied values.
        test_ts=[(datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15),\
                  int(numpy.math.pow(2,10)),time_interval(hours=1)),]

        f1 = 0.76
        f2 = 0.44
        f3 = 0.95
        f4 = 1.23
        av1 = f1 * pi / 12
        av2 = f2 * pi / 12
        av3 = f3 * pi / 12
        av4 = f4 * pi / 12
        import pylab as plt
        for (st, num, delta) in test_ts:
            ## this day contains components of with frequecies of 0.76/day,
            ## 0.44/day, 0.95/day, 1.23/day
            data = [
                numpy.math.sin(av1 * k) + 0.7 * numpy.math.cos(av2 * k) +
                2.4 * numpy.math.sin(av3 * k) + 0.1 * numpy.math.sin(av4 * k)
                for k in range(num)
            ]

            # This ts is the orignial one.
            ts0 = rts(data, st, delta, {})
            ts = butterworth(ts0)
            self.assert_(ts.is_regular())
Exemple #9
0
 def test_butterworth_noevenorder(self):
     """ test a butterworth with non even order input
     """
     st = datetime.datetime(year=2000, month=2, day=3)
     delta = time_interval(hours=1)
     data = arange(100)
     order = 7
     ts0 = rts(data, st, delta, {})
     self.assertRaises(ValueError, butterworth, ts0, order)
Exemple #10
0
    def test_merge_rts_intersect2(self):
        """ Test merging two rts with intersection."""
        
        d1=[1.0]*4
        d2=[2.0]*4
        d1[2]=spy.nan
        
        st1=datetime.datetime(year=1990,month=2,day=1)
        st2=datetime.datetime(year=1990,month=3,day=1)
        
        ts1=rts(d1,st1,time_interval(months=1))
        ts2=rts(d2,st2,time_interval(months=1))
               
        nt=merge(ts1,ts2)
        ## test ts1 data take priority on valid data
        self.assertEqual(nt.data[1],ts1.data[1])

        ## test ts2 data take prioirty on invalid data of ts1
        self.assertEqual(nt.data[2],ts2.data[2])
Exemple #11
0
    def test_period_time_stamp(self):

        st = datetime.datetime(year=1990, month=1, day=1)
        num = 31
        delta = time_interval(days=1)
        op_delta = time_interval(months=1)
        aligned_start = datetime.datetime(year=1990, month=1, day=1)

        data=[random.uniform(self.min_val,self.max_val) \
              for k in range(31)]

        ts_sum = sum(data)

        ts = rts(data, st, delta, {TIMESTAMP: PERIOD_START, AGGREGATION: MEAN})

        ts_op = period_op(ts, op_delta, SUM)

        self.assertEqual(ts_op.start, aligned_start)
        self.assertEqual(len(ts_op), 1)
        self.assertEqual(ts_op.data[0], ts_sum)
Exemple #12
0
    def test_period_op_nan(self):
        """ Test the behaviour of period operation on data with NaN."""

        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        num = 3005
        delta = time_interval(minutes=5)
        interval = '1 hour'
        op_delta = time_interval(hours=1)
        aligned_start = datetime.datetime(year=1990,
                                          month=2,
                                          day=3,
                                          hour=12,
                                          minute=0)

        data=[random.uniform(self.min_val,self.max_val) \
              for k in range(num)]

        # Reformalize raw data, insert known mini_val and max_val
        # and calcuate hourly mean to use later.
        i0 = 9  # this is the first index with aligned calendar
        num_interval = (num - i0 + 1) // 12

        # Here a nan is insert into ts.
        data[9 + 12 * 2] = nan

        times = time_sequence(st, delta, num)
        ts = rts(data, st, delta, {})

        nt = period_op(ts, interval, MEAN)
        self.assert_(isnan(nt.data[2]))

        nt = period_op(ts, interval, SUM)
        self.assert_(isnan(nt.data[2]))

        nt = period_op(ts, interval, MIN)
        if not isnan(nt.data[2]):
            print "period_op omits nan during period_min"

        nt = period_op(ts, interval, MAX)
        if not isnan(nt.data[2]):
            print "period_op omits nan during period_min"
Exemple #13
0
    def test_daily_average(self):
        """ Test godin filter on 2-dimensional data set."""

        d1 = [1.0] * 800 + [2.0] * 400 + [1.0] * 400
        d2 = [1.0] * 800 + [2.0] * 400 + [1.0] * 400
        data = numpy.array([d1, d2])
        data = numpy.transpose(data)
        data[336, :] = numpy.nan
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(minutes=15)
        test_ts = rts(data, st, delta, {})
        nt3 = daily_average(test_ts)
Exemple #14
0
def fill_gaps(tss, max_gap_to_fill=time_interval(hours=1)):
    tss_filled = []
    for ts in tss:
        if ts is not None:
            max_gap = int(max_gap_to_fill.total_seconds() / ts.interval.total_seconds())
            if max_gap > 0:
                tss_filled.append(interpolate_ts_nan(ts, max_gap=max_gap))
            else:
                tss_filled.append(ts)
        else:
            tss_filled.append(None)
    return tss_filled
Exemple #15
0
    def test_merge_rts_intersect(self):
        """ Test merging two rts with intersection."""
        
        d1=[1.0]*int(spy.math.pow(2,10))
        d2=[2.0]*int(spy.math.pow(2,11))
        
        st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
        st2=datetime.datetime(year=1990,month=3,day=3,hour=11, minute=15)
        
        ts1=rts(d1,st1,time_interval(hours=1))
        ts2=rts(d2,st2,time_interval(hours=1))
        
        ## Intentionally put some nan into ts1 to see effect.
        ii=ts1.index_after(st2)
        num_nan=10
        ts1.data[ii+1:ii+num_nan+2]=spy.nan               
        
        
        nt=merge(ts1,ts2)
        
        self.assertEqual(nt.start,ts1.start)
        self.assertEqual(nt.end,ts2.end)
        
        total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
        
        self.assertEqual(len(nt.data),total_n)
        
        s1=nt.index_after(ts1.end)
        s2=nt.index_after(ts2.start)
        s3=ts2.index_after(ts1.end)
        self.assert_(spy.allclose(nt.data[0:ii+1],ts1.data[0:ii+1]))
        self.assert_(spy.allclose(nt.data[ii+1:ii+num_nan+2],ts2.data[0:num_nan+1]))
        self.assert_(spy.allclose(nt.data[ii+num_nan+2:s1+1],ts1.data[ii+num_nan+2:s1+1]))
        self.assert_(spy.allclose(nt.data[s1+1:len(nt.data)],ts2.data[s3+1:len(ts2.data)]))
        
        ## a small test
        d1=[1.0]*4
        d2=[2.0]*2
        
        st1=datetime.datetime(year=1990,month=2,day=1)
        st2=datetime.datetime(year=1990,month=3,day=1)
        
        ts1=rts(d1,st1,time_interval(months=1))
        ts2=rts(d2,st2,time_interval(months=1))
        nt=merge(ts1,ts2)
        self.assertEqual(len(nt),len(ts1))
        self.assertEqual(nt.data[-1],ts1.data[-1])
    

        d1=[1.0,1.0,1.0,spy.nan]
        d2=[2.0,3.0,4.0]
        
        st1=datetime.datetime(year=1990,month=2,day=1)
        st2=datetime.datetime(year=1990,month=3,day=1)
        
        ts1=rts(d1,st1,time_interval(months=1))
        ts2=rts(d2,st2,time_interval(months=1))
        nt=merge(ts1,ts2)
        self.assertEqual(len(nt),len(ts1))
        self.assertEqual(nt.data[-1],ts2.data[-1])
Exemple #16
0
    def test_boxcar_nan(self):
        """ test boxcar performance with a nan in data"""

        data = [1.0] * 200 + [2.0] * 100 + [1.0] * 100
        data = numpy.array(data)
        data[201] = numpy.nan
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        test_ts = rts(data, st, delta, {})
        nt = boxcar(test_ts, 11, 12)

        ## besides the leading and trailing nan, there will be
        ## 24 nan in the middle
        self.assert_(numpy.alltrue(numpy.isnan(nt.data[190:213])))
Exemple #17
0
 def test_decimate_rts(self):
     
     # Test operations on ts of varied values.
     test_ts=[(datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15),
                  int(scipy.math.pow(2,10)),time_interval(minutes=5)),]
     
     for (st,num,delta) in test_ts:
         data=[scipy.math.sin(0.01*scipy.math.pi*k) for k in range(num)]            
         # This ts is the orignial one
         ts0=rts(data,st,delta,{})              
         for interval in self.test_interval:
             sample_num=number_intervals(ts0.start,ts0.end,interval)+1
             ts=decimate(ts0,interval)
             self.assert_(ts.is_regular())
             self.assert_(len(ts.data)==sample_num)
Exemple #18
0
    def test_gaussian_filter(self):
        """ Test the nan data with gaussian filter"""
        data=[2.0*numpy.math.cos(2*pi*i/5+0.8)+3.0*numpy.math.cos(2*pi*i/45+0.1)\
              +7.0*numpy.math.cos(2*pi*i/55+0.3) for i in range(1000)]
        data = numpy.array(data)
        nanloc = 336
        data[nanloc] = numpy.nan
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(data, st, delta, {})
        sigma = 2

        nts = []
        for order in range(4):
            nts.append(ts_gaussian_filter(ts, sigma, order=order))
Exemple #19
0
    def __init__(self, methodName="runTest"):

        super(TestDecimate, self).__init__(methodName)

        # Number of data in a time series.
        self.num_ts = 1000
        self.max_val = 1000
        self.min_val = 0.01
        self.large_data_size = 100000
        self.test_interval = [time_interval(hours=1)]
        t = scipy.pi / 3 + scipy.arange(self.num_ts) * scipy.pi / 12
        data = scipy.sin(t)
        start_time = "1/1/1991"
        interval = "15min"
        props = {}
        self.ts = rts(data, start_time, interval, props)
    def make_depth_average(self, time_basis):
        """ Make depth averaged values from numpy array of outputs

            Parameters
            ----------
            time_basis: datetime.datetime
                time base of the outputs

            Returns
            -------
            lists of a set vtools.data.timeseries of depth and depth-averaged values
                For scalars, the list has only one set of time series.
                For vectors, the list has two sets of time series.
                Each time series has multiple columns of data, and each column
                is for stations.
        """
        nvrt = self._nvrt
        n_casts = self._n_casts
        # collect time stamp first
        times = list()
        output = self._outputs[0]
        for cast_i in range(n_casts):
            i_begin = cast_i * nvrt
            time = time_basis + time_interval(days=output[i_begin, 0])
            times.append(time)
        # collect data
        values = list()
        for output in self._outputs:
            values_at_point = list()
            depths_at_point = list()
            for xy_i in range(self._nxy):
                depths_at_cast = list()
                values_at_cast = list()
                for cast_i in range(n_casts):
                    i_begin = cast_i * nvrt + xy_i * (n_casts * nvrt)
                    depth = -output[i_begin + nvrt - 1, 2]
                    x = -output[i_begin:i_begin + nvrt, 2]
                    y = output[i_begin:i_begin + nvrt, 1]
                    avg = scipy.integrate.simps(y, x) / depth
                    depths_at_cast.append(depth)
                    values_at_cast.append(avg)
                depths_at_point.append(depths_at_cast)
                values_at_point.append(values_at_cast)
            ts_depths = its(times, numpy.array(depths_at_point).transpose())
            ts_values = its(times, numpy.array(values_at_point).transpose())
            values.append((ts_depths, ts_values))
        return values
Exemple #21
0
    def test_boxcar(self):
        """ test boxcar performance in each steps of a godin filter"""

        data = [1.0] * 200 + [2.0] * 100 + [1.0] * 100
        data = numpy.array(data)
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        test_ts = rts(data, st, delta, {})

        ## first round of boxcar in 24 (11,12).
        nt = boxcar(test_ts, 11, 12)
        self.assert_(numpy.alltrue(numpy.isnan(nt.data[0:11])))
        assert_array_almost_equal(nt.data[11:188], [1] * 177, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt.data[188:211], 1)))
        self.assertAlmostEqual(nt.data[196], 1.375)
        assert_array_almost_equal(nt.data[211:288], [2] * 77, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt.data[288:311], 1)))
        self.assertAlmostEqual(nt.data[301], 1.4166666667)
        assert_array_almost_equal(nt.data[311:388], [1] * 77, 12)
        self.assert_(numpy.alltrue(numpy.isnan(nt.data[388:400])))

        ## second round of boxcar in 24 (12,11)
        nt2 = boxcar(nt, 12, 11)
        self.assert_(numpy.alltrue(numpy.isnan(nt2.data[0:23])))
        assert_array_almost_equal(nt2.data[23:177], [1] * 154, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt2.data[177:223], 1)))
        self.assertAlmostEqual(nt2.data[196], 1.364583333)
        assert_array_almost_equal(nt2.data[223:277], [2] * 54, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt2.data[277:323], 1)))
        self.assertAlmostEqual(nt2.data[301], 1.439236111)
        assert_array_almost_equal(nt2.data[323:377], [1] * 54, 12)
        self.assert_(numpy.alltrue(numpy.isnan(nt2.data[377:400])))

        ## third round of boxcar.
        nt3 = boxcar(nt2, 12, 12)
        self.assert_(numpy.alltrue(numpy.isnan(nt3.data[0:35])))
        assert_array_almost_equal(nt3.data[35:165], [1] * 130, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt3.data[165:235], 1)))
        self.assertAlmostEqual(nt3.data[196], 1.393055556)
        assert_array_almost_equal(nt3.data[235:265], [2] * 30, 12)
        self.assert_(numpy.alltrue(numpy.greater(nt3.data[265:335], 1)))
        self.assertAlmostEqual(nt3.data[301], 1.453819444)
        assert_array_almost_equal(nt3.data[335:365], [1] * 30, 12)
        self.assert_(numpy.alltrue(numpy.isnan(nt3.data[365:400])))
Exemple #22
0
    def test_lanczos_cos_filter_phase_neutral(self):
        """ Test the phase neutriality of cosine lanczos filter"""

        ## a signal that is sum of two sine waves with frequency of
        ## 5 and 250HZ, sampled at 2000HZ
        t = numpy.linspace(0, 1.0, 2001)
        xlow = numpy.sin(2 * numpy.pi * 5 * t)
        xhigh = numpy.sin(2 * numpy.pi * 250 * t)
        x = xlow + xhigh
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(x, st, delta, {})

        ## cutoff period is 30 hours, filterd result should be xlow
        ## approximately
        nt1 = cosine_lanczos(ts,
                             cutoff_period=hours(30),
                             filter_len=20,
                             padtype="odd")
        self.assertAlmostEqual(numpy.abs(nt1.data - xlow).max(), 0, places=1)
Exemple #23
0
    def test_lanczos_cos_filter_nan(self):
        """ Test the data with a nan filtered by cosine lanczos filter"""
        data=[2.0*numpy.math.cos(2*pi*i/5+0.8)+3.0*numpy.math.cos(2*pi*i/45+0.1)\
             +7.0*numpy.math.cos(2*pi*i/55+0.3) for i in range(1000)]
        data = numpy.array(data)
        nanloc = 336
        data[nanloc] = numpy.nan
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(data, st, delta, {})
        m = 20

        nt1 = cosine_lanczos(ts,
                             cutoff_period=hours(30),
                             filter_len=m,
                             padtype="even")
        ## result should have nan from nanidx-2*m+2 to nanidx+2*m-1
        nanidx = numpy.where(numpy.isnan(nt1.data))[0]
        nanidx_should_be = numpy.arange(nanloc - 2 * m, nanloc + 2 * m + 1)
        assert_array_equal(nanidx, nanidx_should_be)
Exemple #24
0
    def test_period_op_large(self):
        """ Test performance of period operation on very large size of TS.
            Print out time used also.
        """
        st = datetime.datetime(year=10, month=2, day=3, hour=11, minute=15)
        num = self.large_data_size
        delta = time_interval(hours=1)
        dimension2 = 3
        interval = '1 day'

        data=[[random.uniform(self.min_val,self.max_val) for i in range(dimension2)] \
              for k in range(num)]
        data = sciarray(data)
        ts = rts(data, st, delta, {TIMESTAMP: INST})

        for op in [MIN, MAX, MEAN, SUM]:

            ##### time profile ####
            #debug_timeprofiler.mark()
            ######################
            nt = period_op(ts, interval, op)
Exemple #25
0
    def test_lanczos_cos_filter_len(self):
        """ test cosine lanczos input filter length api"""

        data=[2.0*numpy.math.cos(2*pi*i/5+0.8)+3.0*numpy.math.cos(2*pi*i/45+0.1)\
             +7.0*numpy.math.cos(2*pi*i/55+0.3) for i in range(1000)]
        data = numpy.array(data)
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(data, st, delta, {})

        filter_len = 24
        t1 = cosine_lanczos(ts, cutoff_period=hours(30), filter_len=filter_len)

        filter_len = days(1)
        t2 = cosine_lanczos(ts, cutoff_period=hours(30), filter_len=filter_len)

        assert_array_equal(t1.data, t2.data)

        filter_len = "invalid"
        self.assertRaises(TypeError,cosine_lanczos,ts,cutoff_period=hours(30),\
                          filter_len=filter_len)
Exemple #26
0
 def test_godin_15min(self):
     """ test godin filtering on a 15min constant values 
         data series with a nan.
     """
     data = [1.0] * 800 + [2.0] * 400 + [1.0] * 400
     data = numpy.array(data)
     data[336] = numpy.nan
     st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
     delta = time_interval(minutes=15)
     test_ts = rts(data, st, delta, {})
     nt3 = godin(test_ts)
     self.assert_(numpy.alltrue(numpy.isnan(nt3.data[0:144])))
     assert_array_almost_equal(nt3.data[144:192], [1] * 48, 12)
     self.assert_(numpy.alltrue(numpy.isnan(nt3.data[192:481])))
     assert_array_almost_equal(nt3.data[481:656], [1] * 175, 12)
     self.assert_(numpy.alltrue(numpy.greater(nt3.data[656:944], 1)))
     self.assertAlmostEqual(nt3.data[868], 1.916618441)
     assert_array_almost_equal(nt3.data[944:1056], [2] * 112, 12)
     self.assert_(numpy.alltrue(numpy.greater(nt3.data[1056:1344], 1)))
     self.assertAlmostEqual(nt3.data[1284], 1.041451845)
     assert_array_almost_equal(nt3.data[1344:1456], [1] * 112, 12)
     self.assert_(numpy.alltrue(numpy.isnan(nt3.data[1456:1600])))
Exemple #27
0
    def test_godin_2d(self):
        """ Test godin filter on 2-dimensional data set."""

        d1 = [1.0] * 800 + [2.0] * 400 + [1.0] * 400
        d2 = [1.0] * 800 + [2.0] * 400 + [1.0] * 400
        data = numpy.array([d1, d2])
        data = numpy.transpose(data)
        data[336, :] = numpy.nan
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(minutes=15)
        test_ts = rts(data, st, delta, {})

        nt3 = godin(test_ts)
        d1 = nt3.data[:, 0]
        d2 = nt3.data[:, 1]
        self.assert_(numpy.alltrue(numpy.isnan(d1[0:144])))
        assert_array_almost_equal(d1[144:192], [1] * 48, 12)
        self.assert_(numpy.alltrue(numpy.isnan(d1[192:481])))
        assert_array_almost_equal(d1[481:656], [1] * 175, 12)
        self.assert_(numpy.alltrue(numpy.greater(d1[656:944], 1)))
        self.assertAlmostEqual(d1[868], 1.916618441)
        assert_array_almost_equal(d1[944:1056], [2] * 112, 12)
        self.assert_(numpy.alltrue(numpy.greater(d1[1056:1344], 1)))
        self.assertAlmostEqual(d1[1284], 1.041451845)
        assert_array_almost_equal(d1[1344:1456], [1] * 112, 12)
        self.assert_(numpy.alltrue(numpy.isnan(d1[1456:1600])))

        self.assert_(numpy.alltrue(numpy.isnan(d2[0:144])))
        assert_array_almost_equal(d2[144:192], [1] * 48, 12)
        self.assert_(numpy.alltrue(numpy.isnan(d2[192:481])))
        assert_array_almost_equal(d2[481:656], [1] * 175, 12)
        self.assert_(numpy.alltrue(numpy.greater(d2[656:944], 1)))
        self.assertAlmostEqual(d2[868], 1.916618441)
        assert_array_almost_equal(d2[944:1056], [2] * 112, 12)
        self.assert_(numpy.alltrue(numpy.greater(d2[1056:1344], 1)))
        self.assertAlmostEqual(d2[1284], 1.041451845)
        assert_array_almost_equal(d2[1344:1456], [1] * 112, 12)
        self.assert_(numpy.alltrue(numpy.isnan(d2[1456:1600])))
Exemple #28
0
    def test_lanczos_cos_filter_len_api(self):
        """ Test the filter len api of the cosine filter"""

        ## a signal that is sum of two sine waves with frequency of
        ## 5 and 250HZ, sampled at 2000HZ
        t = numpy.linspace(0, 1.0, 2001)
        xlow = numpy.sin(2 * numpy.pi * 5 * t)
        xhigh = numpy.sin(2 * numpy.pi * 250 * t)
        x = xlow + xhigh
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(x, st, delta, {})

        ## filter len is none
        nt1 = cosine_lanczos(ts, cutoff_period=hours(40), padtype="even")
        self.assertTrue(nt1.is_regular())
        ## filter len by defaut lis 40*1.25=50, use it explicitly and
        ## see if the result is the same as the nt1
        nt2 = cosine_lanczos(ts,
                             cutoff_period=hours(40),
                             filter_len=50,
                             padtype="even")
        self.assertEqual(numpy.abs(nt1.data - nt2.data).max(), 0)
Exemple #29
0
    def test_lanczos_cos_filter_period_freq_api(self):
        """ Test the cutoff period and frequency of filter"""

        ## a signal that is sum of two sine waves with frequency of
        ## 5 and 250HZ, sampled at 2000HZ
        t = numpy.linspace(0, 1.0, 2001)
        xlow = numpy.sin(2 * numpy.pi * 5 * t)
        xhigh = numpy.sin(2 * numpy.pi * 250 * t)
        x = xlow + xhigh
        st = datetime.datetime(year=1990, month=2, day=3, hour=11, minute=15)
        delta = time_interval(hours=1)
        ts = rts(x, st, delta, {})


        nt1=cosine_lanczos(ts,cutoff_period=hours(30),filter_len=20,\
                          padtype="even")

        ## cutoff_frequency is expressed as ratio of nyquist frequency
        ## ,which is 1/0.5/hours
        cutoff_frequency = 2.0 / 30
        nt2=cosine_lanczos(ts,cutoff_frequency=cutoff_frequency,filter_len=20,\
                           padtype="even")

        self.assertEqual(numpy.abs(nt1.data - nt2.data).max(), 0)
Exemple #30
0
    """ Exception tracking error happend in dss catalog operations"""

    ########################################################################### 
    # Object interface.
    ###########################################################################  
    def __init__(self,st="Error in cataloging dss file"):

        self.description_str=st
        
    def __str__(self):

        return self.description_str
        

## Block size dictionary, used for adjust etime later.
one_day=time_interval(days=1)
one_month=time_interval(months=1)
one_year=time_interval(years=1)
one_decade=time_interval(years=10)
one_century=time_interval(years=100)

block_dic={"1MIN":one_day,"2MIN":one_day,"3MIN":one_day,\
           "4MIN":one_day,"5MIN":one_day,"10MIN":one_day,\
           "15MIN":one_month,"20MIN":one_month,"30MIN":one_month,\
           "1HOUR":one_month,"2HOUR":one_month,"3HOUR":one_month,\
           "4HOUR":one_month,"6HOUR":one_month,"8HOUR":one_month,\
           "12HOUR":one_month,"1DAY":one_year,"1WEEK":one_decade,\
           "1MON":one_decade,"1YEAR":one_century,"IR-DAY":one_day,\
           "IR-MONTH":one_month,"IR-YEAR":one_year,"IR-DECADE":one_decade}

NUM_OF_CHANGES_CAUSE_AUTO_UPDATE=10