def test_fromtextfile(self):
"Tests reading from a text file."
fcontent = """#
'Dates', 'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'2007-01', 'strings',1,1.0,'mixed column',,1
'2007-02', 'with embedded "double quotes"',2,2.0,1.0,,1
'2007-03', 'strings',3,3.0E5,3,,1
'2007-05','strings',4,-1e-10,,,1
"""
import os
from datetime import datetime
fname = 'tmp%s' % datetime.now().strftime("%y%m%d%H%M%S%s")
f = open(fname, 'w')
f.write(fcontent)
f.close()
mrectxt = fromtextfile(fname,delimitor=',',varnames='ABCDEFG',
dates_column=0)
os.unlink(fname)
#
dlist = ['2007-%02i' % i for i in (1,2,3,5)]
assert(isinstance(mrectxt, MultiTimeSeries))
assert_equal(mrectxt._dates, date_array(dlist,'M'))
assert_equal(mrectxt.dtype.names, ['B','C','D','E','F','G'])
assert_equal(mrectxt.G, [1,1,1,1])
assert_equal(mrectxt.F._mask, [1,1,1,1])
assert_equal(mrectxt.D, [1,2,3.e+5,-1e-10])
def test_empty_timeseries(self):
"Tests that empty TimeSeries are handled properly"
empty_ts = time_series([], freq='b')
assert_array_equal(empty_ts, empty_ts + 1)
assert_array_equal(empty_ts, empty_ts + empty_ts)
assert_equal(empty_ts.start_date, None)
assert_equal(empty_ts.end_date, None)
def _test_dict_scalars(self):
"test writing multiple scalars at once using write_scalar_dict"
self.db.write_scalar_dict({'$scalar_1':data['scalars']['float32'],
'$scalar_2':data['scalars']['float32']})
result = self.db.read(['$scalar_1', '$scalar_2'])
assert_equal(result['$scalar_1'], data['scalars']['float32'])
assert_equal(result['$scalar_2'], data['scalars']['float32'])
def _test_wildlist(self):
"test wildlist method"
wl1 = self.db.wildlist("$cser_?")
wl2 = self.db.wildlist("$cser_?", wildonly=True)
res1 = sorted(["$CSER_"+x.upper() for x in list(data['cser'])])
res2 = sorted([x.upper() for x in list(data['cser'])])
assert_equal(wl1, res1)
assert_equal(wl2, res2)
def test_consistent_value(self):
"Tests that values don't get mutated when constructing dates from a value"
print "starting test_consistent_value..."
freqs = [x[0] for x in freq_dict.values() if x[0] != 'U']
for f in freqs:
today = thisday(f)
assert_equal(Date(freq=f, value=today.value), today)
print "finished test_consistent_value"
def test_2d_modflags_tot(self):
"2D - modification of model flags"
(x, y, results, _, _, madeup) = self.d
madeup = loess(x,y)
madeup.model.span = 0.8
madeup.model.drop_square_flags = [True, False]
madeup.model.parametric_flags = [True, False]
assert_equal(madeup.model.parametric_flags[:2],[1,0])
madeup.fit()
assert_almost_equal(madeup.outputs.fitted_values, results[1], 5)
assert_almost_equal(madeup.outputs.enp, 6.9, 1)
assert_almost_equal(madeup.outputs.s, 1.4804, 4)
def test_hardmask(self):
"Test hardmask"
[d, m, mrec, dlist, dates, ts, mts] = self.data
mts.harden_mask()
assert(mts._hardmask)
mts._mask = nomask
assert_equal(mts._mask, N.r_[[m,m[::-1]]].all(0))
mts.soften_mask()
assert(not mts._hardmask)
mts._mask = nomask
assert(mts['f1']._mask is nomask)
assert_equal(mts['f0']._mask,mts['f1']._mask)
def test_shortcuts(self):
"Tests some creation shortcuts. Because I'm lazy like that."
print "starting test_shortcuts..."
# Dates shortcuts
assert_equal(Date('D','2007-01'), Date('D',string='2007-01'))
assert_equal(Date('D','2007-01'), Date('D', value=732677))
assert_equal(Date('D',732677), Date('D', value=732677))
# DateArray shortcuts
n = today('M')
d = date_array(start_date=n, length=3)
assert_equal(date_array(n,length=3), d)
assert_equal(date_array(n, n+2), d)
print "finished test_shortcuts"
def test_updatemask(self):
"Checks modification of mask."
(series, data) =self.d
assert_equal(series._mask, [1,0,0,0,0]*3)
series.mask = nomask
assert(series._mask is nomask)
assert(series._series._mask is nomask)
#series._series.mask = [1,0,0]*5
series.mask = [1,0,0]*5
assert_equal(series._mask, [1,0,0]*5)
assert_equal(series._series._mask, [1,0,0]*5)
series[2] = masked
assert_equal(series._mask, [1,0,1]+[1,0,0]*4)
assert_equal(series._series._mask, [1,0,1]+[1,0,0]*4)
def test_on2d(self):
"Tests getitem on a 2D series"
(a,b,d) = ([1,2,3],[3,2,1], date_array(thisday('M'),length=3))
ser_x = time_series(N.column_stack((a,b)), dates=d)
assert_equal(ser_x[0,0], time_series(a[0],d[0]))
assert_equal(ser_x[0,:], time_series([(a[0],b[0])], d[0]))
assert_equal(ser_x[:,0], time_series(a, d))
assert_equal(ser_x[:,:], ser_x)
def test_fromrecords(self):
"Test from recarray."
[d, m, mrec, dlist, dates, ts, mts] = self.data
nrec = N.core.records.fromarrays(N.r_[[d,d[::-1]]])
mrecfr = fromrecords(nrec.tolist(), dates=dates)
assert_equal(mrecfr.f0, mrec.f0)
assert_equal(mrecfr.dtype, mrec.dtype)
#....................
altrec = [tuple([d,]+list(r)) for (d,r) in zip(dlist,nrec)]
mrecfr = fromrecords(altrec, names='dates,f0,f1')
assert_equal(mrecfr.f0, mrec.f0)
assert_equal(mrecfr.dtype, mrec.dtype)
#....................
tmp = MultiTimeSeries(mts._series[::-1], dates=mts.dates)
mrecfr = fromrecords(tmp)
assert_equal(mrecfr.f0, mrec.f0[::-1])
def test_fromrange (self):
"Base data definition."
(dlist, dates, data) = self.d
series = time_series(data, start_date=dates[0], length=15)
assert(isinstance(series, TimeSeries))
assert_equal(series._mask, [1,0,0,0,0]*3)
assert_equal(series._series, data)
assert_equal(series._dates, dates)
assert_equal(series.freqstr, 'D')
def test_fromlist (self):
"Base data definition."
(dlist, dates, data) = self.d
series = time_series(data, dlist)
assert(isinstance(series, TimeSeries))
assert_equal(series._mask, [1,0,0,0,0]*3)
assert_equal(series._series, data)
assert_equal(series._dates, date_array_fromlist(dlist))
assert_equal(series.freqstr, 'D')
def _test_db_attribs(self):
"test setting and retrieving database attributes"
self.db.set_db_desc(_desc)
self.db.set_db_doc(_doc)
created = self.db.db_created()
modified = self.db.db_modified()
desc = self.db.db_desc()
doc = self.db.db_doc()
assert(abs(ts.thisday('s') - created) < 100)
assert(abs(ts.thisday('s') - modified) < 100)
assert_equal(desc, _desc)
assert_equal(doc, _doc)
assert(self.db.db_is_open())
self.db.close()
assert(not self.db.db_is_open())
self.db = fame.FameDb("testdb.db",'s')
def test_wtimeseries(self):
"Tests getitem w/ TimeSeries as index"
(series, data, dates) = self.d
# Testing a basic condition on data
cond = (series<8).filled(False)
dseries = series[cond]
assert_equal(dseries._data, [1,2,3,4,6,7])
assert_equal(dseries._dates, series._dates[[1,2,3,4,6,7]])
assert_equal(dseries._mask, nomask)
# Testing a basic condition on dates
series[series._dates < Date('D',string='2007-01-06')] = masked
assert_equal(series[:5]._series._mask, [1,1,1,1,1])
def _test_initialize_obj_and_post(self):
"""test initializing an object and posting of database"""
self.db.initialize_obj("$postobj", ts.thisday('B'))
exist_script = "from timeseries.io import fame;"
exist_script += "db = fame.FameDb('testdb.db', 'r');"
exist_script += "print db.obj_exists('$postobj');"
proc = os.popen('python -c "'+exist_script+'"')
exists = proc.readlines()[0].strip('\n')
proc.close()
assert_equal(exists, "False")
self.db.post()
proc = os.popen('python -c "'+exist_script+'"')
exists = proc.readlines()[0].strip('\n')
proc.close()
assert_equal(exists, "True")
def _test_read_scalars(self):
"read scalars of every data type"
for s in data['scalars']:
sclr = self.db.read('$scalar_'+s)
orig = data['scalars'][s]
if s == 'int32':
assert_equal(sclr, orig.astype(np.float32))
elif s in ('pyInt', 'pyFloat', 'int64'):
assert_equal(sclr, np.float64(orig))
elif s == 'namelist':
assert_equal(sclr, [x.upper() for x in orig])
else:
assert_equal(sclr, orig)
def test_unsorted(self):
"Tests that the data are properly sorted along the dates."
dlist = ['2007-01-%02i' % i for i in (3,2,1)]
data = [10,20,30]
series = time_series(data,dlist)
assert_equal(series._data,[30,20,10])
#
series = TimeSeries(data, dlist)
assert_equal(series._data,[30,20,10])
#
series = TimeSeries(data, dlist, mask=[1,0,0])
assert_equal(series._mask,[0,0,1])
#
data = masked_array([10,20,30],mask=[1,0,0])
series = TimeSeries(data, dlist)
assert_equal(series._mask,[0,0,1])
def test_fromsobjects(self):
"Tests creation from list of objects."
print "starting test_fromsobjects..."
dlist = ['2007-01-%02i' % i for i in (1,2,4,5,7,8,10,11,13)]
dates = date_array_fromlist(dlist)
dobj = [datetime.datetime.fromordinal(d) for d in dates.toordinal()]
odates = date_array_fromlist(dobj)
assert_equal(dates,odates)
dobj = [DateFromString(d) for d in dlist]
odates = date_array_fromlist(dobj)
assert_equal(dates,odates)
#
D = date_array_fromlist(dlist=['2006-01'])
assert_equal(D.tovalue(), [732312, ])
assert_equal(D.freq, C.FR_UND)
print "finished test_fromsobjects"
def test_maskperiod(self):
"Test mask_period"
(series, data, dates) = self.d
series.mask = nomask
(start, end) = ('2007-01-06', '2007-01-12')
mask = mask_period(series, start, end, inside=True, include_edges=True,
inplace=False)
assert_equal(mask._mask, N.array([0,0,0,0,0,1,1,1,1,1,1,1,0,0,0]))
mask = mask_period(series, start, end, inside=True, include_edges=False,
inplace=False)
assert_equal(mask._mask, [0,0,0,0,0,0,1,1,1,1,1,0,0,0,0])
mask = mask_period(series, start, end, inside=False, include_edges=True,
inplace=False)
assert_equal(mask._mask, [1,1,1,1,1,1,0,0,0,0,0,1,1,1,1])
mask = mask_period(series, start, end, inside=False, include_edges=False,
inplace=False)
assert_equal(mask._mask, [1,1,1,1,1,0,0,0,0,0,0,0,1,1,1])
# Now w/ multivariables
data = masked_array(numeric.arange(30).reshape(-1,2), dtype=float_)
series = time_series(data, dates=dates)
mask = mask_period(series, start, end, inside=True, include_edges=True,
inplace=False)
result = N.array([0,0,0,0,0,1,1,1,1,1,1,1,0,0,0])
assert_equal(mask._mask, result.repeat(2).reshape(-1,2))
def test_fromstrings_wmissing(self):
"Tests creation from list of strings w/ missing dates"
print "starting test_fromstrings_wmissing..."
dlist = ['2007-01-%02i' % i for i in (1,2,4,5,7,8,10,11,13)]
dates = date_array_fromlist(dlist)
assert_equal(dates.freqstr,'U')
assert(not dates.isfull())
assert(not dates.has_duplicated_dates())
assert_equal(dates.tovalue(),732676+numpy.array([1,2,4,5,7,8,10,11,13]))
#
ddates = date_array_fromlist(dlist, 'D')
assert_equal(ddates.freqstr,'D')
assert(not ddates.isfull())
assert(not ddates.has_duplicated_dates())
#
mdates = date_array_fromlist(dlist, 'M')
assert_equal(mdates.freqstr,'M')
assert(not dates.isfull())
assert(mdates.has_duplicated_dates())
print "finished test_fromstrings_wmissing"
def test_fromdatearray(self):
"Tests the creation of a series from a datearray"
_, dates, _ = self.d
data = dates
#
series = time_series(data, dates)
assert(isinstance(series, TimeSeries))
assert_equal(series._dates, dates)
assert_equal(series._data, data)
assert_equal(series.freqstr, 'D')
#
series[5] = MA.masked
# ensure that series can be represented by a string after masking a value
# (there was a bug before that prevented this from working when using a
# DateArray for the data)
strrep = str(series)
"""
current = getattr(dates, period)
previous = getattr(dates-1, period)
return (current - previous).nonzero()[0]
################################################################################
if __name__ == '__main__':
import maskedarray.testutils
from maskedarray.testutils import assert_equal
if 0:
dlist = ['2007-%02i' % i for i in range(1,5)+range(7,13)]
mdates = date_array_fromlist(dlist, 'M')
# Using an integer
assert_equal(mdates[0].value, 24073)
assert_equal(mdates[-1].value, 24084)
# Using a date
lag = mdates.find_dates(mdates[0])
print mdates[lag]
assert_equal(mdates[lag], mdates[0])
if 0:
hodie = today('D')
D = DateArray(today('D'))
assert_equal(D.freq, 6000)
if 0:
freqs = [x[0] for x in corelib.freq_dict.values() if x[0] != 'U']
print freqs
for f in freqs:
print f
today = thisday(f)
def test_set(self):
"Tests setting fields/attributes."
[d, m, mrec, dlist, dates, ts, mts] = self.data
mts.f0._data[:] = 5
assert_equal(mts['f0']._data, [5,5,5,5,5])
mts.f0 = 1
assert_equal(mts['f0']._data, [1]*5)
assert_equal(getmaskarray(mts['f0']), [0]*5)
mts.f1 = MA.masked
assert_equal(mts.f1.mask, [1]*5)
assert_equal(getmaskarray(mts['f1']), [1]*5)
mts._mask = MA.masked
assert_equal(getmaskarray(mts['f1']), [1]*5)
assert_equal(mts['f0']._mask, mts['f1']._mask)
mts._mask = MA.nomask
assert_equal(getmaskarray(mts['f1']), [0]*5)
assert_equal(mts['f0']._mask, mts['f1']._mask)
def test_get(self):
"Tests fields retrieval"
[d, m, mrec, dlist, dates, ts, mts] = self.data
assert(isinstance(mts['f0'], TimeSeries))
assert_equal(mts['f0']._dates, dates)
assert_equal(mts['f0']._data, d)
assert_equal(mts['f0']._mask, m)
#
assert(isinstance(mts[0], MultiTimeSeries))
assert_equal(mts._data[0], mrec[0])
# We can't use assert_equal here, as it tries to convert the tuple into a singleton
assert(mts[0]._data.view(N.ndarray) == mrec[0])
assert_equal(mts._dates[0], dates[0])
assert_equal(mts[0]._dates, dates[0])
#
assert(isinstance(mts['2007-01'], MultiTimeSeries))
assert(mts['2007-01']._data == mrec[0])
assert_equal(mts['2007-01']._dates, dates[0])
#
assert(isinstance(mts.f0, TimeSeries))
assert_equal(mts.f0, time_series(d, dates=dates, mask=m))
assert_equal(mts.f1, time_series(d[::-1], dates=dates, mask=m[::-1]))
assert((mts._fieldmask == N.core.records.fromarrays([m, m[::-1]])).all())
assert_equal(mts._mask, N.r_[[m,m[::-1]]].all(0))
assert_equal(mts.f0[1], mts[1].f0)
#
assert(isinstance(mts[:2], MultiTimeSeries))
assert_equal(mts[:2]._data.f0, mrec[:2].f0)
assert_equal(mts[:2]._data.f1, mrec[:2].f1)
assert_equal(mts[:2]._dates, dates[:2])
def test_concatenate(self):
"Tests concatenate"
dlist = ['2007-%02i' % i for i in range(1,6)]
dates = date_array_fromlist(dlist)
data = masked_array(numeric.arange(5), mask=[1,0,0,0,0], dtype=float_)
#
ser_1 = time_series(data, dates)
ser_2 = time_series(data, dates=dates+10)
newseries = concatenate_series(ser_1, ser_2)
assert_equal(newseries._data,[0,1,2,3,4,0,0,0,0,0,0,1,2,3,4])
assert_equal(newseries._mask,[1,0,0,0,0]+[1]*5+[1,0,0,0,0])
#
ser_1 = time_series(data, dates)
ser_2 = time_series(data, dates=dates+10)
newseries = concatenate_series(ser_1, ser_2, keep_gap=False)
assert_equal(newseries._data,[0,1,2,3,4,0,1,2,3,4])
assert_equal(newseries._mask,[1,0,0,0,0]+[1,0,0,0,0])
assert newseries.has_missing_dates()
#
ser_2 = time_series(data, dates=dates+3)
newseries = concatenate_series(ser_1, ser_2)
assert_equal(newseries._data,[0,1,2,0,1,2,3,4])
assert_equal(newseries._mask,[1,0,0,1,0,0,0,0])
def test_addfield(self):
"Tests addfield"
[d, m, mrec, dlist, dates, ts, mts] = self.data
mts = addfield(mts, masked_array(d+10, mask=m[::-1]))
assert_equal(mts.f2, d+10)
assert_equal(mts.f2._mask, m[::-1])
def test_setslices(self):
"Tests setting slices."
[d, m, mrec, dlist, dates, ts, mts] = self.data
#
mts[:2] = 5
assert_equal(mts.f0._data, [5,5,2,3,4])
assert_equal(mts.f1._data, [5,5,2,1,0])
assert_equal(mts.f0._mask, [0,0,0,1,1])
assert_equal(mts.f1._mask, [0,0,0,0,1])
mts.harden_mask()
mts[-2:] = 5
assert_equal(mts.f0._data, [5,5,2,3,4])
assert_equal(mts.f1._data, [5,5,2,5,0])
assert_equal(mts.f0._mask, [0,0,0,1,1])
assert_equal(mts.f1._mask, [0,0,0,0,1])
if nd > 2:
raise NotImplementedError,"Currently limited to atmost 2D array."
if axis is None or nd == 1:
return flatnotmasked_contiguous(a)
#
result = []
#
other = (axis+1)%2
idx = [0,0]
idx[axis] = slice(None,None)
#
for i in range(a.shape[other]):
idx[other] = i
result.append( flatnotmasked_contiguous(a[idx]) )
return result
################################################################################
if __name__ == '__main__':
#
import numpy as N
from maskedarray.testutils import assert_equal
if 1:
n = N.arange(1,7)
#
m = [1,0,0,0,0,0]
a = masked_array(n, mask=m).reshape(2,3)
b = masked_array(n, mask=m).reshape(3,2)
c = dot(a,b, True)
assert_equal(c.mask, [[1,1],[1,0]])
c = dot(a,b,False)
assert_equal(c, N.dot(a.filled(0), b.filled(0)))
def test_datafromlist(self):
"Check the creation of a time series from a list of data."
(_, dates, _) = self.d
data = list(range(15))
series = time_series(data, dates)
assert_equal(series._data.size, 15)
