class Datatest(unittest.TestCase):
"""Path to sample Data File"""
datadir=path.join(pth,"sample-data")
def setUp(self):
self.d=Data(path.join(path.dirname(__file__),"CoreTest.dat"),setas="xy")
self.d2=Data(path.join(__home__,"..","sample-data","TDI_Format_RT.txt"))
self.d3=Data(path.join(__home__,"..","sample-data","New-XRay-Data.dql"))
self.d4=Data(path.join(__home__,"..","sample-data","Cu_resistivity_vs_T.txt"))
def test_repr(self):
header="""============== ====== ======
TDI Format 1.5 X-Data Y-Data
index 0 (x) 1 (y)
============== ====== ======"""
repr_header="""============================= ======== ========
TDI Format 1.5 X-Data Y-Data
index 0 (x) 1 (y)
============================= ======== ========"""
self.assertEqual(self.d.header,header,"Header output changed.")
self.assertEqual("\n".join(repr(self.d).split("\n")[:4]),repr_header,"Representation gave unexpcted header.")
def test_base_class(self):
m=metadataObject()
m.update(self.d2.metadata)
self.assertTrue(isinstance(m,MutableMapping),"metadataObject is not a MutableMapping")
self.assertEqual(len(m),len(self.d2.metadata),"metadataObject length failure.")
self.assertEqual(len(m[".*"]),len(m),"Failure to index by regexp.")
for k in self.d2.metadata:
self.assertEqual(m[k],self.d2.metadata[k],"Failure to have equal keys for {}".format(k))
m[k]=k
self.assertEqual(m[k],k,"Failure to set keys for {}".format(k))
del m[k]
self.assertEqual(len(m),0,"Failed to delete from metadataObject.")
def test_constructor(self):
"""Constructor Tests"""
d=Data()
self.assertTrue(d.shape==(1,0),"Bare constructor failed")
d=Data(self.d)
self.assertTrue(np.all(d.data==self.d.data),"Constructor from DataFile failed")
d=Data([np.ones(100),np.zeros(100)])
self.assertTrue(d.shape==(100,2),"Constructor from iterable list of nd array failed")
d=Data([np.ones(100),np.zeros(100)],["X","Y"])
self.assertTrue(d.column_headers==["X","Y"],"Failed to set column headers in constructor: {}".format(d.column_headers))
c=np.zeros(100)
d=Data({"X-Data":c,"Y-Data":c,"Z-Data":c})
self.assertEqual(d.shape,(100,3),"Construction from dictionary of columns failed.")
d=Data(False)
self.assertEqual(self.d,d,"Faked file-dialog test")
d=self.d.clone
df=d.to_pandas()
e=Data(df)
self.assertEqual(d,e,"Roundtripping through Pandas DataFrame failed.")
d=self.d.clone
e=Data(d.dict_records)
e.metadata=d.metadata
self.assertEqual(d,e,"Copy from dict records failed.")
def test_column(self):
for i,c in enumerate(self.d.column_headers):
# Column function checks
self.assertTrue(all(self.d.data[:,i]==self.d.column(i)),"Failed to Access column {} by index".format(i))
self.assertTrue(all(self.d.data[:,i]==self.d.column(c)),"Failed to Access column {} by name".format(i))
self.assertTrue(all(self.d.data[:,i]==self.d.column(c[0:3])),"Failed to Access column {} by partial name".format(i))
# Check that access by list of strings returns multpiple columns
self.assertTrue(all(self.d.column(self.d.column_headers)==self.d.data),"Failed to access all columns by list of string indices")
self.assertTrue(all(self.d.column([0,self.d.column_headers[1]])==self.d.data),"Failed to access all columns by mixed list of indices")
# Check regular expression column access
self.assertTrue(all(self.d.column(re.compile(r"^X\-"))[:,0]==self.d.column(0)),"Failed to access column by regular expression")
# Check attribute column access
self.assertTrue(all(self.d.X==self.d.column(0)),"Failed to access column by attribute name")
def test_deltions(self):
ch=["{}-Data".format(chr(x)) for x in range(65,91)]
data=np.zeros((100,26))
metadata=OrderedDict([("Key 1",True),("Key 2",12),("Key 3","Hellow world")])
self.dd=Data(metadata)
self.dd.data=data
self.dd.column_headers=ch
self.dd.setas="3.x3.y3.z"
self.repr_string="""=========================== ======== ======= ======== ======= ======== ========
TDI Format 1.5 D-Data .... H-Data .... Y-Data Z-Data
index 3 (x) 7 (y) 24 25
=========================== ======== ======= ======== ======= ======== ========
Key 1{Boolean}= True 0 0 0 0
Key 2{I32}= 12 0 ... 0 ... 0 0
Key 3{String}= Hellow world 0 ... 0 ... 0 0
Stoner.class{String}= Data 0 ... 0 ... 0 0
... 0 ... 0 ... 0 0"""
self.assertEqual("\n".join(repr(self.dd).split("\n")[:9]),self.repr_string,"Representation with interesting columns failed.")
del self.dd["Key 1"]
self.assertEqual(len(self.dd.metadata),3,"Deletion of metadata failed.")
del self.dd[20:30]
self.assertEqual(self.dd.shape,(90,26),"Deleting rows directly failed.")
self.dd.del_column("Q")
self.assertEqual(self.dd.shape,(90,25),"Deleting rows directly failed.")
self.dd%=3
self.assertEqual(self.dd.shape,(90,24),"Deleting rows directly failed.")
self.dd.setas="x..y..z"
self.dd.del_column(self.dd.setas.not_set)
self.assertEqual(self.dd.shape,(90,3),"Deleting rows directly failed.")
self.dd.mask[50,1]=True
self.dd.del_column()
self.assertEqual(self.dd.shape,(90,2),"Deletion of masked rows failed.")
self.d5=Data(np.ones((10,10)))
self.d5.column_headers=["A"]*5+["B"]*5
self.d5.del_column("A",duplicates=True)
self.assertEqual(self.d5.shape,(10,6),"Failed to delete columns with duplicates True and col specified.")
self.d5=Data(np.ones((10,10)))
self.d5.column_headers=list("ABCDEFGHIJ")
self.d5.setas="..x..y"
self.d5.del_column(True)
self.assertEqual(self.d5.column_headers,["C","F"],"Failed to delete columns with col=True")
def test_indexing(self):
#Check all the indexing possibilities
data=np.array(self.d.data)
colname=self.d.column_headers[0]
self.assertTrue(all(self.d.column(colname)==self.d[:,0]),"Failed direct indexing versus column method")
self.assertTrue(all(self.d[:,0]==data[:,0]),"Failed direct idnexing versusus direct array index")
self.assertTrue(all(self.d[:,[0,1]]==data),"Failed direct list indexing")
self.assertTrue(all(self.d[::2,:]==data[::2]),"Failed slice indexing rows")
self.assertTrue(all(self.d[colname]==data[:,0]),"Failed direct indexing by column name")
self.assertTrue(all(self.d[:,colname]==data[:,0]),"Failed fallback indexing by column name")
self.assertEqual(self.d[25,1],645.0,"Failed direct single cell index")
self.assertEqual(self.d[25,"Y-Data"],645.0,"Failoed single cell index direct")
self.assertEqual(self.d["Y-Data",25],645.0,"Failoed single cell fallback index order")
self.d["X-Dat"]=[11,12,13,14,15]
self.assertEqual(self.d["X-Dat",2],13,"Failed indexing of metadata lists with tuple")
self.assertEqual(self.d["X-Dat"][2],13,"Failed indexing of metadata lists with double indices")
d=Data(np.ones((10,10)))
d[0,0]=5 #Index by tuple into data
d["Column_1",0]=6 # Index by column name, row into data
d[0,"Column_2"]=7 #Index by row, column name into data
d["Column_3"]=[1,2,3,4] # Create a metadata
d["Column_3",2]=2 # Index existing metadata via tuple
d.metadata[0,5]=10
d[0,5]=12 # Even if tuple, index metadata if already existing.
self.assertTrue(np.all(d[0]==np.array([5,6,7,1,1,1,1,1,1,1])),"setitem on Data to index into Data.data failed.\n{}".format(d[0]))
self.assertEqual(d.metadata["Column_3"],[1,2,2,4],"Tuple indexing into metadata Failed.")
self.assertEqual(d.metadata[0,5],12,"Indexing of pre-existing metadta keys rather than Data./data failed.")
self.assertEqual(d.metadata[1],[1, 2, 2, 4],"Indexing metadata by integer failed.")
def test_len(self):
# Check that length of the column is the same as length of the data
self.assertEqual(len(Data()),0,"Empty DataFile not length zero")
self.assertEqual(len(self.d.column(0)),len(self.d),"Column 0 length not equal to DataFile length")
self.assertEqual(len(self.d),self.d.data.shape[0],"DataFile length not equal to data.shape[0]")
# Check that self.column_headers returns the right length
self.assertEqual(len(self.d.column_headers),self.d.data.shape[1],"Length of column_headers not equal to data.shape[1]")
def test_attributes(self):
"""Test various atribute accesses,"""
header='============== ====== ======\nTDI Format 1.5 X-Data Y-Data\n index 0 (x) 1 (y)\n============== ====== ======'
self.assertEqual(self.d.shape,(100,2),"shape attribute not correct.")
self.assertEqual(self.d.dims,2,"dims attribute not correct.")
self.assertEqual(self.d.header,header,"Shorter header not as expected")
self.assertTrue(all(self.d.T==self.d.data.T),"Transpose attribute not right")
self.assertTrue(all(self.d.x==self.d.column(0)),"x attribute quick access not right.")
self.assertTrue(all(self.d.y==self.d.column(1)),"y attribute not right.")
self.assertTrue(all(self.d.q==np.arctan2(self.d.data[:,0],self.d.data[:,1])),"Calculated theta attribute not right.")
self.assertTrue(all(sqrt(self.d[:,0]**2+self.d[:,1]**2)==self.d.r),"Calculated r attribute not right.")
self.assertTrue(self.d2.records._[5]["Column 2"]==self.d2.data[5,2],"Records and as array attributes problem")
d=Data(np.ones((10,2)),["A","A"])
self.assertEqual(d.records.dtype,np.dtype([('A_1', '<f8'), ('A', '<f8')]),".records with identical column headers handling failed")
e=self.d.clone
e.data=np.array([1,1,1])
e.T=self.d.T
e.column_headers=self.d.column_headers
self.assertEqual(e,self.d,"Writingto transposed data failed")
def test_iterators(self):
for i,c in enumerate(self.d.columns()):
self.assertTrue(all(self.d.column(i)==c),"Iterating over DataFile.columns not the same as direct indexing column")
for j,r in enumerate(self.d.rows()):
self.assertTrue(all(self.d[j]==r),"Iteratinf over DataFile.rows not the same as indexed access")
for k,r in enumerate(self.d):
pass
self.assertEqual(j,k,"Iterating over DataFile not the same as DataFile.rows")
self.assertEqual(self.d.data.shape,(j+1,i+1),"Iterating over rows and columns not the same as data.shape")
def test_metadata(self):
self.d["Test"]="This is a test"
self.d["Int"]=1
self.d["Float"]=1.0
keys,values=zip(*self.d.items())
self.assertEqual(tuple(self.d.keys()),keys,"Keys from items() not equal to keys from keys()")
self.assertEqual(tuple(self.d.values()),values,"Values from items() not equal to values from values()")
self.assertEqual(self.d["Int"],1)
self.assertEqual(self.d["Float"],1.0)
self.assertEqual(self.d["Test"],self.d.metadata["Test"])
self.assertEqual(self.d.metadata._typehints["Int"],"I32")
self.assertEqual(len(self.d.dir()),6,"Failed meta data directory listing ({})".format(len(self.d.dir())))
self.assertEqual(len(self.d3["Temperature"]),7,"Regular experssion metadata search failed")
def test_dir(self):
self.assertTrue(self.d.dir("S")==["Stoner.class"],"Dir method failed: dir was {}".format(self.d.dir()))
bad_keys=set(['__metaclass__', 'iteritems', 'iterkeys', 'itervalues','__ge__', '__gt__', '__init_subclass__',
'__le__', '__lt__', '__reversed__', '__slots__',"_abc_negative_cache","_abc_registry",
"_abc_negative_cache_version","_abc_cache","_abc_impl"])
self.attrs=set(dir(self.d))-bad_keys
if len(self.attrs)!=227:
expected={'_conv_string', '__str__', 'clear', 'scale', '__add__', 'popitem', 'priority', '_init_single', 'pop', 'reorder_columns', '_col_label', 'subclasses', '__sizeof__', 'rows', 'plot_matrix', '_PlotMixin__SurfPlotter', '_showfig', '__and__', '_repr_html_', '_pop_mask', 'filename', 'smooth', '__weakref__', 'dir', '_PlotMixin__mpl3DQuiver', 'spline', '__format__', 'plot_xy', 'labels', '_fix_kargs', 'span', '__getattr__', '_push_mask', 'normalise', 'mean', 'sort', '_set_mask', 'shape', 'x2', 'clone', 'save', 'setdefault', 'update', 'plot', 'colormap_xyz', 'plot_xyzuvw', '_patterns', 'section', 'count', 'extrapolate', 'fig', 'mime_type', 'find_col', 'SG_Filter', '__isub__', 'clip', '_fix_fig', '_AnalysisMixin__get_math_val', 'get_filename', 'griddata', 'axes', 'setas', '__sub__', '__floordiv__', '_baseclass', '_subplots', '__add_core__', 'max', 'rename', '__setstate__', 'rolling_window', '_labels', '__dict__', 'adjust_setas', 'column', '__mod__', 'xlim', '_repr_table_', '__abstractmethods__', '__call__', '_public_attrs_real', '__eq__', '_pyplot_proxy', 'stitch', 'lmfit', '_filename', '__lshift__', '_fix_titles', 'multiple', '__dir__', 'column_headers', 'header', '_public_attrs', 'swap_column', 'annotate_fit', '_init_many', 'min', '_load', '__sub_core__', 'dtype', '__doc__', '_col_args', 'filter', '__new__', '__len__', 'format', 'ylim', 'ax', '__hash__', '_PlotMixin__figure', 'polyfit', '__repr__', 'subtract', '__iand__', 'debug', 'diffsum', 'split', 'ylabel', '__iter__', '_vector_color', '__invert__', '__repr_core__', 'del_nan', 'y2', '__contains__', '__reduce__', 'plot_xyuv', 'plot_xyuvw', '__delattr__', 'curve_fit', '__module__', '_conv_float', '__getattribute__', 'keys', 'legend', 'quiver_plot', 'metadata', 'plot_xyz', '__regexp_meta__', 'data', 'figure', 'records', '_DataFile__parse_metadata', 'fignum', '__setattr__', 'insert_rows', 'add', 'make_bins', '_DataFile__setattr_col', '_repr_short_', '__getitem__', '_repr_limits', '_data', '_template', 'outlier_detection', 'template', '__imod__', 'image_plot', '__setitem__', '_get_curve_fit_data', '_DataFile__search_index', 'no_fmt', '__class__', '_AnalysisMixin__threshold', 'dims', 'threshold', 'basename', '_AnalysisMixin__lmfit_one', 'del_rows', 'patterns', 'del_column', '_metadata', '__deepcopy__', 'search', '_record_curve_fit_result', 'positional_fmt', '__getstate__', 'interpolate', 'dict_records', '_span_slice', 'columns', 'title', '__delitem__', '_repr_html_private', '_DataFile__file_dialog', '_getattr_col', 'mask', 'add_column', 'subplot', 'subplots', 'peaks', '_MutableMapping__marker', 'subplot2grid', 'get', '_DataFile__read_iterable', 'contour_xyz', 'inset', '__meta__', '_VectorFieldPlot', '__init__', '_masks', 'select', 'unique', 'xlabel', '_Plot', '__iadd__', 'values', 'multiply', '_raise_type_error', 'divide', 'odr', '__reduce_ex__', 'cmap', 'showfig', '__subclasshook__', 'items', '_interesting_cols', '_init_double', '__ne__', '_fix_cols', 'integrate', 'decompose', 'bin', 'closest', '__and_core__', 'T', 'load', 'apply'}
print("="*120,"\n","Warning=====>",self.attrs-expected,expected-self.attrs)
self.assertEqual(len(self.attrs),227,"DataFile.__dir__ failed.")
def test_filter(self):
self.d._push_mask()
ix=np.argmax(self.d.x)
self.d.filter(lambda r:r.x<=50)
self.assertTrue(np.max(self.d.x)==50,"Failure of filter method to set mask")
self.assertTrue(np.isnan(self.d.x[ix]),"Failed to mask maximum value")
self.d._pop_mask()
self.assertEqual(self.d2.select(Temp__not__gt=150).shape,(839,3),"Seect method failure.")
self.assertEqual(self.d.select(lambda r:r.x<30),self.d.select(X__lt=30),"Select method as callable failed.")
self.assertEqual(self.d.select(__=lambda r:r.x<30),self.d.select(X__lt=30),"Select method as callable failed.")
def test_operators(self):
#Test Column Indexer
self.assertTrue(all(self.d//0==self.d.column(0)),"Failed the // operator with integer")
self.assertTrue(all(self.d//"X-Data"==self.d.column(0)),"Failed the // operator with string")
self.assertTrue(all((self.d//re.compile(r"^X\-"))[:,0]==self.d.column(0)),"Failed the // operator with regexp")
t=[self.d%1,self.d%"Y-Data",(self.d%re.compile(r"Y\-"))]
for ix,tst in enumerate(["integer","string","regexp"]):
self.assertTrue(all(t[ix].data==np.atleast_2d(self.d.column(0)).T),"Failed % operator with {} index".format(tst))
d=self.d&self.d.x
self.assertTrue(d.shape[1]==3,"& operator failed.")
self.assertTrue(len(d.setas)==3,"& messed up setas")
self.assertTrue(len(d.column_headers)==3,"& messed up setas")
d&=d.x
self.assertTrue(d.shape[1]==4,"Inplace & operator failed")
empty=Data()
empty=empty+np.zeros(10)
self.assertEqual(empty.shape,(1,10),"Adding to an empty array failed")
d=self.d+np.array([0,0])
self.assertTrue(len(d)==len(self.d)+1,"+ operator failed.")
d+=np.array([0,0])
self.assertTrue(len(d)==len(self.d)+2,"Inplace + operator failed.")
d=d-(-1)
self.assertTrue(len(d)==len(self.d)+1,"Delete with integer index failed. {} vs {}".format(len(d),len(self.d)+1))
d-=-1
self.assertTrue(len(d)==len(self.d),"Inplace delete with integer index failed. {} vs {}".format(len(d),len(self.d)))
d-=slice(0,-1,2)
self.assertTrue(len(d)==len(self.d)/2,"Inplace delete with slice index failed. {} vs {}".format(len(d),len(self.d)/2))
d=d%1
self.assertTrue(d.shape[1]==1,"Column division failed")
self.assertTrue(len(d.setas)==d.shape[1],"Column removal messed up setas")
self.assertTrue(len(d.column_headers)==d.shape[1],"Column removal messed up column headers")
e=self.d.clone
f=self.d2.clone
g=e+f
self.assertTrue(g.shape==(1776,2),"Add of 2 column and 3 column failed")
g=f+e
self.assertTrue(g.shape==(1776,3),"Add of 3 column and 2 column failed.")
g=e&f
h=f&e
self.assertTrue(g.shape==h.shape,"Anding unequal column lengths faile!")
e=~self.d
self.assertTrue(e.setas[0]=="y","failed to invert setas columns")
f.setas="xyz"
f=self.d2.clone
f+={"Res":0.3,"Temp":1.2,"New_Column":25}
f1,f2=f.shape
self.assertEqual(f1,self.d2.shape[0]+1,"Failed to add a row by providing a dictionary")
self.assertEqual(f2,self.d2.shape[1]+1,"Failed to add an extra columns by adding a dictionary with a new key")
self.assertTrue(np.isnan(f[-1,2]) and np.isnan(f[0,-1]),"Unset values when adding a dictionary not NaN")
d=Data()
d+=np.ones(5)
d+=np.zeros(5)
self.assertTrue(np.all(d//1==np.array([1,0])),"Adding 1D arrays should add new rows")
try:
d+=True
except TypeError:
pass
else:
self.assertTrue(False,"Failed to raise TypeError on adding unsupported type")
with open(self.d.filename,"r") as data:
lines=data.readlines()
e=Data()<<lines
e.setas=self.d.setas
e["Loaded as"]=self.d["Loaded as"]
self.e=e
self.assertEqual(e,self.d,"Failed to read and create from << operator")
d=Data(np.array([[1,2,3,4]]),setas="xdyz")
self.assertEqual((~d).setas,["y","e","z","x"],"Inverting multi-axis array failed")
def test_properties(self):
self.little=Data()
p=np.linspace(0,np.pi,91)
q=np.linspace(0,2*np.pi,91)
r=np.cos(p)
x=r*np.sin(q)
y=r*np.cos(q)
self.little.data=np.column_stack((x,y,r))
self.little.setas="xyz"
q_ang=np.round(self.little.q/np.pi,decimals=2)
p_ang=np.round(self.little.p/np.pi,decimals=2)
self.assertTrue(np.max(q_ang)==1.0,"Data.q failure")
self.assertTrue(np.max(p_ang)==0.5,"Data.p failure")
self.assertTrue(np.min(p_ang)==-0.5,"Data.p failure")
self.little.mask=mask_func
self.little.mask[:,2]=True
self.assertEqual(len(repr(self.little).split("\n")),len(self.little)+5,"Non shorterned representation failed.")
def test_methods(self):
d=self.d.clone
d&=np.where(d.x<50,1.0,0.0)
d.rename(2,"Z-Data")
d.setas="xyz"
self.assertTrue(all(d.unique(2)==np.array([0,1])),"Unique values failed: {}".format(d.unique(2)))
d=self.d.clone
d.insert_rows(10,np.zeros((2,2)))
self.assertEqual(len(d),102,"Failed to inert extra rows")
self.assertTrue(d[9,0]==10 and d[10,0]==0 and d[12,0]==11, "Failed to insert rows properly.")
d=self.d.clone
d.add_column(np.ones(len(d)),replace=False,header="added")
self.assertTrue(d.shape[1]==self.d.shape[1]+1,"Adding a column with replace=False did not add a column.")
self.assertTrue(np.all(d.data[:,-1]==np.ones(len(d))),"Didn't add the new column to the end of the data.")
self.assertTrue(len(d.column_headers)==len(self.d.column_headers)+1,"Column headers isn't bigger by one")
self.assertTrue(d.column_headers==self.d.column_headers+["added",],"Column header not added correctly")
d=self.d.clone
d.add_column(self.d.x)
self.assertTrue(np.all(d.x==d[:,2]),"Adding a column as a DataArray with column headers didn't work")
self.assertTrue(d.x.column_headers[0]==d.column_headers[2],"Adding a column as a DataArray with column headers didn't work")
e=d.clone
d.swap_column([(0,1),(0,2)])
self.assertTrue(d.column_headers==[e.column_headers[x] for x in [2,0,1]],"Swap column test failed: {}".format(d.column_headers))
e=self.d(setas="yx")
self.assertTrue(e.shape==self.d.shape and e.setas[0]=="y","Failed on a DataFile.__call__ test")
spl=len(repr(self.d).split("\n"))
self.assertEqual(spl,105,"Failed to do repr function got {} lines".format(spl))
e=self.d.clone
e=e.add_column(e.x,header=e.column_headers[0])
e.del_column(duplicates=True)
self.assertTrue(e.shape==(100,2),"Deleting duplicate columns failed")
e=self.d2.clone
e.reorder_columns([2,0,1])
self.assertTrue(e.column_headers==[self.d2.column_headers[x] for x in [2,0,1]],"Failed to reorder columns: {}".format(e.column_headers))
d=self.d.clone
d.del_rows(0,10.0)
self.assertEqual(d.shape,(99,2),"Del Rows with value and column failed - actual shape {}".format(d.shape))
d=self.d.clone
d.del_rows(0,(10.0,20.0))
self.assertEqual(d.shape,(89,2),"Del Rows with tuple and column failed - actual shape {}".format(d.shape))
d=self.d.clone
d.mask[::2,0]=True
d.del_rows()
self.assertEqual(d.shape,(50,2),"Del Rows with mask set - actual shape {}".format(d.shape))
d=self.d.clone
d[::2,1]=nan
d.del_nan(1)
self.assertEqual(d.shape,(50,2),"del_nan with explicit column set failed shape was {}".format(d.shape))
d=self.d.clone
d[::2,1]=nan
d.del_nan(0)
self.assertEqual(d.shape,(100,2),"del_nan with explicit column set and not nans failed shape was {}".format(d.shape))
d=self.d.clone
d[::2,1]=nan
d.setas=".y"
d.del_nan()
self.assertEqual(d.shape,(50,2),"del_nan with columns from setas failed shape was {}".format(d.shape))
d=self.d.clone
d2=self.d.clone
d2.data=d2.data[::-1,:]
self.assertEqual(d.sort(reverse=True),d2,"Sorting revserse not the same as manually reversed data.")
def test_metadata_save(self):
local = path.dirname(__file__)
t = np.arange(12).reshape(3,4) #set up a test data file with mixed metadata
t = Data(t)
t.column_headers = ["1","2","3","4"]
metitems = [True,1,0.2,{"a":1, "b":"abc"},(1,2),np.arange(3),[1,2,3], "abc", #all types accepted
r"\\abc\cde", 1e-20, #extra tests
[1,(1,2),"abc"], #list with different types
[[[1]]], #nested list
None, #None value
]
metnames = ["t"+str(i) for i in range(len(metitems))]
for k,v in zip(metnames,metitems):
t[k] = v
t.save(path.join(local, "mixedmetatest.dat"))
tl = Data(path.join(local, "mixedmetatest.txt")) #will change extension to txt if not txt or tdi, is this what we want?
t2 = self.d4.clone #check that python tdi save is the same as labview tdi save
t2.save(path.join(local, "mixedmetatest2.txt"))
t2l = Data(path.join(local, "mixedmetatest2.txt"))
for orig, load in [(t,tl), (t2, t2l)]:
for k in ['Loaded as', 'TDI Format']:
orig[k]=load[k]
self.assertTrue(np.allclose(orig.data, load.data))
self.assertTrue(orig.column_headers==load.column_headers)
self.res=load.metadata^orig.metadata
self.assertTrue(load.metadata==orig.metadata,"Metadata not the same on round tripping to disc")
os.remove(path.join(local, "mixedmetatest.txt")) #clear up
os.remove(path.join(local, "mixedmetatest2.txt"))
d.del_rows(0, lambda x, r: np.any(np.isnan(r)))
popt, pcov = d.curve_fit(bg_wrapper,
xcol=t_col,
ycol=r_col,
p0=p0,
absolute_sigma=False)
perr = np.sqrt(np.diag(pcov))
labels = [r"\theta_D", r"\rho_0", r"A"]
units = ["K", r"\Omega m", r"\Omega m"]
annotation = [
"${}$: {}\n".format(l, format_error(v, e, latex=True, mode="eng", units=u))
for l, v, e, u in zip(labels, popt, perr, units)
]
annotation = "\n".join(annotation)
popt = np.append(popt, 5)
T = d.column(t_col)
d.add_column(blochGrueneisen(T, *popt), header=r"Bloch")
d.plot_xy(
t_col,
[r_col, "Bloch"],
["ro", "b-"],
label=["Data", r"$Bloch-Gr\"ueisen Fit$"],
)
d.xlabel = "Temperature (K)"
d.ylabel = "Resistance ($\Omega$)"
text(0.05, 0.65, annotation, transform=d.axes[0].transAxes)
class Datatest(unittest.TestCase):
"""Path to sample Data File"""
datadir = path.join(pth, "sample-data")
def setUp(self):
self.d = Data(path.join(path.dirname(__file__), "CoreTest.dat"),
setas="xy")
self.d2 = Data(
path.join(__home__, "..", "sample-data", "TDI_Format_RT.txt"))
self.d3 = Data(
path.join(__home__, "..", "sample-data", "New-XRay-Data.dql"))
def test_constructor(self):
"""Constructor Tests"""
d = Data()
self.assertTrue(d.shape == (1, 0), "Bare constructor failed")
d = Data(self.d)
self.assertTrue(np.all(d.data == self.d.data),
"Constructor from DataFile failed")
d = Data([np.ones(100), np.zeros(100)])
self.assertTrue(d.shape == (100, 2),
"Constructor from iterable list of nd array failed")
d = Data([np.ones(100), np.zeros(100)], ["X", "Y"])
self.assertTrue(
d.column_headers == ["X", "Y"],
"Failed to set column headers in constructor: {}".format(
d.column_headers))
def test_column(self):
for i, c in enumerate(self.d.column_headers):
# Column function checks
self.assertTrue(all(self.d.data[:, i] == self.d.column(i)),
"Failed to Access column {} by index".format(i))
self.assertTrue(all(self.d.data[:, i] == self.d.column(c)),
"Failed to Access column {} by name".format(i))
self.assertTrue(
all(self.d.data[:, i] == self.d.column(c[0:3])),
"Failed to Access column {} by partial name".format(i))
# Check that access by list of strings returns multpiple columns
self.assertTrue(
all(self.d.column(self.d.column_headers) == self.d.data),
"Failed to access all columns by list of string indices")
self.assertTrue(
all(self.d.column([0, self.d.column_headers[1]]) == self.d.data),
"Failed to access all columns by mixed list of indices")
# Check regular expression column access
self.assertTrue(
all(self.d.column(re.compile(r"^X\-"))[:, 0] == self.d.column(0)),
"Failed to access column by regular expression")
# Check attribute column access
self.assertTrue(all(self.d.X == self.d.column(0)),
"Failed to access column by attribute name")
def test_indexing(self):
#Check all the indexing possibilities
data = np.array(self.d.data)
colname = self.d.column_headers[0]
self.assertTrue(all(self.d.column(colname) == self.d[:, 0]),
"Failed direct indexing versus column method")
self.assertTrue(all(self.d[:, 0] == data[:, 0]),
"Failed direct idnexing versusus direct array index")
self.assertTrue(all(self.d[:, [0, 1]] == data),
"Failed direct list indexing")
self.assertTrue(all(self.d[::2, :] == data[::2]),
"Failed slice indexing rows")
self.assertTrue(all(self.d[colname] == data[:, 0]),
"Failed direct indexing by column name")
self.assertTrue(all(self.d[:, colname] == data[:, 0]),
"Failed fallback indexing by column name")
self.assertEqual(self.d[25, 1], 645.0,
"Failed direct single cell index")
self.assertEqual(self.d[25, "Y-Data"], 645.0,
"Failoed single cell index direct")
self.assertEqual(self.d["Y-Data", 25], 645.0,
"Failoed single cell fallback index order")
self.d["X-Dat"] = [11, 12, 13, 14, 15]
self.assertEqual(self.d["X-Dat", 2], 13,
"Failed indexing of metadata lists with tuple")
self.assertEqual(
self.d["X-Dat"][2], 13,
"Failed indexing of metadata lists with double indices")
d = Data(np.ones((10, 10)))
d[0, 0] = 5 #Index by tuple into data
d["Column_1", 0] = 6 # Index by column name, row into data
d[0, "Column_2"] = 7 #Index by row, column name into data
d["Column_3"] = [1, 2, 3, 4] # Create a metadata
d["Column_3", 2] = 2 # Index existing metadata via tuple
d.metadata[0, 5] = 10
d[0, 5] = 12 # Even if tuple, index metadata if already existing.
self.assertTrue(
np.all(d[0] == np.array([5, 6, 7, 1, 1, 1, 1, 1, 1, 1])),
"setitem on Data to index into Data.data failed.\n{}".format(d[0]))
self.assertEqual(d.metadata["Column_3"], [1, 2, 2, 4],
"Tuple indexing into metadata Failed.")
self.assertEqual(
d.metadata[0, 5], 12,
"Indexing of pre-existing metadta keys rather than Data./data failed."
)
def test_len(self):
# Check that length of the column is the same as length of the data
self.assertEqual(len(Data()), 0, "Empty DataFile not length zero")
self.assertEqual(len(self.d.column(0)), len(self.d),
"Column 0 length not equal to DataFile length")
self.assertEqual(len(self.d), self.d.data.shape[0],
"DataFile length not equal to data.shape[0]")
# Check that self.column_headers returns the right length
self.assertEqual(
len(self.d.column_headers), self.d.data.shape[1],
"Length of column_headers not equal to data.shape[1]")
def test_attributes(self):
"""Test various atribute accesses,"""
self.assertEqual(self.d.shape, (100, 2),
"shape attribute not correct.")
self.assertTrue(all(self.d.T == self.d.data.T),
"Transpose attribute not right")
self.assertTrue(all(self.d.x == self.d.column(0)),
"x attribute quick access not right.")
self.assertTrue(all(self.d.y == self.d.column(1)),
"y attribute not right.")
self.assertTrue(
all(self.d.q == np.arctan2(self.d.data[:, 0], self.d.data[:, 1])),
"Calculated theta attribute not right.")
self.assertTrue(
all(sqrt(self.d[:, 0]**2 + self.d[:, 1]**2) == self.d.r),
"Calculated r attribute not right.")
self.assertTrue(self.d2.records._[5]["Column 2"] == self.d2.data[5, 2],
"Records and as array attributes problem")
def test_setas(self):
#Check readback of setas
self.assertEqual(self.d.setas[:], ["x", "y"],
"setas attribute not set in constructor")
self.assertEqual(str(self.d.setas), "xy",
"setas attribute not not converted to string")
self.assertTrue(all(self.d.x == self.d.column(0)),
"Attribute setas column axis fails")
self.d.setas(x="Y-Data", reset=False)
self.assertEqual(
self.d.setas[:], ["x", "x"],
"Failed to set setas by type=column keyword assignment")
self.d.setas(Y="y", reset=False)
self.assertEqual(
self.d.setas[:], ["x", "y"],
"Failed to set setas by column=type keyword assignment")
self.assertEqual(
self.d.setas["x"], "X-Data",
"Failed to return column name from setas dict reading")
self.assertEqual(
self.d.setas["#x"], 0,
"Failed to return column index from setas dict reading")
e = ~self.d
self.assertEqual(str(e.setas), "yx",
"Failed the invert xy columns operator ~")
def test_iterators(self):
for i, c in enumerate(self.d.columns()):
self.assertTrue(
all(self.d.column(i) == c),
"Iterating over DataFile.columns not the same as direct indexing column"
)
for j, r in enumerate(self.d.rows()):
self.assertTrue(
all(self.d[j] == r),
"Iteratinf over DataFile.rows not the same as indexed access")
for k, r in enumerate(self.d):
pass
self.assertEqual(
j, k, "Iterating over DataFile not the same as DataFile.rows")
self.assertEqual(
self.d.data.shape, (j + 1, i + 1),
"Iterating over rows and columns not the same as data.shape")
def test_metadata(self):
self.d["Test"] = "This is a test"
self.d["Int"] = 1
self.d["Float"] = 1.0
self.assertEqual(self.d["Int"], 1)
self.assertEqual(self.d["Float"], 1.0)
self.assertEqual(self.d["Test"], self.d.metadata["Test"])
self.assertEqual(self.d.metadata._typehints["Int"], "I32")
self.assertEqual(
len(self.d.dir()),
6, "Failed meta data directory listing ({})".format(
len(self.d.dir())))
self.assertEqual(len(self.d3["Temperature"]), 7,
"Regular experssion metadata search failed")
def test_dir(self):
self.assertTrue(
self.d.dir("S") == ["Stoner.class"],
"Dir method failed: dir was {}".format(self.d.dir()))
def test_filter(self):
self.d._push_mask()
ix = np.argmax(self.d.x)
self.d.filter(lambda r: r.x <= 50)
self.assertTrue(
np.max(self.d.x) == 50, "Failure of filter method to set mask")
self.assertTrue(np.isnan(self.d.x[ix]), "Failed to mask maximum value")
self.d._pop_mask()
self.assertEqual(
self.d2.select(Temp__not__gt=150).shape, (839, 3),
"Seect method failure.")
def test_operators(self):
#Test Column Indexer
self.assertTrue(all(self.d // 0 == self.d.column(0)),
"Failed the // operator with integer")
self.assertTrue(all(self.d // "X-Data" == self.d.column(0)),
"Failed the // operator with string")
self.assertTrue(
all((self.d // re.compile(r"^X\-"))[:, 0] == self.d.column(0)),
"Failed the // operator with regexp")
t = [self.d % 1, self.d % "Y-Data", (self.d % re.compile(r"Y\-"))]
for ix, tst in enumerate(["integer", "string", "regexp"]):
self.assertTrue(
all(t[ix].data == np.atleast_2d(self.d.column(0)).T),
"Failed % operator with {} index".format(tst))
d = self.d & self.d.x
self.assertTrue(d.shape[1] == 3, "& operator failed.")
self.assertTrue(len(d.setas) == 3, "& messed up setas")
self.assertTrue(len(d.column_headers) == 3, "& messed up setas")
d &= d.x
self.assertTrue(d.shape[1] == 4, "Inplace & operator failed")
empty = Data()
empty = empty + np.zeros(10)
self.assertEqual(empty.shape, (1, 10),
"Adding to an empty array failed")
d = self.d + np.array([0, 0])
self.assertTrue(len(d) == len(self.d) + 1, "+ operator failed.")
d += np.array([0, 0])
self.assertTrue(
len(d) == len(self.d) + 2, "Inplace + operator failed.")
d = d - (-1)
self.assertTrue(
len(d) == len(self.d) + 1,
"Delete with integer index failed. {} vs {}".format(
len(d),
len(self.d) + 1))
d -= -1
self.assertTrue(
len(d) == len(self.d),
"Inplace delete with integer index failed. {} vs {}".format(
len(d), len(self.d)))
d -= slice(0, -1, 2)
self.assertTrue(
len(d) == len(self.d) / 2,
"Inplace delete with slice index failed. {} vs {}".format(
len(d),
len(self.d) / 2))
d = d % 1
self.assertTrue(d.shape[1] == 1, "Column division failed")
self.assertTrue(
len(d.setas) == d.shape[1], "Column removal messed up setas")
self.assertTrue(
len(d.column_headers) == d.shape[1],
"Column removal messed up column headers")
e = self.d.clone
f = self.d2.clone
g = e + f
self.assertTrue(g.shape == (1776, 2),
"Add of 2 column and 3 column failed")
g = f + e
self.assertTrue(g.shape == (1776, 3),
"Add of 3 column and 2 column failed.")
g = e & f
h = f & e
self.assertTrue(g.shape == h.shape,
"Anding unequal column lengths faile!")
e = ~self.d
self.assertTrue(e.setas[0] == "y", "failed to invert setas columns")
f.setas = "xyz"
def test_methods(self):
d = self.d.clone
d &= np.where(d.x < 50, 1.0, 0.0)
d.rename(2, "Z-Data")
d.setas = "xyz"
self.assertTrue(all(d.unique(2) == np.array([0, 1])),
"Unique values failed: {}".format(d.unique(2)))
d = self.d.clone
d.insert_rows(10, np.zeros((2, 2)))
self.assertEqual(len(d), 102, "Failed to inert extra rows")
self.assertTrue(d[9, 0] == 10 and d[10, 0] == 0 and d[12, 0] == 11,
"Failed to insert rows properly.")
d = self.d.clone
d.add_column(np.ones(len(d)), replace=False, header="added")
self.assertTrue(
d.shape[1] == self.d.shape[1] + 1,
"Adding a column with replace=False did add a column.")
self.assertTrue(np.all(d.data[:, -1] == np.ones(len(d))),
"Didn't add the new column to the end of the data.")
self.assertTrue(
len(d.column_headers) == len(self.d.column_headers) + 1,
"Column headers isn't bigger by one")
self.assertTrue(
d.column_headers == self.d.column_headers + [
"added",
], "Column header not added correctly")
e = d.clone
d.swap_column([(0, 1), (0, 2)])
self.assertTrue(
d.column_headers == [e.column_headers[x] for x in [2, 0, 1]],
"Swap column test failed: {}".format(d.column_headers))
e = self.d(setas="yx")
self.assertTrue(e.shape == self.d.shape and e.setas[0] == "y",
"Failed on a DataFile.__call__ test")
spl = len(repr(self.d).split("\n"))
self.assertEqual(spl, 104,
"Failed to do repr function got {} lines".format(spl))
e = self.d.clone
e = e.add_column(e.x, header=e.column_headers[0])
e.del_column(duplicates=True)
self.assertTrue(e.shape == (100, 2),
"Deleting duplicate columns failed")
e = self.d2.clone
e.reorder_columns([2, 0, 1])
self.assertTrue(
e.column_headers == [self.d2.column_headers[x] for x in [2, 0, 1]],
"Failed to reorder columns: {}".format(e.column_headers))
def test_setas_metadata(self):
d = self.d.clone
d.setas = "xyz"
self.assertEqual(repr(d.setas), "['x', 'y']",
"setas __repr__ failure {}".format(repr(d.setas)))
self.assertEqual(
d.find_col(slice(5)), [0, 1],
"findcol with a slice failed {}".format(d.find_col(slice(5))))
d = self.d2.clone
d.setas = "xyz"
self.assertTrue(d["Column 2", 5] == d[5, "Column 2"],
"Indexing with mixed integer and string failed.")
self.assertEqual(d.metadata.type(["User",
"Timestamp"]), ['String', 'String'],
"Metadata.type with slice failed")
d.data["Column 2", :] = np.zeros(
len(d)) #TODO make this work with d["Column 2",:] as well
self.assertTrue(d.z.max() == 0.0 and d.z.min() == 0.0,
"Failed to set Dataarray using string indexing")
class Datatest(unittest.TestCase):
"""Path to sample Data File"""
datadir = path.join(pth, "sample-data")
def setUp(self):
self.d = Data(path.join(path.dirname(__file__), "CoreTest.dat"),
setas="xy")
self.d2 = Data(
path.join(__home__, "..", "sample-data", "TDI_Format_RT.txt"))
self.d3 = Data(
path.join(__home__, "..", "sample-data", "New-XRay-Data.dql"))
self.d4 = Data(
path.join(__home__, "..", "sample-data",
"Cu_resistivity_vs_T.txt"))
def test_constructor(self):
"""Constructor Tests"""
d = Data()
self.assertTrue(d.shape == (1, 0), "Bare constructor failed")
d = Data(self.d)
self.assertTrue(np.all(d.data == self.d.data),
"Constructor from DataFile failed")
d = Data([np.ones(100), np.zeros(100)])
self.assertTrue(d.shape == (100, 2),
"Constructor from iterable list of nd array failed")
d = Data([np.ones(100), np.zeros(100)], ["X", "Y"])
self.assertTrue(
d.column_headers == ["X", "Y"],
"Failed to set column headers in constructor: {}".format(
d.column_headers))
def test_column(self):
for i, c in enumerate(self.d.column_headers):
# Column function checks
self.assertTrue(all(self.d.data[:, i] == self.d.column(i)),
"Failed to Access column {} by index".format(i))
self.assertTrue(all(self.d.data[:, i] == self.d.column(c)),
"Failed to Access column {} by name".format(i))
self.assertTrue(
all(self.d.data[:, i] == self.d.column(c[0:3])),
"Failed to Access column {} by partial name".format(i))
# Check that access by list of strings returns multpiple columns
self.assertTrue(
all(self.d.column(self.d.column_headers) == self.d.data),
"Failed to access all columns by list of string indices")
self.assertTrue(
all(self.d.column([0, self.d.column_headers[1]]) == self.d.data),
"Failed to access all columns by mixed list of indices")
# Check regular expression column access
self.assertTrue(
all(self.d.column(re.compile(r"^X\-"))[:, 0] == self.d.column(0)),
"Failed to access column by regular expression")
# Check attribute column access
self.assertTrue(all(self.d.X == self.d.column(0)),
"Failed to access column by attribute name")
def test_indexing(self):
#Check all the indexing possibilities
data = np.array(self.d.data)
colname = self.d.column_headers[0]
self.assertTrue(all(self.d.column(colname) == self.d[:, 0]),
"Failed direct indexing versus column method")
self.assertTrue(all(self.d[:, 0] == data[:, 0]),
"Failed direct idnexing versusus direct array index")
self.assertTrue(all(self.d[:, [0, 1]] == data),
"Failed direct list indexing")
self.assertTrue(all(self.d[::2, :] == data[::2]),
"Failed slice indexing rows")
self.assertTrue(all(self.d[colname] == data[:, 0]),
"Failed direct indexing by column name")
self.assertTrue(all(self.d[:, colname] == data[:, 0]),
"Failed fallback indexing by column name")
self.assertEqual(self.d[25, 1], 645.0,
"Failed direct single cell index")
self.assertEqual(self.d[25, "Y-Data"], 645.0,
"Failoed single cell index direct")
self.assertEqual(self.d["Y-Data", 25], 645.0,
"Failoed single cell fallback index order")
self.d["X-Dat"] = [11, 12, 13, 14, 15]
self.assertEqual(self.d["X-Dat", 2], 13,
"Failed indexing of metadata lists with tuple")
self.assertEqual(
self.d["X-Dat"][2], 13,
"Failed indexing of metadata lists with double indices")
d = Data(np.ones((10, 10)))
d[0, 0] = 5 #Index by tuple into data
d["Column_1", 0] = 6 # Index by column name, row into data
d[0, "Column_2"] = 7 #Index by row, column name into data
d["Column_3"] = [1, 2, 3, 4] # Create a metadata
d["Column_3", 2] = 2 # Index existing metadata via tuple
d.metadata[0, 5] = 10
d[0, 5] = 12 # Even if tuple, index metadata if already existing.
self.assertTrue(
np.all(d[0] == np.array([5, 6, 7, 1, 1, 1, 1, 1, 1, 1])),
"setitem on Data to index into Data.data failed.\n{}".format(d[0]))
self.assertEqual(d.metadata["Column_3"], [1, 2, 2, 4],
"Tuple indexing into metadata Failed.")
self.assertEqual(
d.metadata[0, 5], 12,
"Indexing of pre-existing metadta keys rather than Data./data failed."
)
def test_len(self):
# Check that length of the column is the same as length of the data
self.assertEqual(len(Data()), 0, "Empty DataFile not length zero")
self.assertEqual(len(self.d.column(0)), len(self.d),
"Column 0 length not equal to DataFile length")
self.assertEqual(len(self.d), self.d.data.shape[0],
"DataFile length not equal to data.shape[0]")
# Check that self.column_headers returns the right length
self.assertEqual(
len(self.d.column_headers), self.d.data.shape[1],
"Length of column_headers not equal to data.shape[1]")
def test_attributes(self):
"""Test various atribute accesses,"""
self.assertEqual(self.d.shape, (100, 2),
"shape attribute not correct.")
self.assertTrue(all(self.d.T == self.d.data.T),
"Transpose attribute not right")
self.assertTrue(all(self.d.x == self.d.column(0)),
"x attribute quick access not right.")
self.assertTrue(all(self.d.y == self.d.column(1)),
"y attribute not right.")
self.assertTrue(
all(self.d.q == np.arctan2(self.d.data[:, 0], self.d.data[:, 1])),
"Calculated theta attribute not right.")
self.assertTrue(
all(sqrt(self.d[:, 0]**2 + self.d[:, 1]**2) == self.d.r),
"Calculated r attribute not right.")
self.assertTrue(self.d2.records._[5]["Column 2"] == self.d2.data[5, 2],
"Records and as array attributes problem")
def test_setas(self):
#Check readback of setas
self.assertEqual(self.d.setas[:], ["x", "y"],
"setas attribute not set in constructor")
self.assertEqual(str(self.d.setas), "xy",
"setas attribute not not converted to string")
self.assertTrue(all(self.d.x == self.d.column(0)),
"Attribute setas column axis fails")
self.d.setas(x="Y-Data", reset=False)
self.assertEqual(
self.d.setas[:], ["x", "x"],
"Failed to set setas by type=column keyword assignment")
self.d.setas(Y="y", reset=False)
self.assertEqual(
self.d.setas[:], ["x", "y"],
"Failed to set setas by column=type keyword assignment")
self.assertEqual(
self.d.setas["x"], "X-Data",
"Failed to return column name from setas dict reading")
self.assertEqual(
self.d.setas["#x"], 0,
"Failed to return column index from setas dict reading")
e = ~self.d
self.assertEqual(str(e.setas), "yx",
"Failed the invert xy columns operator ~")
def test_iterators(self):
for i, c in enumerate(self.d.columns()):
self.assertTrue(
all(self.d.column(i) == c),
"Iterating over DataFile.columns not the same as direct indexing column"
)
for j, r in enumerate(self.d.rows()):
self.assertTrue(
all(self.d[j] == r),
"Iteratinf over DataFile.rows not the same as indexed access")
for k, r in enumerate(self.d):
pass
self.assertEqual(
j, k, "Iterating over DataFile not the same as DataFile.rows")
self.assertEqual(
self.d.data.shape, (j + 1, i + 1),
"Iterating over rows and columns not the same as data.shape")
def test_metadata(self):
self.d["Test"] = "This is a test"
self.d["Int"] = 1
self.d["Float"] = 1.0
self.assertEqual(self.d["Int"], 1)
self.assertEqual(self.d["Float"], 1.0)
self.assertEqual(self.d["Test"], self.d.metadata["Test"])
self.assertEqual(self.d.metadata._typehints["Int"], "I32")
self.assertEqual(
len(self.d.dir()),
6, "Failed meta data directory listing ({})".format(
len(self.d.dir())))
self.assertEqual(len(self.d3["Temperature"]), 7,
"Regular experssion metadata search failed")
def test_dir(self):
self.assertTrue(
self.d.dir("S") == ["Stoner.class"],
"Dir method failed: dir was {}".format(self.d.dir()))
def test_filter(self):
self.d._push_mask()
ix = np.argmax(self.d.x)
self.d.filter(lambda r: r.x <= 50)
self.assertTrue(
np.max(self.d.x) == 50, "Failure of filter method to set mask")
self.assertTrue(np.isnan(self.d.x[ix]), "Failed to mask maximum value")
self.d._pop_mask()
self.assertEqual(
self.d2.select(Temp__not__gt=150).shape, (839, 3),
"Seect method failure.")
def test_operators(self):
#Test Column Indexer
self.assertTrue(all(self.d // 0 == self.d.column(0)),
"Failed the // operator with integer")
self.assertTrue(all(self.d // "X-Data" == self.d.column(0)),
"Failed the // operator with string")
self.assertTrue(
all((self.d // re.compile(r"^X\-"))[:, 0] == self.d.column(0)),
"Failed the // operator with regexp")
t = [self.d % 1, self.d % "Y-Data", (self.d % re.compile(r"Y\-"))]
for ix, tst in enumerate(["integer", "string", "regexp"]):
self.assertTrue(
all(t[ix].data == np.atleast_2d(self.d.column(0)).T),
"Failed % operator with {} index".format(tst))
d = self.d & self.d.x
self.assertTrue(d.shape[1] == 3, "& operator failed.")
self.assertTrue(len(d.setas) == 3, "& messed up setas")
self.assertTrue(len(d.column_headers) == 3, "& messed up setas")
d &= d.x
self.assertTrue(d.shape[1] == 4, "Inplace & operator failed")
empty = Data()
empty = empty + np.zeros(10)
self.assertEqual(empty.shape, (1, 10),
"Adding to an empty array failed")
d = self.d + np.array([0, 0])
self.assertTrue(len(d) == len(self.d) + 1, "+ operator failed.")
d += np.array([0, 0])
self.assertTrue(
len(d) == len(self.d) + 2, "Inplace + operator failed.")
d = d - (-1)
self.assertTrue(
len(d) == len(self.d) + 1,
"Delete with integer index failed. {} vs {}".format(
len(d),
len(self.d) + 1))
d -= -1
self.assertTrue(
len(d) == len(self.d),
"Inplace delete with integer index failed. {} vs {}".format(
len(d), len(self.d)))
d -= slice(0, -1, 2)
self.assertTrue(
len(d) == len(self.d) / 2,
"Inplace delete with slice index failed. {} vs {}".format(
len(d),
len(self.d) / 2))
d = d % 1
self.assertTrue(d.shape[1] == 1, "Column division failed")
self.assertTrue(
len(d.setas) == d.shape[1], "Column removal messed up setas")
self.assertTrue(
len(d.column_headers) == d.shape[1],
"Column removal messed up column headers")
e = self.d.clone
f = self.d2.clone
g = e + f
self.assertTrue(g.shape == (1776, 2),
"Add of 2 column and 3 column failed")
g = f + e
self.assertTrue(g.shape == (1776, 3),
"Add of 3 column and 2 column failed.")
g = e & f
h = f & e
self.assertTrue(g.shape == h.shape,
"Anding unequal column lengths faile!")
e = ~self.d
self.assertTrue(e.setas[0] == "y", "failed to invert setas columns")
f.setas = "xyz"
def test_methods(self):
d = self.d.clone
d &= np.where(d.x < 50, 1.0, 0.0)
d.rename(2, "Z-Data")
d.setas = "xyz"
self.assertTrue(all(d.unique(2) == np.array([0, 1])),
"Unique values failed: {}".format(d.unique(2)))
d = self.d.clone
d.insert_rows(10, np.zeros((2, 2)))
self.assertEqual(len(d), 102, "Failed to inert extra rows")
self.assertTrue(d[9, 0] == 10 and d[10, 0] == 0 and d[12, 0] == 11,
"Failed to insert rows properly.")
d = self.d.clone
d.add_column(np.ones(len(d)), replace=False, header="added")
self.assertTrue(
d.shape[1] == self.d.shape[1] + 1,
"Adding a column with replace=False did add a column.")
self.assertTrue(np.all(d.data[:, -1] == np.ones(len(d))),
"Didn't add the new column to the end of the data.")
self.assertTrue(
len(d.column_headers) == len(self.d.column_headers) + 1,
"Column headers isn't bigger by one")
self.assertTrue(
d.column_headers == self.d.column_headers + [
"added",
], "Column header not added correctly")
e = d.clone
d.swap_column([(0, 1), (0, 2)])
self.assertTrue(
d.column_headers == [e.column_headers[x] for x in [2, 0, 1]],
"Swap column test failed: {}".format(d.column_headers))
e = self.d(setas="yx")
self.assertTrue(e.shape == self.d.shape and e.setas[0] == "y",
"Failed on a DataFile.__call__ test")
spl = len(repr(self.d).split("\n"))
self.assertEqual(spl, 105,
"Failed to do repr function got {} lines".format(spl))
e = self.d.clone
e = e.add_column(e.x, header=e.column_headers[0])
e.del_column(duplicates=True)
self.assertTrue(e.shape == (100, 2),
"Deleting duplicate columns failed")
e = self.d2.clone
e.reorder_columns([2, 0, 1])
self.assertTrue(
e.column_headers == [self.d2.column_headers[x] for x in [2, 0, 1]],
"Failed to reorder columns: {}".format(e.column_headers))
d = self.d.clone
d.del_rows(0, 10.0)
self.assertEqual(
d.shape, (99, 2),
"Del Rows with value and column failed - actual shape {}".format(
d.shape))
d = self.d.clone
d.del_rows(0, (10.0, 20.0))
self.assertEqual(
d.shape, (89, 2),
"Del Rows with tuple and column failed - actual shape {}".format(
d.shape))
d = self.d.clone
d.mask[::2, 0] = True
d.del_rows()
self.assertEqual(
d.shape, (50, 2),
"Del Rows with mask set - actual shape {}".format(d.shape))
d = self.d.clone
d[::2, 1] = nan
d.del_nan(1)
self.assertEqual(
d.shape, (50, 2),
"del_nan with explicit column set failed shape was {}".format(
d.shape))
d = self.d.clone
d[::2, 1] = nan
d.del_nan(0)
self.assertEqual(
d.shape, (100, 2),
"del_nan with explicit column set and not nans failed shape was {}"
.format(d.shape))
d = self.d.clone
d[::2, 1] = nan
d.setas = ".y"
d.del_nan()
self.assertEqual(
d.shape, (50, 2),
"del_nan with columns from setas failed shape was {}".format(
d.shape))
def test_metadata_save(self):
local = path.dirname(__file__)
t = np.arange(12).reshape(
3, 4) #set up a test data file with mixed metadata
t = Data(t)
t.column_headers = ["1", "2", "3", "4"]
metitems = [
True,
1,
0.2,
{
"a": 1,
"b": "abc"
},
(1, 2),
np.arange(3),
[1, 2, 3],
"abc", #all types accepted
r"\\abc\cde",
1e-20, #extra tests
[1, (1, 2), "abc"], #list with different types
[[[1]]] #nested list
]
metnames = ["t" + str(i) for i in range(len(metitems))]
for k, v in zip(metnames, metitems):
t[k] = v
t.save(path.join(local, "mixedmetatest.dat"))
tl = Data(
path.join(local, "mixedmetatest.txt")
) #will change extension to txt if not txt or tdi, is this what we want?
t2 = self.d4.clone #check that python tdi save is the same as labview tdi save
t2.save(path.join(local, "mixedmetatest2.txt"))
t2l = Data(path.join(local, "mixedmetatest2.txt"))
for orig, load in [(t, tl), (t2, t2l)]:
self.assertTrue(np.allclose(orig.data, load.data))
self.assertTrue(orig.column_headers == load.column_headers)
self.assertTrue(
all([i in load.metadata.keys() for i in orig.metadata.keys()]))
for k in orig.metadata.keys():
if isinstance(orig[k], np.ndarray):
self.assertTrue(np.allclose(load[k], orig[k]))
elif isinstance(orig[k], float) and np.isnan(orig[k]):
self.assertTrue(np.isnan(load[k]))
else:
self.assertTrue(
load[k] == orig[k],
"Not equal for metadata: {}".format(load[k]))
os.remove(path.join(local, "mixedmetatest.txt")) #clear up
os.remove(path.join(local, "mixedmetatest2.txt"))
def test_setas_metadata(self):
d2 = self.d2.clone
d2.setas += {"x": 0, "y": 1}
self.assertEqual(
d2.setas.to_dict(), {
'x': 'Temperature',
'y': 'Resistance'
}, "__iadd__ failure {}".format(repr(d2.setas.to_dict())))
d3 = self.d2.clone
d3.setas = "..e"
d2.setas.update(d3.setas)
self.assertEqual(
d2.setas.to_dict(), {
'x': 'Temperature',
'y': 'Resistance',
'e': 'Column 2'
}, "__iadd__ failure {}".format(repr(d2.setas.to_dict())))
auto_setas = {
'axes': 2,
'has_axes': True,
'has_ucol': False,
'has_uvw': False,
'has_vcol': False,
'has_wcol': False,
'has_xcol': True,
'has_xerr': False,
'has_ycol': True,
'has_yerr': True,
'has_zcol': False,
'has_zerr': False,
'ucol': None,
'vcol': None,
'wcol': None,
'xcol': 0,
'xerr': None,
'ycol': 1,
'yerr': 2,
'zcol': None,
'zerr': None
}
self.assertEqual(self.d2.setas._get_cols(), auto_setas,
"Automatic guessing of setas failed!")
d2.setas.clear()
self.assertEqual(list(d2.setas), ["."] * 3, "Failed to clear() setas")
d2.setas[[0, 1, 2]] = "x", "y", "z"
self.assertEqual("".join(d2.setas), "xyz",
"Failed to set setas with a list")
d2.setas[[0, 1, 2]] = "x"
self.assertEqual("".join(d2.setas), "xxx",
"Failed to set setas with an element from a list")
d = self.d.clone
d.setas = "xyz"
self.assertEqual(repr(d.setas), "['x', 'y']",
"setas __repr__ failure {}".format(repr(d.setas)))
self.assertEqual(
d.find_col(slice(5)), [0, 1],
"findcol with a slice failed {}".format(d.find_col(slice(5))))
d = self.d2.clone
d.setas = "xyz"
self.assertTrue(d["Column 2", 5] == d[5, "Column 2"],
"Indexing with mixed integer and string failed.")
self.assertEqual(d.metadata.type(["User",
"Timestamp"]), ['String', 'String'],
"Metadata.type with slice failed")
d.data["Column 2", :] = np.zeros(
len(d)) #TODO make this work with d["Column 2",:] as well
self.assertTrue(d.z.max() == 0.0 and d.z.min() == 0.0,
"Failed to set Dataarray using string indexing")
