def test_metadata_save(self):
local = path.dirname(__file__)
t = np.arange(12).reshape(3,4) #set up a test data file with mixed metadata
t = Data(t)
t.column_headers = ["1","2","3","4"]
metitems = [True,1,0.2,{"a":1, "b":"abc"},(1,2),np.arange(3),[1,2,3], "abc", #all types accepted
r"\\abc\cde", 1e-20, #extra tests
[1,(1,2),"abc"], #list with different types
[[[1]]], #nested list
None, #None value
]
metnames = ["t"+str(i) for i in range(len(metitems))]
for k,v in zip(metnames,metitems):
t[k] = v
t.save(path.join(local, "mixedmetatest.dat"))
tl = Data(path.join(local, "mixedmetatest.txt")) #will change extension to txt if not txt or tdi, is this what we want?
t2 = self.d4.clone #check that python tdi save is the same as labview tdi save
t2.save(path.join(local, "mixedmetatest2.txt"))
t2l = Data(path.join(local, "mixedmetatest2.txt"))
for orig, load in [(t,tl), (t2, t2l)]:
for k in ['Loaded as', 'TDI Format']:
orig[k]=load[k]
self.assertTrue(np.allclose(orig.data, load.data))
self.assertTrue(orig.column_headers==load.column_headers)
self.res=load.metadata^orig.metadata
self.assertTrue(load.metadata==orig.metadata,"Metadata not the same on round tripping to disc")
os.remove(path.join(local, "mixedmetatest.txt")) #clear up
os.remove(path.join(local, "mixedmetatest2.txt"))
def collate(grp, trail, **kargs):
"""Gather all the data up again."""
grp.sort()
final = Data()
final.add_column(grp[0].column("Energy"), "Energy")
for g in grp:
final.add_column(g.column("Asym"), g.title)
if "group_key" in kargs:
final[kargs["group_key"]] = grp.key
final["path"] = trail
if "save" in kargs and kargs["save"]:
final.save(kargs["filename"])
return final
def collate(grp,trail,**kargs):
"""Gather all the data up again."""
grp.sort()
final=Data()
final.add_column(grp[0].column('Energy'),'Energy')
for g in grp:
final.add_column(g.column('Asym'),g.title)
if "group_key" in kargs:
final[kargs["group_key"]]=grp.key
final["path"]=trail
if "save" in kargs and kargs["save"]:
final.save(kargs["filename"])
return final
def test_loaders(self):
d = None
skip_files = [] # HDF5 loader not working Python 3.5
tmpdir = tempfile.mkdtemp()
print("Exporting to {}".format(tmpdir))
print("Data files {}".format(self.datadir))
incfiles = [
x for x in os.listdir(self.datadir)
if os.path.isfile(os.path.join(self.datadir, x))
and not x.endswith("tdms_index")
]
if not hyperspy_ok:
print("hyperspy too old, skupping emd file for test")
incfiles.remove("1449 37.0 kx.emd")
for i, f in enumerate(incfiles):
if f.strip().lower() in skip_files: # Known bad files to load
print("Skipping {}".format(f))
continue
else:
print("Testing {}".format(f))
try:
del d
fname = path.join(self.datadir, f)
d = Data(fname, debug=False)
self.assertTrue(
isinstance(d, DataFile),
"Failed to load {} correctly.".format(fname))
if "save" in d.subclasses[d["Loaded as"]].__dict__:
print("Checking save routine for {}".format(
d["Loaded as"]))
pth = os.path.join(tmpdir, f)
name, ext = os.path.splitext(pth)
pth2 = "{}-2.{}".format(name, ext)
d.save(pth, as_loaded=True)
self.assertTrue(
os.path.exists(pth) or os.path.exists(d.filename),
"Failed to save as {}".format(pth))
os.remove(d.filename)
d.save(pth2, as_loaded=d["Loaded as"])
self.assertTrue(
os.path.exists(pth2) or os.path.exists(d.filename),
"Failed to save as {}".format(pth))
os.remove(d.filename)
except Exception as e:
self.assertTrue(
False, "Failed in loading <{}>\n{}".format(
path.join(self.datadir, f), format_exc()))
os.rmdir(tmpdir)
def test_metadata_save(self):
local = path.dirname(__file__)
t = np.arange(12).reshape(
3, 4) #set up a test data file with mixed metadata
t = Data(t)
t.column_headers = ["1", "2", "3", "4"]
metitems = [
True,
1,
0.2,
{
"a": 1,
"b": "abc"
},
(1, 2),
np.arange(3),
[1, 2, 3],
"abc", #all types accepted
r"\\abc\cde",
1e-20, #extra tests
[1, (1, 2), "abc"], #list with different types
[[[1]]] #nested list
]
metnames = ["t" + str(i) for i in range(len(metitems))]
for k, v in zip(metnames, metitems):
t[k] = v
t.save(path.join(local, "mixedmetatest.dat"))
tl = Data(
path.join(local, "mixedmetatest.txt")
) #will change extension to txt if not txt or tdi, is this what we want?
t2 = self.d4.clone #check that python tdi save is the same as labview tdi save
t2.save(path.join(local, "mixedmetatest2.txt"))
t2l = Data(path.join(local, "mixedmetatest2.txt"))
for orig, load in [(t, tl), (t2, t2l)]:
self.assertTrue(np.allclose(orig.data, load.data))
self.assertTrue(orig.column_headers == load.column_headers)
self.assertTrue(
all([i in load.metadata.keys() for i in orig.metadata.keys()]))
for k in orig.metadata.keys():
if isinstance(orig[k], np.ndarray):
self.assertTrue(np.allclose(load[k], orig[k]))
elif isinstance(orig[k], float) and np.isnan(orig[k]):
self.assertTrue(np.isnan(load[k]))
else:
self.assertTrue(
load[k] == orig[k],
"Not equal for metadata: {}".format(load[k]))
os.remove(path.join(local, "mixedmetatest.txt")) #clear up
os.remove(path.join(local, "mixedmetatest2.txt"))
