def test_metadata_save(self): local = path.dirname(__file__) t = np.arange(12).reshape(3,4) #set up a test data file with mixed metadata t = Data(t) t.column_headers = ["1","2","3","4"] metitems = [True,1,0.2,{"a":1, "b":"abc"},(1,2),np.arange(3),[1,2,3], "abc", #all types accepted r"\\abc\cde", 1e-20, #extra tests [1,(1,2),"abc"], #list with different types [[[1]]], #nested list None, #None value ] metnames = ["t"+str(i) for i in range(len(metitems))] for k,v in zip(metnames,metitems): t[k] = v t.save(path.join(local, "mixedmetatest.dat")) tl = Data(path.join(local, "mixedmetatest.txt")) #will change extension to txt if not txt or tdi, is this what we want? t2 = self.d4.clone #check that python tdi save is the same as labview tdi save t2.save(path.join(local, "mixedmetatest2.txt")) t2l = Data(path.join(local, "mixedmetatest2.txt")) for orig, load in [(t,tl), (t2, t2l)]: for k in ['Loaded as', 'TDI Format']: orig[k]=load[k] self.assertTrue(np.allclose(orig.data, load.data)) self.assertTrue(orig.column_headers==load.column_headers) self.res=load.metadata^orig.metadata self.assertTrue(load.metadata==orig.metadata,"Metadata not the same on round tripping to disc") os.remove(path.join(local, "mixedmetatest.txt")) #clear up os.remove(path.join(local, "mixedmetatest2.txt"))
def collate(grp, trail, **kargs): """Gather all the data up again.""" grp.sort() final = Data() final.add_column(grp[0].column("Energy"), "Energy") for g in grp: final.add_column(g.column("Asym"), g.title) if "group_key" in kargs: final[kargs["group_key"]] = grp.key final["path"] = trail if "save" in kargs and kargs["save"]: final.save(kargs["filename"]) return final
def collate(grp,trail,**kargs): """Gather all the data up again.""" grp.sort() final=Data() final.add_column(grp[0].column('Energy'),'Energy') for g in grp: final.add_column(g.column('Asym'),g.title) if "group_key" in kargs: final[kargs["group_key"]]=grp.key final["path"]=trail if "save" in kargs and kargs["save"]: final.save(kargs["filename"]) return final
def test_loaders(self): d = None skip_files = [] # HDF5 loader not working Python 3.5 tmpdir = tempfile.mkdtemp() print("Exporting to {}".format(tmpdir)) print("Data files {}".format(self.datadir)) incfiles = [ x for x in os.listdir(self.datadir) if os.path.isfile(os.path.join(self.datadir, x)) and not x.endswith("tdms_index") ] if not hyperspy_ok: print("hyperspy too old, skupping emd file for test") incfiles.remove("1449 37.0 kx.emd") for i, f in enumerate(incfiles): if f.strip().lower() in skip_files: # Known bad files to load print("Skipping {}".format(f)) continue else: print("Testing {}".format(f)) try: del d fname = path.join(self.datadir, f) d = Data(fname, debug=False) self.assertTrue( isinstance(d, DataFile), "Failed to load {} correctly.".format(fname)) if "save" in d.subclasses[d["Loaded as"]].__dict__: print("Checking save routine for {}".format( d["Loaded as"])) pth = os.path.join(tmpdir, f) name, ext = os.path.splitext(pth) pth2 = "{}-2.{}".format(name, ext) d.save(pth, as_loaded=True) self.assertTrue( os.path.exists(pth) or os.path.exists(d.filename), "Failed to save as {}".format(pth)) os.remove(d.filename) d.save(pth2, as_loaded=d["Loaded as"]) self.assertTrue( os.path.exists(pth2) or os.path.exists(d.filename), "Failed to save as {}".format(pth)) os.remove(d.filename) except Exception as e: self.assertTrue( False, "Failed in loading <{}>\n{}".format( path.join(self.datadir, f), format_exc())) os.rmdir(tmpdir)
def test_metadata_save(self): local = path.dirname(__file__) t = np.arange(12).reshape( 3, 4) #set up a test data file with mixed metadata t = Data(t) t.column_headers = ["1", "2", "3", "4"] metitems = [ True, 1, 0.2, { "a": 1, "b": "abc" }, (1, 2), np.arange(3), [1, 2, 3], "abc", #all types accepted r"\\abc\cde", 1e-20, #extra tests [1, (1, 2), "abc"], #list with different types [[[1]]] #nested list ] metnames = ["t" + str(i) for i in range(len(metitems))] for k, v in zip(metnames, metitems): t[k] = v t.save(path.join(local, "mixedmetatest.dat")) tl = Data( path.join(local, "mixedmetatest.txt") ) #will change extension to txt if not txt or tdi, is this what we want? t2 = self.d4.clone #check that python tdi save is the same as labview tdi save t2.save(path.join(local, "mixedmetatest2.txt")) t2l = Data(path.join(local, "mixedmetatest2.txt")) for orig, load in [(t, tl), (t2, t2l)]: self.assertTrue(np.allclose(orig.data, load.data)) self.assertTrue(orig.column_headers == load.column_headers) self.assertTrue( all([i in load.metadata.keys() for i in orig.metadata.keys()])) for k in orig.metadata.keys(): if isinstance(orig[k], np.ndarray): self.assertTrue(np.allclose(load[k], orig[k])) elif isinstance(orig[k], float) and np.isnan(orig[k]): self.assertTrue(np.isnan(load[k])) else: self.assertTrue( load[k] == orig[k], "Not equal for metadata: {}".format(load[k])) os.remove(path.join(local, "mixedmetatest.txt")) #clear up os.remove(path.join(local, "mixedmetatest2.txt"))