def test_table(): ts = sdata.Data(name="testseries A1", uuid="a6fc7decdb1441518f762e3b5d798ba8") df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"]) table = sdata.Data(name="Table2", uuid="a6fc7decdb1441518f762e3b5d798ba9") table.table = df table.data = df table.metadata.set_attr(name="a", value="Column a", description="bar") table.metadata.set_attr(name="b", value="Column a", description="bar", unit="kN") table.metadata.set_attr(name="c", value="Column a", description="bar", unit="mm") print(table.data) print(table.metadata) ts.add_data(table) print(ts.group) ts.to_folder("/tmp/sdata_table") table.to_xlsx("/tmp/sdata_table.xlsx") print(table.metadata.to_dataframe())
def test_data(): testname = "testdata" data = sdata.Data(name=testname) print(data) print(data.uuid) assert data.name == testname uuidstr = "b1fd2643af554b33b04422070a0dc7c7" uuidstr2 = "b1fd2643-af55-4b33-b044-22070a0dc7c7" uuidobj = uuid.UUID(uuidstr2) data2 = sdata.Data(name=testname + "2", uuid=uuidstr2) print(data2) print(data2.uuid) assert data2.uuid == uuidobj.hex
def test_data_to_html(): df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]}) data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", table=df, description="abc") data.to_html("/tmp/data.html")
def test_blob(): wb = Workbook( name="workbook", description="A excel like workbook", ) print(wb) assert wb.name == "workbook" s0 = wb.create_sheet(1) assert s0 is None df1 = pd.DataFrame([1, 2, 3]) s1 = wb.create_sheet("df1") s1.df = df1 assert s1.name == "df1" s1.describe() data2 = sdata.Data(name="data2", df=pd.DataFrame({"a": [1, 2, 3]})) wb.add_sheet(data2) assert data2 in wb assert "data2" in wb.sheetnames s3 = wb.add_sheet(1) assert s3 is None print(wb.sheetnames) assert wb.sheetnames == ['df1', 'data2'] assert len(wb.sheets) == 2
def test_project(): project = "my_project_id" data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description="data for a project", project=project) assert data.project == project
def get_dummy_table(dim=(10, 3)): df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"]) table = sdata.Data(uuid=six.next(fixed_uuid)) table.data = df table.metadata.set_attr(name="a", value="Column a", description="bar") table.metadata.set_attr(name="b", value="Column a", description="bar", unit="kN") table.metadata.set_attr(name="c", value="Column a", description="bar", unit="mm") return table
def gen_dummy_testprogram(): ts1 = sdata.Data(name="testseries A1", uuid="a6fc7decdb1441518f762e3b5d798ba7") t1 = sdata.Data(name="test 001", uuid="8796c35b2e3a4f8a82af181698c15861") ts1.add_data(t1) t1.add_data(get_dummy_table()) print(t1._get_table()) t2 = sdata.Data(name="test 002", uuid="b62195ac49b64c9e8cedb7dba52bd539") t2.add_data(get_dummy_table()) t2.add_data(get_dummy_table()) ts1.add_data(t2) t3 = sdata.Data(name="test 003", uuid="ddc82782f5f0455895145682fe0a70f2") t3.add_data(get_dummy_table()) ts1.add_data(t3) ts2 = sdata.Data(name="testseries A2", uuid="c3c63f8094464325bd57623cb5bbe58f") t1b = sdata.Data(name="test 001b", uuid="bb507e40663d49cca8264c0ed6751692") t1b.add_data(get_dummy_table()) ts2.add_data(t1b) ts1.add_data(t3) t2b = sdata.Data(name="test 002b", uuid="e574e000f1404f5ebb9aaceb4183dc4c") # t2b.add_result(get_dummy_table()) ts2.add_data(t2b) tp = TestProgram(name="testprogram FOO", uuid="43880975d9b745f1b853c31b691e67a9") tp.add_data(ts1) tp.add_data(ts2) return tp
def test_uuid(): u0 = "38b26864e7794f5182d38459bab8584d" print(u0) data = sdata.Data(name="data", uuid=u0) assert data.uuid == u0 u = uuid.UUID("38b26864e7794f5182d38459bab8584d") print(u) data = sdata.Data(name="data", uuid=u) assert data.uuid == u0 u = "38b26864-e779-4f51-82d3-8459bab8584d" print(u) data = sdata.Data(name="data", uuid=u) assert data.uuid == u0 u = "38b26864-e779-4f51-82d3-8459bab8584d" print(u) data = sdata.Data(name="data") data.uuid = u assert data.uuid == u0
def test_flatstoreexample(): store = FlatHDFDataStore(filepath="/tmp/mystoreexample.h5") data = sdata.Data(name="otto", uuid="d4e97cedca6238bea16732ce88c1922f", table=pd.DataFrame({"a": [1, 2, 3]}), description="Hallo\nSpencer") store.put(data) loaded_data = store.get_data_by_uuid("d4e97cedca6238bea16732ce88c1922f") assert data.sha3_256 == loaded_data.sha3_256 store.close()
def test_to_csv(): df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]}) data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", table=df, description="abc") data.metadata.add("a", 1.23, unit="N", description="force", required=True) data.to_csv(filepath="/tmp/data1.csv") data2 = sdata.Data.from_csv(filepath="/tmp/data1.csv") print(data.name, data2.name) assert data.name == data2.name assert data.uuid == data2.uuid assert data.df.shape == data2.df.shape
def test_to_json(): df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]}) data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", table=df, description="abc") js = data.to_json() data2 = sdata.Data.from_json(s=js) print(data.name, data2.name) assert data.name == data2.name assert data.uuid == data2.uuid assert data.description == data2.description assert data.sha3_256 == data2.sha3_256
def test_group(): data1 = sdata.Data(name="data1", uuid="38b26864e7794f5182d38459bab8584f") data2 = sdata.Data(name="data2", uuid="b1fd2643-af55-4b33-b044-22070a0dc7c7") data3 = sdata.Data(name="data3", uuid=uuid.UUID("664577c2d3134b598bc4d6c13f20b71a")) group1 = sdata.Group(name="group1", uuid="dbc894745fb04f7e87a990bdd4ba97c4") print(group1) group1.add_data(data1) group1.add_data(data2) group1.add_data(data3) print(group1.group) data1a = group1.get_data(uuid="38b26864e7794f5182d38459bab8584f") assert data1a.name == "data1" assert data1a.uuid == "38b26864e7794f5182d38459bab8584f" data3a = group1.get_data(uuid="664577c2d3134b598bc4d6c13f20b71a") assert data3a.name == "data3" assert data3a.uuid == "664577c2d3134b598bc4d6c13f20b71a"
def test_description(): data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description="abc") assert data.description == "abc" text1 = """abc test ende""" data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description=text1) assert data.description == text1 data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description=text1) df = data.description_to_df() print(df) data.description_from_df(df) assert data.description == text1 text2 = text1 + "\n" data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description=text2) assert data.description == text2 data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab8584d", description=text2) df = data.description_to_df() print(df) data.description_from_df(df) assert data.description == text1 assert data.description != text2
def test_hdf5vault(): rootpath = "/tmp/test_hdf5vault.h5" if os.path.exists(rootpath): os.remove(rootpath) vault = Hdf5Vault(rootpath=rootpath) print(vault) assert os.path.exists(rootpath) data = sdata.Data(name="otto", uuid=sdata.uuid_from_str("otto"), table=pd.DataFrame({"a": [1, 2, 3]}), description="Hallo\nSpencer") datac = data.copy(uuid='18b26864e7794f5182d38459bab85842', name="copy") print(datac) ret = vault.dump_blob(data) print(ret) ret = vault.dump_blob(datac) print(ret) ldata = vault.load_blob(data.uuid) assert data == ldata ldatac = vault.load_blob(datac.uuid) assert datac == ldatac keys = vault.keys() print(keys) assert sorted(keys) == sorted([ '18b26864e7794f5182d38459bab85842', '21b83703d98e38a7be2e50e38326d0ce' ]) items = vault.items() print("items", items) assert len(items) == 2 for k, v in items: print(f"{k}:{v}") item_dict = dict(items) assert item_dict['18b26864e7794f5182d38459bab85842'].uuid == datac.uuid assert item_dict['18b26864e7794f5182d38459bab85842'].name == datac.name values = vault.values() print("values", values) assert len(values) == 2 for v in values: print(f"{v.name}:{v}")
def test_hdf5(): df = pd.DataFrame({"a": [1.1, 2.1, 3.5], "b": [2.4, 1.2, 2.2]}) d = sdata.Data(name="basic example", uuid="38b26864e7794f5182d38459bab85842", table=df, description="hallo") d.metadata.add("Temperatur", value=25.4, dtype="float", unit="degC", description="Temperatur") export_path = "/tmp/{}.h5".format(d.osname) d.to_hdf5(export_path) d2 = sdata.Data.from_hdf5(export_path) print(d2.describe()) assert d2.sha3_256 == d.sha3_256
def test_flathdfstore(): store = FlatHDFDataStore("/tmp/flatstore1.h5", mode="w") print(store) assert len(store.keys()) == 0 data = sdata.Data(name="otto", uuid=sdata.uuid_from_str("otto"), table=pd.DataFrame({"a": [1, 2, 3]}), description="Hallo\nSpencer") store.put(data) print(store.keys()) datac = data.copy() datac.name = "otto2" datac.uuid = 'b8315be85d9945579cf8dc6a80a62524' datac.df["b"] = datac.df["a"]**2 datac.df datac.metadata.add("force", 1.23, dtype="float", description="a force", label="F") datac.metadata.add("runid", 123, dtype="int", description="a int", label="r") print(datac) store.put(datac) store.keys() ldata = store.get_data_by_uuid(data.uuid) assert data.sha3_256 == ldata.sha3_256 ldatac = store.get_data_by_uuid('b8315be85d9945579cf8dc6a80a62524') print(datac) ldatac.metadata.df print([ldatac.description]) print(ldatac.describe()) assert datac.sha3_256 == ldatac.sha3_256 store.close()
def test_filename(): data = sdata.Data(name="data1#2/3!4") print(data.filename) assert data.filename == "data1_2_3_4"
def test_asciiname(): assert sdata.Data( name="ö-ü-ä-Ö-Ü-Ä-?- -\\-/").osname == 'oe-ue-ae-oe-ue-ae-?-_-_-_' assert sdata.Data( name="ö-ü-ä-Ö-Ü-Ä-?- -\\-/").asciiname == 'oe-ue-ae-Oe-Ue-Ae-?-_-_-_'
def atest_ks2_full(): tp = sdata.Data("what") tp.to_folder("/tmp/tp_ks2") # sdata.testprogram.TestProgram.clear_folder("/tmp/tp_ks2") tp = sdata.Data.from_folder("/tmp/tp_ks2") tp.clear_group() tp.add_data(sdata.Data(name="materials")) tp.add_data(sdata.Data(name="parts")) print(tp.group) materials = tp.get_data_by_name("materials") parts = tp.get_data_by_name("parts") print(parts) # Material 1 mat1 = sdata.Data(name="HX340LAD", default_attributes=material_default_attributes) mat1.metadata.set_attr("material_type", "steel") # print(mat1.metadata) invalid_attrs = mat1.verify_attributes() print(invalid_attrs) assert len(invalid_attrs) == 0 # print(mat1.metadata.to_dataframe()) materials.add_data(mat1) # Material 2 mat2 = sdata.Data(name="AW6016", default_attributes=material_default_attributes) mat2.metadata.set_attr("material_type", "alu") mat2.metadata.set_attr("material_grade", "T4") materials.add_data(mat2) # Part 1 part1 = sdata.Data(name="upper sheet", default_attributes=ks2specimen_default_attributes) part1.metadata.set_attr("material_uuid", mat1.uuid) invalid_attrs = part1.verify_attributes() print(invalid_attrs) # assert len(invalid_attrs)>0 part1.metadata.set_attr("rd", 0) part1.metadata.set_attr("t", 1.1) part1.metadata.set_attr("r", 2) part1.metadata.set_attr("bi", 34.) part1.metadata.set_attr("hi", 29.) part1.metadata.set_attr("l", 50.) invalid_attrs = part1.verify_attributes() print(invalid_attrs) assert len(invalid_attrs) == 0 parts.add_data(part1) # Part 2 part2 = sdata.Data(name="bottom sheet", default_attributes=ks2specimen_default_attributes) print("!!!", part2.osname) part2.metadata.set_attr("material_uuid", mat1.uuid) print(part2.metadata.get_attr("t")) part2.metadata.set_attr("material_uuid", mat2.uuid) part2.metadata.set_attr("rd", 0) # part2.metadata.get_attr("t").value=2.0 part2.metadata.set_attr("r", 4) part2.metadata.set_attr("bi", 34.) part2.metadata.set_attr("hi", 29.) part2.metadata.set_attr("l", 50.) invalid_attrs = part2.verify_attributes() print(invalid_attrs) assert len(invalid_attrs) == 0 parts.add_data(part2) # KS2 Test Series 1 ts1 = sdata.Data(name="KS2 Series A1") tp.add_data(ts1) # KS2 Test 1 ks2 = sdata.Data(name="KS2 test A1", parts=[part1, part2], default_attributes=ks2test_default_attributes) ks2.name = "KS2 Test A1" print("ks2test", ks2) assert ks2.name == "KS2 Test A1" print("ks2test", ks2.metadata) ks2.metadata.set_attr("angle", 30.) print(ks2.metadata.get_attr("t3")) ts1.add_data(ks2) # Test Result Table for KS2 Test 1 df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"]) table = sdata.Data(name="Fs", desciption="Force Displacement Relation") table.table = df table.metadata.set_attr(name="a", value="Column a", description="bar") table.metadata.set_attr(name="b", value="Column a", description="bar", unit="kN") table.metadata.set_attr(name="c", value="Column a", description="bar", unit="mm") ks2.add_data(table) tp.to_folder("/tmp/tp_ks2") tp2 = sdata.Data.from_folder("/tmp/tp_ks2") tp2.to_folder("/tmp/tp_ks2a") tp2.tree_folder("/tmp/tp_ks2a") print(tp.dir()) # def test_ks2(): tp = sdata.Data.from_folder("/tmp/tp_ks2") tp.name = "KS2-testprogram" tp.to_folder("/tmp/tp_ks2") tp2 = sdata.Data.from_folder("/tmp/tp_ks2") tp2.to_folder("/tmp/tp_ks2a") tp2.tree_folder("/tmp/tp_ks2a")