def test_table():
ts = sdata.Data(name="testseries A1",
uuid="a6fc7decdb1441518f762e3b5d798ba8")
df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"])
table = sdata.Data(name="Table2", uuid="a6fc7decdb1441518f762e3b5d798ba9")
table.table = df
table.data = df
table.metadata.set_attr(name="a", value="Column a", description="bar")
table.metadata.set_attr(name="b",
value="Column a",
description="bar",
unit="kN")
table.metadata.set_attr(name="c",
value="Column a",
description="bar",
unit="mm")
print(table.data)
print(table.metadata)
ts.add_data(table)
print(ts.group)
ts.to_folder("/tmp/sdata_table")
table.to_xlsx("/tmp/sdata_table.xlsx")
print(table.metadata.to_dataframe())
def test_data():
testname = "testdata"
data = sdata.Data(name=testname)
print(data)
print(data.uuid)
assert data.name == testname
uuidstr = "b1fd2643af554b33b04422070a0dc7c7"
uuidstr2 = "b1fd2643-af55-4b33-b044-22070a0dc7c7"
uuidobj = uuid.UUID(uuidstr2)
data2 = sdata.Data(name=testname + "2", uuid=uuidstr2)
print(data2)
print(data2.uuid)
assert data2.uuid == uuidobj.hex
def test_data_to_html():
df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]})
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
table=df,
description="abc")
data.to_html("/tmp/data.html")
def test_blob():
wb = Workbook(
name="workbook",
description="A excel like workbook",
)
print(wb)
assert wb.name == "workbook"
s0 = wb.create_sheet(1)
assert s0 is None
df1 = pd.DataFrame([1, 2, 3])
s1 = wb.create_sheet("df1")
s1.df = df1
assert s1.name == "df1"
s1.describe()
data2 = sdata.Data(name="data2", df=pd.DataFrame({"a": [1, 2, 3]}))
wb.add_sheet(data2)
assert data2 in wb
assert "data2" in wb.sheetnames
s3 = wb.add_sheet(1)
assert s3 is None
print(wb.sheetnames)
assert wb.sheetnames == ['df1', 'data2']
assert len(wb.sheets) == 2
def test_project():
project = "my_project_id"
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description="data for a project",
project=project)
assert data.project == project
def get_dummy_table(dim=(10, 3)):
df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"])
table = sdata.Data(uuid=six.next(fixed_uuid))
table.data = df
table.metadata.set_attr(name="a", value="Column a", description="bar")
table.metadata.set_attr(name="b", value="Column a", description="bar", unit="kN")
table.metadata.set_attr(name="c", value="Column a", description="bar", unit="mm")
return table
def gen_dummy_testprogram():
ts1 = sdata.Data(name="testseries A1", uuid="a6fc7decdb1441518f762e3b5d798ba7")
t1 = sdata.Data(name="test 001", uuid="8796c35b2e3a4f8a82af181698c15861")
ts1.add_data(t1)
t1.add_data(get_dummy_table())
print(t1._get_table())
t2 = sdata.Data(name="test 002", uuid="b62195ac49b64c9e8cedb7dba52bd539")
t2.add_data(get_dummy_table())
t2.add_data(get_dummy_table())
ts1.add_data(t2)
t3 = sdata.Data(name="test 003", uuid="ddc82782f5f0455895145682fe0a70f2")
t3.add_data(get_dummy_table())
ts1.add_data(t3)
ts2 = sdata.Data(name="testseries A2", uuid="c3c63f8094464325bd57623cb5bbe58f")
t1b = sdata.Data(name="test 001b", uuid="bb507e40663d49cca8264c0ed6751692")
t1b.add_data(get_dummy_table())
ts2.add_data(t1b)
ts1.add_data(t3)
t2b = sdata.Data(name="test 002b", uuid="e574e000f1404f5ebb9aaceb4183dc4c")
# t2b.add_result(get_dummy_table())
ts2.add_data(t2b)
tp = TestProgram(name="testprogram FOO", uuid="43880975d9b745f1b853c31b691e67a9")
tp.add_data(ts1)
tp.add_data(ts2)
return tp
def test_uuid():
u0 = "38b26864e7794f5182d38459bab8584d"
print(u0)
data = sdata.Data(name="data", uuid=u0)
assert data.uuid == u0
u = uuid.UUID("38b26864e7794f5182d38459bab8584d")
print(u)
data = sdata.Data(name="data", uuid=u)
assert data.uuid == u0
u = "38b26864-e779-4f51-82d3-8459bab8584d"
print(u)
data = sdata.Data(name="data", uuid=u)
assert data.uuid == u0
u = "38b26864-e779-4f51-82d3-8459bab8584d"
print(u)
data = sdata.Data(name="data")
data.uuid = u
assert data.uuid == u0
def test_flatstoreexample():
store = FlatHDFDataStore(filepath="/tmp/mystoreexample.h5")
data = sdata.Data(name="otto",
uuid="d4e97cedca6238bea16732ce88c1922f",
table=pd.DataFrame({"a": [1, 2, 3]}),
description="Hallo\nSpencer")
store.put(data)
loaded_data = store.get_data_by_uuid("d4e97cedca6238bea16732ce88c1922f")
assert data.sha3_256 == loaded_data.sha3_256
store.close()
def test_to_csv():
df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]})
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
table=df,
description="abc")
data.metadata.add("a", 1.23, unit="N", description="force", required=True)
data.to_csv(filepath="/tmp/data1.csv")
data2 = sdata.Data.from_csv(filepath="/tmp/data1.csv")
print(data.name, data2.name)
assert data.name == data2.name
assert data.uuid == data2.uuid
assert data.df.shape == data2.df.shape
def test_to_json():
df = pd.DataFrame({'a': ['x', 'y', '', 'z'], 'b': [1, 2, 2, 3.2]})
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
table=df,
description="abc")
js = data.to_json()
data2 = sdata.Data.from_json(s=js)
print(data.name, data2.name)
assert data.name == data2.name
assert data.uuid == data2.uuid
assert data.description == data2.description
assert data.sha3_256 == data2.sha3_256
def test_group():
data1 = sdata.Data(name="data1", uuid="38b26864e7794f5182d38459bab8584f")
data2 = sdata.Data(name="data2",
uuid="b1fd2643-af55-4b33-b044-22070a0dc7c7")
data3 = sdata.Data(name="data3",
uuid=uuid.UUID("664577c2d3134b598bc4d6c13f20b71a"))
group1 = sdata.Group(name="group1",
uuid="dbc894745fb04f7e87a990bdd4ba97c4")
print(group1)
group1.add_data(data1)
group1.add_data(data2)
group1.add_data(data3)
print(group1.group)
data1a = group1.get_data(uuid="38b26864e7794f5182d38459bab8584f")
assert data1a.name == "data1"
assert data1a.uuid == "38b26864e7794f5182d38459bab8584f"
data3a = group1.get_data(uuid="664577c2d3134b598bc4d6c13f20b71a")
assert data3a.name == "data3"
assert data3a.uuid == "664577c2d3134b598bc4d6c13f20b71a"
def test_description():
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description="abc")
assert data.description == "abc"
text1 = """abc
test
ende"""
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description=text1)
assert data.description == text1
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description=text1)
df = data.description_to_df()
print(df)
data.description_from_df(df)
assert data.description == text1
text2 = text1 + "\n"
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description=text2)
assert data.description == text2
data = sdata.Data(name="data",
uuid="38b26864e7794f5182d38459bab8584d",
description=text2)
df = data.description_to_df()
print(df)
data.description_from_df(df)
assert data.description == text1
assert data.description != text2
def test_hdf5vault():
rootpath = "/tmp/test_hdf5vault.h5"
if os.path.exists(rootpath):
os.remove(rootpath)
vault = Hdf5Vault(rootpath=rootpath)
print(vault)
assert os.path.exists(rootpath)
data = sdata.Data(name="otto",
uuid=sdata.uuid_from_str("otto"),
table=pd.DataFrame({"a": [1, 2, 3]}),
description="Hallo\nSpencer")
datac = data.copy(uuid='18b26864e7794f5182d38459bab85842', name="copy")
print(datac)
ret = vault.dump_blob(data)
print(ret)
ret = vault.dump_blob(datac)
print(ret)
ldata = vault.load_blob(data.uuid)
assert data == ldata
ldatac = vault.load_blob(datac.uuid)
assert datac == ldatac
keys = vault.keys()
print(keys)
assert sorted(keys) == sorted([
'18b26864e7794f5182d38459bab85842', '21b83703d98e38a7be2e50e38326d0ce'
])
items = vault.items()
print("items", items)
assert len(items) == 2
for k, v in items:
print(f"{k}:{v}")
item_dict = dict(items)
assert item_dict['18b26864e7794f5182d38459bab85842'].uuid == datac.uuid
assert item_dict['18b26864e7794f5182d38459bab85842'].name == datac.name
values = vault.values()
print("values", values)
assert len(values) == 2
for v in values:
print(f"{v.name}:{v}")
def test_hdf5():
df = pd.DataFrame({"a": [1.1, 2.1, 3.5], "b": [2.4, 1.2, 2.2]})
d = sdata.Data(name="basic example",
uuid="38b26864e7794f5182d38459bab85842",
table=df,
description="hallo")
d.metadata.add("Temperatur",
value=25.4,
dtype="float",
unit="degC",
description="Temperatur")
export_path = "/tmp/{}.h5".format(d.osname)
d.to_hdf5(export_path)
d2 = sdata.Data.from_hdf5(export_path)
print(d2.describe())
assert d2.sha3_256 == d.sha3_256
def test_flathdfstore():
store = FlatHDFDataStore("/tmp/flatstore1.h5", mode="w")
print(store)
assert len(store.keys()) == 0
data = sdata.Data(name="otto",
uuid=sdata.uuid_from_str("otto"),
table=pd.DataFrame({"a": [1, 2, 3]}),
description="Hallo\nSpencer")
store.put(data)
print(store.keys())
datac = data.copy()
datac.name = "otto2"
datac.uuid = 'b8315be85d9945579cf8dc6a80a62524'
datac.df["b"] = datac.df["a"]**2
datac.df
datac.metadata.add("force",
1.23,
dtype="float",
description="a force",
label="F")
datac.metadata.add("runid",
123,
dtype="int",
description="a int",
label="r")
print(datac)
store.put(datac)
store.keys()
ldata = store.get_data_by_uuid(data.uuid)
assert data.sha3_256 == ldata.sha3_256
ldatac = store.get_data_by_uuid('b8315be85d9945579cf8dc6a80a62524')
print(datac)
ldatac.metadata.df
print([ldatac.description])
print(ldatac.describe())
assert datac.sha3_256 == ldatac.sha3_256
store.close()
def test_filename():
data = sdata.Data(name="data1#2/3!4")
print(data.filename)
assert data.filename == "data1_2_3_4"
def test_asciiname():
assert sdata.Data(
name="ö-ü-ä-Ö-Ü-Ä-?- -\\-/").osname == 'oe-ue-ae-oe-ue-ae-?-_-_-_'
assert sdata.Data(
name="ö-ü-ä-Ö-Ü-Ä-?- -\\-/").asciiname == 'oe-ue-ae-Oe-Ue-Ae-?-_-_-_'
def atest_ks2_full():
tp = sdata.Data("what")
tp.to_folder("/tmp/tp_ks2")
# sdata.testprogram.TestProgram.clear_folder("/tmp/tp_ks2")
tp = sdata.Data.from_folder("/tmp/tp_ks2")
tp.clear_group()
tp.add_data(sdata.Data(name="materials"))
tp.add_data(sdata.Data(name="parts"))
print(tp.group)
materials = tp.get_data_by_name("materials")
parts = tp.get_data_by_name("parts")
print(parts)
# Material 1
mat1 = sdata.Data(name="HX340LAD",
default_attributes=material_default_attributes)
mat1.metadata.set_attr("material_type", "steel")
# print(mat1.metadata)
invalid_attrs = mat1.verify_attributes()
print(invalid_attrs)
assert len(invalid_attrs) == 0
# print(mat1.metadata.to_dataframe())
materials.add_data(mat1)
# Material 2
mat2 = sdata.Data(name="AW6016",
default_attributes=material_default_attributes)
mat2.metadata.set_attr("material_type", "alu")
mat2.metadata.set_attr("material_grade", "T4")
materials.add_data(mat2)
# Part 1
part1 = sdata.Data(name="upper sheet",
default_attributes=ks2specimen_default_attributes)
part1.metadata.set_attr("material_uuid", mat1.uuid)
invalid_attrs = part1.verify_attributes()
print(invalid_attrs)
# assert len(invalid_attrs)>0
part1.metadata.set_attr("rd", 0)
part1.metadata.set_attr("t", 1.1)
part1.metadata.set_attr("r", 2)
part1.metadata.set_attr("bi", 34.)
part1.metadata.set_attr("hi", 29.)
part1.metadata.set_attr("l", 50.)
invalid_attrs = part1.verify_attributes()
print(invalid_attrs)
assert len(invalid_attrs) == 0
parts.add_data(part1)
# Part 2
part2 = sdata.Data(name="bottom sheet",
default_attributes=ks2specimen_default_attributes)
print("!!!", part2.osname)
part2.metadata.set_attr("material_uuid", mat1.uuid)
print(part2.metadata.get_attr("t"))
part2.metadata.set_attr("material_uuid", mat2.uuid)
part2.metadata.set_attr("rd", 0)
# part2.metadata.get_attr("t").value=2.0
part2.metadata.set_attr("r", 4)
part2.metadata.set_attr("bi", 34.)
part2.metadata.set_attr("hi", 29.)
part2.metadata.set_attr("l", 50.)
invalid_attrs = part2.verify_attributes()
print(invalid_attrs)
assert len(invalid_attrs) == 0
parts.add_data(part2)
# KS2 Test Series 1
ts1 = sdata.Data(name="KS2 Series A1")
tp.add_data(ts1)
# KS2 Test 1
ks2 = sdata.Data(name="KS2 test A1",
parts=[part1, part2],
default_attributes=ks2test_default_attributes)
ks2.name = "KS2 Test A1"
print("ks2test", ks2)
assert ks2.name == "KS2 Test A1"
print("ks2test", ks2.metadata)
ks2.metadata.set_attr("angle", 30.)
print(ks2.metadata.get_attr("t3"))
ts1.add_data(ks2)
# Test Result Table for KS2 Test 1
df = pd.DataFrame(np.random.random((10, 3)), columns=["a", "b", "c"])
table = sdata.Data(name="Fs", desciption="Force Displacement Relation")
table.table = df
table.metadata.set_attr(name="a", value="Column a", description="bar")
table.metadata.set_attr(name="b",
value="Column a",
description="bar",
unit="kN")
table.metadata.set_attr(name="c",
value="Column a",
description="bar",
unit="mm")
ks2.add_data(table)
tp.to_folder("/tmp/tp_ks2")
tp2 = sdata.Data.from_folder("/tmp/tp_ks2")
tp2.to_folder("/tmp/tp_ks2a")
tp2.tree_folder("/tmp/tp_ks2a")
print(tp.dir())
# def test_ks2():
tp = sdata.Data.from_folder("/tmp/tp_ks2")
tp.name = "KS2-testprogram"
tp.to_folder("/tmp/tp_ks2")
tp2 = sdata.Data.from_folder("/tmp/tp_ks2")
tp2.to_folder("/tmp/tp_ks2a")
tp2.tree_folder("/tmp/tp_ks2a")