def test_product():
ctx = Prosto("My Prosto")
t1 = ctx.populate(
table_name="Table 1", attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1.0, 2.0, 3.0]})", tables=[]
)
t2 = ctx.populate(
table_name="Table 2", attributes=["B"],
func="lambda **m: pd.DataFrame({'B': ['x', 'y', 'z']})", tables=[]
)
product = ctx.product(
table_name="Product", attributes=["t1", "t2"],
tables=["Table 1", "Table 2"]
)
t1.evaluate()
t2.evaluate()
product.evaluate()
assert len(product.get_df().columns) == 2
assert len(product.get_df()) == 9
assert product.get_df().columns.to_list() == ["t1", "t2"]
def test_two_keys():
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts", attributes=["A", "B"],
func="lambda **m: pd.DataFrame({'A': ['a', 'b', 'b', 'a'], 'B': ['b', 'c', 'c', 'a']})", tables=[]
)
# Groups
g_tbl = ctx.populate(
table_name="Groups", attributes=["A", "B"],
func="lambda **m: pd.DataFrame({'A': ['a', 'b', 'a'], 'B': ['b', 'c', 'c'], 'C': [1, 2, 3]})", tables=[]
)
# Link
l_clm = ctx.link(
name="Link", table=f_tbl.id, type=g_tbl.id,
columns=["A", "B"], linked_columns=["A", "B"]
)
f_tbl.evaluate()
g_tbl.evaluate()
l_clm.evaluate()
f_tbl_data = f_tbl.get_df()
assert len(f_tbl_data) == 4 # Same number of rows
assert len(f_tbl_data.columns) == 3
l_data = f_tbl.get_series("Link")
assert l_data[0] == 0
assert l_data[1] == 1
assert l_data[2] == 1
assert pd.isna(l_data[3])
#
# Test topology
#
topology = Topology(ctx)
topology.translate() # All data will be reset
layers = topology.elem_layers
assert len(layers) == 2
assert set([x.id for x in layers[0]]) == {"Facts", "Groups"}
assert set([x.id for x in layers[1]]) == {"Link"}
ctx.run()
l_data = f_tbl.get_series("Link")
assert l_data[0] == 0
assert l_data[1] == 1
assert l_data[2] == 1
assert pd.isna(l_data[3])
def test_merge_path2():
"""
Here we do the same as previous test, but specify complex path using separators (rather than a list of simple segment names).
So only the definition of merge operation changes.
"""
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts",
attributes=["A"],
func="lambda **m: pd.DataFrame({'A': ['a', 'a', 'b', 'b']})",
tables=[])
# Groups
g_tbl = ctx.populate(
table_name="Groups",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [2.0, 3.0, 3.0]})",
tables=[])
# Link
l_clm = ctx.link(name="Link",
table=f_tbl.id,
type=g_tbl.id,
columns=["A"],
linked_columns=["A"])
# SuperGroups
sg_tbl = ctx.populate(
table_name="SuperGroups",
attributes=["B", "C"],
func=
"lambda **m: pd.DataFrame({'B': [2.0, 3.0, 4.0], 'C': ['x', 'y', 'z']})",
tables=[])
# SuperLink
sl_clm = ctx.link(name="SuperLink",
table=g_tbl.id,
type=sg_tbl.id,
columns=["B"],
linked_columns=["B"])
# Merge
m_clm = ctx.merge("Merge", f_tbl.id, ["Link::SuperLink::C"])
ctx.run()
f_tbl_data = f_tbl.get_df()
assert len(f_tbl_data) == 4 # Same number of rows
assert len(f_tbl_data.columns) == 3
m_data = f_tbl.get_series("Merge")
assert m_data.to_list() == ['x', 'x', 'y', 'y']
def test_groll_single():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table",
attributes=["G", "A"],
func=
"lambda **m: pd.DataFrame({'G': [1, 2, 1, 2], 'A': [1.0, 2.0, 3.0, 4.0]})",
tables=[])
clm = ctx.roll(name="Roll",
table=tbl.id,
window="2",
link="G",
func="lambda x: x.sum()",
columns=["A"],
model={})
ctx.run()
clm_data = tbl.get_series('Roll')
assert pd.isna(clm_data[0])
assert pd.isna(clm_data[1])
assert np.isclose(clm_data[2], 4.0)
assert np.isclose(clm_data[3], 6.0)
def test_integers2():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table",
attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8, 9]})",
tables=[])
clm = ctx.discretize(name="My column",
table=tbl.id,
columns=["A"],
model={
"origin": 5,
"step": 3,
"label": "right",
"closed": "right"
})
# 1, 2], 3, 4, 5], 6, 7, 8], 9
# -1 -1 0 1 2
ctx.run()
clm_data = tbl.get_series('My column')
assert list(clm_data) == [-1, -1, 0, 0, 0, 1, 1, 1, 2]
def test_calculate_value():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table", attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1, 2, 3]})", tables=[]
)
clm = ctx.calculate(
name="My column", table=tbl.id,
func="lambda x: float(x)", columns=["A"], model=None
)
tbl.evaluate()
clm.evaluate()
clm_data = tbl.get_series('My column')
v0 = clm_data[0]
v1 = clm_data[1]
v2 = clm_data[2]
assert np.isclose(v0, 1.0)
assert np.isclose(v1, 2.0)
assert np.isclose(v2, 3.0)
assert isinstance(v0, float)
assert isinstance(v1, float)
assert isinstance(v2, float)
def test_two_keys():
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts", attributes=["A", "B"],
func="lambda **m: pd.DataFrame({'A': ['a', 'b', 'b', 'a'], 'B': ['b', 'c', 'c', 'a']})", tables=[]
)
# Groups
g_tbl = ctx.project(
table_name="Groups", attributes=["X", "Y"],
tables=["Facts"], columns=["A", "B"]
)
# Link
l_clm = ctx.link(
name="Link", table=f_tbl.id, type=g_tbl.id,
columns=["A", "B"], linked_columns=["X", "Y"]
)
f_tbl.evaluate()
g_tbl.evaluate()
l_clm.evaluate()
g_tbl_data = g_tbl.get_df()
assert len(g_tbl_data) == 3
assert len(g_tbl_data.columns) == 2
l_data = f_tbl.get_series("Link")
assert l_data[0] == 0
assert l_data[1] == 1
assert l_data[2] == 1
assert l_data[3] == 2
#
# Test topology
#
topology = Topology(ctx)
topology.translate()
layers = topology.elem_layers
assert len(layers) == 3
assert set([x.id for x in layers[0]]) == {"Facts"}
assert set([x.id for x in layers[1]]) == {"Groups"}
assert set([x.id for x in layers[2]]) == {"Link"}
g_tbl_data = g_tbl.get_df()
g_tbl_data.drop(g_tbl_data.index, inplace=True) # Empty
ctx.run()
g_tbl_data = g_tbl.get_df()
assert len(g_tbl_data) == 3
assert len(g_tbl_data.columns) == 2
def test_aggregate_with_path():
"""Aggregation with column paths as measures which have to be automatically produce merge operation."""
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts",
attributes=["A", "M"],
func=
"lambda **m: pd.DataFrame({'A': ['a', 'a', 'b', 'b'], 'M': [1.0, 2.0, 3.0, 4.0]})",
tables=[])
# Groups
df = pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [3.0, 2.0, 1.0]})
g_tbl = ctx.populate(
table_name="Groups",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [3.0, 2.0, 1.0]})",
tables=[])
# Link
l_clm = ctx.link(name="Link",
table=f_tbl.id,
type=g_tbl.id,
columns=["A"],
linked_columns=["A"])
# Aggregation
a_clm = ctx.aggregate(name="Aggregate",
table=g_tbl.id,
tables=["Facts"],
link="Link",
func="lambda x, bias, **model: x.sum() + bias",
columns=["Link::B"],
model={"bias": 0.0})
ctx.run()
a_clm_data = g_tbl.get_series('Aggregate')
assert a_clm_data[0] == 6.0
assert a_clm_data[1] == 4.0
assert a_clm_data[2] == 0.0
def test_calculate_with_path():
"""Test topology augmentation. Calculation with column paths which have to be automatically produce merge operation."""
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts", attributes=["A", "M"],
func="lambda **m: pd.DataFrame({'A': ['a', 'a', 'b', 'b'], 'M': [1.0, 2.0, 3.0, 4.0]})", tables=[]
)
# Groups
df = pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [3.0, 2.0, 1.0]})
g_tbl = ctx.populate(
table_name="Groups", attributes=["A", "B"],
func="lambda **m: pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [3.0, 2.0, 1.0]})", tables=[]
)
# Link
l_clm = ctx.link(
name="Link", table=f_tbl.id, type=g_tbl.id,
columns=["A"], linked_columns=["A"]
)
# Calculate
clm = ctx.calculate(
name="My column", table=f_tbl.id,
func="lambda x: x['M'] + x['Link::B']", columns=["M", "Link::B"], model=None
)
ctx.run()
clm_data = f_tbl.get_series('My column')
assert clm_data[0] == 4.0
assert clm_data[1] == 5.0
assert clm_data[2] == 5.0
assert clm_data[3] == 6.0
def test_populate():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': [1.0, 2.0, 3.0], 'B': ['x', 'y', 'z']})",
tables=[],
model={"nrows": 3})
tbl.evaluate()
assert len(tbl.get_df().columns) == 2
assert len(tbl.get_df()) == 3
def test_product_inheritance():
"""
We add an addition calculate column to the product table which uses a column of a base table.
The system has to automatically insert a new operation by resolving this missing column.
"""
ctx = Prosto("My Prosto")
t1 = ctx.populate(
table_name="Table 1", attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1.0, 2.0, 3.0]})", tables=[]
)
t2 = ctx.populate(
table_name="Table 2", attributes=["B"],
func="lambda **m: pd.DataFrame({'B': ['x', 'y', 'z']})", tables=[]
)
product = ctx.product(
table_name="Product", attributes=["t1", "t2"],
tables=["Table 1", "Table 2"]
)
# In this calculate column, we use a column of the product table which actually exists only in a base table
clm = ctx.calculate(
name="My column", table=product.id,
func="lambda x: x + 1.0", columns=["A"], model=None
)
ctx.run()
# We get two columns in addition to two attributes: one merge (augmented) and one calculate column
assert len(product.get_df().columns) == 4
clm_data = product.get_series('My column')
assert clm_data.to_list() == [2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0]
def test_filter_table():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="Base table",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': [1.0, 2.0, 3.0], 'B': ['x', 'yy', 'zzz']})",
tables=[])
# This (boolean) column will be used for filtering
clm = ctx.compute(
name="filter_column",
table=tbl.id,
func=
"lambda x, param: (x['A'] > param) & (x['B'].str.len() < 3)", # Return a boolean Series
columns=["A", "B"],
model={"param": 1.5})
tbl.evaluate()
clm.evaluate()
tbl = ctx.filter(table_name="Filtered table",
attributes=["super"],
func=None,
tables=["Base table"],
columns=["filter_column"])
tbl.evaluate()
assert len(tbl.get_df().columns) == 1 # Only one link-attribute is created
assert len(tbl.get_df()) == 1
assert tbl.get_df()['super'][0] == 1
#
# Test topology
#
topology = Topology(ctx)
topology.translate()
layers = topology.elem_layers
assert len(layers) == 3
assert set([x.id for x in layers[0]]) == {"Base table"}
assert set([x.id for x in layers[1]]) == {"filter_column"}
assert set([x.id for x in layers[2]]) == {"Filtered table"}
def test_filter_inheritance():
"""Test topology augmentation. Use columns from the parent table by automatically adding the merge operation to topology."""
ctx = Prosto("My Prosto")
base_tbl = ctx.populate(
table_name="Base table",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': [1.0, 2.0, 3.0], 'B': ['x', 'yy', 'zzz']})",
tables=[])
# This (boolean) column will be used for filtering
clm = ctx.compute(
name="filter_column",
table=base_tbl.id,
func=
"lambda x, param: (x['A'] > param) & (x['B'].str.len() < 3)", # Return a boolean Series
columns=["A", "B"],
model={"param": 1.5})
f_tbl = ctx.filter(table_name="Filtered table",
attributes=["super"],
func=None,
tables=["Base table"],
columns=["filter_column"])
# In this calculate column, we use a column of the filtered table which actually exists only in the base table
clm = ctx.calculate(name="My column",
table=f_tbl.id,
func="lambda x: x + 1.0",
columns=["A"],
model=None)
ctx.run()
clm_data = f_tbl.get_series('My column')
assert np.isclose(len(clm_data), 1)
assert np.isclose(clm_data[0], 3.0)
# This column had to be added automatically by the augmentation procedure
# It is inherited from the base table and materialized via merge operation
# It stores original values of the inherited base column
clm_data = f_tbl.get_series('A')
assert np.isclose(clm_data[0], 2)
def test_roll_multiple():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table",
attributes=["A", "B"],
func="lambda **m: pd.DataFrame({'A': [1, 2, 3], 'B': [3, 2, 1]})",
tables=[])
clm = ctx.roll(name="Roll",
table=tbl.id,
window="2",
link=None,
func="lambda x: x['A'].sum() + x['B'].sum()",
columns=["A", "B"],
model={})
tbl.evaluate()
clm.evaluate()
clm_data = tbl.get_series('Roll')
assert pd.isna(clm_data[0])
assert np.isclose(clm_data[1], 8.0)
assert np.isclose(clm_data[2], 8.0)
#
# Test topology
#
topology = Topology(ctx)
topology.translate() # All data will be reset
layers = topology.elem_layers
assert len(layers) == 2
assert set([x.id for x in layers[0]]) == {"My table"}
assert set([x.id for x in layers[1]]) == {"Roll"}
ctx.run()
clm_data = tbl.get_series('Roll')
assert pd.isna(clm_data[0])
assert np.isclose(clm_data[1], 8.0)
assert np.isclose(clm_data[2], 8.0)
def test_compute():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table", attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1, 2, 3]})", tables=[]
)
clm = ctx.compute(
name="My column", table=tbl.id,
func="lambda x, **model: x.shift(**model)", columns=["A"], model={"periods": -1}
)
tbl.evaluate()
clm.evaluate()
clm_data = tbl.get_series('My column')
assert np.isclose(clm_data[0], 2.0)
assert np.isclose(clm_data[1], 3.0)
assert pd.isna(clm_data[2])
#
# Test topology
#
topology = Topology(ctx)
topology.translate() # All data will be reset
layers = topology.elem_layers
assert len(layers) == 2
assert set([x.id for x in layers[0]]) == {"My table"}
assert set([x.id for x in layers[1]]) == {"My column"}
ctx.run()
clm_data = tbl.get_series('My column')
assert np.isclose(clm_data[0], 2.0)
assert np.isclose(clm_data[1], 3.0)
assert pd.isna(clm_data[2])
def test_integers():
ctx = Prosto("My Prosto")
tbl = ctx.populate(
table_name="My table",
attributes=["A"],
func="lambda **m: pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8, 9]})",
tables=[])
clm = ctx.discretize(name="My column",
table=tbl.id,
columns=["A"],
model={
"origin": 5,
"step": 3
})
# 1, [2, 3, 4, [5, 6, 7, [8, 9
# -2 -1 0 1
ctx.run()
clm_data = tbl.get_series('My column')
assert list(clm_data) == [-2, -1, -1, -1, 0, 0, 0, 1, 1]
def test_merge_path():
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts",
attributes=["A"],
func="lambda **m: pd.DataFrame({'A': ['a', 'a', 'b', 'b']})",
tables=[])
# Groups
g_tbl = ctx.populate(
table_name="Groups",
attributes=["A", "B"],
func=
"lambda **m: pd.DataFrame({'A': ['a', 'b', 'c'], 'B': [2.0, 3.0, 3.0]})",
tables=[])
# Link
l_clm = ctx.link(name="Link",
table=f_tbl.id,
type=g_tbl.id,
columns=["A"],
linked_columns=["A"])
# SuperGroups
sg_tbl = ctx.populate(
table_name="SuperGroups",
attributes=["B", "C"],
func=
"lambda **m: pd.DataFrame({'B': [2.0, 3.0, 4.0], 'C': ['x', 'y', 'z']})",
tables=[])
# SuperLink
sl_clm = ctx.link(name="SuperLink",
table=g_tbl.id,
type=sg_tbl.id,
columns=["B"],
linked_columns=["B"])
# Merge
m_clm = ctx.merge("Merge", f_tbl.id, ["Link", "SuperLink", "C"])
f_tbl.evaluate()
g_tbl.evaluate()
sg_tbl.evaluate()
l_clm.evaluate()
sl_clm.evaluate()
m_clm.evaluate()
f_tbl_data = f_tbl.get_df()
assert len(f_tbl_data) == 4 # Same number of rows
assert len(f_tbl_data.columns) == 3
m_data = f_tbl.get_series("Merge")
assert m_data.to_list() == ['x', 'x', 'y', 'y']
#
# Test topology
#
topology = Topology(ctx)
topology.translate() # All data will be reset
layers = topology.elem_layers
assert len(layers) == 3
assert set([x.id for x in layers[0]]) == {"Facts", "Groups", "SuperGroups"}
assert set([x.id for x in layers[1]]) == {"Link", "SuperLink"}
assert set([x.id for x in layers[2]]) == {"Merge"}
ctx.run()
m_data = f_tbl.get_series("Merge")
assert m_data.to_list() == ['x', 'x', 'y', 'y']
def test_aggregate():
ctx = Prosto("My Prosto")
# Facts
f_tbl = ctx.populate(
table_name="Facts",
attributes=["A", "M"],
func=
"lambda **m: pd.DataFrame({'A': ['a', 'a', 'b', 'b'], 'M': [1.0, 2.0, 3.0, 4.0], 'N': [4.0, 3.0, 2.0, 1.0]})",
tables=[])
# Groups
df = pd.DataFrame({'A': ['a', 'b', 'c']})
g_tbl = ctx.populate(
table_name="Groups",
attributes=["A"],
func="lambda **m: pd.DataFrame({'A': ['a', 'b', 'c']})",
tables=[])
# Link
l_clm = ctx.link(name="Link",
table=f_tbl.id,
type=g_tbl.id,
columns=["A"],
linked_columns=["A"])
# Aggregation
a_clm = ctx.aggregate(name="Aggregate",
table=g_tbl.id,
tables=["Facts"],
link="Link",
func="lambda x, bias, **model: x.sum() + bias",
columns=["M"],
model={"bias": 0.0})
f_tbl.evaluate()
g_tbl.evaluate()
l_clm.evaluate()
a_clm.evaluate()
g_tbl_data = g_tbl.get_df()
assert len(g_tbl_data) == 3 # Same number of rows
assert len(
g_tbl_data.columns
) == 2 # One aggregate column was added (and one technical "id" column was added which might be removed in future)
a_clm_data = g_tbl.get_series('Aggregate')
assert a_clm_data[0] == 3.0
assert a_clm_data[1] == 7.0
assert a_clm_data[2] == 0.0
#
# Test topology
#
topology = Topology(ctx)
topology.translate() # All data will be reset
layers = topology.elem_layers
assert len(layers) == 3
assert set([x.id for x in layers[0]]) == {"Facts", "Groups"}
assert set([x.id for x in layers[1]]) == {"Link"}
assert set([x.id for x in layers[2]]) == {"Aggregate"}
ctx.run()
a_clm_data = g_tbl.get_series('Aggregate')
assert a_clm_data[0] == 3.0
assert a_clm_data[1] == 7.0
assert a_clm_data[2] == 0.0
#
# Aggregation of multiple columns
#
# Aggregation
a_clm2 = ctx.aggregate(
name="Aggregate 2",
table=g_tbl.id,
tables=["Facts"],
link="Link",
func=
"lambda x, my_param, **model: x['M'].sum() + x['N'].sum() + my_param",
columns=["M", "N"],
model={"my_param": 0.0})
#a_clm2.evaluate()
ctx.translate() # All data will be reset
ctx.run(
) # A new column is NOT added to the existing data frame (not clear where it is)
a_clm2_data = g_tbl.get_series('Aggregate 2')
assert a_clm2_data[0] == 10.0
assert a_clm2_data[1] == 10.0
assert a_clm2_data[2] == 0.0
