def test_load_with_calculated_and_same_calculated_variable_transformed(
self):
self.create_csv()
# Try with plain calculated variable first
all_cols = self.create_columns()
a, b, c = self.create_variables()
tran = self.get_transform('cell')
d = Variable('d', 'D', calculation=a + b, available_transforms=[tran])
load_vars = [a, b, c, d, d.add_one_cell()]
ds = self.create_source(df=None,
columns=all_cols,
load_variables=load_vars)
assert_frame_equal(
ds.df,
self.expect_loaded_df_with_calculated_and_calculated_transformed)
# Now try with plain calculated variable second
all_cols = self.create_columns()
a, b, c = self.create_variables()
tran = self.get_transform('cell')
d = Variable('d', 'D', calculation=a + b, available_transforms=[tran])
load_vars = [
a,
b,
c,
d.add_one_cell(),
d,
]
ds = self.create_source(df=None,
columns=all_cols,
load_variables=load_vars)
assert_frame_equal(
ds.df,
self.expect_loaded_df_with_calculated_transformed_and_calculated)
def create_variables(self,
transform_data: str = '',
apply_transforms: bool = True
) -> Tuple[Variable, Variable, Variable]:
transform_dict = self.get_transform_dict(
transform_data=transform_data, apply_transforms=apply_transforms)
a = Variable('a', 'A', dtype='int', **transform_dict)
b = Variable('b', 'B', dtype='int', **transform_dict)
c = Variable('c', 'C', dtype='str')
return a, b, c
def test_load_with_calculated_variable(self):
self.create_csv()
all_cols = self.create_columns()
a, b, c = self.create_variables()
d = Variable('d', 'D', calculation=a + b)
ds = self.create_source(df=None,
columns=all_cols,
load_variables=[a, b, c, d])
assert_frame_equal(ds.df, self.expect_loaded_df_with_calculated)
def test_load_with_calculated_transformed_variable(self):
self.create_csv()
all_cols = self.create_columns()
a, b, c = self.create_variables()
tran = self.get_transform('cell')
d = Variable('d', 'D', calculation=a + b, available_transforms=[tran])
ds = self.create_source(df=None,
columns=all_cols,
load_variables=[a, b, c,
d.add_one_cell()])
assert_frame_equal(ds.df,
self.expect_loaded_df_with_calculated_transformed)
def test_load_with_calculated_variable_from_func(self):
self.create_csv()
all_cols = self.create_columns()
a, b, c = self.create_variables()
expr = Expression([a, b],
func=expression_series_func,
summary='Add em up')
d = Variable('d', 'D', calculation=expr)
ds = self.create_source(df=None,
columns=all_cols,
load_variables=[a, b, c, d])
assert_frame_equal(ds.df, self.expect_loaded_df_with_calculated)
def test_load_with_calculate_on_transformed_before_transform(self):
self.create_csv()
all_cols = self.create_columns()
a, b, c = self.create_variables(transform_data='cell',
apply_transforms=False)
d = Variable('d', 'D', calculation=a + b)
ds = self.create_source(
df=None,
columns=all_cols,
load_variables=[a.add_one_cell(),
b.add_one_cell(), c, d])
assert_frame_equal(
ds.df, self.
expect_loaded_df_with_calculate_on_transformed_before_transform)
def test_load_with_datetime(self):
test_df = self.test_df.copy()
test_df['d'] = pd.to_datetime('1/1/2000')
self.create_csv(df=test_df)
expect_df = self.expect_loaded_df_rename_only.copy()
expect_df['Date'] = pd.to_datetime('1/1/2000')
date_var = Variable('Date', dtype='datetime')
date_col = Column(date_var, 'd')
all_cols = self.create_columns()
all_cols.append(date_col)
ds = self.create_source(df=None, columns=all_cols)
assert_frame_equal(ds.df, expect_df)
def create_variables(self, **kwargs) -> Tuple[Variable, Variable]:
config_dict = dict()
config_dict.update(**kwargs)
v = Variable(*VAR1_ARGS, **config_dict)
v2 = Variable(*VAR2_ARGS, **config_dict)
return v, v2