def test_subs(self):
fvw = Field([self.v, self.w])
fxy = Field([self.x, self.y])
assert len(fvw - fxy) == 2
assert isinstance(fvw - fxy, Field)
assert isinstance(fvw - 2, Field)
def __init__(self):
self.X_operator = lambda Field, regressors: Field
self.y_operator = lambda Field: Field
self.domain = Domain()
self.field = Field()
self.regressors = Field()
self.sym_regressors = SymField()
self.sym_field = SymField()
def test_over_variables(self):
datas = DataSplit(axis_percentage_dict={"x": 0.5, "y": 0.5}, axis_init_percentage_dict={"x": 0, "y": 0}) * \
Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 5
assert datas.domain.shape["y"] == 3
datas = DataSplit(axis_percentage_dict={"x": 0.5, "y": 0.5}, axis_init_percentage_dict={"x": 0.2, "y": 0.5}) * \
Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 5
assert datas.domain.shape["y"] == 3
assert datas.domain.lower_limits["x"] == 1
assert datas.domain.lower_limits["y"] == 3
def test_over_fields_len(self):
datas = DataSplitIndexClip(axis_start_dict={
"x": 3,
"y": 3
},
axis_len_dict={
"x": 2,
"y": 2
}) * Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 2
assert datas.domain.shape["y"] == 2
datas = DataSplitIndexClip(axis_start_dict={"x": 1},
axis_len_dict={"x": 2}) * self.x
assert len(datas) == 1
assert datas.domain.shape["x"] == 2
assert np.all(datas.data == np.array([0.5, 1]))
datas = DataSplitIndexClip(axis_end_dict={
"x": 3,
"y": 3
},
axis_len_dict={
"x": 2,
"y": 2
}) * Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 2
assert datas.domain.shape["y"] == 2
datas = DataSplitIndexClip(axis_end_dict={"x": -1},
axis_len_dict={"x": 2}) * self.x
assert len(datas) == 1
assert datas.domain.shape["x"] == 2
assert np.all(datas.data == np.array([4, 4.5]))
datas = DataSplitIndexClip(axis_end_dict={"x": 0},
axis_len_dict={"x": 2}) * self.x
assert len(datas) == 1
assert datas.domain.shape["x"] == 2
assert np.all(datas.data == np.array([0, 0.5]))
datas = DataSplitIndexClip(axis_start_dict={"x": 0},
axis_len_dict={"x": 2}) * self.x
assert len(datas) == 1
assert datas.domain.shape["x"] == 2
assert np.all(datas.data == np.array([0, 0.5]))
def explore(self, x_operator_func, y_operator_func, rational=False):
subfolders = [self.type_of_experiment]
stats = pd.DataFrame([])
for df, country, period in self.get_country_data():
print('\n\n========== ========== ========== ==========')
print('Exploring {}'.format(country))
if country not in self.info.keys():
self.info[country] = {}
self.set_underlying_model(df)
for variable in [self.get_variables()] + [variable for variable in self.get_variables()]:
variable = Field(variable)
base_name = str(variable)
if 'all' not in self.accepted_variables and base_name not in self.accepted_variables:
continue
print('\nVariable {}'.format(base_name))
if base_name not in self.info[country].keys():
self.info[country][base_name] = []
# ---------- fit eqdiff ----------
data_manager = DataManager()
data_manager.add_variables(variable)
# data_manager.add_regressors(self.get_regressors())
data_manager.set_domain()
data_manager.set_X_operator(x_operator_func(rational=rational))
data_manager.set_y_operator(y_operator_func())
pde_finder = self.fit_eqdifff(data_manager)
stats = pd.concat([stats, pd.concat([pd.DataFrame([[country, period.label, period.fecha]],
index=pde_finder.coefs_.index,
columns=['country', 'medidas',
'fecha_final']),
pde_finder.coefs_],
axis=1)], axis=0, sort=True)
# ---------- plot ----------
with savefig('{}_{}_coeficients.png'.format(base_name, country), self.experiment_name,
subfolders=subfolders, format='png'):
self.plot_coefficients(pde_finder)
plt.xscale('log')
with savefig('{}_{}_fitvsreal.png'.format(base_name, country), self.experiment_name,
subfolders=subfolders, format='png'):
self.plot_fitted_and_real(pde_finder, data_manager, col="blue", subinit=None, sublen=None)
# --------- predictions ---------
predictions_temp = self.optimize_predictions(pde_finder, variable, x_operator_func, y_operator_func,
data_manager, period, rational)
self.info[country][base_name].append({'coefs': pde_finder.coefs_,
'period': period,
'data_real': data_manager.field,
'predictions': predictions_temp,
'data_raw': df})
stats.to_csv(config.get_filename(filename='{}_coefs.csv'.format(base_name),
experiment=self.experiment_name,
subfolders=[self.type_of_experiment]))
self.plot_results()
def test_over_fields(self):
datas = DataSplitOnIndex(axis_index_dict={
"x": 3,
"y": 3
}) * Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 3
assert datas.domain.shape["y"] == 3
def test_constructor(self):
fv = Field()
assert len(fv) == 0
fv = Field(self.v)
assert len(fv) == 1
fv.append(Field(self.w))
assert len(fv) == 2
def test_over_variables(self):
polyv = Poly(polynomial_order=2) * Field([self.v, self.w, self.x])
assert len(polyv) == 10
equalto = {
'x(x)**2', 'v(x, y)', 'v(x, y)**2', 'v(x, y)*x(x)',
'1.00000000000000', 'x(x)'
}
equalto = {e.replace(' ', '') for e in equalto}
assert set([e.get_full_name() for e in polyv.data]) == equalto
def test_constructor(self):
data_manager = DataManager()
data_manager.add_variables(self.v)
data_manager.add_variables([self.v, self.w])
data_manager.add_field(Field(self.v))
data_manager.add_regressors(self.x)
assert len(data_manager.field) == 4
assert len(data_manager.regressors) == 1
data_manager.set_domain()
assert len(data_manager.domain)
def __mul__(self, other):
if isinstance(other, Poly):
return Poly(other.polynomial_order + self.polynomial_order)
# in this case the operation over fields is not element by element; it meshes all so it should change.
elif isinstance(other, Field):
return Field(self.successive_multiplications(other.data))
elif isinstance(other, SymField):
return SymField(self.successive_multiplications(other.data))
else:
return Operator.__mul__(self, other)
def test_over_fields(self):
datas = DataSplitIndexClip(axis_start_dict={
"x": 3,
"y": 3
},
axis_end_dict={
"x": 5,
"y": 5
}) * Field([self.v, self.w, self.x])
assert len(datas) == 3
assert datas.domain.shape["x"] == 2
assert datas.domain.shape["y"] == 2
def sym_var_inverse_transform(self, X, copy=None):
if isinstance(X, SymVariable):
X = SymField(X)
if isinstance(X, Variable):
X = Field(X)
if isinstance(X, (SymField, Field)):
new_field = X * 1
if self.with_std:
new_field = new_field * self.scale_
if self.with_mean:
new_field = new_field + self.mean_
return new_field
def x_operator(field, regressors):
new_field = copy.deepcopy(field)
new_field = PolyD(derivative_order) * new_field
new_field.append(regressors)
# if rational:
# new_field.append(new_field.__rtruediv__(1.0))
new_field = (Poly(polynomial_order) * new_field)
new_field = Field([
var for var in new_field.data
if not np.allclose(var.data, 1) or '1.000' in var.get_full_name()
])
return new_field
def successive_multiple_delays(self, other):
if isinstance(other, (Variable)):
temp_other = Field([other])
elif isinstance(other, SymVariable):
temp_other = SymField([other])
else:
temp_other = other
delayed_vars = []
for ax_name, delays_list in self.delays_dict.items():
for var in temp_other.data:
for delay in delays_list:
delayed_vars.append(
Delay(axis_name=ax_name, delay=delay) * var)
return delayed_vars
def __mul__(self, other):
if isinstance(other, Variable):
return self.var_operator_func(other)
elif isinstance(other, SymVariable):
return self.sym_var_operator_func(other)
elif isinstance(other, Field):
return Field([self * var for var in other.data])
elif isinstance(other, SymField):
return SymField([self * sym_var for sym_var in other.data])
elif isinstance(other, Domain):
return self.domain_operator_func(other)
elif isinstance(other, Operator):
o = Operator()
o.var_operator_func = lambda var: self.var_operator_func(
other.var_operator_func(var))
o.sym_var_operator_func = lambda sym_var: self.sym_var_operator_func(
other.sym_var_operator_func(sym_var))
o.backward_lag = other.backward_lag + self.backward_lag # TODO: not necessarily will always work, rethink
o.forward_lag = other.forward_lag + self.forward_lag
return o
def test_pow(self):
fvw = Field([self.v, self.w])
assert len(fvw**fvw) == 2
assert len(fvw**2) == 2
assert len(2**fvw) == 2
def test_dot(self):
fvw = Field([self.v, self.w])
fxy = Field([self.x, self.y])
assert len(fvw.dot(fxy)) == 1
assert isinstance(fvw.dot(fxy), Variable)
def var_operator_func(self, var):
return Field(self.successive_multiple_derivatives(var))
def test_add(self):
fvw = Field([self.v, self.w])
fxy = Field([self.x, self.y])
assert len(fvw + fxy) == 2
assert isinstance(fvw + fxy, Field)
def test_str(self):
fvw = Field([self.v, self.w])
fxy = Field([self.x, self.y])
assert str(fvw) == "[v(x,y), w(x,y)]"
assert str(fxy) == "[x(x), y(y)]"
def test_get_subset_from_index_limits(self):
fv = Field(self.v)
new_fv = fv.get_subset_from_index_limits({"x": [0, 2], "y": [0, 2]})
assert new_fv.domain.get_shape() == {"x": 2, "y": 2}
assert fv.domain.get_shape() != {"x": 2, "y": 2}
def test_topandas(self):
fv = Field(self.v)
fv.append(Field(self.w))
print(fv.to_pandas().shape)
assert fv.to_pandas().shape == (self.domain.get_shape("y")["y"] *
self.domain.get_shape("x")["x"], 2)
def var_operator_func(self, var):
return Field(self.successive_multiplications([var]))
class DataManager:
def __init__(self):
self.X_operator = lambda Field, regressors: Field
self.y_operator = lambda Field: Field
self.domain = Domain()
self.field = Field()
self.regressors = Field()
self.sym_regressors = SymField()
self.sym_field = SymField()
def set_domain(self, domain_info=None):
if domain_info is None:
# in case none is passed then it will get the maximum domain from the variables of the field.
self.field.set_domain()
self.domain = self.field.domain
elif isinstance(domain_info, dict):
if ("lower_limits_dict" in domain_info.keys()) and ("upper_limits_dict" in domain_info.keys()) and \
("step_width_dict" in domain_info.keys()):
lower_limits_dict = domain_info["lower_limits_dict"]
upper_limits_dict = domain_info["upper_limits_dict"]
step_width_dict = domain_info["step_width_dict"]
elif all([isinstance(element, np.ndarray) for element in domain_info.values()]):
lower_limits_dict = {}
upper_limits_dict = {}
step_width_dict = {}
for axis_name, range_vals in domain_info.items():
lower_limits_dict[axis_name] = range_vals[0]
upper_limits_dict[axis_name] = range_vals[-1]
step_width_dict[axis_name] = range_vals[1] - range_vals[0]
else:
Exception("imput should be or dict of ranges or dict of limits and steps")
self.domain = Domain(lower_limits_dict, upper_limits_dict, step_width_dict)
elif isinstance(domain_info, Domain):
self.domain = copy.deepcopy(domain_info)
def add_variables(self, variables): # type:((list| Variable)) -> None
self.field.append(variables)
self.sym_field.append(variables)
def add_field(self, field):
self.field.append(field)
self.sym_field.append(field)
def add_regressors(self, regressors): # : (Variable, Field)
self.regressors.append(regressors)
self.sym_regressors.append(regressors)
def set_X_operator(self, operator):
self.X_operator = operator
def set_y_operator(self, operator):
self.y_operator = operator
@staticmethod
def filter_yvars_in_Xvars(y_field, x_field): # type: (Field, Field) -> Field
y_var_names = [var.get_full_name() for var in y_field.data]
return Field([var for var in x_field.data if var.get_full_name() not in y_var_names])
@staticmethod
def filter_ysymvars_in_Xsymvars(y_field, x_field): # type: (SymField, SymField) -> SymField
y_sym_expressions = [str(sym_var) for sym_var in y_field.data]
return SymField([symvar for symvar in x_field.data if str(symvar) not in y_sym_expressions])
def get_X_sym(self, split_operator=Identity()):
"""
gets the simbolic expression of X
:return:
"""
sym_X = self.X_operator(split_operator * self.sym_field,
split_operator * self.sym_regressors if self.sym_regressors != [] else self.sym_regressors)
return self.filter_ysymvars_in_Xsymvars(self.get_y_sym(split_operator), x_field=sym_X)
def get_y_sym(self, split_operator=Identity()):
"""
gets the simbolic expression of y
:return:
"""
return self.y_operator(split_operator * self.sym_field)
def get_X(self, split_operator=Identity()):
X = self.X_operator(split_operator * self.field,
split_operator * self.regressors if self.sym_regressors != [] else self.regressors)
return self.filter_yvars_in_Xvars(self.get_y(split_operator), x_field=X)
def get_y(self, split_operator=Identity()):
return self.y_operator(split_operator * self.field)
def get_X_dframe(self, split_operator=Identity()):
return self.get_X(split_operator).to_pandas()
def get_y_dframe(self, split_operator=Identity()):
return self.get_y(split_operator).to_pandas()
def get_Xy_eq(self):
X = self.get_X()
X = SymField([SymVariable(x.name, SymVariable.get_init_info_from_variable(x)[1], x.domain) for x in X.data])
Y = self.get_y()
Y = SymField([SymVariable(y.name, SymVariable.get_init_info_from_variable(y)[1], y.domain) for y in Y.data])
return X, Y
def filter_yvars_in_Xvars(y_field, x_field): # type: (Field, Field) -> Field
y_var_names = [var.get_full_name() for var in y_field.data]
return Field([var for var in x_field.data if var.get_full_name() not in y_var_names])