def __init__(self,
model_provider: Model,
summary_provider: Callable[[pd.DataFrame], Summary] = Summary,
loss_alpha: float = 0.5,
target_kwargs: Dict[str, Dict[str, Any]] = None,
**kwargs: Dict):
assert isinstance(model_provider.features_and_labels.labels,
(TargetLabelEncoder, Dict))
super().__init__(
# if we have a target args and a target encoder then we need generate multiple targets with different kwargs
model_provider.features_and_labels.with_labels(
{target: model_provider.features_and_labels.labels.with_kwargs(**kwargs) for target, kwargs in target_kwargs.items()}) \
if target_kwargs and isinstance(model_provider.features_and_labels.labels, TargetLabelEncoder) \
else model_provider.features_and_labels,
summary_provider,
**join_kwargs(model_provider.kwargs, kwargs))
if isinstance(model_provider, MultiModel):
raise ValueError(
"Nesting Multi Models is not supported, you might use a flat structure of all your models"
)
self.models = {
target: model_provider()
for target in self.features_and_labels.labels.keys()
}
self.model_provider = model_provider
self.target_kwargs = target_kwargs
self.loss_alpha = loss_alpha
def f(args):
sampled_parameters = {k: args[i] for i, k in enumerate(keys)}
model = model_provider(**join_kwargs(sampled_parameters, constants))
loss = __train_loop(model, cross_validation, train, test)
if loss is None:
raise ValueError("Can not hyper tune if model loss is None")
return {'status': 'ok', 'loss': loss, 'parameter': sampled_parameters}
def __call__(self, *args, **kwargs):
new_model = KerasModel(self.keras_model_provider,
self.features_and_labels, self.summary_provider,
self.epochs, deepcopy(self.callbacks),
**join_kwargs(deepcopy(self.kwargs), kwargs))
# copy weights before return
new_model.set_weights(self.get_weights())
return new_model
def __init__(self, df: pd.DataFrame,
features_and_labels: FeaturesAndLabels, **kwargs):
# prepare fields
labels = features_and_labels.labels
encoder = lambda frame, **kwargs: frame
label_columns = None
joined_kwargs = join_kwargs(features_and_labels.kwargs, kwargs)
# eventually transform callable labels to its expected structure
if callable(labels):
labels = call_callable_dynamic_args(labels, df, **joined_kwargs)
# unfold labels, currently supported types are:
# Union[List[str], TargetLabelEncoder, Dict[str, Union[List[str], TargetLabelEncoder]]]
if isinstance(labels, list):
label_columns = labels
elif isinstance(labels, TargetLabelEncoder):
encoder = labels.encode
label_columns = labels.labels_source_columns
elif isinstance(labels, Dict):
# this is our multi model case, here we add an extra dimension to the labels array
label_columns = [
l for ls in labels.values() for l in (
ls if isinstance(ls, list) else ls.labels_source_columns)
]
# we need a special encoder which is wrapping all encoder for each target
encoder = MultipleTargetEncodingWrapper({
t:
l if isinstance(l, TargetLabelEncoder) else IdentityEncoder(l)
for t, l in labels.items()
}).encode
# assign all fields
self._features_and_labels = features_and_labels # depricated copy all fields here
self._features = features_and_labels.features
self._labels_columns = label_columns
self._labels = labels
self._label_type = features_and_labels.label_type
self._targets = features_and_labels.targets
self._gross_loss = features_and_labels.gross_loss
self._encoder = encoder
self._joined_kwargs = joined_kwargs
# pre assign this variable
# but notice that it get overwritten by an engineered data frame later on
self._df = df
# this function uses clojures
def call_dynamic(func, *args):
joined_kwargs = join_kwargs(self.__dict__, self._joined_kwargs)
return call_callable_dynamic_args(func, *args, **joined_kwargs)
self._df = call_dynamic(features_and_labels.pre_processor, df)
self.__call_dynamic = call_dynamic
def encode(self, df: pd.DataFrame, **kwargs) -> pd.DataFrame:
# eventually pre-process data
joined_kwargs = join_kwargs(self.kwargs, kwargs)
sf = (call_callable_dynamic_args(self.pre_processor, df, **
joined_kwargs)
if self.pre_processor else df)
# extract single series for one hot encoding
if isinstance(sf, pd.Series):
s = sf.rename(self.label)
else:
s = sf[self.label]
# one hot encode and return
return s.to_frame().apply(
lambda r: one_hot(r.values.sum(), self.nr_of_categories),
axis=1,
result_type='expand')
def call_dynamic(func, *args):
joined_kwargs = join_kwargs(self.__dict__, self._joined_kwargs)
return call_callable_dynamic_args(func, *args, **joined_kwargs)
def with_kwargs(self, **kwargs):
copy = deepcopy(self)
copy.kwargs = join_kwargs(copy.kwargs, kwargs)
return copy