def __init__(self,
wrapped,
hashers: List[BaseHasher] = None,
cache_folder_when_no_handle=None):
MetaStep.__init__(self, wrapped, hashers=hashers)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
_DidProcessInputOutputHandlerMixin.__init__(self)
def __init__(self,
hyperparameter_optimizer: BaseHyperparameterOptimizer,
higher_score_is_better=True,
cache_folder_when_no_handle=None):
MetaStep.__init__(self, None)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
self.higher_score_is_better = higher_score_is_better
self.hyperparameter_optimizer = hyperparameter_optimizer
def __init__(self, columns_selection, n_dimension=3):
col_selector: ColumnSelector2D = ColumnSelector2D(
columns_selection=columns_selection)
for _ in range(min(0, n_dimension - 2)):
col_selector = ForEachDataInput(col_selector)
MetaStep.__init__(self, col_selector)
self.n_dimension = n_dimension
def __init__(self,
wrapped: BaseStep,
is_train_only=True,
cache_folder_when_no_handle=None):
MetaStep.__init__(self, wrapped=wrapped)
ForceHandleOnlyMixin.__init__(self,
cache_folder=cache_folder_when_no_handle)
self.is_train_only = is_train_only
def __init__(self, columns_selection, n_dimension=2):
assert n_dimension >= 2
col_selector: ColumnSelector2D = ColumnSelector2D(
columns_selection=columns_selection)
for _ in range(max(0, n_dimension - 2)):
col_selector = ForEach(col_selector)
MetaStep.__init__(self, col_selector)
self.n_dimension = n_dimension
def __init__(self, wrapped: BaseTransformer, then_unflatten: bool = True):
MetaStep.__init__(self, wrapped)
ResumableStepMixin.__init__(self)
ForceHandleMixin.__init__(self)
self.then_unflatten = then_unflatten
self.len_di = []
self.len_eo = []
def __init__(self, wrapped):
"""
Wrap a scikit-learn MetaEstimatorMixin for usage in Neuraxle.
This class is similar to the SKLearnWrapper class of Neuraxle that can wrap a scikit-learn BaseEstimator.
:param wrapped: a scikit-learn object of type "MetaEstimatorMixin".
"""
MetaStep.__init__(self)
self.wrapped_sklearn_metaestimator = wrapped # TODO: use self.set_step of the MetaStepMixin instead?
def __init__(self,
pipeline: Union[BaseStep, NamedTupleList],
validation_size: float = None,
batch_size: int = None,
batch_metrics: Dict[str, Callable] = None,
shuffle_in_each_epoch_at_train: bool = True,
seed: int = None,
n_epochs: int = 1,
epochs_metrics: Dict[str, Callable] = None,
scoring_function: Callable = None,
cache_folder: str = None,
print_epoch_metrics=False,
print_batch_metrics=False):
"""
:param pipeline: pipeline to wrap with an epoch repeater, a validation split wrapper, and a mini batch sequential pipeline
:param validation_size: ratio for validation size between 0 and 1
:param batch_size: batch size for the mini batch sequential pipeline
:param batch_metrics: metrics to calculate for each processed mini batch
:param shuffle_in_each_epoch_at_train:
:param seed: random seed for the data shuffling that can be done at each epoch when the param shuffle_in_each_epoch_at_train is True
:param n_epochs: number of epochs
:param epochs_metrics: metrics to calculate for each epoch
:param scoring_function: scoring function with two arguments (y_true, y_pred)
:param cache_folder: cache folder to be used inside the pipeline
:param print_epoch_metrics: whether or not to print epoch metrics
:param print_batch_metrics: whether or not to print batch metrics
"""
if epochs_metrics is None:
epochs_metrics = {}
if batch_metrics is None:
batch_metrics = {}
self.final_scoring_metric = scoring_function
self.epochs_metrics = epochs_metrics
self.n_epochs = n_epochs
self.shuffle_in_each_epoch_at_train = shuffle_in_each_epoch_at_train
self.batch_size = batch_size
self.batch_metrics = batch_metrics
self.validation_size = validation_size
self.print_batch_metrics = print_batch_metrics
self.print_epoch_metrics = print_epoch_metrics
wrapped: BaseStep = pipeline
wrapped: MiniBatchSequentialPipeline = self._create_mini_batch_pipeline(
wrapped)
if shuffle_in_each_epoch_at_train:
wrapped = TrainShuffled(wrapped=wrapped, seed=seed)
wrapped = self._create_validation_split(wrapped)
wrapped = self._create_epoch_repeater(wrapped)
MetaStep.__init__(self, wrapped)
EvaluableStepMixin.__init__(self)
ForceHandleMixin.__init__(self, cache_folder)
def __init__(self, wrapped=None, scoring_function: Callable = r2_score):
"""
Base class For validation wrappers.
It has a scoring function to calculate the score for the validation split.
:param scoring_function: scoring function with two arguments (y_true, y_pred)
:type scoring_function: Callable
"""
BaseStep.__init__(self)
MetaStep.__init__(self, wrapped)
self.scoring_function = scoring_function
def __init__(self,
wrapped: BaseTransformer,
copy_op=copy.deepcopy,
cache_folder_when_no_handle=None):
MetaStep.__init__(self, wrapped=wrapped)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
self.savers.append(TruncableJoblibStepSaver())
self.set_step(wrapped)
self.steps_as_tuple: List[NamedTupleList] = []
self.copy_op = copy_op
def __init__(self,
wrapped,
epochs,
repeat_in_test_mode=False,
cache_folder_when_no_handle=None):
BaseStep.__init__(self)
MetaStep.__init__(self, wrapped)
ForceHandleOnlyMixin.__init__(self,
cache_folder=cache_folder_when_no_handle)
self.repeat_in_test_mode = repeat_in_test_mode
self.epochs = epochs
def __init__(self,
wrapped,
transform_callback_function,
fit_callback_function,
inverse_transform_callback_function=None,
more_arguments: List = tuple(),
hyperparams=None):
MetaStep.__init__(self, wrapped=wrapped, hyperparams=hyperparams)
self.inverse_transform_callback_function = inverse_transform_callback_function
self.more_arguments = more_arguments
self.fit_callback_function = fit_callback_function
self.transform_callback_function = transform_callback_function
def __init__(
self,
wrapped: BaseStep,
cache_folder: str = DEFAULT_CACHE_FOLDER,
value_hasher: 'BaseValueHasher' = None,
):
MetaStep.__init__(self, wrapped)
self.value_hasher = value_hasher
if self.value_hasher is None:
self.value_hasher = Md5Hasher()
self.value_caching_folder = cache_folder
def __init__(self,
wrapped: BaseTransformer,
max_queue_size: int,
n_workers: int,
use_threading: bool,
additional_worker_arguments=None,
use_savers=False):
if not additional_worker_arguments:
additional_worker_arguments = [[] for _ in range(n_workers)]
MetaStep.__init__(self, wrapped)
ObservableQueueMixin.__init__(self, Queue(maxsize=max_queue_size))
self.use_threading: bool = use_threading
self.workers: List[Process] = []
self.n_workers: int = n_workers
self.observers: List[Queue] = []
self.additional_worker_arguments = additional_worker_arguments
self.use_savers = use_savers
def get_children(self) -> List[BaseStep]:
"""
Get the list of all the children for that step.
:return: list of children
"""
children: List[BaseStep] = MetaStep.get_children(self)
cloned_children = [step for _, step in self.steps_as_tuple]
children.extend(cloned_children)
return children
def __init__(self,
wrapped: BaseStep,
auto_ml_algorithm: AutoMLAlgorithm,
hyperparams_repository: HyperparamsRepository = None,
n_iters: int = 100,
refit=True,
cache_folder_when_no_handle=None):
if not isinstance(wrapped, EvaluableStepMixin):
raise ValueError(
'AutoML algorithm needs evaluable steps that implement the function get_score. Please use a validation technique, or implement EvaluableStepMixin.'
)
MetaStep.__init__(self, auto_ml_algorithm.set_step(wrapped))
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
if hyperparams_repository is None:
hyperparams_repository = InMemoryHyperparamsRepository()
self.hyperparams_repository = hyperparams_repository
self.n_iters = n_iters
self.refit = refit
def __init__(self, wrapped: BaseTransformer, enabled: bool = True, nullified_return_value=None,
cache_folder_when_no_handle=None, use_hyperparameter_space=True, nullify_hyperparams=True):
hyperparameter_space = HyperparameterSpace({
OPTIONAL_ENABLED_HYPERPARAM: Boolean()
}) if use_hyperparameter_space else {}
MetaStep.__init__(
self,
hyperparams=HyperparameterSamples({
OPTIONAL_ENABLED_HYPERPARAM: enabled
}),
hyperparams_space=hyperparameter_space,
wrapped=wrapped
)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
if nullified_return_value is None:
nullified_return_value = []
self.nullified_return_value = nullified_return_value
self.nullify_hyperparams = nullify_hyperparams
def __init__(self, wrapped, cache_folder_when_no_handle=None):
MetaStep.__init__(self, wrapped)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
def __init__(self, condition_function: Callable, wrapped: BaseStep):
MetaStep.__init__(self, wrapped)
HandleOnlyMixin.__init__(self)
self.condition_function: Callable = condition_function
def _transform_data_container(self, data_container: DataContainer,
context: ExecutionContext):
if self.condition_function(self, data_container, context):
return MetaStep._transform_data_container(self, data_container,
context)
return data_container
def __init__(self, wrapped: BaseTransformer):
MetaStep.__init__(self, wrapped)
ResumableStepMixin.__init__(self)
def __init__(self):
MetaStep.__init__(self, wrapped=SomeStepWithHyperparams())
