def join_fit_transform(self, step: Pipeline, data_container: DataContainer, context: ExecutionContext) -> \
Tuple['Any', DataContainer]:
"""
Concatenate the pipeline fit transform output of each batch of self.batch_size together.
:param step: pipeline to fit transform on
:type step: Pipeline
:param data_container: data container to fit transform on
:type data_container: DataContainer
:param context: execution context
:return: fitted self, transformed data inputs
:rtype: Tuple[Any, DataContainer]
"""
context = context.push(step)
data_container_batches = data_container.minibatches(
batch_size=self.batch_size,
include_incomplete_batch=self.include_incomplete_batch,
default_value_data_inputs=self.default_value_data_inputs,
default_value_expected_outputs=self.default_value_expected_outputs)
output_data_container = ListDataContainer.empty()
for data_container_batch in data_container_batches:
step, data_container_batch = step._fit_transform_data_container(
data_container_batch, context)
output_data_container.concat(data_container_batch)
return step, output_data_container
def read_checkpoint(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Read data container data inputs checkpoint with :py:attr:`~data_input_checkpointer`.
Read data container expected outputs checkpoint with :py:attr:`~expected_output_checkpointer`.
:param data_container: data container to read checkpoint for
:type data_container: neuraxle.data_container.DataContainer
:param context: execution context to read checkpoint from
:type context: ExecutionContext
:return: data container checkpoint
:rtype: neuraxle.data_container.DataContainer
"""
data_container_checkpoint = ListDataContainer.empty(
original_data_container=data_container)
current_ids = self.summary_checkpointer.read_summary(
checkpoint_path=context.get_path(), data_container=data_container)
for current_id in current_ids:
data_input = self.data_input_checkpointer.read_checkpoint(
checkpoint_path=self._get_data_input_checkpoint_path(context),
current_id=current_id)
expected_output = self.expected_output_checkpointer.read_checkpoint(
checkpoint_path=self._get_expected_output_checkpoint_path(
context),
current_id=current_id)
data_container_checkpoint.append(current_id, data_input,
expected_output)
return data_container_checkpoint
def join_transform(self, step: Pipeline, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Concatenate the pipeline transform output of each batch of self.batch_size together.
:param step: pipeline to transform on
:type step: Pipeline
:param data_container: data container to transform
:type data_container: DataContainer
:param context: execution context
:return: transformed data container
:rtype: DataContainer
"""
context = context.push(step)
data_container_batches = data_container.minibatches(
batch_size=self.batch_size,
include_incomplete_batch=self.include_incomplete_batch,
default_value_data_inputs=self.default_value_data_inputs,
default_value_expected_outputs=self.default_value_expected_outputs)
output_data_container = ListDataContainer.empty()
for data_container_batch in data_container_batches:
output_data_container.concat(
step._transform_data_container(data_container_batch, context))
return output_data_container
def _fit_transform_data_container(self, data_container: DataContainer,
context: ExecutionContext):
"""
Fit transform each step for each data inputs, and expected outputs
:param data_container: data container to fit transform
:type data_container: DataContainer
:param context: execution context
:type context: ExecutionContext
:return: self, transformed_data_container
"""
output_data_container = ListDataContainer.empty()
for current_id, di, eo in data_container:
self.wrapped, output = self.wrapped.handle_fit_transform(
DataContainer(current_ids=None,
data_inputs=di,
expected_outputs=eo), context)
output_data_container.append(current_id, output.data_inputs,
output.expected_outputs)
output_data_container.summary_id = data_container.summary_id
return self, output_data_container
def join_fit_transform(
self, step: Pipeline, data_container: DataContainer,
context: ExecutionContext) -> Tuple['Any', DataContainer]:
"""
Concatenate the pipeline fit transform output of each batch of self.batch_size together.
:param step: pipeline to fit transform on
:type step: Pipeline
:param data_container: data container to fit transform on
:type data_container: DataContainer
:param context: execution context
:return: fitted self, transformed data inputs
:rtype: Tuple[Any, DataContainer]
"""
context = context.push(step)
data_container_batches = data_container.convolved_1d(
stride=self.batch_size, kernel_size=self.batch_size)
output_data_container = ListDataContainer.empty()
for data_container_batch in data_container_batches:
step, data_container_batch = step._fit_transform_data_container(
data_container_batch, context)
output_data_container.concat(data_container_batch)
return step, output_data_container
def join_transform(self, step: Pipeline, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
"""
Concatenate the pipeline transform output of each batch of self.batch_size together.
:param step: pipeline to transform on
:type step: Pipeline
:param data_container: data container to transform
:type data_container: DataContainer
:param context: execution context
:return: transformed data container
:rtype: DataContainer
"""
context = context.push(step)
data_container_batches = data_container.convolved_1d(
stride=self.batch_size,
kernel_size=self.batch_size
)
output_data_container = ListDataContainer.empty()
for data_container_batch in data_container_batches:
output_data_container.concat(
step._transform_data_container(data_container_batch, context)
)
return output_data_container
def _transform_data_container(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Transform each step for each data inputs.
:param data_container: data container
:type data_container: DataContainer
:param context: execution context
:type context: ExecutionContext
:return: self
"""
output_data_container: ListDataContainer = ListDataContainer.empty(
original_data_container=data_container)
for current_id, di, eo in data_container:
output: DataContainer = self.wrapped.handle_transform(
DataContainer(data_inputs=di,
current_ids=None,
expected_outputs=eo), context)
output_data_container.append(current_id, output.data_inputs,
output.expected_outputs)
output_data_container.summary_id = data_container.summary_id
return output_data_container
def handle_transform(self, data_container: DataContainer,
context: ExecutionContext):
"""
Transform each step for each data inputs.
:param data_container: data container
:type data_container: DataContainer
:param context: execution context
:type context: ExecutionContext
:return: self
"""
output_data_container = ListDataContainer.empty()
for current_id, di, eo in data_container:
output = self.wrapped.handle_transform(
DataContainer(current_ids=None,
data_inputs=di,
expected_outputs=eo), context)
output_data_container.append(current_id, output.data_inputs,
output.expected_outputs)
current_ids = self.hash(data_container)
output_data_container.set_current_ids(current_ids)
return output_data_container
def _fit_transform_data_container(
self, data_container: DataContainer,
context: ExecutionContext) -> Tuple[BaseStep, DataContainer]:
"""
Fit transform each step for each data inputs, and expected outputs
:param data_container: data container to fit transform
:type data_container: DataContainer
:param context: execution context
:type context: ExecutionContext
:return: self, transformed_data_container
"""
output_data_container: DataContainer = ListDataContainer.empty(
original_data_container=data_container)
for current_id, di, eo in data_container:
try:
self.wrapped, output = self.wrapped.handle_fit_transform(
DataContainer(data_inputs=di,
current_ids=None,
expected_outputs=eo), context)
output_data_container.append(current_id, output.data_inputs,
output.expected_outputs)
except ContinueInterrupt:
continue
except BreakInterrupt:
break
output_data_container.summary_id = data_container.summary_id
return self, output_data_container
def _join_step_results(self, data_containers):
# reorder results by summary id
data_containers.data_inputs.sort(key=lambda dc: self.summary_ids.index(dc.summary_id))
step_results = ListDataContainer.empty()
for data_container in data_containers.data_inputs:
data_container = data_container.set_summary_id(data_containers.data_inputs[-1].summary_id)
step_results.concat(data_container)
return step_results
def _inverse_transform_data_container(
self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
inverse_transform_results = []
for i, (current_ids, data_inputs,
expected_outputs) in enumerate(data_container):
inverse_transform_result = self[i].handle_inverse_transform(
DataContainer(current_ids=current_ids,
data_inputs=data_inputs,
expected_outputs=expected_outputs), context)
inverse_transform_results.append(inverse_transform_result)
output_data_container = ListDataContainer.empty()
for data_container_batch in inverse_transform_results:
output_data_container.append_data_container(data_container_batch)
return output_data_container
def _fit_transform_data_container(
self, data_container: DataContainer,
context: ExecutionContext) -> ('BaseStep', DataContainer):
fitted_steps_data_containers = []
for i, (current_ids, data_inputs,
expected_outputs) in enumerate(data_container):
fitted_step_data_container = self.steps[i].handle_fit_transform(
DataContainer(current_ids=current_ids,
data_inputs=data_inputs,
expected_outputs=expected_outputs), context)
fitted_steps_data_containers.append(fitted_step_data_container)
self.steps = [step for step, _ in fitted_steps_data_containers]
output_data_container = ListDataContainer.empty()
for _, data_container_batch in fitted_steps_data_containers:
output_data_container.append_data_container(data_container_batch)
return self, output_data_container
