def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
data_inputs = data_container.data_inputs
expected_outputs = data_container.expected_outputs
with tf.device(self.device_name):
output = self._transform_model(data_inputs, expected_outputs)
data_container.set_data_inputs(output.numpy())
return data_container
def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
past = data_container.data_inputs[0].data_inputs
predicted = data_container.data_inputs[1].data_inputs
expected = data_container.expected_outputs
if self.enabled:
self._plot_predictions(expected, past, predicted)
data_container.set_data_inputs(predicted)
return data_container
def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
"""
Handle inverse transform by updating the data inputs, and expected outputs inside the data container.
:param context: execution context
:param data_container:
:return:
"""
di_eo = (data_container.data_inputs, data_container.expected_outputs)
new_data_inputs, new_expected_outputs = self.transform(di_eo)
data_container.set_data_inputs((new_data_inputs, new_expected_outputs))
return data_container
def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer):
"""
Handle transform by fitting the step,
and updating the data inputs, and expected outputs inside the data container.
:param context: execution context
:param data_container:
:return:
"""
new_self, (new_data_inputs, new_expected_outputs) = self.fit_transform(
(data_container.data_inputs, data_container.expected_outputs), None)
data_container.set_data_inputs((new_data_inputs, new_expected_outputs))
return new_self, data_container
def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext):
"""
Handle transform.
:param data_container: the data container to join
:param context: execution context
:return: transformed data container
"""
data_inputs = self.transform([dc.data_inputs for dc in data_container.data_inputs if len(dc.data_inputs) > 0])
data_container = DataContainer(data_inputs=data_inputs, current_ids=data_container.current_ids,
expected_outputs=data_container.expected_outputs)
data_container.set_data_inputs(data_inputs)
return data_container
def join(self, original_data_container: DataContainer) -> DataContainer:
"""
Return the accumulated results received by the on next method of this observer.
:return: transformed data container
:rtype: DataContainer
"""
while self.n_batches_left_to_do > 0:
task: QueuedPipelineTask = self.queue.get()
self.n_batches_left_to_do -= 1
step_name = task.step_name
if step_name not in self.result:
self.result[step_name] = ListDataContainer(
current_ids=[],
data_inputs=[],
expected_outputs=[],
summary_id=task.data_container.summary_id
)
self.result[step_name].append_data_container_in_data_inputs(task.data_container)
data_containers = self._join_all_step_results()
self.result = {}
return original_data_container.set_data_inputs(data_containers)
def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> (
'BaseStep', DataContainer):
"""
Fit cross validation with wrapped step, and return the score.
:param data_container: data container to fit transform
:param context: execution context
:return: self, step score
:rtype: (AutoMLAlgorithm, float)
"""
step: BaseCrossValidationWrapper = copy.deepcopy(self.wrapped)
step = step.handle_fit(data_container, context)
score = step.get_score()
data_container.set_data_inputs(score)
return self, data_container
def _did_process(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
di, eo = data_container.data_inputs
if len(di) != len(eo):
raise AssertionError(
'{}: Found different len for data inputs, and expected outputs. Please return the same the same amount of data inputs, and expected outputs, or otherwise create your own handler methods to do more funky things.'.format(
self.name))
data_container.set_data_inputs(data_inputs=di)
data_container.set_expected_outputs(expected_outputs=eo)
data_container = super()._did_process(data_container, context)
if len(data_container.current_ids) != len(data_container.data_inputs):
raise AssertionError(
'{}: Caching broken because there is a different len of current ids, and data inputs. Please use InputAndOutputTransformerWrapper if you plan to change the len of the data inputs.'.format(
self.name))
return data_container
def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer):
"""
Fit transform data container.
:param context: execution context
:param data_container: the data container to transform
:type data_container: neuraxle.data_container.DataContainer
:return: tuple(fitted pipeline, data_container)
"""
self.create_checkpoint_path(context.get_path())
self.flush_cache()
self.wrapped = self.wrapped.fit(data_container.data_inputs, data_container.expected_outputs)
outputs = self._transform_with_cache(data_container)
data_container.set_data_inputs(outputs)
return self, data_container
def handle_transform(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Handle transform by updating the data inputs, and expected outputs inside the data container.
:param context: execution context
:param data_container:
:return:
"""
di_eo = (data_container.data_inputs, data_container.expected_outputs)
new_data_inputs, new_expected_outputs = self.transform(di_eo)
data_container.set_data_inputs(new_data_inputs)
data_container.set_expected_outputs(new_expected_outputs)
current_ids = self.hash(data_container)
data_container.set_current_ids(current_ids)
return data_container
def handle_transform(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Transform data container.
:param context: execution context
:param data_container: the data container to transform
:type data_container: neuraxle.data_container.DataContainer
:return: transformed data container
"""
self.create_checkpoint_path(context.get_path())
outputs = self._transform_with_cache(data_container)
data_container.set_data_inputs(outputs)
current_ids = self.hash(data_container)
data_container.set_current_ids(current_ids)
return data_container
def _fit_data_container(
self, data_container: DataContainer,
context: ExecutionContext) -> (BaseStep, DataContainer):
"""
Handle fit by passing the data inputs, and the expected outputs to the wrapped step fit method.
:param context: execution context
:type context: ExecutionContext
:param data_container: data container to fit on
:return: self, data container
:rtype: (BaseStep, DataContainer)
"""
self.wrapped = self.wrapped.handle_fit(
DataContainer(data_inputs=(copy.copy(data_container.data_inputs),
copy.copy(
data_container.expected_outputs)),
current_ids=data_container.current_ids,
expected_outputs=None), context)
data_container.set_data_inputs(
(data_container.data_inputs, data_container.expected_outputs))
data_container.set_expected_outputs(expected_outputs=None)
return self
