def save_checkpoint(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
"""
Save data container data inputs with :py:attr:`~data_input_checkpointer`.
Save data container expected outputs with :py:attr:`~expected_output_checkpointer`.
:param data_container: data container to checkpoint
:type data_container: neuraxle.data_container.DataContainer
:param context: execution context to checkpoint from
:type context: ExecutionContext
:return:
"""
if not self.is_for_execution_mode(context.get_execution_mode()):
return data_container
context.mkdir()
self.summary_checkpointer.save_summary(
checkpoint_path=context.get_path(), data_container=data_container)
for current_id, data_input, expected_output in data_container:
self.data_input_checkpointer.save_checkpoint(
checkpoint_path=self._get_data_input_checkpoint_path(context),
current_id=current_id,
data=data_input)
self.expected_output_checkpointer.save_checkpoint(
checkpoint_path=self._get_expected_output_checkpoint_path(
context),
current_id=current_id,
data=expected_output)
return data_container
def save_checkpoint(self, data_container: DataContainer,
context: ExecutionContext) -> DataContainer:
if self.is_for_execution_mode(context.get_execution_mode()):
# TODO: save the context by execution mode AND data container ids / summary
context.copy().save()
return data_container
def read_checkpoint(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
"""
Read data checkpoint for the current execution mode using self.data_checkpointers.
:param data_container: data container to load checkpoint from
:param context: execution mode to load checkpoint from
:return: loaded data container checkpoint
:rtype: neuraxle.data_container.DataContainer
"""
for checkpointer in self.all_checkpointers:
if checkpointer.is_for_execution_mode(context.get_execution_mode()):
data_container = checkpointer.read_checkpoint(data_container, context)
return data_container
def should_resume(self, data_container: DataContainer, context: ExecutionContext) -> bool:
"""
Returns True if all of the execution mode data checkpointers can be resumed.
:param context: execution context
:param data_container: data to resume
:return: if we can resume the checkpoint
:rtype: bool
"""
for checkpointer in self.all_checkpointers:
if checkpointer.is_for_execution_mode(context.get_execution_mode()):
if not checkpointer.should_resume(data_container, context):
return False
return True
