def test_file_name(self):
#Arrange
input_data_file = InputDataFile()
file_name = Path('input_data.batch.0.epoch.{}.csv'.format(epoch_id))
#Act & Assert
self.assertEqual(input_data_file.file_name(batch_id, epoch_id), file_name)
def test_save(self):
#Arrange
input_data = get_input_data()
input_data_file = InputDataFile()
#Mocks
input_data.to_csv = MagicMock()
#Act
input_data_file.save(input_data, batch_id, epoch_id)
#Assert
input_data.to_csv.assert_called_with(input_data_file.file_name(batch_id, epoch_id))
def on_input_data_save(self, checkpoint):
#Arrange
input_data_file = InputDataFile().file_name(checkpoint._batch_id,
checkpoint._epoch_id)
#Assert
checkpoint._input_data.to_csv.assert_called_once()
checkpoint._input_data.to_csv.assert_called_with(input_data_file)
def test_dropbox_called(self):
#Arrange
checkpoint, _, _, _ = get_checkpoint(dropbox=get_dropbox())
checkpoint._dropbox.upload = MagicMock()
checkpoint._model = MagicMock()
input_data = get_input_data()
input_data.to_csv = MagicMock()
checkpoint.set_input_data(input_data)
input_file = InputDataFile().file_name(0, epoch_id)
with mock_patch.object(Path, 'unlink') as mock_unlink:
#Act
checkpoint.on_epoch_begin(epoch_id)
#Assert
checkpoint._dropbox.upload.assert_called_with(input_file)
mock_unlink.assert_called_once()
def get_checkpoint(dropbox=None,
checkpoint_batch_interval=checkpoint_batch_interval):
#Arrange
batch_input = ModelInput(model_name)
input_data_file = InputDataFile()
result_file = ResultFile()
batch_input_files = [batch_input, result_file]
epoch_begin_input_files = [input_data_file]
epoch_end_input_files = [result_file]
checkpoint = BatchTrainStateCheckpoint(
batch_input_files=batch_input_files,
checkpoint_batch_interval=checkpoint_batch_interval,
epoch_begin_input_files=epoch_begin_input_files,
epoch_end_input_files=epoch_end_input_files,
dropbox=dropbox)
return checkpoint, batch_input_files, epoch_begin_input_files, epoch_end_input_files
def batch_train_state_callback(model_name, checkpoint_batch_interval, dropbox):
"""It creates the state checkpoint callback that provides callbacks to training events.
Arguments:
model_name {string} -- The name of the model.
checkpoint_batch_interval {int} -- It specifies the number of batches after which to take a checkpoint
dropbox {client.dropbox.DropboxConnection} -- The dropbox client.
"""
#Initialize input files
model_input = ModelInput(model_name)
input_data_file = InputDataFile()
result_file = ResultFile()
state_checkpoint_callback = BatchTrainStateCheckpoint(
batch_input_files = [model_input, result_file],
checkpoint_batch_interval = checkpoint_batch_interval,
epoch_begin_input_files = [input_data_file],
epoch_end_input_files = [result_file],
dropbox = dropbox)
return state_checkpoint_callback
#Dropbox
dropbox = None
if dropbox_parameters:
dropbox_params = DropboxConnection.Parameters(dropbox_parameters[0], dropbox_parameters[1])
dropbox = DropboxConnection(dropbox_params)
logger.info('Dropbox parameters:: dropbox_params: %s', dropbox_params)
#Model file
model_file = ModelInput(input_params.model_name)
model_file_name = model_file.file_name(0, 0)
#Input data file
input_data_file = InputDataFile(constants.PREDICTION_INPUT_DATA_FILE_NAME_GUIDANCE)
input_data_file_name = input_data_file.file_name(0, 0)
#Prepare input files
input_files_client = InputFiles(dropbox)
input_files = input_files_client.get_all([input_data_file_name, model_file_name])
#Assign input files
input_data_file_path = input_files[input_data_file_name]
model_file_path = input_files[model_file_name]
#Load model
model = load_model(str(model_file_path))
#Input data frame
input_data = read_csv(input_data_file_path, index_col = 0)
dropbox = None
if dropbox_parameters:
dropbox_params = DropboxConnection.Parameters(dropbox_parameters[0], dropbox_parameters[1])
dropbox = DropboxConnection(dropbox_params)
logger.info('Dropbox parameters:: dropbox_params: %s', dropbox_params)
#Predictable randomness
seed = 3
np_seed(seed)
tf_seed(seed)
imgaug_seed(seed)
#Input data file
input_data_file = InputDataFile()
input_data_file_name = input_data_file.file_name(0, training_params.epoch_id)
#Prepare input files
input_files_client = InputFiles(dropbox)
input_data_file_path = input_files_client.get_all([input_data_file_name])[input_data_file_name]
#Input data frame
input_data = read_csv(input_data_file_path, index_col = 0)
#Update input data parameters
num_classes = max(getattr(input_data, image_generation_params.label_col)) + 1
image_generation_params_update = dict(num_classes = num_classes)
update_params(image_generation_params, **image_generation_params_update)
logger.info('Updated input data parameters: %s', input_params)
def test_init(self):
#Valid inputs
_ = InputDataFile()