def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# create input layer
input_layer = self.create_other(
"SingleInputLayer", architecture, metadata,
Configuration({"input_size": architecture.arguments.noise_size}))
# conditional
if "conditional" in architecture.arguments:
# wrap the input layer with a conditional layer
input_layer = ConditionalLayer(
input_layer, metadata, **architecture.arguments.conditional)
# create the hidden layers factory
hidden_layers_factory = self.create_other(
"HiddenLayers", architecture, metadata,
arguments.get("hidden_layers", {}))
# create the output layer factory
output_layer_factory = self.create_output_layer_factory(
architecture, metadata, arguments)
# create the generator
return FeedForward(input_layer,
hidden_layers_factory,
output_layer_factory,
default_hidden_activation=ReLU())
def run(self, configuration: Configuration) -> None:
inputs_queue = Queue()
outputs_queue = Queue()
# queue all the inputs unwrapped
for inputs in configuration.get("inputs", unwrap=True):
inputs_queue.put(inputs)
# outputs worker: we will write in the output file using only one process and a queue
output_path = create_parent_directories_if_needed(configuration.output)
output_fields = self.task_worker_class.output_fields()
output_process = Process(target=write_worker,
args=(outputs_queue, output_path,
output_fields))
output_process.start()
# additional process to log the remaining tasks
count_process = Process(target=count_worker,
args=(inputs_queue,
configuration.get("log_every", 5)))
count_process.start()
# we don't need to join this one
# workers: we will process tasks in parallel
if type(configuration.workers) == int:
worker_configurations = [{} for _ in range(configuration.workers)]
elif type(configuration.workers) == list:
worker_configurations = configuration.workers
else:
raise Exception("Invalid worker configuration.")
worker_processes = []
for worker_number, worker_configuration in enumerate(
worker_configurations):
worker_process = Process(target=task_worker_wrapper,
args=(self.task_worker_class,
worker_number, worker_configuration,
inputs_queue, outputs_queue))
worker_process.start()
worker_processes.append(worker_process)
# wait for all the workers to finish
multiprocessing_logger.info("Waiting for the workers...")
for worker_process in worker_processes:
worker_process.join()
multiprocessing_logger.info("Workers finished.")
# the workers stopped queuing rows
# add to stop event for the writing worker
outputs_queue.put({"event_type": EVENT_TYPE_EXIT})
# wait until the writing worker actually stops
output_process.join()
def timed_run(self, configuration: Configuration) -> None:
logging.basicConfig(**configuration.get("logging", default={}, transform_default=False))
self.logger.info("Starting task...")
start_time = time.time()
self.run(configuration)
elapsed_time = time.time() - start_time
elapsed_time_unit = "seconds"
if elapsed_time_unit == "seconds" and elapsed_time > 60:
elapsed_time /= 60
elapsed_time_unit = "minutes"
if elapsed_time_unit == "minutes" and elapsed_time > 60:
elapsed_time /= 60
elapsed_time_unit = "hours"
if elapsed_time_unit == "hours" and elapsed_time > 24:
elapsed_time /= 24
elapsed_time_unit = "days"
self.logger.info("Task ended in {:02f} {}.".format(elapsed_time, elapsed_time_unit))
def create(self, architecture: Architecture, metadata: Metadata, arguments: Configuration) -> Any:
# create the input layer
input_layer = self.create_input_layer(architecture, metadata, arguments)
# conditional
if "conditional" in architecture.arguments:
# wrap the input layer with a conditional layer
input_layer = ConditionalLayer(input_layer, metadata, **architecture.arguments.conditional)
# create the hidden layers factory
hidden_layers_factory = self.create_other("HiddenLayers", architecture, metadata,
arguments.get("hidden_layers", {}))
# create the output activation
if "output_activation" in arguments:
output_activation = self.create_other(arguments.output_activation.factory, architecture,
metadata,
arguments.output_activation.get("arguments", {}))
else:
output_activation = None
# create the output layer factory
output_layer_factory = SingleOutputLayerFactory(architecture.arguments.code_size, activation=output_activation)
# create the encoder
return FeedForward(input_layer, hidden_layers_factory, output_layer_factory, default_hidden_activation=Tanh())
def create_input_layer(self, architecture: Architecture,
metadata: Metadata,
arguments: Configuration) -> InputLayer:
# override the input layer size
return self.create_other(
"SingleInputLayer", architecture, metadata,
Configuration({"input_size": metadata.get_num_features()}))
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# create the input layer
input_layer = self.create_input_layer(architecture, metadata,
arguments)
# wrap the input layer with the special gain input layer (to receive the mask)
input_layer = GAINInputLayer(input_layer, metadata.get_num_features())
# create the hidden layers factory
hidden_layers_factory = self.create_other(
"HiddenLayers", architecture, metadata,
arguments.get("hidden_layers", {}))
# create the output layer factory
# this is different from a normal discriminator
# because the output has the size of the input
# it predicts if each feature is real or fake
output_layer_factory = SingleOutputLayerFactory(
metadata.get_num_features(), activation=Sigmoid())
# create the encoder
return FeedForward(input_layer,
hidden_layers_factory,
output_layer_factory,
default_hidden_activation=Tanh())
def run(self, configuration: Configuration) -> None:
seed_all(configuration.get("seed"))
metadata = load_metadata(configuration.metadata)
inputs = torch.from_numpy(np.load(configuration.inputs))
missing_mask = generate_mask_for(inputs,
configuration.missing_probability,
metadata)
np.save(configuration.outputs, missing_mask.numpy())
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
reconstruction_loss = self.create_other(
arguments.reconstruction_loss.factory, architecture, metadata,
arguments.reconstruction_loss.get("arguments", {}))
return AutoEncoderLoss(reconstruction_loss,
**arguments.get_all_defined(["masked"]))
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# create input layer
input_layer_configuration = {}
if self.code:
input_layer_configuration[
"input_size"] = architecture.arguments.code_size
else:
input_layer_configuration[
"input_size"] = metadata.get_num_features()
input_layer = self.create_other(
"SingleInputLayer", architecture, metadata,
Configuration(input_layer_configuration))
# conditional
if "conditional" in architecture.arguments:
# wrap the input layer with a conditional layer
input_layer = ConditionalLayer(
input_layer, metadata, **architecture.arguments.conditional)
# mini-batch averaging
if arguments.get("mini_batch_averaging", False):
input_layer = MiniBatchAveraging(input_layer)
# create the hidden layers factory
hidden_layers_factory = self.create_other(
"HiddenLayers", architecture, metadata,
arguments.get("hidden_layers", {}))
# create the output activation
if self.critic:
output_activation = View(-1)
else:
output_activation = Sequential(Sigmoid(), View(-1))
# create the output layer factory
output_layer_factory = SingleOutputLayerFactory(
1, activation=output_activation)
# create the discriminator
return FeedForward(input_layer,
hidden_layers_factory,
output_layer_factory,
default_hidden_activation=LeakyReLU(0.2))
def create_component(architecture: Architecture, metadata: Metadata,
configuration: Configuration) -> Any:
if "factory" not in configuration:
raise Exception("Missing factory name while creating component.")
factory = factory_by_name[configuration.factory]
arguments = configuration.get("arguments", {})
return factory.validate_and_create(architecture, metadata, arguments)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# separate arguments
noise_layer_arguments = None
autoencoder_arguments = Configuration()
for argument_name, argument_value in arguments.items():
if argument_name == "noise_layer":
noise_layer_arguments = argument_value
else:
autoencoder_arguments[argument_name] = argument_value
noise_layer = self.create_other(
noise_layer_arguments.factory, architecture, metadata,
noise_layer_arguments.get("arguments", {}))
autoencoder = self.create_other(self.factory_name, architecture,
metadata, autoencoder_arguments)
return DeNoisingAutoencoder(noise_layer, autoencoder)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
reconstruction_loss = self.create_other(
arguments.reconstruction_loss.factory, architecture, metadata,
arguments.reconstruction_loss.get("arguments", {}))
optional_arguments = arguments.get_all_defined(
["reconstruction_loss_weight"])
return GAINGeneratorLoss(reconstruction_loss, **optional_arguments)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
optional = arguments.get_all_defined(["sizes", "bn_decay", "shortcut"])
if "activation" in arguments:
activation_configuration = arguments.activation
optional["activation"] = self.create_other(
activation_configuration.factory, architecture, metadata,
activation_configuration.get("arguments", {}))
return HiddenLayersFactory(**optional)
def run(self, configuration: Configuration) -> None:
seed_all(configuration.get("seed"))
metadata = load_metadata(configuration.metadata)
architecture_configuration = load_configuration(configuration.architecture)
self.validate_architecture_configuration(architecture_configuration)
architecture = create_architecture(metadata, architecture_configuration)
architecture.to_gpu_if_available()
checkpoints = Checkpoints()
checkpoint = checkpoints.load(configuration.checkpoint)
if "best_architecture" in checkpoint:
checkpoints.load_states(checkpoint["best_architecture"], architecture)
else:
checkpoints.load_states(checkpoint["architecture"], architecture)
# pre-processing
imputation = create_component(architecture, metadata, configuration.imputation)
pre_processing = PreProcessing(imputation)
# post-processing
if "scale_transform" in configuration:
scale_transform = load_scale_transform(configuration.scale_transform)
else:
scale_transform = None
post_processing = PostProcessing(metadata, scale_transform)
# load the features
features = to_gpu_if_available(torch.from_numpy(np.load(configuration.features)).float())
missing_mask = to_gpu_if_available(torch.from_numpy(np.load(configuration.missing_mask)).float())
# initial imputation
batch = pre_processing.transform({"features": features, "missing_mask": missing_mask})
# generate the model outputs
output = self.impute(configuration, metadata, architecture, batch)
# imputation
output = compose_with_mask(mask=missing_mask, differentiable=False, where_one=output, where_zero=features)
# post-process
output = post_processing.transform(output)
# save the imputation
output = to_cpu_if_was_in_gpu(output)
output = output.numpy()
np.save(configuration.output, output)
def create_output_layer_factory(
self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> OutputLayerFactory:
# override the output layer size
output_layer_configuration = {
"output_size": metadata.get_num_features()
}
# copy activation arguments only if defined
if "output_layer" in arguments and "activation" in arguments.output_layer:
output_layer_configuration[
"activation"] = arguments.output_layer.activation
# create the output layer factory
return self.create_other("SingleOutputLayer", architecture, metadata,
Configuration(output_layer_configuration))
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# collect the parameters from the indicated models
parameters = []
for module_name in arguments.parameters:
parameters.extend(architecture[module_name].parameters())
# copy the rest of the arguments
arguments = {}
for key, value in arguments.items():
if key != "parameters":
arguments[key] = value
# create the optimizer
return self.optimizer_class(parameters, **arguments)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# override the reduction argument
reconstruction_loss_configuration = arguments.reconstruction_loss.get(
"arguments", {})
if "reduction" in reconstruction_loss_configuration:
assert reconstruction_loss_configuration["reduction"] == "sum"
else:
reconstruction_loss_configuration["reduction"] = "sum"
# create the reconstruction loss
reconstruction_loss = self.create_other(
arguments.reconstruction_loss.factory, architecture, metadata,
reconstruction_loss_configuration)
# create the vae loss
return VAELoss(reconstruction_loss,
**arguments.get_all_defined(["masked"]))
def validate_arguments(self, arguments: Configuration) -> None:
# keep the remaining arguments here
remaining_arguments = set(arguments.keys())
# mandatory arguments
for mandatory_argument in self.mandatory_arguments():
if mandatory_argument in remaining_arguments:
remaining_arguments.remove(mandatory_argument)
else:
raise MissingArgument(mandatory_argument)
# optional arguments
for optional_argument in self.optional_arguments():
if optional_argument in remaining_arguments:
remaining_arguments.remove(optional_argument)
# invalid arguments
for remaining_argument in remaining_arguments:
raise InvalidArgument(remaining_argument)
def run(self, configuration: Configuration) -> None:
seed_all(configuration.get("seed"))
metadata = load_metadata(configuration.metadata)
architecture_configuration = load_configuration(
configuration.architecture)
self.validate_architecture_configuration(architecture_configuration)
architecture = create_architecture(metadata,
architecture_configuration)
architecture.to_gpu_if_available()
checkpoints = Checkpoints()
checkpoint = checkpoints.load(configuration.checkpoint)
if "best_architecture" in checkpoint:
checkpoints.load_states(checkpoint["best_architecture"],
architecture)
else:
checkpoints.load_states(checkpoint["architecture"], architecture)
# load the features
features = to_gpu_if_available(
torch.from_numpy(np.load(configuration.features)).float())
# conditional
if "labels" in configuration:
condition = to_gpu_if_available(
torch.from_numpy(np.load(configuration.labels)).float())
else:
condition = None
# encode
with torch.no_grad():
code = architecture.autoencoder.encode(features,
condition=condition)["code"]
# save the code
code = to_cpu_if_was_in_gpu(code)
code = code.numpy()
np.save(configuration.output, code)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
# create the input layer
input_layer = self.create_input_layer(architecture, metadata,
arguments)
# wrap the input layer with the special gain input layer (to receive the mask)
input_layer = GAINInputLayer(input_layer, metadata.get_num_features())
# create the hidden layers factory
hidden_layers_factory = self.create_other(
"HiddenLayers", architecture, metadata,
arguments.get("hidden_layers", {}))
# create the output layer factory
output_layer_factory = self.create_output_layer_factory(
architecture, metadata, arguments)
# create the encoder
return FeedForward(input_layer,
hidden_layers_factory,
output_layer_factory,
default_hidden_activation=Tanh())
def run(self, configuration: Configuration) -> None:
inputs = torch.from_numpy(np.load(configuration.inputs))
missing_mask = torch.from_numpy(np.load(configuration.missing_mask))
assert inputs.shape == missing_mask.shape
# the model need np.nan in the missing values to work
inputs = compose_with_mask(missing_mask,
where_one=torch.empty_like(inputs).fill_(np.nan),
where_zero=inputs,
differentiable=False) # cannot be differentiable with nans!
# create the model
model = IterativeImputer(random_state=configuration.get("seed", 0),
estimator=ExtraTreeRegressor(),
missing_values=np.nan)
# go back to torch (annoying)
model.fit(inputs.numpy())
# save the model
with open(create_parent_directories_if_needed(configuration.outputs), "wb") as model_file:
pickle.dump(model, model_file)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
return MeanAndModesImputationLayer(
to_gpu_if_available(
torch.from_numpy(np.load(arguments.path)).float()),
**arguments.get_all_defined(["differentiable"]))
def run(self, configuration: Configuration) -> None:
seed_all(configuration.get("seed"))
metadata = load_metadata(configuration.metadata)
if "scale_transform" in configuration:
scale_transform = load_scale_transform(
configuration.scale_transform)
else:
scale_transform = None
post_processing = PostProcessing(metadata, scale_transform)
architecture_configuration = load_configuration(
configuration.architecture)
self.validate_architecture_configuration(architecture_configuration)
architecture = create_architecture(metadata,
architecture_configuration)
architecture.to_gpu_if_available()
checkpoints = Checkpoints()
checkpoint = checkpoints.load(configuration.checkpoint)
if "best_architecture" in checkpoint:
checkpoints.load_states(checkpoint["best_architecture"],
architecture)
else:
checkpoints.load_states(checkpoint["architecture"], architecture)
samples = []
# create the strategy if defined
if "strategy" in configuration:
# validate strategy name is present
if "factory" not in configuration.strategy:
raise Exception(
"Missing factory name while creating sample strategy.")
# validate strategy name
strategy_name = configuration.strategy.factory
if strategy_name not in strategy_class_by_name:
raise Exception(
"Invalid factory name '{}' while creating sample strategy."
.format(strategy_name))
# create the strategy
strategy_class = strategy_class_by_name[strategy_name]
strategy = strategy_class(**configuration.strategy.get(
"arguments", default={}, transform_default=False))
# use the default strategy
else:
strategy = DefaultSampleStrategy()
# this is only to pass less parameters back and forth
sampler = Sampler(self, configuration, metadata, architecture,
post_processing)
# while more samples are needed
start = 0
while start < configuration.sample_size:
# do not calculate gradients
with torch.no_grad():
# sample:
# the task delegates to the strategy and passes the sampler object to avoid passing even more parameters
# the strategy may prepare additional sampling arguments (e.g. condition)
# the strategy delegates to the sampler object
# the sampler object delegates back to the task adding parameters that it was keeping
# the task child class does the actual sampling depending on the model
# the sampler object applies post-processing
# the strategy may apply filtering to the samples (e.g. rejection)
# the task finally gets the sample
batch_samples = strategy.generate_sample(
sampler, configuration, metadata)
# transform back the samples
batch_samples = to_cpu_if_was_in_gpu(batch_samples)
batch_samples = batch_samples.numpy()
# if the batch is not empty
if len(batch_samples) > 0:
# do not go further than the desired number of samples
end = min(start + len(batch_samples),
configuration.sample_size)
# limit the samples taken from the batch based on what is missing
batch_samples = batch_samples[:min(len(batch_samples), end -
start), :]
# if it is the first batch
if len(samples) == 0:
samples = batch_samples
# if its not the first batch we have to concatenate
else:
samples = np.concatenate((samples, batch_samples), axis=0)
# move to next batch
start = end
# save the samples
np.save(configuration.output, samples)
def process(self, inputs: Any) -> None:
AugmentationTask(self).run(Configuration(inputs))
def impute(self, configuration: Configuration, metadata: Metadata,
scaled_inputs: Tensor, missing_mask: Tensor) -> Tensor:
optional = configuration.get_all_defined(["noise_mean", "noise_std"])
optional["differentiable"] = False
return NormalNoiseImputationLayer(**optional)(scaled_inputs,
missing_mask)
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
return MultiInputDropout(
metadata, **arguments.get_all_defined(self.optional_arguments()))
def create(self, architecture: Architecture, metadata: Metadata,
arguments: Configuration) -> Any:
return self.wrapped_class(
**arguments.get_all_defined(self.optional_class_arguments))
def impute(self, configuration: Configuration, metadata: Metadata,
architecture: Architecture, batch: Dict[str, Tensor]) -> Tensor:
# loss function
loss_function = create_component(architecture, metadata,
configuration.reconstruction_loss)
masked_loss_function = MaskedReconstructionLoss(loss_function)
batch_size = batch["features"].shape[0] * batch["features"].shape[1]
# we need the non missing mask for the loss
non_missing_mask = inverse_mask(batch["missing_mask"])
# initial noise
noise = to_gpu_if_available(
FloatTensor(len(batch["features"]),
architecture.arguments.noise_size).normal_())
noise.requires_grad_()
# it is not the generator what we are updating
# it is the noise
optimizer = Adam([noise],
weight_decay=0,
lr=configuration.noise_learning_rate)
architecture.generator.eval()
# logger
log_path = create_parent_directories_if_needed(configuration.logs)
logger = TrainLogger(self.logger, log_path, False)
# initial generation
logger.start_timer()
generated = architecture.generator(noise,
condition=batch.get("labels"))
# iterate until we reach the maximum number of iterations or until the non missing loss is too small
max_iterations = configuration.max_iterations
for iteration in range(1, max_iterations + 1):
# compute the loss on the non-missing values
non_missing_loss = masked_loss_function(generated,
batch["features"],
non_missing_mask)
logger.log(iteration, max_iterations, "non_missing_loss",
to_cpu_if_was_in_gpu(non_missing_loss).item())
# this loss only makes sense if the ground truth is present
# only used for debugging
if configuration.get("log_missing_loss", False):
# this part should not affect the gradient calculation
with torch.no_grad():
missing_loss = masked_loss_function(
generated, batch["raw_features"],
batch["missing_mask"])
logger.log(iteration, max_iterations, "missing_loss",
to_cpu_if_was_in_gpu(missing_loss).item())
loss = loss_function(generated,
batch["raw_features"]) / batch_size
logger.log(iteration, max_iterations, "loss",
to_cpu_if_was_in_gpu(loss).item())
# if the generation is good enough we stop
if to_cpu_if_was_in_gpu(non_missing_loss).item(
) < configuration.get("tolerance", 1e-5):
break
# clear previous gradients
optimizer.zero_grad()
# compute the gradients
non_missing_loss.backward()
# update the noise
optimizer.step()
# generate next
logger.start_timer()
generated = architecture.generator(noise,
condition=batch.get("labels"))
return generated
def create(self, architecture: Architecture, metadata: Metadata, arguments: Configuration) -> Any:
return MultiReconstructionLoss(metadata, **arguments.get_all_defined(self.optional_arguments()))
def run(self, configuration: Configuration) -> None:
seed_all(configuration.get("seed"))
datasets = Datasets()
for dataset_name, dataset_path in configuration.data.items():
datasets[dataset_name] = to_gpu_if_available(torch.from_numpy(np.load(dataset_path)).float())
metadata = load_metadata(configuration.metadata)
architecture_configuration = load_configuration(configuration.architecture)
self.validate_architecture_configuration(architecture_configuration)
architecture = create_architecture(metadata, architecture_configuration)
architecture.to_gpu_if_available()
create_parent_directories_if_needed(configuration.checkpoints.output)
checkpoints = Checkpoints()
# no input checkpoint by default
checkpoint = None
# continue from an output checkpoint (has priority over input checkpoint)
if configuration.checkpoints.get("continue_from_output", default=False) \
and checkpoints.exists(configuration.checkpoints.output):
checkpoint = checkpoints.load(configuration.checkpoints.output)
# continue from an input checkpoint
elif "input" in configuration.checkpoints:
checkpoint = checkpoints.load(configuration.checkpoints.input)
if configuration.checkpoints.get("ignore_input_epochs", default=False):
checkpoint["epoch"] = 0
if configuration.checkpoints.get("use_best_input", default=False):
checkpoint["architecture"] = checkpoint.pop("best_architecture")
checkpoint.pop("best_epoch")
checkpoint.pop("best_metric")
# if there is no starting checkpoint then initialize
if checkpoint is None:
architecture.initialize()
checkpoint = {
"architecture": checkpoints.extract_states(architecture),
"epoch": 0
}
# if there is a starting checkpoint then load it
else:
checkpoints.load_states(checkpoint["architecture"], architecture)
log_path = create_parent_directories_if_needed(configuration.logs)
logger = TrainLogger(self.logger, log_path, checkpoint["epoch"] > 0)
# pre-processing
if "imputation" in configuration:
imputation = create_component(architecture, metadata, configuration.imputation)
else:
imputation = None
pre_processing = PreProcessing(imputation)
# post-processing
if "scale_transform" in configuration:
scale_transform = load_scale_transform(configuration.scale_transform)
else:
scale_transform = None
post_processing = PostProcessing(metadata, scale_transform)
for epoch in range(checkpoint["epoch"] + 1, configuration.epochs + 1):
# train discriminator and generator
logger.start_timer()
metrics = self.train_epoch(configuration, metadata, architecture, datasets, pre_processing, post_processing)
for metric_name, metric_value in metrics.items():
logger.log(epoch, configuration.epochs, metric_name, metric_value)
# update the checkpoint
checkpoint["architecture"] = checkpoints.extract_states(architecture)
checkpoint["epoch"] = epoch
# if the best architecture parameters should be kept
if "keep_checkpoint_by_metric" in configuration:
# get the metric used to compare checkpoints
checkpoint_metric = metrics[configuration.keep_checkpoint_by_metric]
# check if this is the best checkpoint (or the first)
if "best_metric" not in checkpoint or checkpoint_metric < checkpoint["best_metric"]:
checkpoint["best_architecture"] = checkpoint["architecture"]
checkpoint["best_epoch"] = epoch
checkpoint["best_metric"] = checkpoint_metric
# save checkpoint
checkpoints.delayed_save(checkpoint, configuration.checkpoints.output, configuration.checkpoints.max_delay)
# force save of last checkpoint
checkpoints.save(checkpoint, configuration.checkpoints.output)
# finish
logger.close()