def test_experiment_outputs_path_creation_deletion(self):
experiment_outputs_path = get_experiment_outputs_path(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is False
create_experiment_outputs_path(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is True
delete_experiment_outputs(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is False
def experiment_pre_deleted(sender, **kwargs):
instance = kwargs['instance']
# Delete outputs and logs
delete_experiment_outputs(instance.unique_name)
delete_experiment_logs(instance.unique_name)
# Delete clones
for experiment in instance.clones.filter(cloning_strategy=CloningStrategy.RESUME):
experiment.delete()
def new_experiment(sender, **kwargs):
instance = kwargs['instance']
instance.set_status(ExperimentLifeCycle.CREATED)
# Clean outputs and logs
delete_experiment_logs(instance.unique_name)
delete_experiment_outputs(instance.unique_name)
# Create logs path
create_experiment_logs_path(instance.unique_name)
def experiment_deleted(sender, **kwargs):
instance = kwargs['instance']
try:
_ = instance.experiment_group
# Delete all jobs from DB before sending a signal to k8s,
# this way no statuses will be updated in the meanwhile
instance.jobs.all().delete()
experiment_scheduler.stop_experiment(instance, update_status=False)
except ExperimentGroup.DoesNotExist:
# The experiment was already stopped when the group was deleted
pass
# Delete outputs and logs
delete_experiment_outputs(instance.unique_name)
delete_experiment_logs(instance.unique_name)
def new_experiment(sender, **kwargs):
instance = kwargs['instance']
created = kwargs.get('created', False)
# Check if the experiment is newly created and that we can start it independently
if not created:
return
instance.set_status(ExperimentLifeCycle.CREATED)
# Clean outputs and logs
delete_experiment_logs(instance.unique_name)
delete_experiment_outputs(instance.unique_name)
# Create logs path
create_experiment_logs_path(instance.unique_name)
if instance.is_independent:
# Start building the experiment and then Schedule it to be picked by the spawners
build_experiment.apply_async((instance.id, ), countdown=1)
def experiment_deleted(sender, **kwargs):
instance = kwargs['instance']
# Delete outputs and logs
delete_experiment_outputs(instance.unique_name)
delete_experiment_logs(instance.unique_name)