def experiment_group_deleted(sender, **kwargs):
"""Stop all experiments before deleting the group."""
instance = kwargs['instance']
for experiment in instance.running_experiments:
scheduler.stop_experiment(experiment, update_status=False)
# Delete outputs
delete_experiment_group_outputs(instance.unique_name)
def experiment_group_deleted(sender, **kwargs):
"""Stop all experiments before deleting the group."""
instance = kwargs['instance']
for experiment in instance.running_experiments:
# Delete all jobs from DB before sending a signal to k8s,
# this way no statuses will be updated in the meanwhile
experiment.jobs.all().delete()
scheduler.stop_experiment(experiment, update_status=False)
# Delete outputs and logs
delete_experiment_group_outputs(instance.unique_name)
delete_experiment_group_logs(instance.unique_name)
def test_experiment_group_outputs_path_creation_deletion(self):
with patch('experiments.tasks.build_experiment.apply_async') as _:
experiment = ExperimentFactory(user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
create_experiment_outputs_path(experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(experiment.unique_name)
experiment_group_outputs_path = get_experiment_group_outputs_path(
self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(experiment_group_outputs_path) is True
delete_experiment_group_outputs(self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(experiment_group_outputs_path) is False
def new_experiment_group(sender, **kwargs):
instance = kwargs['instance']
created = kwargs.get('created', False)
if not created:
return
# Clean outputs and logs
delete_experiment_group_outputs(instance.unique_name)
delete_experiment_group_logs(instance.unique_name)
# Parse polyaxonfile content and create the experiments
specification = instance.specification
for xp in range(specification.matrix_space):
Experiment.objects.create(project=instance.project,
user=instance.user,
experiment_group=instance,
config=specification.parsed_data[xp])
start_group_experiments.apply_async((instance.id,), countdown=1)