def stop_experiment(experiment_name, experiment_config_filename): """Stop the experiment specified by |experiment_config_filename|.""" experiment_config = yaml_utils.read(experiment_config_filename) if experiment_config.get('local_experiment', False): raise NotImplementedError( 'Local experiment stop logic is not implemented.') cloud_project = experiment_config['cloud_project'] cloud_compute_zone = experiment_config['cloud_compute_zone'] gce.initialize() instances = list(gce.get_instances(cloud_project, cloud_compute_zone)) experiment_instances = [] dispatcher_instance = experiment_utils.get_dispatcher_instance_name( experiment_name) if dispatcher_instance not in instances: logger.warning('Dispatcher instance not running, skip.') else: experiment_instances.append(dispatcher_instance) trial_prefix = 'r-' + experiment_name experiment_instances.extend([ instance for instance in instances if instance.startswith(trial_prefix) ]) if not experiment_instances: logger.warning('No experiment instances found, no work to do.') return True if not gcloud.delete_instances(experiment_instances, cloud_compute_zone): logger.error('Failed to stop experiment instances.') return False logger.info('Successfully stopped experiment.') return True
def delete_instances(instances, experiment_config): """Deletes |instances|.""" cloud_project = experiment_config['cloud_project'] cloud_compute_zone = experiment_config['cloud_compute_zone'] instances_to_delete = [ i for i in gce.get_instances(cloud_project, cloud_compute_zone) if i in instances ] return gcloud.delete_instances(instances_to_delete, experiment_config['cloud_compute_zone'])