def assign_gpus(free_gpus, resources_on_most_free_gpu, cost): gpu_ids = [] while resources_on_most_free_gpu >= cost: free_gpus, resources_on_most_free_gpu, gpu_number = allocate_gpu_resources( free_gpus, cost) if gpu_number not in gpu_ids: gpu_ids.append(gpu_number) return free_gpus, resources_on_most_free_gpu, gpu_ids
def folder_execute(params=None): """ Execute a folder of experiments. It will execute trainings and all the selected evaluations for each of the models present on the folder. Args params: a dictionary containing: gpus: the gpu numbers that are going to be allocated for the experiment gpu_value: the "value" of each gpu, depending on the value more or less experiments will be allocated per GPU folder: the folder where all the experiment configuration files are validation_datasets: the validation datasets that are going to be validated per experiment driving_environments: The driving environments where the models are going to be tested. """ folder = params['folder'] allocated_gpus = params['gpus'] validation_datasets = params['validation_datasets'] driving_environments = params['driving_environments'] allocation_parameters = params['allocation_parameters'] experiments_list = os.listdir(os.path.join('configs', folder)) experiments_list = [ experiment.split('.')[-2] for experiment in experiments_list ] allocated_gpus = { gpu: allocation_parameters['gpu_value'] for gpu in allocated_gpus } executing_processes = [] free_gpus, resources_on_most_free_gpu, executing_processes = get_gpu_resources( allocated_gpus, executing_processes, allocation_parameters) # Is a queue of tasks to be executed. The priority is always train. # then test then val. tasks_queue = mount_experiment_heap(folder, experiments_list, params['is_training'], [], [], validation_datasets, driving_environments) # No process is executing right now. while True: # if not done or executing get to the list # If amount of resources is smaller than a threshold. while resources_on_most_free_gpu >= min([allocation_parameters['train_cost'], allocation_parameters['validation_cost'], allocation_parameters['drive_cost']]) \ and tasks_queue != []: # Allocate all the gpus popped_thing = heapq.heappop(tasks_queue) process_specs = popped_thing[2] # To get directly the dict # Get the train status, that will affect in scheduling a validation or drive process train_status = monitorer.get_status(folder, process_specs['experiment'], 'train')[0] # ADD TRAIN TO EXECUTE if process_specs['type'] == 'train' and resources_on_most_free_gpu >= \ allocation_parameters['train_cost']: free_gpus, resources_on_most_free_gpu, gpu_number = allocate_gpu_resources( free_gpus, allocation_parameters['train_cost']) execute_train(gpu_number, process_specs['folder'], process_specs['experiment'], params['number_of_workers']) process_specs.update({'gpu': gpu_number}) executing_processes.append(process_specs) # ADD DRIVE TO EXECUTE elif process_specs['type'] == 'drive' and resources_on_most_free_gpu >= \ allocation_parameters['drive_cost'] \ and (train_status == 'Iterating' or train_status == 'Loading' or train_status == 'Finished'): print(process_specs['type']) free_gpus, resources_on_most_free_gpu, gpu_number = allocate_gpu_resources( free_gpus, allocation_parameters['drive_cost']) execute_drive(gpu_number, process_specs['folder'], process_specs['experiment'], process_specs['environment'], params['driving_parameters']) process_specs.update({'gpu': gpu_number}) executing_processes.append(process_specs) # ADD VALIDATION TO EXECUTE elif process_specs['type'] == 'validation' and resources_on_most_free_gpu >= \ allocation_parameters['validation_cost'] \ and (train_status == 'Iterating' or train_status == 'Loading' or train_status == 'Finished'): free_gpus, resources_on_most_free_gpu, gpu_number = allocate_gpu_resources( free_gpus, allocation_parameters['validation_cost']) execute_validation(gpu_number, process_specs['folder'], process_specs['experiment'], process_specs['dataset']) process_specs.update({'gpu': gpu_number}) executing_processes.append(process_specs) tasks_queue = mount_experiment_heap(folder, experiments_list, params['is_training'], executing_processes, tasks_queue, validation_datasets, driving_environments, False) printer.plot_folder_summaries(folder, params['is_training'], validation_datasets, driving_environments) # Check allocated process, and look which ones finished. if len(tasks_queue) == 0 and len(executing_processes) == 0: break free_gpus, resources_on_most_free_gpu, executing_processes = get_gpu_resources( allocated_gpus, executing_processes, allocation_parameters) time.sleep(10) print("ALL EXPERIMENTS EXECUTED")