# execution queue to add experiments to params['execution_queue_name'] = 'default' # experiment template to optimize with random parameter search params['experiment_template_name'] = 'Keras HP optimization base' # Select base template task # Notice we can be more imaginative and use task_id which will eliminate the need to use project name template_task = Task.get_task(project_name='examples', task_name=params['experiment_template_name']) for i in range(params['total_number_of_experiments']): # clone the template task into a new write enabled task (where we can change parameters) cloned_task = Task.clone(source_task=template_task, name=template_task.name + ' {}'.format(i), parent=template_task.id) # get the original template parameters cloned_task_parameters = cloned_task.get_parameters() # override with random samples form grid for k in space.keys(): cloned_task_parameters[k] = space[k]() # put back into the new cloned task cloned_task.set_parameters(cloned_task_parameters) print('Experiment {} set with parameters {}'.format( i, cloned_task_parameters)) # enqueue the task for execution
# This is the parameter value in the next task we want to change param["param_name_new_value"] = 3 # The queue where we want the template task (clone) to be sent to param["execution_queue_name"] = "default" # Simulate the work of a Task print("Processing....") sleep(2.0) print("Done processing :)") # Get a reference to the task to pipe to. next_task = Task.get_task(project_name=task.get_project_name(), task_name=param["next_task_name"]) # Clone the task to pipe to. This creates a task with status Draft whose parameters can be modified. cloned_task = Task.clone(source_task=next_task, name="Auto generated cloned task") # Get the original parameters of the Task, modify the value of one parameter, # and set the parameters in the next Task cloned_task_parameters = cloned_task.get_parameters() cloned_task_parameters[param["param_name"]] = param["param_name_new_value"] cloned_task.set_parameters(cloned_task_parameters) # Enqueue the Task for execution. The enqueued Task must already exist in the clearml platform print("Enqueue next step in pipeline to queue: {}".format( param["execution_queue_name"])) Task.enqueue(cloned_task.id, queue_name=param["execution_queue_name"]) # We are done. The next step in the pipe line is in charge of the pipeline now. print("Done")