# execution queue to add experiments to
params['execution_queue_name'] = 'default'
# experiment template to optimize with random parameter search
params['experiment_template_name'] = 'Keras HP optimization base'
# Select base template task
# Notice we can be more imaginative and use task_id which will eliminate the need to use project name
template_task = Task.get_task(project_name='examples',
task_name=params['experiment_template_name'])
for i in range(params['total_number_of_experiments']):
# clone the template task into a new write enabled task (where we can change parameters)
cloned_task = Task.clone(source_task=template_task,
name=template_task.name + ' {}'.format(i),
parent=template_task.id)
# get the original template parameters
cloned_task_parameters = cloned_task.get_parameters()
# override with random samples form grid
for k in space.keys():
cloned_task_parameters[k] = space[k]()
# put back into the new cloned task
cloned_task.set_parameters(cloned_task_parameters)
print('Experiment {} set with parameters {}'.format(
i, cloned_task_parameters))
# enqueue the task for execution
# This is the parameter value in the next task we want to change
param["param_name_new_value"] = 3
# The queue where we want the template task (clone) to be sent to
param["execution_queue_name"] = "default"
# Simulate the work of a Task
print("Processing....")
sleep(2.0)
print("Done processing :)")
# Get a reference to the task to pipe to.
next_task = Task.get_task(project_name=task.get_project_name(),
task_name=param["next_task_name"])
# Clone the task to pipe to. This creates a task with status Draft whose parameters can be modified.
cloned_task = Task.clone(source_task=next_task,
name="Auto generated cloned task")
# Get the original parameters of the Task, modify the value of one parameter,
# and set the parameters in the next Task
cloned_task_parameters = cloned_task.get_parameters()
cloned_task_parameters[param["param_name"]] = param["param_name_new_value"]
cloned_task.set_parameters(cloned_task_parameters)
# Enqueue the Task for execution. The enqueued Task must already exist in the clearml platform
print("Enqueue next step in pipeline to queue: {}".format(
param["execution_queue_name"]))
Task.enqueue(cloned_task.id, queue_name=param["execution_queue_name"])
# We are done. The next step in the pipe line is in charge of the pipeline now.
print("Done")
