def archive(project, experiment_ids):
'''Archive SigOpt Experiments.'''
factory = SigOptFactory(project)
factory.set_up_cli()
for experiment_id in experiment_ids:
try:
factory.connection.experiments(experiment_id).delete()
except Exception as e:
raise click.ClickException(
f"experiment_id: {experiment_id}, {e}") from e
def create(project, project_name):
'''Create a SigOpt Project.'''
factory = SigOptFactory(project)
try:
factory.create_project(name=project_name)
except ConflictingProjectException as cpe:
raise click.ClickException(cpe) from cpe
print_logger.info("Project '%s' created.", project)
print_logger.info("To use this project, set the SIGOPT_PROJECT environment variable:")
print_logger.info("> export SIGOPT_PROJECT='%s'", project)
def iterate_runs_by_filters(self, filters, project=None, client=None):
if project is None:
client, project = SigOptFactory.from_default_project(
).ensure_project_exists()
return (
self.conn.clients(client).projects(project).training_runs().fetch(
filters=json.dumps(filters)).iterate_pages())
def start_worker(experiment_id, command, run_options, source_file):
'''Start a worker for the given Experiment.'''
factory = SigOptFactory.from_default_project()
factory.set_up_cli()
try:
experiment = factory.get_experiment(experiment_id)
except ValueError as ve:
raise click.ClickException(str(ve))
cli_experiment_loop(config, experiment, command, run_options, source_file)
def refill_experiment_pods(
self,
k8s_settings,
sigopt_settings,
experiment_id,
run_states,
refill_pods_event,
):
logger = self.logger
logger.info("starting management loop")
project = sigopt_settings.project
run_factory = SigOptFactory(
project,
connection=sigopt_settings.conn).get_experiment(experiment_id)
while True:
refill_pods_event.wait()
refill_pods_event.clear()
if self.stop_threads_event.is_set():
break
logger.debug("checking pod count")
active_run_count = count_active_runs(run_states)
logger.debug("active run count %s", active_run_count)
missing_pod_count = get_missing_experiment_pod_count(
sigopt_settings.conn, active_run_count, experiment_id)
logger.debug("missing pod count %s", missing_pod_count)
if missing_pod_count + active_run_count == 0:
logger.info("no more pods to add and no runs are active")
break
if missing_pod_count:
logger.info("adding %s pods", missing_pod_count)
for _ in range(missing_pod_count):
run_name = random_run_name()
logger.debug("creating run %s", run_name)
run_context = run_factory.create_run(name=run_name)
run_context.log_metadata('cluster_name',
k8s_settings.cluster_name)
run_states[run_name] = RunState(run_context, sigopt_settings,
k8s_settings, run_name)
pod = create_run_pod(
k8s_settings=k8s_settings,
run_context=run_context,
)
logger.info("added pod %s", pod.metadata.name)
logger.info("exited refill loop")
def unarchive(project, run_ids):
'''Unarchive SigOpt Runs.'''
factory = SigOptFactory(project)
factory.set_up_cli()
for run_id in run_ids:
try:
factory.unarchive_run(run_id)
except Exception as e:
raise click.ClickException(f'run_id: {run_id}, {e}') from e
def unarchive(project, experiment_ids):
'''Unarchive SigOpt Experiments.'''
factory = SigOptFactory(project)
factory.set_up_cli()
for experiment_id in experiment_ids:
try:
factory.unarchive_experiment(experiment_id)
except Exception as e:
raise click.ClickException(
f'experiment_id: {experiment_id}, {e}') from e
def create_experiment_from_validated_data(experiment_file, project):
assert isinstance(experiment_file, ValidatedData)
factory = SigOptFactory(project)
return factory.create_prevalidated_experiment(experiment_file.data)
def factory(self):
factory = SigOptFactory("test-project")
return factory
def run(command, run_options, source_file, project):
'''Create a SigOpt Run.'''
factory = SigOptFactory(project)
factory.set_up_cli()
with factory.create_run(name=run_options.get("name")) as run_context:
run_user_program(config, run_context, command, source_file)
def ensure_project_exists(self, project_id):
factory = SigOptFactory(project_id)
return factory.ensure_project_exists()
def create_run(self, run_name, cluster, project_id):
factory = SigOptFactory(project_id)
return factory.create_run(
name=run_name,
metadata={'cluster_name': cluster.name},
)
def fetch_experiment(self, experiment_id):
factory = SigOptFactory.from_default_project()
return factory.get_experiment(experiment_id)
def create_experiment(self, experiment_body, project_id):
factory = SigOptFactory(project_id)
return factory.create_prevalidated_experiment(experiment_body)