Python get_running_experiment_group Examples

Example #1

File: experiment_groups.py Project: nwittstruck/polyaxon

def experiments_group_stop_experiments(experiment_group_id,
                                       pending,
                                       message=None):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if pending:
        for experiment in experiment_group.pending_experiments:
            # Update experiment status to show that its stopped
            experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
    else:
        experiments = experiment_group.experiments.exclude(
            status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
        for experiment in experiments:
            if experiment.is_running:
                celery_app.send_task(
                    SchedulerCeleryTasks.EXPERIMENTS_STOP,
                    kwargs={
                        'project_name': experiment.project.unique_name,
                        'project_uuid': experiment.project.uuid.hex,
                        'experiment_name': experiment.unique_name,
                        'experiment_uuid': experiment.uuid.hex,
                        'experiment_group_name': experiment_group.unique_name,
                        'experiment_group_uuid': experiment_group.uuid.hex,
                        'specification': experiment.config,
                        'update_status': True
                    })
            else:
                # Update experiment status to show that its stopped
                experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)

    experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED, message=message)

Example #2

File: random.py Project: mouradmourafiq/polyaxon

def hp_random_search_create(experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group=experiment_group)

Example #3

def hp_hyperband_iterate(self, experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if experiment_group.non_done_experiments.count() > 0:
        # Schedule another task, because all experiment must be done
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration):
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_CREATE,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    if search_manager.should_reduce_configs(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration):
        iteration_manager.reduce_configs()
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_START,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    base.check_group_experiments_finished(experiment_group_id)

Example #4

File: experiment_groups.py Project: mashroomxl/polyaxon

def experiments_group_stop_experiments(experiment_group_id,
                                       pending,
                                       message=None):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if pending:
        for experiment in experiment_group.pending_experiments:
            # Update experiment status to show that its stopped
            experiment.set_status(status=ExperimentLifeCycle.STOPPED,
                                  message=message)
    else:
        experiments = experiment_group.experiments.exclude(
            status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
        for experiment in experiments:
            if experiment.is_running:
                celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP,
                                     kwargs={'experiment_id': experiment.id})
            else:
                # Update experiment status to show that its stopped
                experiment.set_status(status=ExperimentLifeCycle.STOPPED,
                                      message=message)

    experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)

Example #5

def hp_bo_create(experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group=experiment_group)

Example #6

File: hyperband.py Project: ttsvetanov/polyaxon

def hp_hyperband_iterate(self, experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if experiment_group.non_done_experiments.count() > 0:
        # Schedule another task, because all experiment must be done
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(iteration=iteration_config.iteration,
                                        bracket_iteration=iteration_config.bracket_iteration):
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_CREATE,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
                                            bracket_iteration=iteration_config.bracket_iteration):
        iteration_manager.reduce_configs()
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_START,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    base.check_group_experiments_finished(experiment_group_id)

Example #7

def hp_bo_iterate(self, experiment_group_id, auto_retry=False):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if experiment_group.non_done_experiments.count() > 0:
        if auto_retry:
            # Schedule another task, because all experiment must be done
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(iteration=iteration_config.iteration):
        celery_app.send_task(
            HPCeleryTasks.HP_BO_CREATE,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    base.check_group_experiments_finished(experiment_group_id,
                                          auto_retry=auto_retry)

Example #8

File: __init__.py Project: mohithg/polyaxon

def hp_start(experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    start(experiment_group)

Example #9

File: bo.py Project: yutiansut/polyaxon

def hp_bo_start(self, experiment_group_id, auto_retry=False):
    if not base.should_group_start(experiment_group_id=experiment_group_id,
                                   task=HPCeleryTasks.HP_BO_START,
                                   auto_retry=auto_retry):
        return

    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(
        experiment_group=experiment_group)
    if should_retry:
        if auto_retry:
            # Schedule another task
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    workers.send(HPCeleryTasks.HP_BO_ITERATE,
                 kwargs={
                     'experiment_group_id': experiment_group_id,
                     'auto_retry': auto_retry
                 },
                 countdown=None)

Example #10

File: __init__.py Project: mohithg/polyaxon

def hp_create(self, experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group and self.request.retries < 2:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    create(experiment_group)

Example #11

File: experiment_groups.py Project: himoutoumaru/polyaxon

def experiments_group_stop_experiments(experiment_group_id,
                                       pending,
                                       collect_logs=True,
                                       update_status=True,
                                       message=None):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id, include_deleted=True)
    if not experiment_group:
        return

    if pending:
        # this won't work for archived groups anyways!
        for experiment in experiment_group.pending_experiments.iterator():
            # Update experiment status to show that its stopped
            experiment.set_status(status=ExperimentLifeCycle.STOPPED,
                                  message=message)
    else:
        experiments = experiment_group.all_experiments.exclude(
            status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct(
            ).iterator()
        for experiment in experiments:
            if experiment.is_stoppable:
                celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP,
                                     kwargs={
                                         'project_name':
                                         experiment.project.unique_name,
                                         'project_uuid':
                                         experiment.project.uuid.hex,
                                         'experiment_name':
                                         experiment.unique_name,
                                         'experiment_uuid':
                                         experiment.uuid.hex,
                                         'experiment_group_name':
                                         experiment_group.unique_name,
                                         'experiment_group_uuid':
                                         experiment_group.uuid.hex,
                                         'specification':
                                         experiment.content,
                                         'update_status':
                                         True,
                                         'collect_logs':
                                         collect_logs,
                                         'is_managed':
                                         experiment.is_managed,
                                     },
                                     countdown=conf.get('GLOBAL_COUNTDOWN'))
            else:
                # Update experiment status to show that its stopped
                experiment.set_status(status=ExperimentLifeCycle.STOPPED,
                                      message=message)

    if update_status:
        experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED,
                                    message=message)

Example #12

File: grid.py Project: ttsvetanov/polyaxon

def hp_grid_search_start(self, experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    base.check_group_experiments_finished(experiment_group_id)

Example #13

File: random.py Project: bigdatasciencegroup/polyaxon

def hp_random_search_start(self, experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    base.check_group_experiments_finished(experiment_group_id)

Example #14

File: bo.py Project: himoutoumaru/polyaxon

def hp_bo_create_experiments(experiment_group_id, suggestions):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    try:
        experiments = base.create_group_experiments(experiment_group=experiment_group,
                                                    suggestions=suggestions)
    except ExperimentGroupException:  # The experiments will be stopped
        return
    experiment_group.iteration_manager.add_iteration_experiments(
        experiment_ids=[xp.id for xp in experiments])

Example #15

File: hyperband.py Project: ttsvetanov/polyaxon

def hp_hyperband_start(self, experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    celery_app.send_task(
        HPCeleryTasks.HP_HYPERBAND_ITERATE,
        kwargs={'experiment_group_id': experiment_group_id})

Example #16

def hp_grid_search_create_experiments(experiment_group_id, suggestions):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    experiments = base.create_group_experiments(
        experiment_group=experiment_group, suggestions=suggestions)
    experiment_group.iteration_manager.add_iteration_experiments(
        experiment_ids=[xp.id for xp in experiments])

    celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_START,
                         kwargs={'experiment_group_id': experiment_group.id},
                         countdown=1)

Example #17

def hp_hyperband_start(self, experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(
        experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    celery_app.send_task(HPCeleryTasks.HP_HYPERBAND_ITERATE,
                         kwargs={'experiment_group_id': experiment_group_id})

Example #18

File: bo.py Project: yu-iskw/polyaxon

def hp_bo_create_experiments(experiment_group_id, suggestions):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    try:
        experiments = base.create_group_experiments(experiment_group=experiment_group,
                                                    suggestions=suggestions)
    except ExperimentGroupException:  # The experiments will be stopped
        return
    experiment_group.iteration_manager.add_iteration_experiments(
        experiment_ids=[xp.id for xp in experiments])

    celery_app.send_task(
        HPCeleryTasks.HP_BO_START,
        kwargs={'experiment_group_id': experiment_group.id},
        countdown=1)

Example #19

def experiments_group_stop(experiment_group_id,
                           collect_logs=True,
                           update_status=True,
                           message=None):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id, include_deleted=True)
    if not experiment_group:
        return

    experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING)
    workers.send(SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS,
                 kwargs={
                     'experiment_group_id': experiment_group_id,
                     'pending': False,
                     'collect_logs': collect_logs,
                     'update_status': update_status,
                     'message': message
                 })

Example #20

File: experiment_groups.py Project: waltsims/polyaxon

def experiments_group_stop(experiment_group_id,
                           collect_logs=True,
                           message=None):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id,
                                                    include_deleted=True)
    if not experiment_group:
        return

    experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING)
    celery_app.send_task(
        SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS,
        kwargs={
            'experiment_group_id': experiment_group_id,
            'pending': False,
            'collect_logs': collect_logs,
            'message': message
        },
        countdown=conf.get('GLOBAL_COUNTDOWN'))

Example #21

File: grid.py Project: himoutoumaru/polyaxon

def hp_grid_search_start(self, experiment_group_id, auto_retry=False):
    if not base.should_group_start(experiment_group_id=experiment_group_id,
                                   task=HPCeleryTasks.HP_GRID_SEARCH_START,
                                   auto_retry=auto_retry):
        return

    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        if auto_retry:
            # Schedule another task
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    base.check_group_experiments_done(experiment_group_id, auto_retry=auto_retry)

Example #22

def hp_hyperband_iterate(self, experiment_group_id, auto_retry=False):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if experiment_group.non_done_experiments.count() > 0:
        if auto_retry:
            # Schedule another task, because all experiment must be done
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration):
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_CREATE,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    if search_manager.should_reduce_configs(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration):
        try:
            iteration_manager.reduce_configs()
        except ExperimentGroupException:
            experiment_group.set_status(
                ExperimentGroupLifeCycle.FAILED,
                message='Experiment group could not create new iteration.')
            return
        celery_app.send_task(
            HPCeleryTasks.HP_HYPERBAND_START,
            kwargs={'experiment_group_id': experiment_group_id})
        return

    base.check_group_experiments_finished(experiment_group_id,
                                          auto_retry=auto_retry)

Example #23

File: experiment_groups.py Project: ttsvetanov/polyaxon

def experiments_group_stop_experiments(experiment_group_id, pending, message=None):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    if pending:
        for experiment in experiment_group.pending_experiments:
            # Update experiment status to show that its stopped
            experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
    else:
        experiments = experiment_group.experiments.exclude(
            status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
        for experiment in experiments:
            if experiment.is_running:
                celery_app.send_task(
                    SchedulerCeleryTasks.EXPERIMENTS_STOP,
                    kwargs={'experiment_id': experiment.id})
            else:
                # Update experiment status to show that its stopped
                experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)

    experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)

Example #24

File: __init__.py Project: ttsvetanov/polyaxon

def hp_create(experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    create(experiment_group)

Example #25

File: grid.py Project: ttsvetanov/polyaxon

def hp_grid_search_create(experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group)

Example #26

File: hyperband.py Project: ttsvetanov/polyaxon

def hp_hyperband_create(experiment_group_id):
    experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group)

Example #27

File: __init__.py Project: saiprashanths/polyaxon

def hp_create(experiment_group_id):
    experiment_group = get_running_experiment_group(
        experiment_group_id=experiment_group_id)
    create(experiment_group)