def end_expired_trials(experiment_config: dict):
"""Get all expired trials, end them and return them."""
trials_past_expiry = get_expired_trials(
experiment_config['experiment'], experiment_config['max_total_time'])
expired_instances = []
current_dt = datetime_now()
for trial in trials_past_expiry:
expired_instances.append(
experiment_utils.get_trial_instance_name(
experiment_config['experiment'], trial.id))
trial.time_ended = current_dt
# Bail out here because trials_past_expiry will be truthy until evaluated.
if not expired_instances:
return
# Delete instances for expired trials.
running_instances = gcloud.list_instances()
instances_to_delete = [
i for i in expired_instances if i in running_instances
]
if instances_to_delete and not gcloud.delete_instances(
instances_to_delete,
experiment_config['cloud_compute_zone'],
write_to_stdout=False):
# If we failed to delete some instances, then don't update the status
# of expired trials in database as we don't know which instances were
# successfully deleted. Wait for next iteration of end_expired_trials.
return
db_utils.bulk_save(trials_past_expiry)
def end_expired_trials(experiment_config: dict):
"""Get all expired trials, end them and return them."""
trials_past_expiry = get_expired_trials(
experiment_config['experiment'], experiment_config['max_total_time'])
expired_instances = []
current_dt = datetime_now()
for trial in trials_past_expiry:
expired_instances.append(
experiment_utils.get_trial_instance_name(
experiment_config['experiment'], trial.id))
trial.time_ended = current_dt
# Bail out here because trials_past_expiry will be truthy until evaluated.
if not expired_instances:
return
if not experiment_utils.is_local_experiment() and not delete_instances(
expired_instances, experiment_config):
# If we failed to delete some instances, then don't update the status
# of expired trials in database as we don't know which instances were
# successfully deleted. Wait for next iteration of end_expired_trials.
logger.error('Failed to delete instances after trial expiry.')
return
db_utils.bulk_save(trials_past_expiry)
def save_snapshots():
"""Saves measured snapshots if there were any, resets |snapshots| to an
empty list and records the fact that snapshots have been measured."""
if not snapshots:
return
db_utils.bulk_save(snapshots)
snapshots.clear()
nonlocal snapshots_measured
snapshots_measured = True
def _initialize_experiment_in_db(experiment: str, git_hash: str,
trials: List[models.Trial]):
"""Initializes |experiment| in the database by creating the experiment
entity and entities for each trial in the experiment."""
db_utils.add_all([
db_utils.get_or_create(models.Experiment,
name=experiment,
git_hash=git_hash)
])
# TODO(metzman): Consider doing this without sqlalchemy. This can get
# slow with SQLalchemy (it's much worse with add_all).
db_utils.bulk_save(trials)
def _initialize_experiment_in_db(experiment: str, benchmarks: List[str],
fuzzers: List[str], num_trials: int):
"""Initializes |experiment| in the database by creating the experiment
entity and entities for each trial in the experiment."""
db_utils.add_all(
[db_utils.get_or_create(models.Experiment, name=experiment)])
trials_args = itertools.product(sorted(benchmarks), range(num_trials),
sorted(fuzzers))
trials = [
models.Trial(fuzzer=fuzzer, experiment=experiment, benchmark=benchmark)
for benchmark, _, fuzzer in trials_args
]
# TODO(metzman): Consider doing this without sqlalchemy. This can get
# slow with SQLalchemy (it's much worse with add_all).
db_utils.bulk_save(trials)
def _initialize_trials_in_db(trials: List[models.Trial]):
"""Initializes entities for each trial in the experiment."""
# TODO(metzman): Consider doing this without sqlalchemy. This can get
# slow with SQLalchemy (it's much worse with add_all).
db_utils.bulk_save(trials)
