def dispatcher_main():
"""Do the experiment and report results."""
logs.info('Starting experiment.')
# Set this here because we get failures if we do it in measurer for some
# reason.
multiprocessing.set_start_method('spawn')
db_utils.initialize()
if os.getenv('LOCAL_EXPERIMENT'):
models.Base.metadata.create_all(db_utils.engine)
experiment_config_file_path = os.path.join(fuzzer_config_utils.get_dir(),
'experiment.yaml')
experiment = Experiment(experiment_config_file_path)
preemptible = experiment.preemptible
trials = build_images_for_trials(experiment.fuzzers, experiment.benchmarks,
experiment.num_trials, preemptible)
_initialize_experiment_in_db(experiment.experiment_name,
experiment.git_hash, trials)
create_work_subdirs(['experiment-folders', 'measurement-folders'])
# Start measurer and scheduler in seperate threads/processes.
scheduler_loop_thread = threading.Thread(target=scheduler.schedule_loop,
args=(experiment.config, ))
scheduler_loop_thread.start()
max_total_time = experiment.config['max_total_time']
measurer_loop_process = multiprocessing.Process(
target=measurer.measure_loop,
args=(experiment.experiment_name, max_total_time))
measurer_loop_process.start()
is_complete = False
while True:
time.sleep(LOOP_WAIT_SECONDS)
if not scheduler_loop_thread.is_alive():
is_complete = not measurer_loop_process.is_alive()
# Generate periodic output reports.
reporter.output_report(experiment.web_bucket,
in_progress=not is_complete)
if is_complete:
# Experiment is complete, bail out.
break
logs.info('Dispatcher finished.')
scheduler_loop_thread.join()
measurer_loop_process.join()
def dispatcher_main():
"""Do the experiment and report results."""
logs.info('Starting experiment.')
# Set this here because we get failures if we do it in measurer for some
# reason.
multiprocessing.set_start_method('spawn')
db_utils.initialize()
if os.getenv('LOCAL_EXPERIMENT'):
models.Base.metadata.create_all(db_utils.engine)
builder.build_base_images()
experiment_config_file_path = os.path.join(fuzzer_config_utils.get_dir(),
'experiment.yaml')
experiment = Experiment(experiment_config_file_path)
# When building, we only care about the underlying fuzzer rather than the
# display name that we use to identify a specific configuration.
unique_fuzzers = list({
fuzzer_config_utils.get_underlying_fuzzer_name(f)
for f in experiment.fuzzers
})
builder.build_all_fuzzer_benchmarks(unique_fuzzers, experiment.benchmarks)
create_work_subdirs(['experiment-folders', 'measurement-folders'])
# Start measurer and scheduler in threads.
scheduler_loop_thread = threading.Thread(target=scheduler.schedule_loop,
args=(experiment.config, ))
scheduler_loop_thread.start()
measurer_loop_thread = multiprocessing.Process(
target=measurer.measure_loop,
args=(
experiment.config['experiment'],
experiment.config['max_total_time'],
))
measurer_loop_thread.start()
while True:
time.sleep(LOOP_WAIT_SECONDS)
is_complete = (not scheduler_loop_thread.is_alive()
and not measurer_loop_thread.is_alive())
# Generate periodic output reports.
reporter.output_report(experiment.web_bucket,
in_progress=not is_complete)
if is_complete:
# Experiment is complete, bail out.
break