def _consume(self, trial, workdirname):
config_file = tempfile.NamedTemporaryFile(mode='w',
prefix='trial_',
suffix='.conf',
dir=workdirname,
delete=False)
config_file.close()
log.debug("## New temp config file: %s", config_file.name)
results_file = tempfile.NamedTemporaryFile(mode='w',
prefix='results_',
suffix='.log',
dir=workdirname,
delete=False)
results_file.close()
log.debug("## New temp results file: %s", results_file.name)
log.debug(
"## Building command line argument and configuration for trial.")
env = self.get_execution_environment(trial, results_file.name)
cmd_args = self.template_builder.format(config_file.name, trial,
self.experiment)
log.debug(
"## Launch user's script as a subprocess and wait for finish.")
self.pacemaker = TrialPacemaker(trial)
self.pacemaker.start()
try:
self.execute_process(cmd_args, env)
finally:
# merciless
self.pacemaker.stop()
return results_file
def test_trial_heartbeat_not_updated(exp, trial):
"""Test that the heartbeat of a trial is not updated when trial is not longer reserved."""
trial_monitor = TrialPacemaker(trial, wait_time=1)
trial_monitor.start()
time.sleep(2)
trials = exp.fetch_trials_by_status("reserved")
assert trial.heartbeat != trials[0].heartbeat
get_storage().set_trial_status(trial, status="interrupted")
time.sleep(2)
# `join` blocks until all thread have finish executing. So, the test will hang if it fails.
trial_monitor.join()
assert 1
def test_trial_heartbeat_not_updated_inbetween(exp, trial):
"""Test that the heartbeat of a trial is not updated before wait time."""
trial_monitor = TrialPacemaker(trial, wait_time=5)
trial_monitor.start()
time.sleep(1)
trials = exp.fetch_trials_by_status('reserved')
assert trial.heartbeat.replace(microsecond=0) == trials[0].heartbeat.replace(microsecond=0)
heartbeat = trials[0].heartbeat
time.sleep(6)
trials = exp.fetch_trials_by_status(status='reserved')
assert heartbeat != trials[0].heartbeat
trial_monitor.stop()
def test_trial_update_heartbeat(exp, trial):
"""Test that the heartbeat of a trial has been updated."""
trial_monitor = TrialPacemaker(trial, wait_time=1)
trial_monitor.start()
time.sleep(2)
trials = exp.fetch_trials_by_status("reserved")
assert trial.heartbeat != trials[0].heartbeat
heartbeat = trials[0].heartbeat
time.sleep(2)
trials = exp.fetch_trials_by_status(status="reserved")
assert heartbeat != trials[0].heartbeat
trial_monitor.stop()
def test_trial_heartbeat_not_updated(exp, trial):
"""Test that the heartbeat of a trial is not updated when trial is not longer reserved."""
trial_monitor = TrialPacemaker(exp, trial.id, wait_time=1)
trial_monitor.start()
time.sleep(2)
trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'})
assert trial.heartbeat != trials[0].heartbeat
data = {'status': 'interrupted'}
Database().write('trials', data, query=dict(_id=trial.id))
time.sleep(2)
# `join` blocks until all thread have finish executing. So, the test will hang if it fails.
trial_monitor.join()
assert 1
def test_trial_update_heartbeat(exp, trial):
"""Test that the heartbeat of a trial has been updated."""
trial_monitor = TrialPacemaker(exp, trial.id, wait_time=1)
trial_monitor.start()
time.sleep(2)
trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'})
assert trial.heartbeat != trials[0].heartbeat
heartbeat = trials[0].heartbeat
time.sleep(2)
trials = exp.fetch_trials({'_id': trial.id, 'status': 'reserved'})
assert heartbeat != trials[0].heartbeat
trial_monitor.stop()
def _maintain_reservation(self, trial):
self._pacemakers[trial.id] = TrialPacemaker(trial)
self._pacemakers[trial.id].start()