def test_cancel_analysis(self):
"""Test canceling experiment analysis."""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
return self._get_job_result(1)
def _analysis(*args): # pylint: disable = unused-argument
event.wait(timeout=15)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis)
exp_data.cancel_analysis()
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.RUNNING)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING)
# Test status after job finishes
event.set()
self.assertEqual(exp_data.job_status(), JobStatus.DONE)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_cancel(self):
"""Test canceling experiment jobs and analysis."""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _analysis(*args): # pylint: disable = unused-argument
event.wait(timeout=15)
def _status():
if event.is_set():
return JobStatus.CANCELLED
return JobStatus.RUNNING
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = _status
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis)
exp_data.cancel()
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_status_job_pending(self):
"""Test experiment status when job is pending."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
event = threading.Event()
job2 = mock.create_autospec(Job, instance=True)
job2.result = lambda *args, **kwargs: event.wait(timeout=15)
job2.status = lambda: JobStatus.CANCELLED if event.is_set(
) else JobStatus.RUNNING
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job1)
exp_data.add_jobs(job2)
exp_data.add_analysis_callback(
lambda *args, **kwargs: event.wait(timeout=15))
self.assertEqual(ExperimentStatus.RUNNING, exp_data.status())
self.assertEqual(JobStatus.RUNNING, exp_data.job_status())
self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status())
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
event = threading.Event()
cancel_count = 0
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _job_cancel():
nonlocal cancel_count
cancel_count += 1
event.set()
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.cancel = _job_cancel
job.result = _job_result
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data.add_jobs(job)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.cancel_jobs()
self.assertEqual(cancel_count, 1)
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_partial_cancel_analysis(self):
"""Test canceling experiment analysis."""
event = threading.Event()
self.addCleanup(event.set)
run_analysis = []
def _job_result():
event.wait(timeout=3)
return self._get_job_result(1)
def _analysis(expdata, name=None, timeout=0): # pylint: disable = unused-argument
event.wait(timeout=timeout)
run_analysis.append(name)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis, name=1, timeout=1)
exp_data.add_analysis_callback(_analysis, name=2, timeout=30)
cancel_id = exp_data._analysis_callbacks.keys()[-1]
exp_data.add_analysis_callback(_analysis, name=3, timeout=1)
exp_data.cancel_analysis(cancel_id)
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.RUNNING)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING)
# Test status after job finishes
event.set()
self.assertEqual(exp_data.job_status(), JobStatus.DONE)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
# Check that correct analysis callback was cancelled
exp_data.block_for_results()
self.assertEqual(run_analysis, [1, 3])
for cid, analysis in exp_data._analysis_callbacks.items():
if cid == cancel_id:
self.assertEqual(analysis.status, AnalysisStatus.CANCELLED)
else:
self.assertEqual(analysis.status, AnalysisStatus.DONE)
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_jobs(job)
self.assertExperimentDone(exp_data)
self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_data(job)
exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
exp_data.block_for_results()
self.assertEqual("DONE", exp_data.status())
def test_block_for_results(self):
"""Test blocking for jobs"""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
jobs = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_jobs(job)
jobs.append(job)
exp_data.block_for_results()
self.assertTrue(all(job.status() == JobStatus.DONE for job in jobs))
self.assertEqual("DONE", exp_data.status())
def test_status_post_processing(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=15)))
status = exp_data.status()
self.assertEqual(ExperimentStatus.POST_PROCESSING, status)
def test_status_job_error(self):
"""Test experiment status when job failed."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs([job1, job2])
self.assertIn("Adding a job from a backend", ",".join(cm.output))
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
def test_status_cancelled_analysis(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=2)))
# Add second callback because the first can't be cancelled once it has started
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=20)))
exp_data.cancel_analysis()
status = exp_data.status()
self.assertEqual(ExperimentStatus.CANCELLED, status)
def test_status_post_processing_error(self):
"""Test experiment status when post processing failed."""
def _post_processing(*args, **kwargs):
raise ValueError("Kaboom!")
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data(job)
exp_data.add_analysis_callback(_post_processing)
exp_data.block_for_results()
self.assertEqual("ERROR", exp_data.status())
self.assertIn("Kaboom!", ",".join(cm.output))
def test_errors(self):
"""Test getting experiment error message."""
def _post_processing(*args, **kwargs): # pylint: disable=unused-argument
raise ValueError("Kaboom!")
job1 = mock.create_autospec(Job, instance=True)
job1.job_id.return_value = "1234"
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
job2.job_id.return_value = "5678"
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs(job1)
exp_data.add_analysis_callback(_post_processing)
exp_data.add_jobs(job2)
exp_data.block_for_results()
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
self.assertIn("Kaboom", ",".join(cm.output))
self.assertTrue(re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL))
def test_add_jobs_timeout(self):
"""Test timeout kwarg of add_jobs"""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job, timeout=0.5)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.block_for_results()
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
