def test_add_data_callback(self):
"""Test add data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
nonlocal called_back_count, expected_data, subtests
expected_data.extend(subtests[called_back_count][1])
self.assertEqual([dat["counts"] for dat in _exp_data.data()],
expected_data)
called_back_count += 1
a_result = self._get_job_result(1)
results = [self._get_job_result(1), self._get_job_result(1)]
a_dict = {"counts": {"01": 518}}
dicts = [{"counts": {"00": 284}}, {"counts": {"00": 14}}]
subtests = [
(a_result, [a_result.get_counts()]),
(results, [res.get_counts() for res in results]),
(a_dict, [a_dict["counts"]]),
(dicts, [dat["counts"] for dat in dicts]),
]
called_back_count = 0
expected_data = []
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
for data, _ in subtests:
with self.subTest(data=data):
exp_data.add_data(data)
exp_data.add_analysis_callback(_callback)
exp_data.block_for_results()
self.assertEqual(len(subtests), called_back_count)
def test_status_job_pending(self):
"""Test experiment status when job is pending."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
event = threading.Event()
job2 = mock.create_autospec(Job, instance=True)
job2.result = lambda *args, **kwargs: event.wait(timeout=15)
job2.status = lambda: JobStatus.CANCELLED if event.is_set(
) else JobStatus.RUNNING
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job1)
exp_data.add_jobs(job2)
exp_data.add_analysis_callback(
lambda *args, **kwargs: event.wait(timeout=15))
self.assertEqual(ExperimentStatus.RUNNING, exp_data.status())
self.assertEqual(JobStatus.RUNNING, exp_data.job_status())
self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status())
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_recursive_callback_raises(self):
"""Test handling of excepting callbacks"""
def callback1(exp_data):
"""Callback function that call add_analysis_callback"""
time.sleep(1)
exp_data.add_analysis_callback(callback2)
result = DbAnalysisResult("RESULT1", True, ["Q0"], exp_data.experiment_id)
exp_data.add_analysis_results(result)
def callback2(exp_data):
"""Callback function that exercises status lookups"""
time.sleep(1)
exp_data.add_analysis_callback(callback3)
raise RuntimeError("YOU FAIL")
def callback3(exp_data):
"""Callback function that exercises status lookups"""
time.sleep(1)
result = DbAnalysisResult("RESULT2", True, ["Q0"], exp_data.experiment_id)
exp_data.add_analysis_results(result)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_analysis_callback(callback1)
exp_data.block_for_results(timeout=10)
results = exp_data.analysis_results(block=False)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.ERROR)
self.assertTrue("RuntimeError: YOU FAIL" in exp_data.analysis_errors())
self.assertEqual(len(results), 2)
def test_copy_metadata_pending_job(self):
"""Test copy metadata with a pending job."""
event = threading.Event()
self.addCleanup(event.set)
job_results1 = self._get_job_result(1)
job_results2 = self._get_job_result(1)
def _job1_result():
event.wait(timeout=15)
return job_results1
def _job2_result():
event.wait(timeout=15)
return job_results2
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = mock.create_autospec(Job, instance=True)
job.result = _job1_result
exp_data.add_jobs(job)
copied = exp_data.copy(copy_results=False)
job2 = mock.create_autospec(Job, instance=True)
job2.result = _job2_result
copied.add_jobs(job2)
event.set()
exp_data.block_for_results()
copied.block_for_results()
self.assertEqual(1, len(exp_data.data()))
self.assertEqual(2, len(copied.data()))
self.assertIn(
exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]]
)
def test_simple_methods_from_callback(self):
"""Test that simple methods used in call back function don't hang
This test runs through many of the public methods of DbExperimentData
from analysis callbacks to make sure that they do not raise exceptions
or hang the analysis thread. Hangs have occurred in the past when one
of these methods blocks waiting for analysis to complete.
These methods are not tested because they explicitly assume they are
run from the main thread:
+ copy
+ block_for_results
These methods are not tested because they require additional setup.
They could be tested in separate tests:
+ save
+ save_metadata
+ add_jobs
+ cancel
+ cancel_analysis
+ cancel_jobs
"""
def callback1(exp_data):
"""Callback function that call add_analysis_callback"""
exp_data.add_analysis_callback(callback2)
result = DbAnalysisResult("result_name", 0, [Qubit(0)], "experiment_id")
exp_data.add_analysis_results(result)
figure = get_non_gui_ax().get_figure()
exp_data.add_figures(figure, "figure.svg")
exp_data.add_data({"key": 1.2})
exp_data.data()
def callback2(exp_data):
"""Callback function that exercises status lookups"""
exp_data.figure("figure.svg")
exp_data.jobs()
exp_data.analysis_results("result_name", block=False)
exp_data.delete_figure("figure.svg")
exp_data.delete_analysis_result("result_name")
exp_data.status()
exp_data.job_status()
exp_data.analysis_status()
exp_data.errors()
exp_data.job_errors()
exp_data.analysis_errors()
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_analysis_callback(callback1)
exp_data.block_for_results(timeout=3)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.DONE)
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_data(job)
exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
exp_data.block_for_results()
self.assertEqual("DONE", exp_data.status())
def test_block_for_results(self):
"""Test blocking for jobs"""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
jobs = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_jobs(job)
jobs.append(job)
exp_data.block_for_results()
self.assertTrue(all(job.status() == JobStatus.DONE for job in jobs))
self.assertEqual("DONE", exp_data.status())
def test_add_data_job(self):
"""Test add job to experiment data."""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
transpiled = transpile(ReferenceCircuits.bell(), self.backend)
transpiled.metadata = {"foo": "bar"}
job = self._run_circuit(transpiled)
exp_data.add_jobs(job)
self.assertEqual([job.job_id()], exp_data.job_ids)
result = job.result()
exp_data.block_for_results()
circuit_data = exp_data.data(0)
self.assertEqual(result.get_counts(0), circuit_data["counts"])
self.assertEqual(job.job_id(), circuit_data["job_id"])
self.assertEqual(transpiled.metadata, circuit_data["metadata"])
def test_block_for_jobs(self):
"""Test blocking for jobs."""
def _sleeper(*args, **kwargs): # pylint: disable=unused-argument
time.sleep(2)
nonlocal sleep_count
sleep_count += 1
return self._get_job_result(1)
sleep_count = 0
job = mock.create_autospec(Job, instance=True)
job.result = _sleeper
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_sleeper)
exp_data.block_for_results()
self.assertEqual(2, sleep_count)
def test_partial_cancel_analysis(self):
"""Test canceling experiment analysis."""
event = threading.Event()
self.addCleanup(event.set)
run_analysis = []
def _job_result():
event.wait(timeout=3)
return self._get_job_result(1)
def _analysis(expdata, name=None, timeout=0): # pylint: disable = unused-argument
event.wait(timeout=timeout)
run_analysis.append(name)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis, name=1, timeout=1)
exp_data.add_analysis_callback(_analysis, name=2, timeout=30)
cancel_id = exp_data._analysis_callbacks.keys()[-1]
exp_data.add_analysis_callback(_analysis, name=3, timeout=1)
exp_data.cancel_analysis(cancel_id)
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.RUNNING)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING)
# Test status after job finishes
event.set()
self.assertEqual(exp_data.job_status(), JobStatus.DONE)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
# Check that correct analysis callback was cancelled
exp_data.block_for_results()
self.assertEqual(run_analysis, [1, 3])
for cid, analysis in exp_data._analysis_callbacks.items():
if cid == cancel_id:
self.assertEqual(analysis.status, AnalysisStatus.CANCELLED)
else:
self.assertEqual(analysis.status, AnalysisStatus.DONE)
def test_status_post_processing_error(self):
"""Test experiment status when post processing failed."""
def _post_processing(*args, **kwargs):
raise ValueError("Kaboom!")
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data(job)
exp_data.add_analysis_callback(_post_processing)
exp_data.block_for_results()
self.assertEqual("ERROR", exp_data.status())
self.assertIn("Kaboom!", ",".join(cm.output))
def test_add_data_job_callback_kwargs(self):
"""Test add job data with callback and additional arguments."""
def _callback(_exp_data, **kwargs):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual({"foo": callback_kwargs}, kwargs)
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
called_back = False
callback_kwargs = "foo"
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.add_analysis_callback(_callback, foo=callback_kwargs)
exp_data.block_for_results()
self.assertTrue(called_back)
def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.result = _job_result
exp_data.add_jobs(job)
exp_data.cancel_jobs()
job.cancel.assert_called_once()
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_add_data_job_callback(self):
"""Test add job data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual([dat["counts"] for dat in _exp_data.data()],
a_job.result().get_counts())
exp_data.add_figures(str.encode("hello world"))
exp_data.add_analysis_results(mock.MagicMock())
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
called_back = False
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.add_analysis_callback(_callback)
exp_data.block_for_results()
self.assertTrue(called_back)
def test_add_data_job(self):
"""Test add job data."""
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(3)
jobs = []
for _ in range(2):
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(2)
jobs.append(job)
expected = a_job.result().get_counts()
for job in jobs:
expected.extend(job.result().get_counts())
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.block_for_results()
exp_data.add_data(jobs)
exp_data.block_for_results()
self.assertEqual(expected,
[sdata["counts"] for sdata in exp_data.data()])
self.assertIn(a_job.job_id(), exp_data.job_ids)
def test_errors(self):
"""Test getting experiment error message."""
def _post_processing(*args, **kwargs): # pylint: disable=unused-argument
raise ValueError("Kaboom!")
job1 = mock.create_autospec(Job, instance=True)
job1.job_id.return_value = "1234"
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
job2.job_id.return_value = "5678"
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs(job1)
exp_data.add_analysis_callback(_post_processing)
exp_data.add_jobs(job2)
exp_data.block_for_results()
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
self.assertIn("Kaboom", ",".join(cm.output))
self.assertTrue(re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL))
def test_add_jobs_timeout(self):
"""Test timeout kwarg of add_jobs"""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job, timeout=0.5)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.block_for_results()
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
