def test_add_data_callback(self):
"""Test add data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
nonlocal called_back_count, expected_data, subtests
expected_data.extend(subtests[called_back_count][1])
self.assertEqual([dat["counts"] for dat in _exp_data.data()], expected_data)
called_back_count += 1
a_result = self._get_job_result(1)
results = [self._get_job_result(1), self._get_job_result(1)]
a_dict = {"counts": {"01": 518}}
dicts = [{"counts": {"00": 284}}, {"counts": {"00": 14}}]
subtests = [
(a_result, [a_result.get_counts()]),
(results, [res.get_counts() for res in results]),
(a_dict, [a_dict["counts"]]),
(dicts, [dat["counts"] for dat in dicts]),
]
called_back_count = 0
expected_data = []
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
for data, _ in subtests:
with self.subTest(data=data):
exp_data.add_data(data)
exp_data.add_analysis_callback(_callback)
self.assertExperimentDone(exp_data)
self.assertEqual(len(subtests), called_back_count)
def test_different_backend(self):
"""Test setting a different backend."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
a_job = mock.create_autospec(Job, instance=True)
self.assertNotEqual(exp_data.backend, a_job.backend())
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.add_jobs(a_job)
def test_set_tags(self):
"""Test updating experiment tags."""
exp_data = DbExperimentData(experiment_type="qiskit_test",
tags=["foo"])
self.assertEqual(["foo"], exp_data.tags)
exp_data.tags = ["bar"]
self.assertEqual(["bar"], exp_data.tags)
def test_save_all(self):
"""Test saving all."""
exp_data = self._create_experiment_data()
exp_data.tags = ["foo", "bar"]
aresult = AnalysisResult(
value={},
name="qiskit_test",
device_components=self.device_components,
experiment_id=exp_data.experiment_id,
)
exp_data.add_analysis_results(aresult)
hello_bytes = str.encode("hello world")
exp_data.add_figures(hello_bytes, figure_names="hello.svg")
exp_data.save()
rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment)
self.assertEqual(["foo", "bar"], rexp.tags)
self.assertEqual(aresult.result_id, rexp.analysis_results(0).result_id)
self.assertEqual(hello_bytes, rexp.figure(0))
exp_data.delete_analysis_result(0)
exp_data.delete_figure(0)
with mock.patch("builtins.input", lambda _: "y"):
exp_data.save()
rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment)
self.assertRaises(IBMExperimentEntryNotFound, rexp.figure, "hello.svg")
self.assertRaises(DbExperimentEntryNotFound, rexp.analysis_results,
aresult.result_id)
def test_auto_save(self):
"""Test auto save."""
service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.auto_save = True
mock_result = mock.MagicMock()
subtests = [
# update function, update parameters, service called
(exp_data.add_analysis_results, (mock_result, ), mock_result.save),
(exp_data.add_figures, (str.encode("hello world"), ),
service.create_figure),
(exp_data.delete_figure, (0, ), service.delete_figure),
(exp_data.delete_analysis_result, (0, ),
service.delete_analysis_result),
(setattr, (exp_data, "tags", ["foo"]), service.update_experiment),
(setattr, (exp_data, "notes", "foo"), service.update_experiment),
(setattr, (exp_data, "share_level", "hub"),
service.update_experiment),
]
for func, params, called in subtests:
with self.subTest(func=func):
func(*params)
if called:
called.assert_called_once()
service.reset_mock()
def test_status_job_pending(self):
"""Test experiment status when job is pending."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
event = threading.Event()
job2 = mock.create_autospec(Job, instance=True)
job2.result = lambda *args, **kwargs: event.wait(timeout=15)
job2.status = lambda: JobStatus.CANCELLED if event.is_set(
) else JobStatus.RUNNING
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job1)
exp_data.add_jobs(job2)
exp_data.add_analysis_callback(
lambda *args, **kwargs: event.wait(timeout=15))
self.assertEqual(ExperimentStatus.RUNNING, exp_data.status())
self.assertEqual(JobStatus.RUNNING, exp_data.job_status())
self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status())
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_cancel(self):
"""Test canceling experiment jobs and analysis."""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _analysis(*args): # pylint: disable = unused-argument
event.wait(timeout=15)
def _status():
if event.is_set():
return JobStatus.CANCELLED
return JobStatus.RUNNING
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = _status
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis)
exp_data.cancel()
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def _create_experiment_data(self):
"""Create an experiment data."""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.save()
self.experiments_to_delete.append(exp_data.experiment_id)
return exp_data
def test_set_service_job(self):
"""Test setting service with a job."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = self._run_circuit()
exp_data.add_jobs(job)
exp_data.save()
rexp = self.experiment.experiment(exp_data.experiment_id)
self.assertEqual([job.job_id()], rexp["job_ids"])
def test_set_service_job(self):
"""Test setting service via adding a job."""
mock_service = self._set_mock_service()
job = mock.create_autospec(Job, instance=True)
job.backend.return_value = self.backend
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.service)
exp_data.add_data(job)
self.assertEqual(mock_service, exp_data.service)
def test_add_data_result_metadata(self):
"""Test add result metadata."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
result1 = self._get_job_result(1, has_metadata=False)
result2 = self._get_job_result(1, has_metadata=True)
exp_data.add_data(result1)
exp_data.add_data(result2)
self.assertNotIn("metadata", exp_data.data(0))
self.assertIn("metadata", exp_data.data(1))
def test_set_service_direct(self):
"""Test setting service directly."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.service)
mock_service = mock.MagicMock()
exp_data.service = mock_service
self.assertEqual(mock_service, exp_data.service)
with self.assertRaises(DbExperimentDataError):
exp_data.service = mock_service
def test_str(self):
"""Test the string representation."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(self._get_job_result(1))
result = mock.MagicMock()
exp_data.add_analysis_results(result)
exp_data_str = str(exp_data)
self.assertIn(exp_data.experiment_type, exp_data_str)
self.assertIn(exp_data.experiment_id, exp_data_str)
self.assertIn(str(result), exp_data_str)
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_jobs(job)
self.assertExperimentDone(exp_data)
self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_add_figure_save(self):
"""Test saving a figure in the database."""
hello_bytes = str.encode("hello world")
service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
exp_data.add_figures(hello_bytes, save_figure=True)
service.create_figure.assert_called_once()
_, kwargs = service.create_figure.call_args
self.assertEqual(kwargs["figure"], hello_bytes)
self.assertEqual(kwargs["experiment_id"], exp_data.experiment_id)
def test_delayed_backend(self):
"""Test initializing experiment data without a backend."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.backend)
self.assertIsNone(exp_data.service)
exp_data.save_metadata()
a_job = mock.create_autospec(Job, instance=True)
exp_data.add_jobs(a_job)
self.assertIsNotNone(exp_data.backend)
self.assertIsNotNone(exp_data.service)
def test_add_get_analysis_results(self):
"""Test adding and getting a list of analysis results."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
results = []
for idx in range(5):
res = mock.MagicMock()
res.result_id = idx
results.append(res)
exp_data.add_analysis_results(results)
self.assertEqual(results, exp_data.analysis_results())
def test_add_figure_plot(self):
"""Test adding a matplotlib figure."""
figure, ax = plt.subplots()
ax.plot([1, 2, 3])
service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
exp_data.add_figures(figure, save_figure=True)
self.assertEqual(figure, exp_data.figure(0))
service.create_figure.assert_called_once()
_, kwargs = service.create_figure.call_args
self.assertIsInstance(kwargs["figure"], bytes)
def test_new_backend_has_service(self):
"""Test changing backend doesn't change existing service."""
orig_service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
self.assertEqual(orig_service, exp_data.service)
job = mock.create_autospec(Job, instance=True)
new_service = self._set_mock_service()
self.assertNotEqual(orig_service, new_service)
job.backend.return_value = self.backend
job.status.return_value = JobStatus.DONE
exp_data.add_jobs(job)
self.assertEqual(orig_service, exp_data.service)
def test_delete_figure(self):
"""Test deleting a figure."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
id_template = "figure_{}.svg"
for idx in range(3):
exp_data.add_figures(str.encode("hello world"), id_template.format(idx))
sub_tests = [(1, id_template.format(1)), (id_template.format(2), id_template.format(2))]
for del_key, figure_name in sub_tests:
with self.subTest(del_key=del_key):
exp_data.delete_figure(del_key)
self.assertRaises(DbExperimentEntryNotFound, exp_data.figure, figure_name)
def test_delete_analysis_result(self):
"""Test deleting analysis result."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
id_template = "result_{}"
for idx in range(3):
res = mock.MagicMock()
res.result_id = id_template.format(idx)
exp_data.add_analysis_results(res)
subtests = [(0, id_template.format(0)), (id_template.format(2), id_template.format(2))]
for del_key, res_id in subtests:
with self.subTest(del_key=del_key):
exp_data.delete_analysis_result(del_key)
self.assertRaises(DbExperimentEntryNotFound, exp_data.analysis_results, res_id)
def test_copy_metadata_pending_job(self):
"""Test copy metadata with a pending job."""
event = threading.Event()
self.addCleanup(event.set)
job_results1 = self._get_job_result(1)
job_results2 = self._get_job_result(1)
def _job1_result():
event.wait(timeout=15)
return job_results1
def _job2_result():
event.wait(timeout=15)
return job_results2
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = mock.create_autospec(Job, instance=True)
job.result = _job1_result
exp_data.add_jobs(job)
copied = exp_data.copy(copy_results=False)
job2 = mock.create_autospec(Job, instance=True)
job2.result = _job2_result
copied.add_jobs(job2)
event.set()
exp_data.block_for_results()
copied.block_for_results()
self.assertEqual(1, len(exp_data.data()))
self.assertEqual(2, len(copied.data()))
self.assertIn(
exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]]
)
def test_status_job_error(self):
"""Test experiment status when job failed."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs([job1, job2])
self.assertIn("Adding a job from a backend", ",".join(cm.output))
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
def test_recursive_callback_raises(self):
"""Test handling of excepting callbacks"""
def callback1(exp_data):
"""Callback function that call add_analysis_callback"""
time.sleep(1)
exp_data.add_analysis_callback(callback2)
result = DbAnalysisResult("RESULT1", True, ["Q0"], exp_data.experiment_id)
exp_data.add_analysis_results(result)
def callback2(exp_data):
"""Callback function that exercises status lookups"""
time.sleep(1)
exp_data.add_analysis_callback(callback3)
raise RuntimeError("YOU FAIL")
def callback3(exp_data):
"""Callback function that exercises status lookups"""
time.sleep(1)
result = DbAnalysisResult("RESULT2", True, ["Q0"], exp_data.experiment_id)
exp_data.add_analysis_results(result)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_analysis_callback(callback1)
exp_data.block_for_results(timeout=10)
results = exp_data.analysis_results(block=False)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.ERROR)
self.assertTrue("RuntimeError: YOU FAIL" in exp_data.analysis_errors())
self.assertEqual(len(results), 2)
def test_block_for_jobs(self):
"""Test blocking for jobs."""
def _sleeper(*args, **kwargs): # pylint: disable=unused-argument
time.sleep(2)
nonlocal sleep_count
sleep_count += 1
return self._get_job_result(1)
sleep_count = 0
job = mock.create_autospec(Job, instance=True)
job.result = _sleeper
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_sleeper)
self.assertExperimentDone(exp_data)
self.assertEqual(2, sleep_count)
def test_status_post_processing_error(self):
"""Test experiment status when post processing failed."""
def _post_processing(*args, **kwargs):
raise ValueError("Kaboom!")
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data(job)
exp_data.add_analysis_callback(_post_processing)
exp_data.block_for_results()
self.assertEqual("ERROR", exp_data.status())
self.assertIn("Kaboom!", ",".join(cm.output))
def test_status_cancelled_analysis(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=2)))
# Add second callback because the first can't be cancelled once it has started
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=20)))
exp_data.cancel_analysis()
status = exp_data.status()
self.assertEqual(ExperimentStatus.CANCELLED, status)
def load(cls, experiment_id: str, service: DatabaseService) -> ExperimentData:
expdata = DbExperimentData.load(experiment_id, service)
expdata.__class__ = ExperimentData
expdata._experiment = None
child_data_ids = expdata.metadata.pop("child_data_ids", [])
child_data = [ExperimentData.load(child_id, service) for child_id in child_data_ids]
expdata._set_child_data(child_data)
return expdata
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_data(job)
exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
exp_data.block_for_results()
self.assertEqual("DONE", exp_data.status())
def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
event = threading.Event()
cancel_count = 0
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _job_cancel():
nonlocal cancel_count
cancel_count += 1
event.set()
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.cancel = _job_cancel
job.result = _job_result
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data.add_jobs(job)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.cancel_jobs()
self.assertEqual(cancel_count, 1)
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
