def test_copy_metadata_pending_job(self):
"""Test copy metadata with a pending job."""
def _job1_result():
event.wait(timeout=15)
return job_results[0]
def _job2_result():
event.wait(timeout=15)
return job_results[1]
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job_results = [self._get_job_result(1), self._get_job_result(1)]
job = mock.create_autospec(Job, instance=True)
job.result = _job1_result
exp_data.add_data(job)
copied = exp_data._copy_metadata()
job2 = mock.create_autospec(Job, instance=True)
job2.result = _job2_result
copied.add_data(job2)
event.set()
exp_data.block_for_results()
copied.block_for_results()
self.assertEqual(1, len(exp_data.data()))
self.assertEqual(2, len(copied.data()))
self.assertIn(
exp_data.data(0)["counts"],
[copied.data(0)["counts"],
copied.data(1)["counts"]])
def test_add_data_callback(self):
"""Test add data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
nonlocal called_back_count, expected_data, subtests
expected_data.extend(subtests[called_back_count][1])
self.assertEqual([dat["counts"] for dat in _exp_data.data()], expected_data)
called_back_count += 1
a_result = self._get_job_result(1)
results = [self._get_job_result(1), self._get_job_result(1)]
a_dict = {"counts": {"01": 518}}
dicts = [{"counts": {"00": 284}}, {"counts": {"00": 14}}]
subtests = [
(a_result, [a_result.get_counts()]),
(results, [res.get_counts() for res in results]),
(a_dict, [a_dict["counts"]]),
(dicts, [dat["counts"] for dat in dicts]),
]
called_back_count = 0
expected_data = []
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
for data, _ in subtests:
with self.subTest(data=data):
exp_data.add_data(data)
exp_data.add_analysis_callback(_callback)
self.assertExperimentDone(exp_data)
self.assertEqual(len(subtests), called_back_count)
def test_new_experiment_data(self):
"""Test creating a new experiment data."""
metadata = {"complex": 2 + 3j, "numpy": np.zeros(2)}
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test",
tags=["foo", "bar"],
share_level="hub",
metadata=metadata,
notes="some notes")
job_ids = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_data(job)
job_ids.append(job.job_id())
exp_data.save()
self.experiments_to_delete.append(exp_data.experiment_id)
credentials = self.backend.provider().credentials
rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment)
self._verify_experiment_data(exp_data, rexp)
self.assertEqual(credentials.hub, rexp.hub) # pylint: disable=no-member
self.assertEqual(credentials.group, rexp.group) # pylint: disable=no-member
self.assertEqual(credentials.project, rexp.project) # pylint: disable=no-member
def test_different_backend(self):
"""Test setting a different backend."""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
a_job = mock.create_autospec(Job, instance=True)
self.assertNotEqual(exp_data.backend, a_job.backend())
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.add_data(a_job)
def test_set_service_job(self):
"""Test setting service via adding a job."""
mock_service = self._set_mock_service()
job = mock.create_autospec(Job, instance=True)
job.backend.return_value = self.backend
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.service)
exp_data.add_data(job)
self.assertEqual(mock_service, exp_data.service)
def test_add_data_dict(self):
"""Test add data in dictionary."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
a_dict = {"counts": {"01": 518}}
dicts = [{"counts": {"00": 284}}, {"counts": {"00": 14}}]
exp_data.add_data(a_dict)
exp_data.add_data(dicts)
self.assertEqual([a_dict] + dicts, exp_data.data())
def test_set_service_job(self):
"""Test setting service with a job."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = self._run_circuit()
exp_data.add_data(job)
exp_data.save()
rexp = self.experiment.experiment(exp_data.experiment_id)
self.assertEqual([job.job_id()], rexp["job_ids"])
def test_copy_metadata(self):
"""Test copy metadata."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(self._get_job_result(1))
result = mock.MagicMock()
exp_data.add_analysis_results(result)
copied = exp_data.copy(copy_results=False)
self.assertEqual(exp_data.data(), copied.data())
self.assertFalse(copied.analysis_results())
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_data(job)
exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
exp_data.block_for_results()
self.assertEqual("DONE", exp_data.status())
def test_str(self):
"""Test the string representation."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(self._get_job_result(1))
result = mock.MagicMock()
exp_data.add_analysis_results(result)
exp_data_str = str(exp_data)
self.assertIn(exp_data.experiment_type, exp_data_str)
self.assertIn(exp_data.experiment_id, exp_data_str)
self.assertIn(str(result), exp_data_str)
def test_delayed_backend(self):
"""Test initializing experiment data without a backend."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.backend)
self.assertIsNone(exp_data.service)
exp_data.save_metadata()
a_job = mock.create_autospec(Job, instance=True)
exp_data.add_data(a_job)
self.assertIsNotNone(exp_data.backend)
self.assertIsNotNone(exp_data.service)
def test_add_data_result_metadata(self):
"""Test add result metadata."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
result1 = self._get_job_result(1, has_metadata=False)
result2 = self._get_job_result(1, has_metadata=True)
exp_data.add_data(result1)
exp_data.add_data(result2)
self.assertNotIn("metadata", exp_data.data(0))
self.assertIn("metadata", exp_data.data(1))
def test_block_for_results(self):
"""Test blocking for jobs"""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
jobs = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_data(job)
jobs.append(job)
exp_data.block_for_results()
self.assertTrue(all(job.status() == JobStatus.DONE for job in jobs))
self.assertEqual("DONE", exp_data.status())
def test_new_backend_has_service(self):
"""Test changing backend doesn't change existing service."""
orig_service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
self.assertEqual(orig_service, exp_data.service)
job = mock.create_autospec(Job, instance=True)
new_service = self._set_mock_service()
self.assertNotEqual(orig_service, new_service)
job.backend.return_value = self.backend
exp_data.add_data(job)
self.assertEqual(orig_service, exp_data.service)
def test_status_post_processing(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_analysis_callback(
(lambda *args, **kwargs: event.wait(timeout=15)))
status = exp_data.status()
self.assertEqual("POST_PROCESSING", status)
def test_add_data_result(self):
"""Test add result data."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
a_result = self._get_job_result(1)
results = [self._get_job_result(2), self._get_job_result(3)]
expected = [a_result.get_counts()]
for res in results:
expected.extend(res.get_counts())
exp_data.add_data(a_result)
exp_data.add_data(results)
self.assertEqual(expected, [sdata["counts"] for sdata in exp_data.data()])
self.assertIn(a_result.job_id, exp_data.job_ids)
def test_add_data_job(self):
"""Test add job to experiment data."""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
transpiled = transpile(ReferenceCircuits.bell(), self.backend)
transpiled.metadata = {"foo": "bar"}
job = self._run_circuit(transpiled)
exp_data.add_data(job)
self.assertEqual([job.job_id()], exp_data.job_ids)
result = job.result()
exp_data.block_for_results()
circuit_data = exp_data.data(0)
self.assertEqual(result.get_counts(0), circuit_data['counts'])
self.assertEqual(job.job_id(), circuit_data['job_id'])
self.assertEqual(transpiled.metadata, circuit_data['metadata'])
def test_get_data(self):
"""Test getting data."""
data1 = []
for _ in range(5):
data1.append({"counts": {"00": randrange(1024)}})
results = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(data1)
exp_data.add_data(results)
self.assertEqual(data1[1], exp_data.data(1))
self.assertEqual(data1[2:4], exp_data.data(slice(2, 4)))
self.assertEqual(
results.get_counts(), [sdata["counts"] for sdata in exp_data.data(results.job_id)]
)
def test_status_job_error(self):
"""Test experiment status when job failed."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data([job1, job2])
self.assertIn("Adding a job from a backend", ",".join(cm.output))
self.assertEqual("ERROR", exp_data.status())
def test_add_data_pending_post_processing(self):
"""Test add job data while post processing is still running."""
def _callback(_exp_data, **kwargs):
kwargs["event"].wait(timeout=3)
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_analysis_callback(_callback, event=event)
exp_data.add_data(a_job)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.add_data({"foo": "bar"})
def test_block_for_jobs(self):
"""Test blocking for jobs."""
def _sleeper(*args, **kwargs): # pylint: disable=unused-argument
time.sleep(2)
nonlocal sleep_count
sleep_count += 1
return self._get_job_result(1)
sleep_count = 0
job = mock.create_autospec(Job, instance=True)
job.result = _sleeper
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
exp_data.add_analysis_callback(_sleeper)
exp_data.block_for_results()
self.assertEqual(2, sleep_count)
def test_status_post_processing_error(self):
"""Test experiment status when post processing failed."""
def _post_processing(*args, **kwargs):
raise ValueError("Kaboom!")
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job)
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data(job)
exp_data.add_analysis_callback(_post_processing)
exp_data.block_for_results()
self.assertEqual("ERROR", exp_data.status())
self.assertIn("Kaboom!", ",".join(cm.output))
def test_add_data_job_callback_kwargs(self):
"""Test add job data with callback and additional arguments."""
def _callback(_exp_data, **kwargs):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual({"foo": callback_kwargs}, kwargs)
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
called_back = False
callback_kwargs = "foo"
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.add_analysis_callback(_callback, foo=callback_kwargs)
exp_data.block_for_results()
self.assertTrue(called_back)
def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.result = _job_result
exp_data.add_data(job)
exp_data.cancel_jobs()
job.cancel.assert_called_once()
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_add_data_job_callback(self):
"""Test add job data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual([dat["counts"] for dat in _exp_data.data()],
a_job.result().get_counts())
exp_data.add_figures(str.encode("hello world"))
exp_data.add_analysis_results(mock.MagicMock())
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
called_back = False
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.add_analysis_callback(_callback)
exp_data.block_for_results()
self.assertTrue(called_back)
def test_add_data_job(self):
"""Test add job data."""
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(3)
jobs = []
for _ in range(2):
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(2)
jobs.append(job)
expected = a_job.result().get_counts()
for job in jobs:
expected.extend(job.result().get_counts())
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_data(a_job)
exp_data.block_for_results()
exp_data.add_data(jobs)
exp_data.block_for_results()
self.assertEqual(expected,
[sdata["counts"] for sdata in exp_data.data()])
self.assertIn(a_job.job_id(), exp_data.job_ids)
def test_status_job_pending(self):
"""Test experiment status when job is pending."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
event = threading.Event()
job2 = mock.create_autospec(Job, instance=True)
job2.result = lambda *args, **kwargs: event.wait(timeout=15)
job2.status.return_value = JobStatus.RUNNING
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_data(job1)
exp_data.add_data(job2)
exp_data.add_analysis_callback(
lambda *args, **kwargs: event.wait(timeout=15))
self.assertEqual("RUNNING", exp_data.status())
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_errors(self):
"""Test getting experiment error message."""
def _post_processing(*args, **kwargs): # pylint: disable=unused-argument
raise ValueError("Kaboom!")
job1 = mock.create_autospec(Job, instance=True)
job1.job_id.return_value = "1234"
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
job2.job_id.return_value = "5678"
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service",
level="WARN") as cm:
exp_data.add_data(job1)
exp_data.add_analysis_callback(_post_processing)
exp_data.add_data(job2)
exp_data.block_for_results()
self.assertEqual("ERROR", exp_data.status())
self.assertIn("Kaboom", ",".join(cm.output))
self.assertTrue(
re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL))
