def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
event = threading.Event()
cancel_count = 0
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _job_cancel():
nonlocal cancel_count
cancel_count += 1
event.set()
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.cancel = _job_cancel
job.result = _job_result
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data.add_jobs(job)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.cancel_jobs()
self.assertEqual(cancel_count, 1)
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_new_experiment_data(self):
"""Test creating a new experiment data."""
metadata = {"complex": 2 + 3j, "numpy": np.zeros(2)}
exp_data = DbExperimentData(
backend=self.backend,
experiment_type="qiskit_test",
tags=["foo", "bar"],
share_level="hub",
metadata=metadata,
notes="some notes",
)
job_ids = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_jobs(job)
job_ids.append(job.job_id())
exp_data.save()
self.experiments_to_delete.append(exp_data.experiment_id)
credentials = self.backend.provider().credentials
rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment)
self._verify_experiment_data(exp_data, rexp)
self.assertEqual(credentials.hub, rexp.hub) # pylint: disable=no-member
self.assertEqual(credentials.group, rexp.group) # pylint: disable=no-member
self.assertEqual(credentials.project, rexp.project) # pylint: disable=no-member
def test_cancel_analysis(self):
"""Test canceling experiment analysis."""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
return self._get_job_result(1)
def _analysis(*args): # pylint: disable = unused-argument
event.wait(timeout=15)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis)
exp_data.cancel_analysis()
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.RUNNING)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING)
# Test status after job finishes
event.set()
self.assertEqual(exp_data.job_status(), JobStatus.DONE)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_cancel(self):
"""Test canceling experiment jobs and analysis."""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
def _analysis(*args): # pylint: disable = unused-argument
event.wait(timeout=15)
def _status():
if event.is_set():
return JobStatus.CANCELLED
return JobStatus.RUNNING
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = _status
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis)
exp_data.cancel()
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_status_job_pending(self):
"""Test experiment status when job is pending."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
event = threading.Event()
job2 = mock.create_autospec(Job, instance=True)
job2.result = lambda *args, **kwargs: event.wait(timeout=15)
job2.status = lambda: JobStatus.CANCELLED if event.is_set(
) else JobStatus.RUNNING
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job1)
exp_data.add_jobs(job2)
exp_data.add_analysis_callback(
lambda *args, **kwargs: event.wait(timeout=15))
self.assertEqual(ExperimentStatus.RUNNING, exp_data.status())
self.assertEqual(JobStatus.RUNNING, exp_data.job_status())
self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status())
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_different_backend(self):
"""Test setting a different backend."""
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
a_job = mock.create_autospec(Job, instance=True)
self.assertNotEqual(exp_data.backend, a_job.backend())
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.add_jobs(a_job)
def test_copy_metadata_pending_job(self):
"""Test copy metadata with a pending job."""
event = threading.Event()
self.addCleanup(event.set)
job_results1 = self._get_job_result(1)
job_results2 = self._get_job_result(1)
def _job1_result():
event.wait(timeout=15)
return job_results1
def _job2_result():
event.wait(timeout=15)
return job_results2
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = mock.create_autospec(Job, instance=True)
job.result = _job1_result
exp_data.add_jobs(job)
copied = exp_data.copy(copy_results=False)
job2 = mock.create_autospec(Job, instance=True)
job2.result = _job2_result
copied.add_jobs(job2)
event.set()
exp_data.block_for_results()
copied.block_for_results()
self.assertEqual(1, len(exp_data.data()))
self.assertEqual(2, len(copied.data()))
self.assertIn(
exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]]
)
def test_set_service_job(self):
"""Test setting service with a job."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
job = self._run_circuit()
exp_data.add_jobs(job)
exp_data.save()
rexp = self.experiment.experiment(exp_data.experiment_id)
self.assertEqual([job.job_id()], rexp["job_ids"])
def test_set_service_job(self):
"""Test setting service via adding a job."""
mock_service = self._set_mock_service()
job = mock.create_autospec(Job, instance=True)
job.backend.return_value = self.backend
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.service)
exp_data.add_jobs(job)
self.assertEqual(mock_service, exp_data.service)
def test_delayed_backend(self):
"""Test initializing experiment data without a backend."""
exp_data = DbExperimentData(experiment_type="qiskit_test")
self.assertIsNone(exp_data.backend)
self.assertIsNone(exp_data.service)
exp_data.save_metadata()
a_job = mock.create_autospec(Job, instance=True)
exp_data.add_jobs(a_job)
self.assertIsNotNone(exp_data.backend)
self.assertIsNotNone(exp_data.service)
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_jobs(job)
self.assertExperimentDone(exp_data)
self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_status_done(self):
"""Test experiment status when all jobs are done."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_jobs(job)
exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
exp_data.block_for_results()
self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_block_for_results(self):
"""Test blocking for jobs"""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
jobs = []
for _ in range(2):
job = self._run_circuit()
exp_data.add_jobs(job)
jobs.append(job)
exp_data.block_for_results()
self.assertTrue(all(job.status() == JobStatus.DONE for job in jobs))
self.assertEqual("DONE", exp_data.status())
def test_new_backend_has_service(self):
"""Test changing backend doesn't change existing service."""
orig_service = self._set_mock_service()
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
self.assertEqual(orig_service, exp_data.service)
job = mock.create_autospec(Job, instance=True)
new_service = self._set_mock_service()
self.assertNotEqual(orig_service, new_service)
job.backend.return_value = self.backend
job.status.return_value = JobStatus.DONE
exp_data.add_jobs(job)
self.assertEqual(orig_service, exp_data.service)
def test_status_post_processing(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=15)))
status = exp_data.status()
self.assertEqual(ExperimentStatus.POST_PROCESSING, status)
def test_status_job_error(self):
"""Test experiment status when job failed."""
job1 = mock.create_autospec(Job, instance=True)
job1.result.return_value = self._get_job_result(3)
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs([job1, job2])
self.assertIn("Adding a job from a backend", ",".join(cm.output))
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
def test_add_data_job(self):
"""Test add job to experiment data."""
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
transpiled = transpile(ReferenceCircuits.bell(), self.backend)
transpiled.metadata = {"foo": "bar"}
job = self._run_circuit(transpiled)
exp_data.add_jobs(job)
self.assertEqual([job.job_id()], exp_data.job_ids)
result = job.result()
exp_data.block_for_results()
circuit_data = exp_data.data(0)
self.assertEqual(result.get_counts(0), circuit_data["counts"])
self.assertEqual(job.job_id(), circuit_data["job_id"])
self.assertEqual(transpiled.metadata, circuit_data["metadata"])
def test_block_for_jobs(self):
"""Test blocking for jobs."""
def _sleeper(*args, **kwargs): # pylint: disable=unused-argument
time.sleep(2)
nonlocal sleep_count
sleep_count += 1
return self._get_job_result(1)
sleep_count = 0
job = mock.create_autospec(Job, instance=True)
job.result = _sleeper
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_sleeper)
exp_data.block_for_results()
self.assertEqual(2, sleep_count)
def test_partial_cancel_analysis(self):
"""Test canceling experiment analysis."""
event = threading.Event()
self.addCleanup(event.set)
run_analysis = []
def _job_result():
event.wait(timeout=3)
return self._get_job_result(1)
def _analysis(expdata, name=None, timeout=0): # pylint: disable = unused-argument
event.wait(timeout=timeout)
run_analysis.append(name)
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_analysis, name=1, timeout=1)
exp_data.add_analysis_callback(_analysis, name=2, timeout=30)
cancel_id = exp_data._analysis_callbacks.keys()[-1]
exp_data.add_analysis_callback(_analysis, name=3, timeout=1)
exp_data.cancel_analysis(cancel_id)
# Test status while job still running
self.assertEqual(exp_data.job_status(), JobStatus.RUNNING)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING)
# Test status after job finishes
event.set()
self.assertEqual(exp_data.job_status(), JobStatus.DONE)
self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
# Check that correct analysis callback was cancelled
exp_data.block_for_results()
self.assertEqual(run_analysis, [1, 3])
for cid, analysis in exp_data._analysis_callbacks.items():
if cid == cancel_id:
self.assertEqual(analysis.status, AnalysisStatus.CANCELLED)
else:
self.assertEqual(analysis.status, AnalysisStatus.DONE)
def test_add_data_pending_post_processing(self):
"""Test add job data while post processing is still running."""
def _callback(_exp_data, **kwargs):
kwargs["event"].wait(timeout=3)
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
a_job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_analysis_callback(_callback, event=event)
exp_data.add_jobs(a_job)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.add_data({"foo": "bar"})
def test_status_cancelled_analysis(self):
"""Test experiment status during post processing."""
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
event = threading.Event()
self.addCleanup(event.set)
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=2)))
# Add second callback because the first can't be cancelled once it has started
exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=20)))
exp_data.cancel_analysis()
status = exp_data.status()
self.assertEqual(ExperimentStatus.CANCELLED, status)
def test_status_post_processing_error(self):
"""Test experiment status when post processing failed."""
def _post_processing(*args, **kwargs):
raise ValueError("Kaboom!")
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(3)
job.status.return_value = JobStatus.DONE
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job)
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs(job)
exp_data.add_analysis_callback(_post_processing)
exp_data.block_for_results()
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
self.assertIn("Kaboom!", ",".join(cm.output))
def test_cancel_jobs(self):
"""Test canceling experiment jobs."""
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
exp_data = DbExperimentData(experiment_type="qiskit_test")
event = threading.Event()
self.addCleanup(event.set)
job = mock.create_autospec(Job, instance=True)
job.result = _job_result
exp_data.add_jobs(job)
exp_data.cancel_jobs()
job.cancel.assert_called_once()
# Cleanup
with self.assertLogs("qiskit_experiments", "WARNING"):
event.set()
exp_data.block_for_results()
def test_add_data_job_callback_kwargs(self):
"""Test add job data with callback and additional arguments."""
def _callback(_exp_data, **kwargs):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual({"foo": callback_kwargs}, kwargs)
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
a_job.status.return_value = JobStatus.DONE
called_back = False
callback_kwargs = "foo"
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
exp_data.add_jobs(a_job)
exp_data.add_analysis_callback(_callback, foo=callback_kwargs)
self.assertExperimentDone(exp_data)
self.assertTrue(called_back)
def test_add_data_job_callback(self):
"""Test add job data with callback."""
def _callback(_exp_data):
self.assertIsInstance(_exp_data, DbExperimentData)
self.assertEqual([dat["counts"] for dat in _exp_data.data()],
a_job.result().get_counts())
exp_data.add_figures(str.encode("hello world"))
exp_data.add_analysis_results(mock.MagicMock())
nonlocal called_back
called_back = True
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(2)
a_job.status.return_value = JobStatus.DONE
called_back = False
exp_data = DbExperimentData(backend=self.backend,
experiment_type="qiskit_test")
exp_data.add_jobs(a_job)
exp_data.add_analysis_callback(_callback)
exp_data.block_for_results()
self.assertTrue(called_back)
def test_add_data_job(self):
"""Test add job data."""
a_job = mock.create_autospec(Job, instance=True)
a_job.result.return_value = self._get_job_result(3)
jobs = []
for _ in range(2):
job = mock.create_autospec(Job, instance=True)
job.result.return_value = self._get_job_result(2)
job.status.return_value = JobStatus.DONE
jobs.append(job)
expected = a_job.result().get_counts()
for job in jobs:
expected.extend(job.result().get_counts())
exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
exp_data.add_jobs(a_job)
self.assertExperimentDone(exp_data)
exp_data.add_jobs(jobs)
self.assertExperimentDone(exp_data)
self.assertEqual(expected, [sdata["counts"] for sdata in exp_data.data()])
self.assertIn(a_job.job_id(), exp_data.job_ids)
def test_errors(self):
"""Test getting experiment error message."""
def _post_processing(*args, **kwargs): # pylint: disable=unused-argument
raise ValueError("Kaboom!")
job1 = mock.create_autospec(Job, instance=True)
job1.job_id.return_value = "1234"
job1.status.return_value = JobStatus.DONE
job2 = mock.create_autospec(Job, instance=True)
job2.status.return_value = JobStatus.ERROR
job2.job_id.return_value = "5678"
exp_data = DbExperimentData(experiment_type="qiskit_test")
with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm:
exp_data.add_jobs(job1)
exp_data.add_analysis_callback(_post_processing)
exp_data.add_jobs(job2)
exp_data.block_for_results()
self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
self.assertIn("Kaboom", ",".join(cm.output))
self.assertTrue(re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL))
def test_add_jobs_timeout(self):
"""Test timeout kwarg of add_jobs"""
event = threading.Event()
self.addCleanup(event.set)
def _job_result():
event.wait(timeout=15)
raise ValueError("Job was cancelled.")
job = mock.create_autospec(Job, instance=True)
job.job_id.return_value = "1234"
job.result = _job_result
job.cancel = event.set
job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING
exp_data = DbExperimentData(experiment_type="qiskit_test")
exp_data.add_jobs(job, timeout=0.5)
with self.assertLogs("qiskit_experiments", "WARNING"):
exp_data.block_for_results()
self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
