def test_add_data_callback(self): """Test add data with callback.""" def _callback(_exp_data): self.assertIsInstance(_exp_data, DbExperimentData) nonlocal called_back_count, expected_data, subtests expected_data.extend(subtests[called_back_count][1]) self.assertEqual([dat["counts"] for dat in _exp_data.data()], expected_data) called_back_count += 1 a_result = self._get_job_result(1) results = [self._get_job_result(1), self._get_job_result(1)] a_dict = {"counts": {"01": 518}} dicts = [{"counts": {"00": 284}}, {"counts": {"00": 14}}] subtests = [ (a_result, [a_result.get_counts()]), (results, [res.get_counts() for res in results]), (a_dict, [a_dict["counts"]]), (dicts, [dat["counts"] for dat in dicts]), ] called_back_count = 0 expected_data = [] exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") for data, _ in subtests: with self.subTest(data=data): exp_data.add_data(data) exp_data.add_analysis_callback(_callback) self.assertExperimentDone(exp_data) self.assertEqual(len(subtests), called_back_count)
def test_different_backend(self): """Test setting a different backend.""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") a_job = mock.create_autospec(Job, instance=True) self.assertNotEqual(exp_data.backend, a_job.backend()) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.add_jobs(a_job)
def test_set_tags(self): """Test updating experiment tags.""" exp_data = DbExperimentData(experiment_type="qiskit_test", tags=["foo"]) self.assertEqual(["foo"], exp_data.tags) exp_data.tags = ["bar"] self.assertEqual(["bar"], exp_data.tags)
def test_save_all(self): """Test saving all.""" exp_data = self._create_experiment_data() exp_data.tags = ["foo", "bar"] aresult = AnalysisResult( value={}, name="qiskit_test", device_components=self.device_components, experiment_id=exp_data.experiment_id, ) exp_data.add_analysis_results(aresult) hello_bytes = str.encode("hello world") exp_data.add_figures(hello_bytes, figure_names="hello.svg") exp_data.save() rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment) self.assertEqual(["foo", "bar"], rexp.tags) self.assertEqual(aresult.result_id, rexp.analysis_results(0).result_id) self.assertEqual(hello_bytes, rexp.figure(0)) exp_data.delete_analysis_result(0) exp_data.delete_figure(0) with mock.patch("builtins.input", lambda _: "y"): exp_data.save() rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment) self.assertRaises(IBMExperimentEntryNotFound, rexp.figure, "hello.svg") self.assertRaises(DbExperimentEntryNotFound, rexp.analysis_results, aresult.result_id)
def test_auto_save(self): """Test auto save.""" service = self._set_mock_service() exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.auto_save = True mock_result = mock.MagicMock() subtests = [ # update function, update parameters, service called (exp_data.add_analysis_results, (mock_result, ), mock_result.save), (exp_data.add_figures, (str.encode("hello world"), ), service.create_figure), (exp_data.delete_figure, (0, ), service.delete_figure), (exp_data.delete_analysis_result, (0, ), service.delete_analysis_result), (setattr, (exp_data, "tags", ["foo"]), service.update_experiment), (setattr, (exp_data, "notes", "foo"), service.update_experiment), (setattr, (exp_data, "share_level", "hub"), service.update_experiment), ] for func, params, called in subtests: with self.subTest(func=func): func(*params) if called: called.assert_called_once() service.reset_mock()
def test_status_job_pending(self): """Test experiment status when job is pending.""" job1 = mock.create_autospec(Job, instance=True) job1.result.return_value = self._get_job_result(3) job1.status.return_value = JobStatus.DONE event = threading.Event() job2 = mock.create_autospec(Job, instance=True) job2.result = lambda *args, **kwargs: event.wait(timeout=15) job2.status = lambda: JobStatus.CANCELLED if event.is_set( ) else JobStatus.RUNNING self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job1) exp_data.add_jobs(job2) exp_data.add_analysis_callback( lambda *args, **kwargs: event.wait(timeout=15)) self.assertEqual(ExperimentStatus.RUNNING, exp_data.status()) self.assertEqual(JobStatus.RUNNING, exp_data.job_status()) self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status()) # Cleanup with self.assertLogs("qiskit_experiments", "WARNING"): event.set() exp_data.block_for_results()
def test_cancel(self): """Test canceling experiment jobs and analysis.""" event = threading.Event() self.addCleanup(event.set) def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") def _analysis(*args): # pylint: disable = unused-argument event.wait(timeout=15) def _status(): if event.is_set(): return JobStatus.CANCELLED return JobStatus.RUNNING job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.result = _job_result job.cancel = event.set job.status = _status exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_analysis) exp_data.cancel() # Test status while job still running self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def _create_experiment_data(self): """Create an experiment data.""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.save() self.experiments_to_delete.append(exp_data.experiment_id) return exp_data
def test_set_service_job(self): """Test setting service with a job.""" exp_data = DbExperimentData(experiment_type="qiskit_test") job = self._run_circuit() exp_data.add_jobs(job) exp_data.save() rexp = self.experiment.experiment(exp_data.experiment_id) self.assertEqual([job.job_id()], rexp["job_ids"])
def test_set_service_job(self): """Test setting service via adding a job.""" mock_service = self._set_mock_service() job = mock.create_autospec(Job, instance=True) job.backend.return_value = self.backend exp_data = DbExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.service) exp_data.add_data(job) self.assertEqual(mock_service, exp_data.service)
def test_add_data_result_metadata(self): """Test add result metadata.""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") result1 = self._get_job_result(1, has_metadata=False) result2 = self._get_job_result(1, has_metadata=True) exp_data.add_data(result1) exp_data.add_data(result2) self.assertNotIn("metadata", exp_data.data(0)) self.assertIn("metadata", exp_data.data(1))
def test_set_service_direct(self): """Test setting service directly.""" exp_data = DbExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.service) mock_service = mock.MagicMock() exp_data.service = mock_service self.assertEqual(mock_service, exp_data.service) with self.assertRaises(DbExperimentDataError): exp_data.service = mock_service
def test_str(self): """Test the string representation.""" exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_data(self._get_job_result(1)) result = mock.MagicMock() exp_data.add_analysis_results(result) exp_data_str = str(exp_data) self.assertIn(exp_data.experiment_type, exp_data_str) self.assertIn(exp_data.experiment_id, exp_data_str) self.assertIn(str(result), exp_data_str)
def test_status_done(self): """Test experiment status when all jobs are done.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_jobs(job) self.assertExperimentDone(exp_data) self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_add_figure_save(self): """Test saving a figure in the database.""" hello_bytes = str.encode("hello world") service = self._set_mock_service() exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.add_figures(hello_bytes, save_figure=True) service.create_figure.assert_called_once() _, kwargs = service.create_figure.call_args self.assertEqual(kwargs["figure"], hello_bytes) self.assertEqual(kwargs["experiment_id"], exp_data.experiment_id)
def test_delayed_backend(self): """Test initializing experiment data without a backend.""" exp_data = DbExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.backend) self.assertIsNone(exp_data.service) exp_data.save_metadata() a_job = mock.create_autospec(Job, instance=True) exp_data.add_jobs(a_job) self.assertIsNotNone(exp_data.backend) self.assertIsNotNone(exp_data.service)
def test_add_get_analysis_results(self): """Test adding and getting a list of analysis results.""" exp_data = DbExperimentData(experiment_type="qiskit_test") results = [] for idx in range(5): res = mock.MagicMock() res.result_id = idx results.append(res) exp_data.add_analysis_results(results) self.assertEqual(results, exp_data.analysis_results())
def test_add_figure_plot(self): """Test adding a matplotlib figure.""" figure, ax = plt.subplots() ax.plot([1, 2, 3]) service = self._set_mock_service() exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.add_figures(figure, save_figure=True) self.assertEqual(figure, exp_data.figure(0)) service.create_figure.assert_called_once() _, kwargs = service.create_figure.call_args self.assertIsInstance(kwargs["figure"], bytes)
def test_new_backend_has_service(self): """Test changing backend doesn't change existing service.""" orig_service = self._set_mock_service() exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") self.assertEqual(orig_service, exp_data.service) job = mock.create_autospec(Job, instance=True) new_service = self._set_mock_service() self.assertNotEqual(orig_service, new_service) job.backend.return_value = self.backend job.status.return_value = JobStatus.DONE exp_data.add_jobs(job) self.assertEqual(orig_service, exp_data.service)
def test_delete_figure(self): """Test deleting a figure.""" exp_data = DbExperimentData(experiment_type="qiskit_test") id_template = "figure_{}.svg" for idx in range(3): exp_data.add_figures(str.encode("hello world"), id_template.format(idx)) sub_tests = [(1, id_template.format(1)), (id_template.format(2), id_template.format(2))] for del_key, figure_name in sub_tests: with self.subTest(del_key=del_key): exp_data.delete_figure(del_key) self.assertRaises(DbExperimentEntryNotFound, exp_data.figure, figure_name)
def test_delete_analysis_result(self): """Test deleting analysis result.""" exp_data = DbExperimentData(experiment_type="qiskit_test") id_template = "result_{}" for idx in range(3): res = mock.MagicMock() res.result_id = id_template.format(idx) exp_data.add_analysis_results(res) subtests = [(0, id_template.format(0)), (id_template.format(2), id_template.format(2))] for del_key, res_id in subtests: with self.subTest(del_key=del_key): exp_data.delete_analysis_result(del_key) self.assertRaises(DbExperimentEntryNotFound, exp_data.analysis_results, res_id)
def test_copy_metadata_pending_job(self): """Test copy metadata with a pending job.""" event = threading.Event() self.addCleanup(event.set) job_results1 = self._get_job_result(1) job_results2 = self._get_job_result(1) def _job1_result(): event.wait(timeout=15) return job_results1 def _job2_result(): event.wait(timeout=15) return job_results2 exp_data = DbExperimentData(experiment_type="qiskit_test") job = mock.create_autospec(Job, instance=True) job.result = _job1_result exp_data.add_jobs(job) copied = exp_data.copy(copy_results=False) job2 = mock.create_autospec(Job, instance=True) job2.result = _job2_result copied.add_jobs(job2) event.set() exp_data.block_for_results() copied.block_for_results() self.assertEqual(1, len(exp_data.data())) self.assertEqual(2, len(copied.data())) self.assertIn( exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]] )
def test_status_job_error(self): """Test experiment status when job failed.""" job1 = mock.create_autospec(Job, instance=True) job1.result.return_value = self._get_job_result(3) job1.status.return_value = JobStatus.DONE job2 = mock.create_autospec(Job, instance=True) job2.status.return_value = JobStatus.ERROR exp_data = DbExperimentData(experiment_type="qiskit_test") with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm: exp_data.add_jobs([job1, job2]) self.assertIn("Adding a job from a backend", ",".join(cm.output)) self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
def test_recursive_callback_raises(self): """Test handling of excepting callbacks""" def callback1(exp_data): """Callback function that call add_analysis_callback""" time.sleep(1) exp_data.add_analysis_callback(callback2) result = DbAnalysisResult("RESULT1", True, ["Q0"], exp_data.experiment_id) exp_data.add_analysis_results(result) def callback2(exp_data): """Callback function that exercises status lookups""" time.sleep(1) exp_data.add_analysis_callback(callback3) raise RuntimeError("YOU FAIL") def callback3(exp_data): """Callback function that exercises status lookups""" time.sleep(1) result = DbAnalysisResult("RESULT2", True, ["Q0"], exp_data.experiment_id) exp_data.add_analysis_results(result) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_analysis_callback(callback1) exp_data.block_for_results(timeout=10) results = exp_data.analysis_results(block=False) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.ERROR) self.assertTrue("RuntimeError: YOU FAIL" in exp_data.analysis_errors()) self.assertEqual(len(results), 2)
def test_block_for_jobs(self): """Test blocking for jobs.""" def _sleeper(*args, **kwargs): # pylint: disable=unused-argument time.sleep(2) nonlocal sleep_count sleep_count += 1 return self._get_job_result(1) sleep_count = 0 job = mock.create_autospec(Job, instance=True) job.result = _sleeper exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_sleeper) self.assertExperimentDone(exp_data) self.assertEqual(2, sleep_count)
def test_status_post_processing_error(self): """Test experiment status when post processing failed.""" def _post_processing(*args, **kwargs): raise ValueError("Kaboom!") job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_data(job) with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm: exp_data.add_data(job) exp_data.add_analysis_callback(_post_processing) exp_data.block_for_results() self.assertEqual("ERROR", exp_data.status()) self.assertIn("Kaboom!", ",".join(cm.output))
def test_status_cancelled_analysis(self): """Test experiment status during post processing.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE event = threading.Event() self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=2))) # Add second callback because the first can't be cancelled once it has started exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=20))) exp_data.cancel_analysis() status = exp_data.status() self.assertEqual(ExperimentStatus.CANCELLED, status)
def load(cls, experiment_id: str, service: DatabaseService) -> ExperimentData: expdata = DbExperimentData.load(experiment_id, service) expdata.__class__ = ExperimentData expdata._experiment = None child_data_ids = expdata.metadata.pop("child_data_ids", []) child_data = [ExperimentData.load(child_id, service) for child_id in child_data_ids] expdata._set_child_data(child_data) return expdata
def test_status_done(self): """Test experiment status when all jobs are done.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_data(job) exp_data.add_data(job) exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1)) exp_data.block_for_results() self.assertEqual("DONE", exp_data.status())
def test_cancel_jobs(self): """Test canceling experiment jobs.""" event = threading.Event() cancel_count = 0 def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") def _job_cancel(): nonlocal cancel_count cancel_count += 1 event.set() exp_data = DbExperimentData(experiment_type="qiskit_test") event = threading.Event() self.addCleanup(event.set) job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.cancel = _job_cancel job.result = _job_result job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING exp_data.add_jobs(job) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.cancel_jobs() self.assertEqual(cancel_count, 1) self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)