def test_cancel_jobs(self): """Test canceling experiment jobs.""" event = threading.Event() cancel_count = 0 def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") def _job_cancel(): nonlocal cancel_count cancel_count += 1 event.set() exp_data = DbExperimentData(experiment_type="qiskit_test") event = threading.Event() self.addCleanup(event.set) job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.cancel = _job_cancel job.result = _job_result job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING exp_data.add_jobs(job) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.cancel_jobs() self.assertEqual(cancel_count, 1) self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_new_experiment_data(self): """Test creating a new experiment data.""" metadata = {"complex": 2 + 3j, "numpy": np.zeros(2)} exp_data = DbExperimentData( backend=self.backend, experiment_type="qiskit_test", tags=["foo", "bar"], share_level="hub", metadata=metadata, notes="some notes", ) job_ids = [] for _ in range(2): job = self._run_circuit() exp_data.add_jobs(job) job_ids.append(job.job_id()) exp_data.save() self.experiments_to_delete.append(exp_data.experiment_id) credentials = self.backend.provider().credentials rexp = DbExperimentData.load(exp_data.experiment_id, self.experiment) self._verify_experiment_data(exp_data, rexp) self.assertEqual(credentials.hub, rexp.hub) # pylint: disable=no-member self.assertEqual(credentials.group, rexp.group) # pylint: disable=no-member self.assertEqual(credentials.project, rexp.project) # pylint: disable=no-member
def test_cancel_analysis(self): """Test canceling experiment analysis.""" event = threading.Event() self.addCleanup(event.set) def _job_result(): event.wait(timeout=15) return self._get_job_result(1) def _analysis(*args): # pylint: disable = unused-argument event.wait(timeout=15) job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.result = _job_result job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_analysis) exp_data.cancel_analysis() # Test status while job still running self.assertEqual(exp_data.job_status(), JobStatus.RUNNING) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING) # Test status after job finishes event.set() self.assertEqual(exp_data.job_status(), JobStatus.DONE) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_cancel(self): """Test canceling experiment jobs and analysis.""" event = threading.Event() self.addCleanup(event.set) def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") def _analysis(*args): # pylint: disable = unused-argument event.wait(timeout=15) def _status(): if event.is_set(): return JobStatus.CANCELLED return JobStatus.RUNNING job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.result = _job_result job.cancel = event.set job.status = _status exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_analysis) exp_data.cancel() # Test status while job still running self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
def test_status_job_pending(self): """Test experiment status when job is pending.""" job1 = mock.create_autospec(Job, instance=True) job1.result.return_value = self._get_job_result(3) job1.status.return_value = JobStatus.DONE event = threading.Event() job2 = mock.create_autospec(Job, instance=True) job2.result = lambda *args, **kwargs: event.wait(timeout=15) job2.status = lambda: JobStatus.CANCELLED if event.is_set( ) else JobStatus.RUNNING self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job1) exp_data.add_jobs(job2) exp_data.add_analysis_callback( lambda *args, **kwargs: event.wait(timeout=15)) self.assertEqual(ExperimentStatus.RUNNING, exp_data.status()) self.assertEqual(JobStatus.RUNNING, exp_data.job_status()) self.assertEqual(AnalysisStatus.QUEUED, exp_data.analysis_status()) # Cleanup with self.assertLogs("qiskit_experiments", "WARNING"): event.set() exp_data.block_for_results()
def test_different_backend(self): """Test setting a different backend.""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") a_job = mock.create_autospec(Job, instance=True) self.assertNotEqual(exp_data.backend, a_job.backend()) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.add_jobs(a_job)
def test_copy_metadata_pending_job(self): """Test copy metadata with a pending job.""" event = threading.Event() self.addCleanup(event.set) job_results1 = self._get_job_result(1) job_results2 = self._get_job_result(1) def _job1_result(): event.wait(timeout=15) return job_results1 def _job2_result(): event.wait(timeout=15) return job_results2 exp_data = DbExperimentData(experiment_type="qiskit_test") job = mock.create_autospec(Job, instance=True) job.result = _job1_result exp_data.add_jobs(job) copied = exp_data.copy(copy_results=False) job2 = mock.create_autospec(Job, instance=True) job2.result = _job2_result copied.add_jobs(job2) event.set() exp_data.block_for_results() copied.block_for_results() self.assertEqual(1, len(exp_data.data())) self.assertEqual(2, len(copied.data())) self.assertIn( exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]] )
def test_set_service_job(self): """Test setting service with a job.""" exp_data = DbExperimentData(experiment_type="qiskit_test") job = self._run_circuit() exp_data.add_jobs(job) exp_data.save() rexp = self.experiment.experiment(exp_data.experiment_id) self.assertEqual([job.job_id()], rexp["job_ids"])
def test_set_service_job(self): """Test setting service via adding a job.""" mock_service = self._set_mock_service() job = mock.create_autospec(Job, instance=True) job.backend.return_value = self.backend job.status.return_value = JobStatus.DONE exp_data = DbExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.service) exp_data.add_jobs(job) self.assertEqual(mock_service, exp_data.service)
def test_delayed_backend(self): """Test initializing experiment data without a backend.""" exp_data = DbExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.backend) self.assertIsNone(exp_data.service) exp_data.save_metadata() a_job = mock.create_autospec(Job, instance=True) exp_data.add_jobs(a_job) self.assertIsNotNone(exp_data.backend) self.assertIsNotNone(exp_data.service)
def test_status_done(self): """Test experiment status when all jobs are done.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_jobs(job) self.assertExperimentDone(exp_data) self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_status_done(self): """Test experiment status when all jobs are done.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_jobs(job) exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1)) exp_data.block_for_results() self.assertEqual(ExperimentStatus.DONE, exp_data.status())
def test_block_for_results(self): """Test blocking for jobs""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") jobs = [] for _ in range(2): job = self._run_circuit() exp_data.add_jobs(job) jobs.append(job) exp_data.block_for_results() self.assertTrue(all(job.status() == JobStatus.DONE for job in jobs)) self.assertEqual("DONE", exp_data.status())
def test_new_backend_has_service(self): """Test changing backend doesn't change existing service.""" orig_service = self._set_mock_service() exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") self.assertEqual(orig_service, exp_data.service) job = mock.create_autospec(Job, instance=True) new_service = self._set_mock_service() self.assertNotEqual(orig_service, new_service) job.backend.return_value = self.backend job.status.return_value = JobStatus.DONE exp_data.add_jobs(job) self.assertEqual(orig_service, exp_data.service)
def test_status_post_processing(self): """Test experiment status during post processing.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE event = threading.Event() self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=15))) status = exp_data.status() self.assertEqual(ExperimentStatus.POST_PROCESSING, status)
def test_status_job_error(self): """Test experiment status when job failed.""" job1 = mock.create_autospec(Job, instance=True) job1.result.return_value = self._get_job_result(3) job1.status.return_value = JobStatus.DONE job2 = mock.create_autospec(Job, instance=True) job2.status.return_value = JobStatus.ERROR exp_data = DbExperimentData(experiment_type="qiskit_test") with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm: exp_data.add_jobs([job1, job2]) self.assertIn("Adding a job from a backend", ",".join(cm.output)) self.assertEqual(ExperimentStatus.ERROR, exp_data.status())
def test_add_data_job(self): """Test add job to experiment data.""" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") transpiled = transpile(ReferenceCircuits.bell(), self.backend) transpiled.metadata = {"foo": "bar"} job = self._run_circuit(transpiled) exp_data.add_jobs(job) self.assertEqual([job.job_id()], exp_data.job_ids) result = job.result() exp_data.block_for_results() circuit_data = exp_data.data(0) self.assertEqual(result.get_counts(0), circuit_data["counts"]) self.assertEqual(job.job_id(), circuit_data["job_id"]) self.assertEqual(transpiled.metadata, circuit_data["metadata"])
def test_block_for_jobs(self): """Test blocking for jobs.""" def _sleeper(*args, **kwargs): # pylint: disable=unused-argument time.sleep(2) nonlocal sleep_count sleep_count += 1 return self._get_job_result(1) sleep_count = 0 job = mock.create_autospec(Job, instance=True) job.result = _sleeper exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_sleeper) exp_data.block_for_results() self.assertEqual(2, sleep_count)
def test_partial_cancel_analysis(self): """Test canceling experiment analysis.""" event = threading.Event() self.addCleanup(event.set) run_analysis = [] def _job_result(): event.wait(timeout=3) return self._get_job_result(1) def _analysis(expdata, name=None, timeout=0): # pylint: disable = unused-argument event.wait(timeout=timeout) run_analysis.append(name) job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.result = _job_result job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_analysis, name=1, timeout=1) exp_data.add_analysis_callback(_analysis, name=2, timeout=30) cancel_id = exp_data._analysis_callbacks.keys()[-1] exp_data.add_analysis_callback(_analysis, name=3, timeout=1) exp_data.cancel_analysis(cancel_id) # Test status while job still running self.assertEqual(exp_data.job_status(), JobStatus.RUNNING) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.RUNNING) # Test status after job finishes event.set() self.assertEqual(exp_data.job_status(), JobStatus.DONE) self.assertEqual(exp_data.analysis_status(), AnalysisStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED) # Check that correct analysis callback was cancelled exp_data.block_for_results() self.assertEqual(run_analysis, [1, 3]) for cid, analysis in exp_data._analysis_callbacks.items(): if cid == cancel_id: self.assertEqual(analysis.status, AnalysisStatus.CANCELLED) else: self.assertEqual(analysis.status, AnalysisStatus.DONE)
def test_add_data_pending_post_processing(self): """Test add job data while post processing is still running.""" def _callback(_exp_data, **kwargs): kwargs["event"].wait(timeout=3) a_job = mock.create_autospec(Job, instance=True) a_job.result.return_value = self._get_job_result(2) a_job.status.return_value = JobStatus.DONE event = threading.Event() self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_analysis_callback(_callback, event=event) exp_data.add_jobs(a_job) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.add_data({"foo": "bar"})
def test_status_cancelled_analysis(self): """Test experiment status during post processing.""" job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE event = threading.Event() self.addCleanup(event.set) exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=2))) # Add second callback because the first can't be cancelled once it has started exp_data.add_analysis_callback((lambda *args, **kwargs: event.wait(timeout=20))) exp_data.cancel_analysis() status = exp_data.status() self.assertEqual(ExperimentStatus.CANCELLED, status)
def test_status_post_processing_error(self): """Test experiment status when post processing failed.""" def _post_processing(*args, **kwargs): raise ValueError("Kaboom!") job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(3) job.status.return_value = JobStatus.DONE exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm: exp_data.add_jobs(job) exp_data.add_analysis_callback(_post_processing) exp_data.block_for_results() self.assertEqual(ExperimentStatus.ERROR, exp_data.status()) self.assertIn("Kaboom!", ",".join(cm.output))
def test_cancel_jobs(self): """Test canceling experiment jobs.""" def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") exp_data = DbExperimentData(experiment_type="qiskit_test") event = threading.Event() self.addCleanup(event.set) job = mock.create_autospec(Job, instance=True) job.result = _job_result exp_data.add_jobs(job) exp_data.cancel_jobs() job.cancel.assert_called_once() # Cleanup with self.assertLogs("qiskit_experiments", "WARNING"): event.set() exp_data.block_for_results()
def test_add_data_job_callback_kwargs(self): """Test add job data with callback and additional arguments.""" def _callback(_exp_data, **kwargs): self.assertIsInstance(_exp_data, DbExperimentData) self.assertEqual({"foo": callback_kwargs}, kwargs) nonlocal called_back called_back = True a_job = mock.create_autospec(Job, instance=True) a_job.result.return_value = self._get_job_result(2) a_job.status.return_value = JobStatus.DONE called_back = False callback_kwargs = "foo" exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.add_jobs(a_job) exp_data.add_analysis_callback(_callback, foo=callback_kwargs) self.assertExperimentDone(exp_data) self.assertTrue(called_back)
def test_add_data_job_callback(self): """Test add job data with callback.""" def _callback(_exp_data): self.assertIsInstance(_exp_data, DbExperimentData) self.assertEqual([dat["counts"] for dat in _exp_data.data()], a_job.result().get_counts()) exp_data.add_figures(str.encode("hello world")) exp_data.add_analysis_results(mock.MagicMock()) nonlocal called_back called_back = True a_job = mock.create_autospec(Job, instance=True) a_job.result.return_value = self._get_job_result(2) a_job.status.return_value = JobStatus.DONE called_back = False exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.add_jobs(a_job) exp_data.add_analysis_callback(_callback) exp_data.block_for_results() self.assertTrue(called_back)
def test_add_data_job(self): """Test add job data.""" a_job = mock.create_autospec(Job, instance=True) a_job.result.return_value = self._get_job_result(3) jobs = [] for _ in range(2): job = mock.create_autospec(Job, instance=True) job.result.return_value = self._get_job_result(2) job.status.return_value = JobStatus.DONE jobs.append(job) expected = a_job.result().get_counts() for job in jobs: expected.extend(job.result().get_counts()) exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test") exp_data.add_jobs(a_job) self.assertExperimentDone(exp_data) exp_data.add_jobs(jobs) self.assertExperimentDone(exp_data) self.assertEqual(expected, [sdata["counts"] for sdata in exp_data.data()]) self.assertIn(a_job.job_id(), exp_data.job_ids)
def test_errors(self): """Test getting experiment error message.""" def _post_processing(*args, **kwargs): # pylint: disable=unused-argument raise ValueError("Kaboom!") job1 = mock.create_autospec(Job, instance=True) job1.job_id.return_value = "1234" job1.status.return_value = JobStatus.DONE job2 = mock.create_autospec(Job, instance=True) job2.status.return_value = JobStatus.ERROR job2.job_id.return_value = "5678" exp_data = DbExperimentData(experiment_type="qiskit_test") with self.assertLogs(logger="qiskit_experiments.database_service", level="WARN") as cm: exp_data.add_jobs(job1) exp_data.add_analysis_callback(_post_processing) exp_data.add_jobs(job2) exp_data.block_for_results() self.assertEqual(ExperimentStatus.ERROR, exp_data.status()) self.assertIn("Kaboom", ",".join(cm.output)) self.assertTrue(re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL))
def test_add_jobs_timeout(self): """Test timeout kwarg of add_jobs""" event = threading.Event() self.addCleanup(event.set) def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") job = mock.create_autospec(Job, instance=True) job.job_id.return_value = "1234" job.result = _job_result job.cancel = event.set job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING exp_data = DbExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job, timeout=0.5) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.block_for_results() self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED) self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)