def test_container_script_param_injection(is_child_job): # Test that child jobs created by the shell executor have the # job and run IDs of the script which created them (if any). job_id, run_id = '123', '13' c = _setup_client_mock(42, 43, n_failures=0) mock_env = {'CIVIS_JOB_ID': job_id, 'CIVIS_RUN_ID': run_id} with mock.patch.dict('os.environ', mock_env): init_kwargs = dict(client=c, polling_interval=0.01) if is_child_job: init_kwargs['params'] = [ {'name': 'CIVIS_PARENT_JOB_ID', 'type': 'integer', 'value': '888'}, {'name': 'CIVIS_PARENT_RUN_ID', 'type': 'integer', 'value': '999'}, ] bpe = _ContainerShellExecutor(**init_kwargs) bpe.submit("foo") params = sorted(c.scripts.post_containers.call_args[1].get('params'), key=itemgetter('name')) assert params == [ {'name': 'CIVIS_PARENT_JOB_ID', 'type': 'integer', 'value': job_id}, {'name': 'CIVIS_PARENT_RUN_ID', 'type': 'integer', 'value': run_id} ], "The parent job parameters were not set correctly."
def _init_civis_backend(self): """init the Civis API client and the executors""" self.using_template = (self.from_template_id is not None) if self.max_submit_retries < 0: raise ValueError( "max_submit_retries cannot be negative (value = %d)" % self.max_submit_retries) self.client = self.client or civis.APIClient() if self.from_template_id: self.executor = CustomScriptExecutor(self.from_template_id, client=self.client, **self.executor_kwargs) else: self.executor = _ContainerShellExecutor(client=self.client, **self.executor_kwargs)
def test_container_script_param_injection(): # Test that child jobs created by the shell executor have the # job and run IDs of the script which created them (if any). job_id, run_id = '123', '13' c = _setup_client_mock(42, 43, n_failures=0) with mock.patch.dict('os.environ', { 'CIVIS_JOB_ID': job_id, 'CIVIS_RUN_ID': run_id }): bpe = _ContainerShellExecutor(client=c, polling_interval=0.01) bpe.submit("foo") params = c.scripts.post_containers.call_args[1].get('params') assert params is not None assert {'name': 'CIVIS_PARENT_JOB_ID', 'type': 'integer', 'value': job_id} in params, \ "The creator's job ID wasn't passed to the child." assert {'name': 'CIVIS_PARENT_RUN_ID', 'type': 'integer', 'value': run_id} in params, \ "The creator's run ID wasn't passed to the child."
def _check_executor(from_template_id=None): job_id, run_id = 42, 43 c = _setup_client_mock(job_id, run_id, n_failures=0) mock_run = c.scripts.post_containers_runs() if from_template_id: bpe = CustomScriptExecutor(from_template_id=from_template_id, client=c, polling_interval=0.01) future = bpe.submit(my_param='spam') else: bpe = _ContainerShellExecutor(client=c, polling_interval=0.01) future = bpe.submit("foo") # Mock and test running, future.job_id, and done() mock_run.state = "running" assert future.running(), "future is incorrectly marked as not running" assert future.job_id == job_id, "job_id not stored properly" assert not future.done(), "future is incorrectly marked as done" future.cancel() # Mock and test cancelled() assert future.cancelled(), "cancelled() did not return True as expected" assert not future.running(), "running() did not return False as expected" # Mock and test done() mock_run.state = "succeeded" assert future.done(), "done() did not return True as expected" # Test cancelling all jobs. mock_run.state = "running" bpe.cancel_all() assert future.cancelled(), "cancel_all() failed" # Test shutdown method. bpe.shutdown(wait=True) assert future.done(), "shutdown() failed" return c
def __init__(self, setup_cmd=_DEFAULT_SETUP_CMD, from_template_id=None, max_submit_retries=0, client=None, **executor_kwargs): if max_submit_retries < 0: raise ValueError( "max_submit_retries cannot be negative (value = %d)" % max_submit_retries) if client is None: client = civis.APIClient(resources='all') self._client = client if from_template_id: self.executor = CustomScriptExecutor(from_template_id, client=client, **executor_kwargs) else: self.executor = _ContainerShellExecutor(client=client, **executor_kwargs) self.setup_cmd = setup_cmd self.max_submit_retries = max_submit_retries self.using_template = (from_template_id is not None)