def _post_job_tokens(self): """Add some job tokens to the master.""" request = ModifyRequest(updates=[]) name = Name(workflow='some_workflow', instance='12345') for job_id in range(0, 2): if job_id % 2 == 0: name.job_state = Name.WAITING_STATE else: name.job_state = Name.RUNNABLE_STATE name.job = 'some_job_%d' % job_id job_token = Token(name=name.get_job_token_name()) request.updates.append(job_token) client = self._factory.get_client() client.modify(request)
def _get_job_names(self, workflow_name, instance, state): """Return list of job names in a given workflow instance and state. E.g., assume the following tokens are stored in the master: /workflow/some_workflow/12345/waiting/some_waiting_job /workflow/some_workflow/12345/waiting/some_other_waiting_job /workflow/some_workflow/12345/runnable/some_runnable_job the method called with workflow_name=some_workflow, instance=12345, state=waiting will return [some_waiting_job, some_other_waiting_job]. """ request = GroupRequest() name = Name() name.workflow = workflow_name name.instance = instance name.job_state = state request.namePrefix = name.get_job_state_prefix() request.groupSuffix = Name.DELIMITER response = self._client.group(request) job_names = [] if response.counts: for job_name in response.counts.keys(): name = Name.from_job_token_name(job_name) job_names.append(name.job) return job_names
def _make_runnable(self, workflow, instance): """Attempt to make jobs in a given workflow instance runnable. Go over all waiting jobs in a given workflow instance and try to make them runnable. Args: workflow: The name of the workflow whose jobs should be considered. instance: The workflow instance whose jobs should be considered. Returns: True if there were no errors during communication with the master, otherwise False. """ name = Name() name.workflow = workflow name.instance = instance name.job_state = Name.WAITING_STATE query = Query(namePrefix=name.get_job_state_prefix()) # TODO(pawel): to prevent multiple workers from trying to make the # same job runnable at the same time, this should be a # QueryAndOwnRequest. Note that the current implementation is correct, # just inefficient. request = QueryRequest(queries=[query]) try: response = self._client.query(request) except TokenMasterException: LOG.exception('error sending request %s', request) return False assert len(response.tokens) == 1 for token in response.tokens[0]: if not self._make_job_runnable(token): return False return True