def _get_job_tokens(self, workflow=None, instance=None, job_state=None, job=None): """Extract job tokens from the store. Args: workflow: The name of the workflow whose jobs we are interested in. instance: The name of the instance whose jobs we are interested in. job_state: The state of the jobs we are interested in. job: The name of the job we are interested in. Returns: List of jobs matching the specification. """ name = Name(workflow=workflow, instance=instance, job_state=job_state, job=job) if name.job: prefix = name.get_job_token_name() elif name.job_state: prefix = name.get_job_state_prefix() elif name.instance: prefix = name.get_job_prefix() elif name.workflow: prefix = name.get_workflow_prefix() else: prefix = '' tokens = self._store.read_tokens(name_prefix=prefix) result = [] for token in tokens: token_name = Name.from_job_token_name(token.name) if token_name.get_job_token_name(): # This is a job token. if not job or job == token_name.job: # We matched the prefix so if we are looking for a specific # job, its names must match exactly. result.append(token) return result
def _query_and_own_runnable_job_token(self, workflow, instance): """Attempt to own a runnable job token from a given workflow instance. Try to own a runnable job token in a given workflow instance. The ownership of the qualifying job token lasts for a limited time so it has to be periodically renewed. Args: workflow: The name of the workflow whose jobs should be considered. instance: The workflow instance whose jobs should be considered. """ assert not self._owned_job_token name = Name(workflow=workflow, instance=instance, job_state=Name.RUNNABLE_STATE) query = Query() query.namePrefix = name.get_job_state_prefix() query.maxTokens = 1 request = QueryAndOwnRequest() request.query = query request.expirationTime = time.time() + Worker._LEASE_TIME_SEC request.owner = self._name try: response = self._client.query_and_own(request) if response.tokens: assert len(response.tokens) == 1 self._owned_job_token = response.tokens[0] except TokenMasterException: LOG.exception('error sending request %s', request)
def _get_job_names(self, workflow_name, instance, state): """Return list of job names in a given workflow instance and state. E.g., assume the following tokens are stored in the master: /workflow/some_workflow/12345/waiting/some_waiting_job /workflow/some_workflow/12345/waiting/some_other_waiting_job /workflow/some_workflow/12345/runnable/some_runnable_job the method called with workflow_name=some_workflow, instance=12345, state=waiting will return [some_waiting_job, some_other_waiting_job]. """ request = GroupRequest() name = Name() name.workflow = workflow_name name.instance = instance name.job_state = state request.namePrefix = name.get_job_state_prefix() request.groupSuffix = Name.DELIMITER response = self._client.group(request) job_names = [] if response.counts: for job_name in response.counts.keys(): name = Name.from_job_token_name(job_name) job_names.append(name.job) return job_names
def _is_done(self, workflow, instance): """Check if the workflow instance is done. A workflow is done if it does not have runnable jobs. Returns: True if we are certain that the workflow is not running. Otherwise False. If there were any errors during communication with the master, the return value is False. """ # Attempt to make the workflow runnable and verify that no WAITING job # tokens were changed in the meantime. name = Name(workflow=workflow, instance=instance, job_state=Name.WAITING_STATE) query = Query(namePrefix=name.get_job_state_prefix()) request = QueryRequest(queries=[query]) try: snapshot = Snapshot(self._client, request) except: LOG.exception('error sending request %s', request) return False if not self._make_runnable(workflow, instance): return False if not self._has_no_runnable_jobs(workflow, instance): return False try: return not snapshot.refresh() except: LOG.exception('error sending request %s', request) return False
def _make_runnable(self, workflow, instance): """Attempt to make jobs in a given workflow instance runnable. Go over all waiting jobs in a given workflow instance and try to make them runnable. Args: workflow: The name of the workflow whose jobs should be considered. instance: The workflow instance whose jobs should be considered. Returns: True if there were no errors during communication with the master, otherwise False. """ name = Name() name.workflow = workflow name.instance = instance name.job_state = Name.WAITING_STATE query = Query(namePrefix=name.get_job_state_prefix()) # TODO(pawel): to prevent multiple workers from trying to make the # same job runnable at the same time, this should be a # QueryAndOwnRequest. Note that the current implementation is correct, # just inefficient. request = QueryRequest(queries=[query]) try: response = self._client.query(request) except TokenMasterException: LOG.exception('error sending request %s', request) return False assert len(response.tokens) == 1 for token in response.tokens[0]: if not self._make_job_runnable(token): return False return True
def _has_no_runnable_jobs(self, workflow, instance): """Check if the workflow instance does not contain runnable jobs. Returns: True if we are certain that the workflow has no runnable jobs. Otherwise False. If there were any errors during communication with the master, the return value is False. """ name = Name(workflow=workflow, instance=instance, job_state=Name.RUNNABLE_STATE) query = Query(namePrefix=name.get_job_state_prefix()) request = QueryRequest(queries=[query]) try: response = self._client.query(request) except TokenMasterException: LOG.exception('error sending request %s', request) return False assert len(response.tokens) == 1 if response.tokens[0]: return False return True