def _post_job_tokens(self):
"""Add some job tokens to the master."""
request = ModifyRequest(updates=[])
name = Name(workflow='some_workflow', instance='12345')
for job_id in range(0, 2):
if job_id % 2 == 0:
name.job_state = Name.WAITING_STATE
else:
name.job_state = Name.RUNNABLE_STATE
name.job = 'some_job_%d' % job_id
job_token = Token(name=name.get_job_token_name())
request.updates.append(job_token)
client = self._factory.get_client()
client.modify(request)
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
