def _get_job_tokens(self, workflow=None, instance=None, job_state=None,
job=None):
"""Extract job tokens from the store.
Args:
workflow: The name of the workflow whose jobs we are interested in.
instance: The name of the instance whose jobs we are interested in.
job_state: The state of the jobs we are interested in.
job: The name of the job we are interested in.
Returns:
List of jobs matching the specification.
"""
name = Name(workflow=workflow, instance=instance, job_state=job_state,
job=job)
if name.job:
prefix = name.get_job_token_name()
elif name.job_state:
prefix = name.get_job_state_prefix()
elif name.instance:
prefix = name.get_job_prefix()
elif name.workflow:
prefix = name.get_workflow_prefix()
else:
prefix = ''
tokens = self._store.read_tokens(name_prefix=prefix)
result = []
for token in tokens:
token_name = Name.from_job_token_name(token.name)
if token_name.get_job_token_name():
# This is a job token.
if not job or job == token_name.job:
# We matched the prefix so if we are looking for a specific
# job, its names must match exactly.
result.append(token)
return result
def _query_and_own_runnable_job_token(self, workflow, instance):
"""Attempt to own a runnable job token from a given workflow instance.
Try to own a runnable job token in a given workflow instance. The
ownership of the qualifying job token lasts for a limited time so it
has to be periodically renewed.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
"""
assert not self._owned_job_token
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query()
query.namePrefix = name.get_job_state_prefix()
query.maxTokens = 1
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = time.time() + Worker._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) == 1
self._owned_job_token = response.tokens[0]
except TokenMasterException:
LOG.exception('error sending request %s', request)
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
def _query_and_own_runnable_job_token(self, workflow, instance):
"""Attempt to own a runnable job token from a given workflow instance.
Try to own a runnable job token in a given workflow instance. The
ownership of the qualifying job token lasts for a limited time so it
has to be periodically renewed.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
"""
assert not self._owned_job_token
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query()
query.namePrefix = name.get_job_state_prefix()
query.maxTokens = 1
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = time.time() + Worker._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) == 1
self._owned_job_token = response.tokens[0]
except TokenMasterException:
LOG.exception('error sending request %s', request)
def _is_done(self, workflow, instance):
"""Check if the workflow instance is done.
A workflow is done if it does not have runnable jobs.
Returns:
True if we are certain that the workflow is not running. Otherwise
False. If there were any errors during communication with the
master, the return value is False.
"""
# Attempt to make the workflow runnable and verify that no WAITING job
# tokens were changed in the meantime.
name = Name(workflow=workflow,
instance=instance,
job_state=Name.WAITING_STATE)
query = Query(namePrefix=name.get_job_state_prefix())
request = QueryRequest(queries=[query])
try:
snapshot = Snapshot(self._client, request)
except:
LOG.exception('error sending request %s', request)
return False
if not self._make_runnable(workflow, instance):
return False
if not self._has_no_runnable_jobs(workflow, instance):
return False
try:
return not snapshot.refresh()
except:
LOG.exception('error sending request %s', request)
return False
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _is_done(self, workflow, instance):
"""Check if the workflow instance is done.
A workflow is done if it does not have runnable jobs.
Returns:
True if we are certain that the workflow is not running. Otherwise
False. If there were any errors during communication with the
master, the return value is False.
"""
# Attempt to make the workflow runnable and verify that no WAITING job
# tokens were changed in the meantime.
name = Name(workflow=workflow,
instance=instance,
job_state=Name.WAITING_STATE)
query = Query(namePrefix=name.get_job_state_prefix())
request = QueryRequest(queries=[query])
try:
snapshot = Snapshot(self._client, request)
except:
LOG.exception('error sending request %s', request)
return False
if not self._make_runnable(workflow, instance):
return False
if not self._has_no_runnable_jobs(workflow, instance):
return False
try:
return not snapshot.refresh()
except:
LOG.exception('error sending request %s', request)
return False
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
def _get_job_tokens(self,
workflow=None,
instance=None,
job_state=None,
job=None):
"""Extract job tokens from the store.
Args:
workflow: The name of the workflow whose jobs we are interested in.
instance: The name of the instance whose jobs we are interested in.
job_state: The state of the jobs we are interested in.
job: The name of the job we are interested in.
Returns:
List of jobs matching the specification.
"""
name = Name(workflow=workflow,
instance=instance,
job_state=job_state,
job=job)
if name.job:
prefix = name.get_job_token_name()
elif name.job_state:
prefix = name.get_job_state_prefix()
elif name.instance:
prefix = name.get_job_prefix()
elif name.workflow:
prefix = name.get_workflow_prefix()
else:
prefix = ''
tokens = self._store.read_tokens(name_prefix=prefix)
result = []
for token in tokens:
token_name = Name.from_job_token_name(token.name)
if token_name.get_job_token_name():
# This is a job token.
if not job or job == token_name.job:
# We matched the prefix so if we are looking for a specific
# job, its names must match exactly.
result.append(token)
return result
def _has_no_runnable_jobs(self, workflow, instance):
"""Check if the workflow instance does not contain runnable jobs.
Returns:
True if we are certain that the workflow has no runnable jobs.
Otherwise False. If there were any errors during communication
with the master, the return value is False.
"""
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query(namePrefix=name.get_job_state_prefix())
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
if response.tokens[0]:
return False
return True
def _has_no_runnable_jobs(self, workflow, instance):
"""Check if the workflow instance does not contain runnable jobs.
Returns:
True if we are certain that the workflow has no runnable jobs.
Otherwise False. If there were any errors during communication
with the master, the return value is False.
"""
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query(namePrefix=name.get_job_state_prefix())
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
if response.tokens[0]:
return False
return True
