def _get_instances_using_cache(self, workflow):
"""Get workflow instances, preferably from the cache.
As a side effect, archived instances that do not exist in the cache
will be added to the cache.
Args:
workflow: The name of the workflow whose instances we are
interested in.
Returns:
List of instances for the given workflow.
"""
name = Name(workflow=workflow)
workflow_prefix = name.get_workflow_prefix()
workflow_token_names = self._store.read_token_names(
name_prefix=workflow_prefix)
instances_prefixes = DataBuilder._get_instance_prefixes(
workflow_token_names)
result = []
for prefix in instances_prefixes:
name = Name.from_instance_prefix(prefix)
assert name.workflow and name.instance, (
'Expected instance prefix, found %s' % prefix)
result.append(self.get_instance(name.workflow, name.instance))
return result
def _get_job_tokens(self, workflow=None, instance=None, job_state=None,
job=None):
"""Extract job tokens from the store.
Args:
workflow: The name of the workflow whose jobs we are interested in.
instance: The name of the instance whose jobs we are interested in.
job_state: The state of the jobs we are interested in.
job: The name of the job we are interested in.
Returns:
List of jobs matching the specification.
"""
name = Name(workflow=workflow, instance=instance, job_state=job_state,
job=job)
if name.job:
prefix = name.get_job_token_name()
elif name.job_state:
prefix = name.get_job_state_prefix()
elif name.instance:
prefix = name.get_job_prefix()
elif name.workflow:
prefix = name.get_workflow_prefix()
else:
prefix = ''
tokens = self._store.read_tokens(name_prefix=prefix)
result = []
for token in tokens:
token_name = Name.from_job_token_name(token.name)
if token_name.get_job_token_name():
# This is a job token.
if not job or job == token_name.job:
# We matched the prefix so if we are looking for a specific
# job, its names must match exactly.
result.append(token)
return result
def _get_jobs(self, workflow, job):
"""Get job definitions from the store across all workflow instances.
Args:
workflow: The name of the job workflow.
instance: The name of the job instance.
job: The name of the job.
Returns:
Matching job definition.
"""
name = Name(workflow=workflow)
name_prefix = name.get_workflow_prefix()
# This is a bit hacky since we bypass the Name module where all the
# token naming logic is supposed to be located.
# TODO(pawel): extend the Name module to support abstractions needed
# here.
name_infix = '/job/'
name_suffix = '/%s' % job
job_tokens = self._store.read_tokens(name_prefix=name_prefix,
name_infix=name_infix,
name_suffix=name_suffix)
result = []
for job_token in job_tokens:
job_record = pickle.loads(job_token.data)
result.append(job_record)
return result
def _get_instances_using_cache(self, workflow):
"""Get workflow instances, preferably from the cache.
As a side effect, archived instances that do not exist in the cache
will be added to the cache.
Args:
workflow: The name of the workflow whose instances we are
interested in.
Returns:
List of instances for the given workflow.
"""
name = Name(workflow=workflow)
workflow_prefix = name.get_workflow_prefix()
workflow_token_names = self._store.read_token_names(
name_prefix=workflow_prefix)
instances_prefixes = DataBuilder._get_instance_prefixes(
workflow_token_names)
result = []
for prefix in instances_prefixes:
name = Name.from_instance_prefix(prefix)
assert name.workflow and name.instance, (
'Expected instance prefix, found %s' % prefix)
result.append(self.get_instance(name.workflow, name.instance))
return result
def _get_jobs(self, workflow, job):
"""Get job definitions from the store across all workflow instances.
Args:
workflow: The name of the job workflow.
instance: The name of the job instance.
job: The name of the job.
Returns:
Matching job definition.
"""
name = Name(workflow=workflow)
name_prefix = name.get_workflow_prefix()
# This is a bit hacky since we bypass the Name module where all the
# token naming logic is supposed to be located.
# TODO(pawel): extend the Name module to support abstractions needed
# here.
name_infix = '/job/'
name_suffix = '/%s' % job
job_tokens = self._store.read_tokens(name_prefix=name_prefix,
name_infix=name_infix,
name_suffix=name_suffix)
result = []
for job_token in job_tokens:
job_record = pickle.loads(job_token.data)
result.append(job_record)
return result
def _read_tokens_from_client(self, client):
"""Read archived job tokens from the client.
Args:
client: The client to read tokens from.
"""
name = Name(workflow=self._workflow, instance=self._instance)
query = Query(namePrefix=name.get_workflow_prefix())
request = QueryRequest(queries=[query])
response = client.query(request)
assert len(response.tokens) == 1
tokens = response.tokens[0]
self._filter_job_tokens(tokens)
self._filter_event_tokens(tokens)
def _read_tokens_from_client(self, client):
"""Read archived job tokens from the client.
Args:
client: The client to read tokens from.
"""
name = Name(workflow=self._workflow, instance=self._instance)
query = Query(namePrefix=name.get_workflow_prefix())
request = QueryRequest(queries=[query])
response = client.query(request)
assert len(response.tokens) == 1
tokens = response.tokens[0]
self._filter_job_tokens(tokens)
self._filter_event_tokens(tokens)
def get_workflow_instances(self, workflow_name):
"""Return list of instances of a given workflow."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
request.namePrefix = name.get_workflow_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
instance_names = []
if response.counts:
for prefix in response.counts.keys():
name = Name.from_instance_prefix(prefix)
if name.instance:
instance_names.append(name.instance)
return instance_names
def get_workflow_instances(self, workflow_name):
"""Return list of instances of a given workflow."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
request.namePrefix = name.get_workflow_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
instance_names = []
if response.counts:
for prefix in response.counts.keys():
name = Name.from_instance_prefix(prefix)
if name.instance:
instance_names.append(name.instance)
return instance_names
def _get_job_tokens(self,
workflow=None,
instance=None,
job_state=None,
job=None):
"""Extract job tokens from the store.
Args:
workflow: The name of the workflow whose jobs we are interested in.
instance: The name of the instance whose jobs we are interested in.
job_state: The state of the jobs we are interested in.
job: The name of the job we are interested in.
Returns:
List of jobs matching the specification.
"""
name = Name(workflow=workflow,
instance=instance,
job_state=job_state,
job=job)
if name.job:
prefix = name.get_job_token_name()
elif name.job_state:
prefix = name.get_job_state_prefix()
elif name.instance:
prefix = name.get_job_prefix()
elif name.workflow:
prefix = name.get_workflow_prefix()
else:
prefix = ''
tokens = self._store.read_tokens(name_prefix=prefix)
result = []
for token in tokens:
token_name = Name.from_job_token_name(token.name)
if token_name.get_job_token_name():
# This is a job token.
if not job or job == token_name.job:
# We matched the prefix so if we are looking for a specific
# job, its names must match exactly.
result.append(token)
return result
