def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def _generate_missing_events(self, job_names):
"""Generate external events required to run all jobs in a set.
For a set of jobs (a subset of all jobs in the workflow), produce
events satisfying upstream dependencies external to that set. E.g.,
for job dependency structure like this:
A1 A2
| /
B1 B2
|
C1 C2
| /
D1
and job_names = (C1, D1) we would generate events satisfying the
following deps: B1->C1, C2->D1.
Args:
job_names: The set of job names whose external deps are to be
satisfied by the generated events.
"""
input_prefixes = set()
for job_name in job_names:
job = self._jobs[job_name]
for job_input in job.inputs:
if job_input not in job_names:
name = Name(workflow=self._workflow,
instance=self._instance,
job=job_name,
input_name=job_input,
event='poison_%d' % len(input_prefixes))
input_prefix = name.get_input_prefix()
if input_prefix not in input_prefixes:
input_prefixes.add(input_prefix)
event_token_name = name.get_event_token_name()
if not event_token_name in self._existing_events:
self._new_events[
name.get_event_token_name()] = Event(
'analyzer')
def _generate_missing_events(self, job_names):
"""Generate external events required to run all jobs in a set.
For a set of jobs (a subset of all jobs in the workflow), produce
events satisfying upstream dependencies external to that set. E.g.,
for job dependency structure like this:
A1 A2
| /
B1 B2
|
C1 C2
| /
D1
and job_names = (C1, D1) we would generate events satisfying the
following deps: B1->C1, C2->D1.
Args:
job_names: The set of job names whose external deps are to be
satisfied by the generated events.
"""
input_prefixes = set()
for job_name in job_names:
job = self._jobs[job_name]
for job_input in job.inputs:
if job_input not in job_names:
name = Name(workflow=self._workflow,
instance=self._instance,
job=job_name,
input_name=job_input,
event='poison_%d' % len(input_prefixes))
input_prefix = name.get_input_prefix()
if input_prefix not in input_prefixes:
input_prefixes.add(input_prefix)
event_token_name = name.get_event_token_name()
if not event_token_name in self._existing_events:
self._new_events[
name.get_event_token_name()] = Event(
'analyzer')
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
