def _refresh_actions(self):
"""Reload actions from the master."""
request = QueryRequest(queries=[])
name = Name()
top_query = Query()
top_query.namePrefix = name.get_signal_prefix()
request.queries.append(top_query)
if self._workflow:
workflow_query = Query()
name.workflow = self._workflow
workflow_query.namePrefix = name.get_signal_prefix()
request.queries.append(workflow_query)
if self._instance:
instance_query = Query()
name.instance = self._instance
instance_query.namePrefix = name.get_signal_prefix()
request.queries.append(instance_query)
response = self._client.query(request)
signal_tokens = []
for tokens in response.tokens:
signal_tokens.extend(tokens)
self._dedup_actions(signal_tokens)
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
def _get_output_event_tokens(self, job):
"""Create output event tokens for the owned job token.
Args:
job: The job which output tokens should be generated.
Returns:
A list of event tokens corresponding to the outputs of the owned
job token.
"""
assert self._owned_job_token
job_name = Name.from_job_token_name(self._owned_job_token.name)
output_name = Name()
output_name.workflow = job_name.workflow
output_name.instance = job_name.instance
output_name.input = job_name.job
event_tokens = []
for output in job.outputs:
output_name.job = output
output_name.event = get_unique_name()
event = Event(creator=self._name)
assert job.history
execution_record = job.history[-1]
event.attributes = execution_record.get_event_attributes()
event_tokens.append(
Token(name=output_name.get_event_token_name(),
data=pickle.dumps(event)))
return event_tokens
def _refresh_actions(self):
"""Reload actions from the master."""
request = QueryRequest(queries=[])
name = Name()
top_query = Query()
top_query.namePrefix = name.get_signal_prefix()
request.queries.append(top_query)
if self._workflow:
workflow_query = Query()
name.workflow = self._workflow
workflow_query.namePrefix = name.get_signal_prefix()
request.queries.append(workflow_query)
if self._instance:
instance_query = Query()
name.instance = self._instance
instance_query.namePrefix = name.get_signal_prefix()
request.queries.append(instance_query)
response = self._client.query(request)
signal_tokens = []
for tokens in response.tokens:
signal_tokens.extend(tokens)
self._dedup_actions(signal_tokens)
def _make_runnable(self, workflow, instance):
"""Attempt to make jobs in a given workflow instance runnable.
Go over all waiting jobs in a given workflow instance and try to make
them runnable.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
name = Name()
name.workflow = workflow
name.instance = instance
name.job_state = Name.WAITING_STATE
query = Query(namePrefix=name.get_job_state_prefix())
# TODO(pawel): to prevent multiple workers from trying to make the
# same job runnable at the same time, this should be a
# QueryAndOwnRequest. Note that the current implementation is correct,
# just inefficient.
request = QueryRequest(queries=[query])
try:
response = self._client.query(request)
except TokenMasterException:
LOG.exception('error sending request %s', request)
return False
assert len(response.tokens) == 1
for token in response.tokens[0]:
if not self._make_job_runnable(token):
return False
return True
def _get_output_event_tokens(self, job):
"""Create output event tokens for the owned job token.
Args:
job: The job which output tokens should be generated.
Returns:
A list of event tokens corresponding to the outputs of the owned
job token.
"""
assert self._owned_job_token
job_name = Name.from_job_token_name(self._owned_job_token.name)
output_name = Name()
output_name.workflow = job_name.workflow
output_name.instance = job_name.instance
output_name.input = job_name.job
event_tokens = []
for output in job.outputs:
output_name.job = output
output_name.event = get_unique_name()
event = Event(creator=self._name)
assert job.history
execution_record = job.history[-1]
event.attributes = execution_record.get_event_attributes()
event_tokens.append(Token(name=output_name.get_event_token_name(),
data=pickle.dumps(event)))
return event_tokens
def _post_signal_tokens(self):
"""Add some signal tokens to the master."""
request = ModifyRequest(updates=[])
signal = Signal(action=Signal.EXIT)
name = Name(signal='exit')
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.DRAIN)
name.signal = 'drain'
name.workflow = 'some_workflow'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
name.instance = '123'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.ABORT)
name.signal = 'abort'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
client = self._factory.get_client()
client.modify(request)
def _get_job_names(self, workflow_name, instance, state):
"""Return list of job names in a given workflow instance and state.
E.g., assume the following tokens are stored in the master:
/workflow/some_workflow/12345/waiting/some_waiting_job
/workflow/some_workflow/12345/waiting/some_other_waiting_job
/workflow/some_workflow/12345/runnable/some_runnable_job
the method called with workflow_name=some_workflow, instance=12345,
state=waiting will return [some_waiting_job, some_other_waiting_job].
"""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job_state = state
request.namePrefix = name.get_job_state_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
job_names = []
if response.counts:
for job_name in response.counts.keys():
name = Name.from_job_token_name(job_name)
job_names.append(name.job)
return job_names
def _post_signal_tokens(self):
"""Add some signal tokens to the master."""
request = ModifyRequest(updates=[])
signal = Signal(action=Signal.EXIT)
name = Name(signal='exit')
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.DRAIN)
name.signal = 'drain'
name.workflow = 'some_workflow'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
name.instance = '123'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.ABORT)
name.signal = 'abort'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
client = self._factory.get_client()
client.modify(request)
def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
