def test_update(self):
Options = collections.namedtuple('args', 'name version owner '
'expiration_time priority data '
'command_args')
options = Options(name='/some_path/some_token',
version=10,
owner='some_other_owner',
expiration_time=100,
priority=10,
data='some_other_data',
command_args=None)
command = Update()
command.prepare(options)
client = mock.Mock()
output_token = Token(version=11,
name='/some_path/some_token',
owner='some_other_owner',
expirationTime=100,
priority=10,
data='some_other_data')
response = ModifyResponse(updates=[output_token])
client.modify.return_value = response
output = command.execute(client, None)
input_token = Token(version=10,
name='/some_path/some_token',
owner='some_other_owner',
expirationTime=100,
priority=10,
data='some_other_data')
request = ModifyRequest(updates=[input_token])
client.modify.assert_called_once_with(request)
self.assertEqual('updated %s\nupdated 1 token\n' % str(output_token),
output)
def set_action(self, action):
"""Send a signal with a specific action to the master.
Local signal store gets updated with the new action if it is
successfully submitted to the master. If the communication with the
master fails, locally stored signals get refreshed.
Args:
action: The action to set.
"""
attributes = {}
if action == Signal.ABORT:
attributes[Signal.TIMESTAMP_ATTR] = time.time()
elif action == Signal.EXIT:
attributes[Signal.GENERATION_ATTR] = PinballConfig.GENERATION
signal = self._signals.get(action)
if signal and signal.attributes == attributes:
return
# A signal with the same action but different data may already exist
# in the master.
signal_token = self._get_signal_token(action)
if not signal_token:
name = Name(workflow=self._workflow,
instance=self._instance,
signal=Signal.action_to_string(action))
signal_token = Token(name=name.get_signal_token_name())
signal = Signal(action, attributes)
signal_token.data = pickle.dumps(signal)
request = ModifyRequest(updates=[signal_token])
if self._send_request(request):
self._signals[action] = signal
def set_action(self, action):
"""Send a signal with a specific action to the master.
Local signal store gets updated with the new action if it is
successfully submitted to the master. If the communication with the
master fails, locally stored signals get refreshed.
Args:
action: The action to set.
"""
attributes = {}
if action == Signal.ABORT:
attributes[Signal.TIMESTAMP_ATTR] = time.time()
elif action == Signal.EXIT:
attributes[Signal.GENERATION_ATTR] = PinballConfig.GENERATION
signal = self._signals.get(action)
if signal and signal.attributes == attributes:
return
# A signal with the same action but different data may already exist
# in the master.
signal_token = self._get_signal_token(action)
if not signal_token:
name = Name(workflow=self._workflow, instance=self._instance,
signal=Signal.action_to_string(action))
signal_token = Token(name=name.get_signal_token_name())
signal = Signal(action, attributes)
signal_token.data = pickle.dumps(signal)
request = ModifyRequest(updates=[signal_token])
if self._send_request(request):
self._signals[action] = signal
def setUp(self):
"""Set up self._trie with 111 tokens, one of them a blessed version."""
self._trie = pytrie.StringTrie()
self._store = EphemeralStore()
blessed_version = BlessedVersion(MasterHandler._BLESSED_VERSION,
MasterHandler._MASTER_OWNER)
for i in range(0, 10):
some_token = Token(blessed_version.advance_version(),
'/some_dir/some_token_%d' % i,
priority=i,
data='some_data_%d' % i)
self._trie[some_token.name] = some_token
self._store.commit_tokens(updates=[some_token])
for j in range(0, 10):
some_other_token = Token(
blessed_version.advance_version(),
'/some_dir/some_token_%d/some_other_token_%d' % (i, j),
priority=j,
data='some_data_%d_%d' % (i, j))
self._trie[some_other_token.name] = some_other_token
self._store.commit_tokens(updates=[some_other_token])
blessed_version.advance_version()
self._trie[MasterHandler._BLESSED_VERSION] = blessed_version
self._store.commit_tokens(updates=[blessed_version])
self._check_version_uniqueness()
def _get_schedule_token():
name = Name(workflow='workflow_0')
now = int(time.time())
token = Token(name=name.get_workflow_schedule_token_name(),
owner='some_owner',
expirationTime=now - 10)
schedule = WorkflowSchedule(next_run_time=now - 10,
recurrence_seconds=10,
workflow='workflow_0')
token.data = pickle.dumps(schedule)
return token
def _get_schedule_token():
name = Name(workflow='workflow_0')
now = int(time.time())
token = Token(name=name.get_workflow_schedule_token_name(),
owner='some_owner',
expirationTime=now - 10)
schedule = WorkflowSchedule(next_run_time=now - 10,
recurrence_seconds=10,
workflow='workflow_0')
token.data = pickle.dumps(schedule)
return token
def _add_active_workflow_tokens(self):
"""Add some active workflow tokens.
The job dependencies form a complete binary tree turned upside down.
I.e., each job has two parents.
"""
self._store = EphemeralStore()
version = 1
for level in range(AnalyzerTestCase._NUM_LEVELS):
jobs_at_level = 2**(AnalyzerTestCase._NUM_LEVELS - level - 1)
for job_index in range(jobs_at_level):
job_name = 'job_%d_%d' % (level, job_index)
event_name = Name(workflow='some_workflow',
instance='123',
job=job_name,
event='some_event')
if level == 0:
inputs = [
Name.WORKFLOW_START_INPUT,
Name.WORKFLOW_START_INPUT + '_prime'
]
event_name.input = Name.WORKFLOW_START_INPUT
else:
inputs = [
'job_%d_%d' % (level - 1, 2 * job_index),
'job_%d_%d' % (level - 1, 2 * job_index + 1)
]
event_name.input = 'job_%d_%d' % (level - 1, 2 * job_index)
if level == AnalyzerTestCase._NUM_LEVELS - 1:
outputs = []
else:
outputs = ['job_%d_%d' % (level + 1, job_index / 2)]
job = ShellJob(name=job_name,
inputs=inputs,
outputs=outputs,
command='some_command')
job.history.append(ExecutionRecord())
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job=job_name)
job_token = Token(version=version,
name=name.get_job_token_name(),
priority=10,
data=pickle.dumps(job))
version += 1
event = Event('some_event')
event_token = Token(version=version,
name=event_name.get_event_token_name(),
priority=10,
data=pickle.dumps(event))
self._store.commit_tokens([job_token, event_token])
def _post_signal_tokens(self):
"""Add some signal tokens to the master."""
request = ModifyRequest(updates=[])
signal = Signal(action=Signal.EXIT)
name = Name(signal='exit')
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.DRAIN)
name.signal = 'drain'
name.workflow = 'some_workflow'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
name.instance = '123'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
signal = Signal(action=Signal.ABORT)
name.signal = 'abort'
signal_token = Token(name=name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
client = self._factory.get_client()
client.modify(request)
def get_job_token(self, workflow_name, workflow_instance):
"""Convert the job to a pinball token representing its properties.
Condition is similar to job. We check the template to decide use
job_templates or condition_templates dynamically.
Args:
workflow_name: The name of the workflow in which context this job
is instantiated.
workflow_instance: The workflow instance of the output job token.
Returns:
A pinball token representing the job.
"""
if self.workflow.name == workflow_name:
inputs = [input_job.name for input_job in self.inputs]
else:
# If it's an external job, do not include its inputs.
inputs = []
if not inputs:
inputs = [Name.WORKFLOW_START_INPUT]
outputs = []
for output_job in self.outputs:
if output_job.workflow.name == workflow_name:
outputs.append(output_job.name)
if issubclass(self.template.__class__, JobTemplateBase):
params = self._get_template_params()
job = self.template.get_pinball_job(inputs, outputs, params)
name = Name(workflow=workflow_name,
instance=workflow_instance,
job_state=Name.WAITING_STATE,
job=self.name)
result = Token(name=name.get_job_token_name(),
data=pickle.dumps(job))
result.priority = self.compute_score()
elif issubclass(self.template.__class__, ConditionTemplateBase):
condition = self.template.get_pinball_condition(outputs)
name = Name(workflow=workflow_name,
instance=workflow_instance,
job_state=Name.WAITING_STATE,
job=self.name)
result = Token(name=name.get_job_token_name(),
data=pickle.dumps(condition))
else:
raise Exception(
"Template must be a subclass of JobTemplateBase or ConditionTemplateBase!"
)
return result
def test_recursive(self):
Options = collections.namedtuple('args',
'recursive force command_args')
options = Options(recursive=True, force=True,
command_args=['/some_path'])
command = Rm()
command.prepare(options)
client = mock.Mock()
token = Token(version=10,
name='/some_path/some_token',
owner='some_owner',
expirationTime=10,
data='some_data')
query_response = QueryResponse(tokens=[[token]])
client.query.return_value = query_response
modify_response = ModifyResponse()
client.modify.return_value = modify_response
output = command.execute(client, None)
query = Query(namePrefix='/some_path')
query_request = QueryRequest(queries=[query])
client.query.assert_called_once_with(query_request)
modify_request = ModifyRequest(deletes=[token])
client.modify.assert_called_once_with(modify_request)
self.assertEqual('removed 1 token(s)\n', output)
def _get_output_event_tokens(self, job):
"""Create output event tokens for the owned job token.
Args:
job: The job which output tokens should be generated.
Returns:
A list of event tokens corresponding to the outputs of the owned
job token.
"""
assert self._owned_job_token
job_name = Name.from_job_token_name(self._owned_job_token.name)
output_name = Name()
output_name.workflow = job_name.workflow
output_name.instance = job_name.instance
output_name.input = job_name.job
event_tokens = []
for output in job.outputs:
output_name.job = output
output_name.event = get_unique_name()
event = Event(creator=self._name)
assert job.history
execution_record = job.history[-1]
event.attributes = execution_record.get_event_attributes()
event_tokens.append(
Token(name=output_name.get_event_token_name(),
data=pickle.dumps(event)))
return event_tokens
def test_recursive(self):
Options = collections.namedtuple('args', 'recursive command_args')
options = Options(recursive=True, command_args=['/some_path'])
command = Cat()
command.prepare(options)
client = mock.Mock()
token = Token(version=10,
name='/some_path/some_token',
owner='some_owner',
expirationTime=10,
data='some_data')
query_response = QueryResponse(tokens=[[token]])
client.query.return_value = query_response
output = command.execute(client, None)
query = Query(namePrefix='/some_path')
query_request = QueryRequest(queries=[query])
client.query.assert_called_once_with(query_request)
self.assertEqual('total 1\nToken(version=10, owner=some_owner, '
'expirationTime=1970-01-01 00:00:10 UTC, '
'priority=0.000000, name=/some_path/some_token, '
'data=some_data)\n',
output)
def to_token(self):
return Token(version=self.version,
name=self.name,
owner=self.owner,
expirationTime=self.expirationTime,
priority=self.priority,
data=self.data)
def _insert_token(self, handler):
request = ModifyRequest()
token = Token(name='/some_other_dir/some_token', data='some data')
request.updates = [token]
response = handler.modify(request)
self.assertEqual(1, len(response.updates))
return response.updates[0]
def get_workflow_tokens(self):
"""Create Pinball tokens representing a workflow instance.
Convert workflow jobs to tokens and create event tokens in inputs of
top-level jobs.
Returns:
A list of job and event tokens representing a workflow instance.
"""
all_jobs = self._get_transitive_deps()
instance = get_unique_workflow_instance()
result = []
for job in all_jobs:
result.append(job.get_job_token(self.name, instance))
top_level_jobs = self._get_top_level_jobs()
for job in top_level_jobs:
event = Event(creator='parser')
event_name = Name(workflow=self.name,
instance=instance,
job=job.name,
input_name=Name.WORKFLOW_START_INPUT,
event='workflow_start_event')
result.append(
Token(name=event_name.get_event_token_name(),
data=pickle.dumps(event)))
return result
def get_job_token(self, workflow_name, workflow_instance):
"""Convert the job to a pinball token representing its properties.
Condition is similar to job. We check the template to decide use
job_templates or condition_templates dynamically.
Args:
workflow_name: The name of the workflow in which context this job
is instantiated.
workflow_instance: The workflow instance of the output job token.
Returns:
A pinball token representing the job.
"""
if self.workflow.name == workflow_name:
inputs = [input_job.name for input_job in self.inputs]
else:
# If it's an external job, do not include its inputs.
inputs = []
if not inputs:
inputs = [Name.WORKFLOW_START_INPUT]
outputs = []
for output_job in self.outputs:
if output_job.workflow.name == workflow_name:
outputs.append(output_job.name)
if issubclass(self.template.__class__, JobTemplateBase):
params = self._get_template_params()
job = self.template.get_pinball_job(inputs, outputs, params)
name = Name(workflow=workflow_name,
instance=workflow_instance,
job_state=Name.WAITING_STATE,
job=self.name)
result = Token(name=name.get_job_token_name(), data=pickle.dumps(job))
result.priority = self.compute_score()
elif issubclass(self.template.__class__, ConditionTemplateBase):
condition = self.template.get_pinball_condition(outputs)
name = Name(workflow=workflow_name,
instance=workflow_instance,
job_state=Name.WAITING_STATE,
job=self.name)
result = Token(name=name.get_job_token_name(), data=pickle.dumps(condition))
else:
raise Exception("Template must be a subclass of JobTemplateBase or ConditionTemplateBase!")
return result
def __init__(self, name=None, owner=None):
"""Create blessed version with a given name and owner.
Name and owner have to either both be set or none should be set.
Blessed version in use should always have name and owner set. The
version of init with name and owner set to None relies on external
initialization of those fields.
Args:
name: The name of the blessed version token.
owner: The owner of the blessed version token.
"""
assert (name and owner) or (not name and not owner)
if name and owner:
now = BlessedVersion._get_timestamp_millis()
data_str = ('blessed version created at %s' %
timestamp_to_str(now / 1000))
Token.__init__(self, now, name, owner, sys.maxint, 0, data_str)
else:
Token.__init__(self)
def get_schedule_token(self):
"""Create a token describing workflow execution schedule."""
self.schedule.advance_next_run_time()
timestamp = self.schedule.next_run_time
token_name = (Name(
workflow=self.name).get_workflow_schedule_token_name())
return Token(name=token_name,
owner='parser',
expirationTime=timestamp,
data=pickle.dumps(self.schedule))
def __init__(self, name=None, owner=None):
"""Create blessed version with a given name and owner.
Name and owner have to either both be set or none should be set.
Blessed version in use should always have name and owner set. The
version of init with name and owner set to None relies on external
initialization of those fields.
Args:
name: The name of the blessed version token.
owner: The owner of the blessed version token.
"""
assert (name and owner) or (not name and not owner)
if name and owner:
now = BlessedVersion._get_timestamp_millis()
data_str = ('blessed version created at %s' %
timestamp_to_str(now / 1000))
Token.__init__(self, now, name, owner, sys.maxint, 0, data_str)
else:
Token.__init__(self)
def _get_child_job_token(self):
name = Name(workflow='some_workflow',
instance='12345',
job_state=Name.WAITING_STATE,
job='child_job')
job = ShellJob(name=name.job,
inputs=['parent_job'],
outputs=[],
command='echo child',
emails=['*****@*****.**'])
return Token(name=name.get_job_token_name(), data=pickle.dumps(job))
def _post_workflow_start_event_token(self):
name = Name(workflow='some_workflow',
instance='12345',
job='parent_job',
input_name=Name.WORKFLOW_START_INPUT,
event='workflow_start_event')
event = Event(creator='SimpleWorkflowTest')
token = Token(name=name.get_event_token_name(),
data=pickle.dumps(event))
request = ModifyRequest(updates=[token])
self._client.modify(request)
def _generate_signal_tokens(workflows):
result = []
for w in range(0, workflows, 2):
workflow = 'workflow_%d' % w
signal = Signal(Signal.DRAIN)
name = Name(workflow=workflow,
signal=Signal.action_to_string(signal.action))
result.append(
Token(name=name.get_signal_token_name(),
version=10000000000 * w,
data=pickle.dumps(signal)))
return result
def test_token_to_string(self):
"""Test conversion of token to string."""
token = Token(version=12345,
name='/some_name',
owner='some_owner',
expirationTime=10,
priority=1.5,
data=pickle.dumps('some_data'))
self.assertEqual(
'Token(version=12345, owner=some_owner, '
'expirationTime=1970-01-01 00:00:10 UTC, '
'priority=1.500000, name=/some_name, data=some_data)',
token_to_str(token))
def get_new_event_tokens(self):
"""Export new event tokens.
Returns:
The list of new event tokens after all transformations performed by
the analyzer.
"""
result = []
for event_name, event in self._new_events.items():
data = pickle.dumps(event)
token = Token(name=event_name, data=data)
result.append(token)
return result
def _move_job_token_to_waiting(self, job, succeeded):
"""Move the owned job token to the waiting group.
If the job succeeded, also post events to job outputs. If the job
failed or it is the final job (a job with no outputs), post an archive
signal to finish the workflow.
Args:
job: The job that should be stored in the data field of the waiting
job token.
succeeded: True if the job succeeded, otherwise False.
"""
assert self._owned_job_token
name = Name.from_job_token_name(self._owned_job_token.name)
name.job_state = Name.WAITING_STATE
waiting_job_token = Token(name=name.get_job_token_name(),
priority=self._owned_job_token.priority,
data=pickle.dumps(job))
request = ModifyRequest(deletes=[self._owned_job_token],
updates=[waiting_job_token])
if succeeded:
request.updates.extend(self._get_output_event_tokens(job))
if not job.outputs or not succeeded:
# This is either the only job in the workflow with no outputs or a
# failed job. In either case, the workflow is done.
signaller = Signaller(self._client,
workflow=name.workflow,
instance=name.instance)
if not signaller.is_action_set(Signal.ARCHIVE):
signal_name = Name(
workflow=name.workflow,
instance=name.instance,
signal=Signal.action_to_string(Signal.ARCHIVE))
signal = Signal(Signal.ARCHIVE)
signal_token = Token(name=signal_name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
self._send_request(request)
def _move_job_token_to_waiting(self, job, succeeded):
"""Move the owned job token to the waiting group.
If the job succeeded, also post events to job outputs. If the job
failed or it is the final job (a job with no outputs), post an archive
signal to finish the workflow.
Args:
job: The job that should be stored in the data field of the waiting
job token.
succeeded: True if the job succeeded, otherwise False.
"""
assert self._owned_job_token
name = Name.from_job_token_name(self._owned_job_token.name)
name.job_state = Name.WAITING_STATE
waiting_job_token = Token(name=name.get_job_token_name(),
priority=self._owned_job_token.priority,
data=pickle.dumps(job))
request = ModifyRequest(deletes=[self._owned_job_token],
updates=[waiting_job_token])
if succeeded:
request.updates.extend(self._get_output_event_tokens(job))
if not job.outputs or not succeeded:
# This is either the only job in the workflow with no outputs or a
# failed job. In either case, the workflow is done.
signaller = Signaller(self._client,
workflow=name.workflow,
instance=name.instance)
if not signaller.is_action_set(Signal.ARCHIVE):
signal_name = Name(workflow=name.workflow,
instance=name.instance,
signal=Signal.action_to_string(
Signal.ARCHIVE))
signal = Signal(Signal.ARCHIVE)
signal_token = Token(name=signal_name.get_signal_token_name())
signal_token.data = pickle.dumps(signal)
request.updates.append(signal_token)
self._send_request(request)
def _generate_schedule_tokens(workflows):
result = []
for w in range(workflows):
next_run_time = time.time() + (365 + w) * 24 * 60 * 60
recurrence = min(365 * 24 * 60 * 60, 60**w)
workflow = 'workflow_%d' % w
schedule = WorkflowSchedule(next_run_time,
recurrence_seconds=recurrence,
overrun_policy=w % 4,
workflow=workflow)
name = Name(workflow=workflow)
result.append(
Token(name=name.get_workflow_schedule_token_name(),
version=100000000 * w,
owner='some_owner',
expirationTime=next_run_time,
data=pickle.dumps(schedule)))
return result
def get_tokens(self):
"""Export all internally stored tokens.
Returns:
The list of tokens after all transformations performed by the
analyzer.
"""
result = []
for job in self._jobs.values():
name = Name(workflow=self._workflow, instance=self._instance,
job_state=Name.WAITING_STATE, job=job.name)
data = pickle.dumps(job)
token = Token(name=name.get_job_token_name(),
priority=self._job_priorities[job.name],
data=data)
result.append(token)
result.extend(self.get_new_event_tokens())
return result
def execute(self, client, store):
output = ''
token = Token(name=self._name,
version=self._version,
owner=self._owner,
expirationTime=self._expiration_time,
priority=self._priority,
data=self._data)
request = ModifyRequest(updates=[token])
response = client.modify(request)
assert len(response.updates) == 1
if token.version is None:
action = 'inserted'
else:
action = 'updated'
output += '%s %s\n' % (action, str(response.updates[0]))
output += 'updated 1 token\n'
return output
def _generate_job_token(workflow, instance, job, executions, max_jobs):
if job == 0:
inputs = [Name.WORKFLOW_START_INPUT]
else:
inputs = ['job_%d' % (job - 1)]
if job == max_jobs - 1:
outputs = []
else:
outputs = ['job_%d' % (job + 1)]
shell_job = ShellJob(name='job_%d' % job,
inputs=inputs,
outputs=outputs,
command='some command %d' % job)
for e in range(0, executions):
start_time = 1000000 * workflow + 10000 * instance + 100 * job + e + 1
end_time = start_time + 10 * e + 1
DIR = '/tmp/pinball/logs'
if not os.path.exists(DIR):
os.makedirs(DIR)
LOG_PATTERN = '%s/%%s.%%d.%%s' % DIR
info_log_file = LOG_PATTERN % (job, start_time, 'info')
with open(info_log_file, 'w') as f:
f.write('some info log of execution %d' % e)
error_log_file = LOG_PATTERN % (job, start_time, 'error')
with open(error_log_file, 'w') as f:
f.write('some error log of execution %d' % e)
record = ExecutionRecord(info='some_command %d some_args %d' % (e, e),
instance='instance_%d' % instance,
start_time=start_time,
end_time=end_time,
exit_code=(workflow + instance + e) % 2,
logs={
'info': info_log_file,
'error': error_log_file
})
shell_job.history.append(record)
name = Name(workflow='workflow_%d' % workflow,
instance='instance_%d' % instance,
job_state=Name.WAITING_STATE,
job='job_%d' % job)
return Token(name=name.get_job_token_name(),
version=1000000 * workflow + 10000 * instance + 100 * job,
priority=job,
data=pickle.dumps(shell_job))
def test_run(self, load_path_mock):
config_parser = mock.Mock()
load_path_mock.return_value = config_parser
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job='some_job')
config_parser.get_workflow_tokens.return_value = [
Token(name=name.get_job_token_name())
]
schedule = WorkflowSchedule(workflow='some_workflow')
store = EphemeralStore()
emailer = Emailer('some_host', '8080')
request = schedule.run(emailer, store)
self.assertEqual(load_path_mock.call_args_list, [
mock.call('pinball_ext.workflow.parser.PyWorkflowParser', {},
'schedule')
])
self.assertEqual(1, len(request.updates))
def test_run(self, load_path_mock):
config_parser = mock.Mock()
def load_path(params):
self.assertEqual([], params.keys())
return config_parser
load_path_mock.return_value = load_path
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job='some_job')
config_parser.get_workflow_tokens.return_value = [
Token(name=name.get_job_token_name())
]
schedule = WorkflowSchedule(workflow='some_workflow')
store = EphemeralStore()
emailer = Emailer('some_host', '8080')
request = schedule.run(emailer, store)
self.assertEqual(1, len(request.updates))
def get_workflow_tokens(self, workflow):
# TODO(pawel): add workflow connectivity check.
job_configs = {}
top_level_job_names = []
job_names = self._repository.get_job_names(workflow)
for job_name in job_names:
job_config = self._repository.get_job(workflow, job_name)
job_configs[job_name] = job_config
if not job_config.parents:
top_level_job_names.append(job_name)
job_config.parents = [Name.WORKFLOW_START_INPUT]
job_outputs = collections.defaultdict(list)
for job_config in job_configs.values():
for parent_job_name in job_config.parents:
job_outputs[parent_job_name].append(job_config.job)
result = []
instance = get_unique_workflow_instance()
# Convert job configs to job tokens.
for job_config in job_configs.values():
token = RepositoryConfigParser._job_config_to_token(
workflow, instance, job_config, job_outputs[job_config.job])
result.append(token)
# Create triggering events for top-level jobs.
for job_name in top_level_job_names:
event = Event(creator='repository_config_parser')
event_name = Name(workflow=workflow,
instance=instance,
job=job_name,
input_name=Name.WORKFLOW_START_INPUT,
event='workflow_start_event')
result.append(
Token(name=event_name.get_event_token_name(),
data=pickle.dumps(event)))
return result
def _move_job_token_to_runnable(self, job_token, triggering_event_tokens):
"""Move a job token to the runnable branch of the token tree.
Token tree is the global, hierarchically structured token namespace.
Args:
job_token: The job token to make runnable.
triggering_event_tokens: The list of events used to trigger the
job. These events will be removed from the master in the same
call to that makes the job token runnable.
Returns:
True on success, otherwise False.
"""
name = Name.from_job_token_name(job_token.name)
name.job_state = Name.RUNNABLE_STATE
job = pickle.loads(job_token.data)
Worker._add_events_to_job(job, triggering_event_tokens)
runnable_job_token = Token(name=name.get_job_token_name(),
priority=job_token.priority,
data=pickle.dumps(job))
request = ModifyRequest(updates=[runnable_job_token],
deletes=triggering_event_tokens + [job_token])
return self._send_request(request)
def test_modify_updates(self):
request = ModifyRequest(updates=[])
n_tokens_before = len(self._trie)
token = copy.copy(self._trie['/some_dir/some_token_0'])
token.data = 'some other data'
request.updates.append(token)
new_token = Token(name='/some_other_dir/some_token', data='some data')
request.updates.append(new_token)
transaction = ModifyTransaction()
transaction.prepare(request)
response = transaction.commit(self._trie, self._get_blessed_version(),
self._store)
self.assertEqual(2, len(response.updates))
self.assertNotEqual(token.version, response.updates[0].version)
self.assertEqual(token.name, response.updates[0].name)
self.assertEqual(token.data, response.updates[0].data)
self.assertLess(0, response.updates[1].version)
self.assertEqual(new_token.name, response.updates[1].name)
self.assertEqual(new_token.data, response.updates[1].data)
n_tokens_after = len(self._trie)
self.assertEqual(n_tokens_before + 1, n_tokens_after)
self._check_version_uniqueness()
def _job_config_to_token(workflow, instance, job_config, job_outputs):
"""Create a job token from a job config.
Args:
workflow: The workflow name.
instance: The workflow instance.
job_config: The job config to create token from.
job_outputs: The names of the job outputs.
Returns:
Job token constructed from the job config.
"""
if job_config.is_condition:
job = RepositoryConfigParser._condition_config_to_condition(
job_config, job_outputs)
else:
job = RepositoryConfigParser._job_config_to_job(
job_config, job_outputs)
name = Name(workflow=workflow,
instance=instance,
job_state=Name.WAITING_STATE,
job=job_config.job)
job_token = Token(name=name.get_job_token_name(),
data=pickle.dumps(job))
return job_token
