def _simulate(self):
"""Simulate execution of active jobs."""
tokens = self._store.read_tokens()
satisfied_deps = set()
executed_jobs = []
jobs = {}
for token in tokens:
event_name = Name.from_event_token_name(token.name)
if event_name.event:
satisfied_deps.add((event_name.input, event_name.job))
else:
job_name = Name.from_job_token_name(token.name)
if job_name.job:
job = pickle.loads(token.data)
jobs[job.name] = job
dep_counts = collections.defaultdict(int)
while satisfied_deps:
last_satisfied_deps = satisfied_deps
satisfied_deps = set()
for (_, job_name) in last_satisfied_deps:
dep_counts[job_name] += 1
if dep_counts[job_name] == 2:
executed_jobs.append(job_name)
job = jobs[job_name]
for output in job.outputs:
satisfied_deps.add((job_name, output))
return executed_jobs
def _simulate(self):
"""Simulate execution of active jobs."""
tokens = self._store.read_tokens()
satisfied_deps = set()
executed_jobs = []
jobs = {}
for token in tokens:
event_name = Name.from_event_token_name(token.name)
if event_name.event:
satisfied_deps.add((event_name.input, event_name.job))
else:
job_name = Name.from_job_token_name(token.name)
if job_name.job:
job = pickle.loads(token.data)
jobs[job.name] = job
dep_counts = collections.defaultdict(int)
while satisfied_deps:
last_satisfied_deps = satisfied_deps
satisfied_deps = set()
for (_, job_name) in last_satisfied_deps:
dep_counts[job_name] += 1
if dep_counts[job_name] == 2:
executed_jobs.append(job_name)
job = jobs[job_name]
for output in job.outputs:
satisfied_deps.add((job_name, output))
return executed_jobs
def change_instance(self, instance):
"""Move all tokens to a specific instance."""
self._instance = instance
new_events = {}
for event_name, event in self._new_events.items():
name = Name.from_event_token_name(event_name)
name.instance = instance
new_events[name.get_event_token_name()] = event
self._new_events = new_events
def test_event_token_name(self):
NAME = "/workflow/some_workflow/some_instance/input/some_job/" "some_input/some_event"
name = Name.from_event_token_name(NAME)
self.assertEqual("some_workflow", name.workflow)
self.assertEqual("some_instance", name.instance)
self.assertEqual("some_job", name.job)
self.assertEqual("some_input", name.input)
self.assertEqual("some_event", name.event)
self.assertEqual(NAME, name.get_event_token_name())
def change_instance(self, instance):
"""Move all tokens to a specific instance."""
self._instance = instance
new_events = {}
for event_name, event in self._new_events.items():
name = Name.from_event_token_name(event_name)
name.instance = instance
new_events[name.get_event_token_name()] = event
self._new_events = new_events
def test_get_workflow_tokens(self, repository_mock):
repository = mock.Mock()
repository_mock.return_value = repository
repository.get_job_names.return_value = ['some_job']
job_config = JobConfig()
job_config.workflow = 'some_workflow'
job_config.job = 'some_job'
job_config.is_condition = False
job_config.template = ('tests.pinball.parser.'
'repository_config_parser_test.SomeJobTemplate')
job_config.template_params = {'some_param': 'some_value'}
job_config.parents = []
job_config.emails = [
'*****@*****.**', '*****@*****.**'
]
job_config.max_attempts = 10
job_config.retry_delay_sec = 20
job_config.priority = 100
repository.get_job.return_value = job_config
parser = RepositoryConfigParser()
workflow_tokens = parser.get_workflow_tokens('some_workflow')
self.assertEqual(2, len(workflow_tokens))
# Verify the triggering event token.
if Name.from_event_token_name(workflow_tokens[0].name).workflow:
event_token = workflow_tokens[0]
else:
event_token = workflow_tokens[1]
event_name = Name.from_event_token_name(event_token.name)
self.assertEqual('some_workflow', event_name.workflow)
self.assertEqual('some_job', event_name.job)
self.assertEqual('__WORKFLOW_START__', event_name.input)
event = pickle.loads(event_token.data)
self.assertEqual('repository_config_parser', event.creator)
repository.get_job_names.assert_called_once_with('some_workflow')
repository.get_job.assert_called_once_with('some_workflow', 'some_job')
def test_event_token_name(self):
NAME = ('/workflow/some_workflow/some_instance/input/some_job/'
'some_input/some_event')
name = Name.from_event_token_name(NAME)
self.assertEqual('some_workflow', name.workflow)
self.assertEqual('some_instance', name.instance)
self.assertEqual('some_job', name.job)
self.assertEqual('some_input', name.input)
self.assertEqual('some_event', name.event)
self.assertEqual(NAME, name.get_event_token_name())
def test_get_workflow_tokens(self, repository_mock):
repository = mock.Mock()
repository_mock.return_value = repository
repository.get_job_names.return_value = ['some_job']
job_config = JobConfig()
job_config.workflow = 'some_workflow'
job_config.job = 'some_job'
job_config.is_condition = False
job_config.template = ('tests.pinball.parser.'
'repository_config_parser_test.SomeJobTemplate')
job_config.template_params = {'some_param': 'some_value'}
job_config.parents = []
job_config.emails = ['*****@*****.**',
'*****@*****.**']
job_config.max_attempts = 10
job_config.retry_delay_sec = 20
job_config.priority = 100
repository.get_job.return_value = job_config
parser = RepositoryConfigParser()
workflow_tokens = parser.get_workflow_tokens('some_workflow')
self.assertEqual(2, len(workflow_tokens))
# Verify the triggering event token.
if Name.from_event_token_name(workflow_tokens[0].name).workflow:
event_token = workflow_tokens[0]
else:
event_token = workflow_tokens[1]
event_name = Name.from_event_token_name(event_token.name)
self.assertEqual('some_workflow', event_name.workflow)
self.assertEqual('some_job', event_name.job)
self.assertEqual('__WORKFLOW_START__', event_name.input)
event = pickle.loads(event_token.data)
self.assertEqual('repository_config_parser', event.creator)
repository.get_job_names.assert_called_once_with('some_workflow')
repository.get_job.assert_called_once_with('some_workflow', 'some_job')
def _filter_event_tokens(self, tokens):
"""Filter out all tokens which are not event tokens.
Args:
tokens: The tokens to filter.
"""
for token in tokens:
name = Name.from_event_token_name(token.name)
if not self._instance and name.instance:
self._instance = name.instance
if name.event:
event = pickle.loads(token.data)
self._existing_events[token.name] = event
def _filter_event_tokens(self, tokens):
"""Filter out all tokens which are not event tokens.
Args:
tokens: The tokens to filter.
"""
for token in tokens:
name = Name.from_event_token_name(token.name)
if not self._instance and name.instance:
self._instance = name.instance
if name.event:
event = pickle.loads(token.data)
self._existing_events[token.name] = event
def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def get_event_names(self, workflow_name, instance, job, input_name):
"""Return names of events under a workflow instance, job, and input."""
request = GroupRequest()
name = Name()
name.workflow = workflow_name
name.instance = instance
name.job = job
name.input = input_name
request.namePrefix = name.get_input_prefix()
request.groupSuffix = Name.DELIMITER
response = self._client.group(request)
events = []
if response.counts:
for event in response.counts.keys():
name = Name.from_event_token_name(event)
events.append(name.event)
return events
def test_get_output_event_tokens(self):
self._post_job_tokens()
self._post_workflow_start_event_token()
self._worker._own_runnable_job_token()
self.assertIsNotNone(self._worker._owned_job_token)
job = pickle.loads(self._worker._owned_job_token.data)
execution_record = ExecutionRecord(start_time=123456,
end_time=1234567,
exit_code=0)
job.history.append(execution_record)
event_tokens = self._worker._get_output_event_tokens(job)
self.assertEqual(1, len(event_tokens))
event_token_name = Name.from_event_token_name(event_tokens[0].name)
expected_prefix = Name(workflow='some_workflow',
instance='12345',
job='child_job',
input_name='parent_job').get_input_prefix()
self.assertEqual(expected_prefix, event_token_name.get_input_prefix())
def test_get_output_event_tokens(self):
self._post_job_tokens()
self._post_workflow_start_event_token()
self._worker._own_runnable_job_token()
self.assertIsNotNone(self._worker._owned_job_token)
job = pickle.loads(self._worker._owned_job_token.data)
execution_record = ExecutionRecord(start_time=123456,
end_time=1234567,
exit_code=0)
job.history.append(execution_record)
event_tokens = self._worker._get_output_event_tokens(job)
self.assertEqual(1, len(event_tokens))
event_token_name = Name.from_event_token_name(event_tokens[0].name)
expected_prefix = Name(workflow='some_workflow',
instance='12345',
job='child_job',
input_name='parent_job').get_input_prefix()
self.assertEqual(expected_prefix, event_token_name.get_input_prefix())
def run(self, emailer, store):
if not self._check_workflow_instances(emailer, self.workflow, store):
LOG.warn('too many instances running for workflow %s', self.workflow)
return None
config_parser = load_path(PinballConfig.PARSER)(self.parser_params)
workflow_tokens = config_parser.get_workflow_tokens(self.workflow)
if not workflow_tokens:
LOG.error('workflow %s not found', self.workflow)
return None
result = ModifyRequest()
result.updates = workflow_tokens
assert result.updates
token = result.updates[0]
name = Name.from_job_token_name(token.name)
if not name.instance:
name = Name.from_event_token_name(token.name)
LOG.info('exporting workflow %s instance %s. Its tokens are under %s',
name.workflow, name.instance, name.get_instance_prefix())
return result
def run(self, emailer, store):
if not self._check_workflow_instances(emailer, self.workflow, store):
LOG.warn('too many instances running for workflow %s',
self.workflow)
return None
config_parser = load_path(PinballConfig.PARSER)(self.parser_params)
workflow_tokens = config_parser.get_workflow_tokens(self.workflow)
if not workflow_tokens:
LOG.error('workflow %s not found', self.workflow)
return None
result = ModifyRequest()
result.updates = workflow_tokens
assert result.updates
token = result.updates[0]
name = Name.from_job_token_name(token.name)
if not name.instance:
name = Name.from_event_token_name(token.name)
LOG.info('exporting workflow %s instance %s. Its tokens are under %s',
name.workflow, name.instance, name.get_instance_prefix())
return result
def _add_events_to_job(job, triggering_event_tokens):
"""Put triggering events inside the job.
Args:
job: The job which should be augmented with the events.
triggering_event_tokens: List of event tokens that triggered the
job.
"""
assert not job.events
for event_token in triggering_event_tokens:
if event_token.data:
event = pickle.loads(event_token.data)
# Optimization to make the job data structure smaller: do not
# append events with no attributes.
if event.attributes:
job.events.append(event)
else:
# This logic is here for backwards compatibility.
# TODO(pawel): remove this logic after the transition to the
# new model has been completed.
name = Name.from_event_token_name(event_token.name)
assert name.input == Name.WORKFLOW_START_INPUT
def _add_events_to_job(job, triggering_event_tokens):
"""Put triggering events inside the job.
Args:
job: The job which should be augmented with the events.
triggering_event_tokens: List of event tokens that triggered the
job.
"""
assert not job.events
for event_token in triggering_event_tokens:
if event_token.data:
event = pickle.loads(event_token.data)
# Optimization to make the job data structure smaller: do not
# append events with no attributes.
if event.attributes:
job.events.append(event)
else:
# This logic is here for backwards compatibility.
# TODO(pawel): remove this logic after the transition to the
# new model has been completed.
name = Name.from_event_token_name(event_token.name)
assert name.input == Name.WORKFLOW_START_INPUT
