def test_send_job_timeout_warning_message(self, smtplib_mock):
emailer = Emailer('some_host', '8080')
execution_data = JobExecutionData(workflow='some_workflow',
instance='123',
job='some_job',
execution=1,
info='some_info',
start_time=10,
logs={
'stdout': '/some/out',
'stderr': '/some/error'
})
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_job_timeout_warning_message(
['*****@*****.**', '*****@*****.**'],
execution_data)
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('Workflow some_workflow\'s job some_job exceeded '
'timeout' in msg)
self.assertTrue('123' in msg)
self.assertTrue('some_job' in msg)
self.assertTrue('stdout' in msg)
self.assertTrue('stderr' in msg)
def test_send_too_many_running_instances_warning_message(self, smtplib_mock):
emailer = Emailer('some_host', '8080')
workflow = "some_workflow"
number_running_instances = 3
max_running_instances = 3
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_too_many_running_instances_warning_message(
['*****@*****.**', '*****@*****.**'],
workflow,
number_running_instances,
max_running_instances)
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('3' in msg)
self.assertTrue('some_workflow' in msg)
def test_send_job_timeout_warning_message(self, smtplib_mock):
emailer = Emailer('some_host', '8080')
execution_data = JobExecutionData(workflow='some_workflow',
instance='123',
job='some_job',
execution=1,
info='some_info',
start_time=10,
logs={'stdout': '/some/out',
'stderr': '/some/error'})
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_job_timeout_warning_message(
['*****@*****.**', '*****@*****.**'],
execution_data)
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('Workflow some_workflow\'s job some_job exceeded '
'timeout' in msg)
self.assertTrue('123' in msg)
self.assertTrue('some_job' in msg)
self.assertTrue('stdout' in msg)
self.assertTrue('stderr' in msg)
def setUp(self):
self._factory = Factory()
store = EphemeralStore()
self._factory.create_master(store)
emailer = Emailer('some_host', '8080')
self._scheduler = Scheduler(self._factory.get_client(), store, emailer)
self._client = self._factory.get_client()
self._post_schedule_token()
def test_send_instance_end_message(self, smtplib_mock):
emailer = Emailer('some_host', '8080')
instance_data = WorkflowInstanceData(workflow='some_workflow',
instance='123',
status=Status.FAILURE,
start_time=10,
end_time=100)
parent_job_data = JobData('some_workflow', '123', 'parent', 'ShellJob',
False, 'some_command', ['workflow_input'],
['parent_output'],
['*****@*****.**'], 1, 2, 100, 200,
1, Status.SUCCESS, 10, 50)
child_job_data = JobData('some_workflow', '123', 'child', 'ShellJob',
False, 'some_command', ['parent_output'], [],
['*****@*****.**'], 1, 2, 100, 200,
2, Status.FAILURE, 50, 100)
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_instance_end_message(
['*****@*****.**', '*****@*****.**'],
instance_data, [child_job_data, parent_job_data])
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('FAILURE for workflow some_workflow' in msg)
self.assertTrue('123' in msg)
self.assertTrue('parent' in msg)
self.assertTrue('child' in msg)
self.assertTrue('SUCCESS' in msg)
# Make sure that jobs are ordered on the end time.
parent_index = msg.find('parent')
child_index = msg.find('child')
self.assertLess(parent_index, child_index)
def test_send_too_many_running_instances_warning_message(
self, smtplib_mock):
emailer = Emailer('some_host', '8080')
workflow = "some_workflow"
number_running_instances = 3
max_running_instances = 3
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_too_many_running_instances_warning_message(
['*****@*****.**', '*****@*****.**'],
workflow, number_running_instances, max_running_instances)
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('3' in msg)
self.assertTrue('some_workflow' in msg)
def test_run(self, load_path_mock):
config_parser = mock.Mock()
load_path_mock.return_value = config_parser
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job='some_job')
config_parser.get_workflow_tokens.return_value = [
Token(name=name.get_job_token_name())
]
schedule = WorkflowSchedule(workflow='some_workflow')
store = EphemeralStore()
emailer = Emailer('some_host', '8080')
request = schedule.run(emailer, store)
self.assertEqual(load_path_mock.call_args_list, [
mock.call('pinball_ext.workflow.parser.PyWorkflowParser', {},
'schedule')
])
self.assertEqual(1, len(request.updates))
def test_run(self, load_path_mock):
config_parser = mock.Mock()
def load_path(params):
self.assertEqual([], params.keys())
return config_parser
load_path_mock.return_value = load_path
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job='some_job')
config_parser.get_workflow_tokens.return_value = [
Token(name=name.get_job_token_name())
]
schedule = WorkflowSchedule(workflow='some_workflow')
store = EphemeralStore()
emailer = Emailer('some_host', '8080')
request = schedule.run(emailer, store)
self.assertEqual(1, len(request.updates))
def main():
_register_signal_listener()
parser = argparse.ArgumentParser(
description='Start Pinball master and workers.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-m',
'--mode',
dest='mode',
choices=['master', 'scheduler', 'workers', 'ui'],
default='master',
help='execution mode')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME:
master_name(PinballConfig.MASTER_NAME)
_pinball_imports()
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
if options.mode == 'ui':
hostport = '%s:%d' % (socket.gethostname(), PinballConfig.UI_PORT)
cache_thread.start_cache_thread(DbStore())
if not PinballConfig.UI_HOST:
hostport = 'localhost:%d' % PinballConfig.UI_PORT
# Disable reloader to prevent auto refresh on file changes. The
# problem with auto-refresh is that it starts multiple processes. Some
# of those processes will become orphans if we kill the UI in a wrong
# way.
management.call_command('runserver', hostport, interactive=False,
use_reloader=False)
return
factory = Factory(master_hostname=PinballConfig.MASTER_HOST,
master_port=PinballConfig.MASTER_PORT)
threads = []
if options.mode == 'master':
factory.create_master(DbStore())
elif options.mode == 'scheduler':
threads.append(_create_scheduler(factory, emailer))
else:
assert options.mode == 'workers'
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
threads = _create_workers(PinballConfig.WORKERS, factory, emailer)
try:
if options.mode == 'master':
factory.run_master_server()
else:
_wait_for_threads(threads)
except KeyboardInterrupt:
LOG.info('Exiting')
sys.exit()
def test_send_instance_end_message(self, smtplib_mock):
emailer = Emailer('some_host', '8080')
instance_data = WorkflowInstanceData(workflow='some_workflow',
instance='123',
status=Status.FAILURE,
start_time=10,
end_time=100)
parent_job_data = JobData('some_workflow',
'123',
'parent',
'ShellJob',
False,
'some_command',
['workflow_input'],
['parent_output'],
['*****@*****.**'],
1,
2,
100,
200,
1,
Status.SUCCESS,
10,
50)
child_job_data = JobData('some_workflow',
'123',
'child',
'ShellJob',
False,
'some_command',
['parent_output'],
[],
['*****@*****.**'],
1,
2,
100,
200,
2,
Status.FAILURE,
50,
100)
smtp = mock.Mock()
smtplib_mock.SMTP.return_value = smtp
sendmail = mock.Mock()
smtp.sendmail = sendmail
emailer.send_instance_end_message(['*****@*****.**',
'*****@*****.**'],
instance_data, [child_job_data,
parent_job_data])
self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0])
self.assertEqual(
['*****@*****.**', '*****@*****.**'],
sendmail.call_args[0][1])
msg = sendmail.call_args[0][2]
self.assertTrue('FAILURE for workflow some_workflow' in msg)
self.assertTrue('123' in msg)
self.assertTrue('parent' in msg)
self.assertTrue('child' in msg)
self.assertTrue('SUCCESS' in msg)
# Make sure that jobs are ordered on the end time.
parent_index = msg.find('parent')
child_index = msg.find('child')
self.assertLess(parent_index, child_index)
