def test_send_job_timeout_warning_message(self, smtplib_mock): emailer = Emailer('some_host', '8080') execution_data = JobExecutionData(workflow='some_workflow', instance='123', job='some_job', execution=1, info='some_info', start_time=10, logs={ 'stdout': '/some/out', 'stderr': '/some/error' }) smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_job_timeout_warning_message( ['*****@*****.**', '*****@*****.**'], execution_data) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('Workflow some_workflow\'s job some_job exceeded ' 'timeout' in msg) self.assertTrue('123' in msg) self.assertTrue('some_job' in msg) self.assertTrue('stdout' in msg) self.assertTrue('stderr' in msg)
def test_send_too_many_running_instances_warning_message(self, smtplib_mock): emailer = Emailer('some_host', '8080') workflow = "some_workflow" number_running_instances = 3 max_running_instances = 3 smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_too_many_running_instances_warning_message( ['*****@*****.**', '*****@*****.**'], workflow, number_running_instances, max_running_instances) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('3' in msg) self.assertTrue('some_workflow' in msg)
def test_send_job_timeout_warning_message(self, smtplib_mock): emailer = Emailer('some_host', '8080') execution_data = JobExecutionData(workflow='some_workflow', instance='123', job='some_job', execution=1, info='some_info', start_time=10, logs={'stdout': '/some/out', 'stderr': '/some/error'}) smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_job_timeout_warning_message( ['*****@*****.**', '*****@*****.**'], execution_data) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('Workflow some_workflow\'s job some_job exceeded ' 'timeout' in msg) self.assertTrue('123' in msg) self.assertTrue('some_job' in msg) self.assertTrue('stdout' in msg) self.assertTrue('stderr' in msg)
def setUp(self): self._factory = Factory() store = EphemeralStore() self._factory.create_master(store) emailer = Emailer('some_host', '8080') self._scheduler = Scheduler(self._factory.get_client(), store, emailer) self._client = self._factory.get_client() self._post_schedule_token()
def test_send_instance_end_message(self, smtplib_mock): emailer = Emailer('some_host', '8080') instance_data = WorkflowInstanceData(workflow='some_workflow', instance='123', status=Status.FAILURE, start_time=10, end_time=100) parent_job_data = JobData('some_workflow', '123', 'parent', 'ShellJob', False, 'some_command', ['workflow_input'], ['parent_output'], ['*****@*****.**'], 1, 2, 100, 200, 1, Status.SUCCESS, 10, 50) child_job_data = JobData('some_workflow', '123', 'child', 'ShellJob', False, 'some_command', ['parent_output'], [], ['*****@*****.**'], 1, 2, 100, 200, 2, Status.FAILURE, 50, 100) smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_instance_end_message( ['*****@*****.**', '*****@*****.**'], instance_data, [child_job_data, parent_job_data]) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('FAILURE for workflow some_workflow' in msg) self.assertTrue('123' in msg) self.assertTrue('parent' in msg) self.assertTrue('child' in msg) self.assertTrue('SUCCESS' in msg) # Make sure that jobs are ordered on the end time. parent_index = msg.find('parent') child_index = msg.find('child') self.assertLess(parent_index, child_index)
def test_send_too_many_running_instances_warning_message( self, smtplib_mock): emailer = Emailer('some_host', '8080') workflow = "some_workflow" number_running_instances = 3 max_running_instances = 3 smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_too_many_running_instances_warning_message( ['*****@*****.**', '*****@*****.**'], workflow, number_running_instances, max_running_instances) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('3' in msg) self.assertTrue('some_workflow' in msg)
def test_run(self, load_path_mock): config_parser = mock.Mock() load_path_mock.return_value = config_parser name = Name(workflow='some_workflow', instance='123', job_state=Name.WAITING_STATE, job='some_job') config_parser.get_workflow_tokens.return_value = [ Token(name=name.get_job_token_name()) ] schedule = WorkflowSchedule(workflow='some_workflow') store = EphemeralStore() emailer = Emailer('some_host', '8080') request = schedule.run(emailer, store) self.assertEqual(load_path_mock.call_args_list, [ mock.call('pinball_ext.workflow.parser.PyWorkflowParser', {}, 'schedule') ]) self.assertEqual(1, len(request.updates))
def test_run(self, load_path_mock): config_parser = mock.Mock() def load_path(params): self.assertEqual([], params.keys()) return config_parser load_path_mock.return_value = load_path name = Name(workflow='some_workflow', instance='123', job_state=Name.WAITING_STATE, job='some_job') config_parser.get_workflow_tokens.return_value = [ Token(name=name.get_job_token_name()) ] schedule = WorkflowSchedule(workflow='some_workflow') store = EphemeralStore() emailer = Emailer('some_host', '8080') request = schedule.run(emailer, store) self.assertEqual(1, len(request.updates))
def main(): _register_signal_listener() parser = argparse.ArgumentParser( description='Start Pinball master and workers.') parser.add_argument( '-c', '--config_file', dest='config_file', required=True, help='full path to the pinball setting configure file') parser.add_argument( '-m', '--mode', dest='mode', choices=['master', 'scheduler', 'workers', 'ui'], default='master', help='execution mode') options = parser.parse_args(sys.argv[1:]) PinballConfig.parse(options.config_file) if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME: master_name(PinballConfig.MASTER_NAME) _pinball_imports() if PinballConfig.UI_HOST: emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT) else: emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT) if options.mode == 'ui': hostport = '%s:%d' % (socket.gethostname(), PinballConfig.UI_PORT) cache_thread.start_cache_thread(DbStore()) if not PinballConfig.UI_HOST: hostport = 'localhost:%d' % PinballConfig.UI_PORT # Disable reloader to prevent auto refresh on file changes. The # problem with auto-refresh is that it starts multiple processes. Some # of those processes will become orphans if we kill the UI in a wrong # way. management.call_command('runserver', hostport, interactive=False, use_reloader=False) return factory = Factory(master_hostname=PinballConfig.MASTER_HOST, master_port=PinballConfig.MASTER_PORT) threads = [] if options.mode == 'master': factory.create_master(DbStore()) elif options.mode == 'scheduler': threads.append(_create_scheduler(factory, emailer)) else: assert options.mode == 'workers' if PinballConfig.UI_HOST: emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT) else: emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT) threads = _create_workers(PinballConfig.WORKERS, factory, emailer) try: if options.mode == 'master': factory.run_master_server() else: _wait_for_threads(threads) except KeyboardInterrupt: LOG.info('Exiting') sys.exit()
def test_send_instance_end_message(self, smtplib_mock): emailer = Emailer('some_host', '8080') instance_data = WorkflowInstanceData(workflow='some_workflow', instance='123', status=Status.FAILURE, start_time=10, end_time=100) parent_job_data = JobData('some_workflow', '123', 'parent', 'ShellJob', False, 'some_command', ['workflow_input'], ['parent_output'], ['*****@*****.**'], 1, 2, 100, 200, 1, Status.SUCCESS, 10, 50) child_job_data = JobData('some_workflow', '123', 'child', 'ShellJob', False, 'some_command', ['parent_output'], [], ['*****@*****.**'], 1, 2, 100, 200, 2, Status.FAILURE, 50, 100) smtp = mock.Mock() smtplib_mock.SMTP.return_value = smtp sendmail = mock.Mock() smtp.sendmail = sendmail emailer.send_instance_end_message(['*****@*****.**', '*****@*****.**'], instance_data, [child_job_data, parent_job_data]) self.assertEqual(PinballConfig.DEFAULT_EMAIL, sendmail.call_args[0][0]) self.assertEqual( ['*****@*****.**', '*****@*****.**'], sendmail.call_args[0][1]) msg = sendmail.call_args[0][2] self.assertTrue('FAILURE for workflow some_workflow' in msg) self.assertTrue('123' in msg) self.assertTrue('parent' in msg) self.assertTrue('child' in msg) self.assertTrue('SUCCESS' in msg) # Make sure that jobs are ordered on the end time. parent_index = msg.find('parent') child_index = msg.find('child') self.assertLess(parent_index, child_index)