def test_validate_all_actions(self): job_dict = { 'name': 'my_job', 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'testservice', 'actions': [ { 'name': 'first', 'command': 'echo first', 'cpus': 'bad string', }, { 'name': 'second', 'command': 'echo second', 'mem': 'not a number', }, ], 'cleanup_action': { 'command': 'rm *', 'cpus': 'also bad', }, } job_config = tron_tools.TronJobConfig(job_dict, 'fake-cluster') errors = job_config.validate() assert len(errors) == 3
def test_validate_action_valid_deploy_group(self, mock_pipeline_config): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'deploy_group': 'deploy_group_1', 'monitoring': { 'team': 'noop', 'page': True, }, 'actions': { 'first': { 'command': 'echo first', 'deploy_group': 'deploy_group_2', }, }, } mock_pipeline_config.return_value = [{ 'step': 'deploy_group_1' }, { 'step': 'deploy_group_2' }] job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') errors = job_config.validate() assert len(errors) == 0
def test_load_tron_service_config(self, mock_load_tron_yaml): mock_load_tron_yaml.return_value = { "_template": { "actions": { "action1": {} } }, "job1": { "actions": { "action1": {} } }, } job_configs = tron_tools.load_tron_service_config( service="service", cluster="test-cluster", load_deployments=False, soa_dir="fake", ) assert job_configs == [ tron_tools.TronJobConfig( name="job1", service="service", cluster="test-cluster", config_dict={"actions": { "action1": {} }}, load_deployments=False, soa_dir="fake", ) ] mock_load_tron_yaml.assert_called_once_with(service="service", cluster="test-cluster", soa_dir="fake")
def test_validate_all_actions(self): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "service": "testservice", "actions": { "first": { "command": "echo first", "cpus": "bad string" }, "second": { "command": "echo second", "mem": "not a number" }, }, "cleanup_action": { "command": "rm *", "cpus": "also bad" }, "monitoring": { "team": "noop" }, } job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert len(errors) == 3
def test_load_tron_service_config_jobs_list(self, mock_load_yaml): job_1 = {'name': 'job_1'} config_dict = { 'jobs': [job_1], } mock_load_yaml.return_value = config_dict soa_dir = '/other/services' job_configs, extra_config = tron_tools.load_tron_service_config( service='foo', cluster='dev', load_deployments=True, soa_dir=soa_dir, ) assert extra_config == {} assert job_configs == [ tron_tools.TronJobConfig( name='job_1', config_dict=job_1, cluster='dev', load_deployments=True, service='foo', soa_dir=soa_dir, ), ]
def test_load_tron_service_config(self, mock_load_tron_yaml): mock_load_tron_yaml.return_value = { '_template': {'actions': {'action1': {}}}, 'extra': 'data', 'jobs': { 'job1': { 'actions': {'action1': {}}, }, }, } job_configs, extra_config = tron_tools.load_tron_service_config( service='service', cluster='test-cluster', load_deployments=False, soa_dir='fake', ) assert job_configs == [ tron_tools.TronJobConfig( name='job1', service='service', cluster='test-cluster', config_dict={'actions': {'action1': {}}}, load_deployments=False, soa_dir='fake', ), ] assert extra_config == {'extra': 'data'} # template filtered out mock_load_tron_yaml.assert_called_once_with( service='service', cluster='test-cluster', soa_dir='fake', )
def test_validate_all_actions(self, mock_get_pipeline_deploy_groups): job_dict = { "node": "paasta", "deploy_group": "test", "schedule": "daily 12:10:00", "service": "testservice", "actions": { "first": { "command": "echo first", "cpus": "bad string" }, "second": { "command": "echo second", "mem": "not a number" }, }, "cleanup_action": { "command": "rm *", "cpus": "also bad" }, "monitoring": { "team": "noop" }, } mock_get_pipeline_deploy_groups.return_value = ["test"] job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert len(errors) == 3
def test_create_complete_config( self, mock_yaml_dump, mock_format_job, mock_tron_service_config, mock_tron_system_config, mock_system_config, service, ): job_config = tron_tools.TronJobConfig("my_job", {}, "fake-cluster") mock_tron_service_config.return_value = [job_config] soa_dir = "/testing/services" cluster = "fake-cluster" assert (tron_tools.create_complete_config( service=service, cluster=cluster, soa_dir=soa_dir) == mock_yaml_dump.return_value) mock_tron_service_config.assert_called_once_with(service=service, cluster=cluster, load_deployments=True, soa_dir=soa_dir) mock_format_job.assert_called_once_with(job_config) complete_config = {"jobs": {"my_job": mock_format_job.return_value}} mock_yaml_dump.assert_called_once_with(complete_config, Dumper=mock.ANY, default_flow_style=mock.ANY)
def test_validate_action_valid_deploy_group( self, mock_get_pipeline_deploy_groups): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "deploy_group": "deploy_group_1", "monitoring": { "team": "noop", "page": True }, "actions": { "first": { "command": "echo first", "deploy_group": "deploy_group_2" } }, } mock_get_pipeline_deploy_groups.return_value = [ "deploy_group_1", "deploy_group_2", ] job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert len(errors) == 0
def test_format_tron_job_dict_with_cleanup_action( self, mock_format_action, mock_get_action_config ): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "service": "my_service", "deploy_group": "prod", "max_runtime": "2h", "actions": {"normal": {"command": "echo first"}}, "cleanup_action": {"command": "rm *"}, "monitoring": {"team": "noop"}, } job_config = tron_tools.TronJobConfig("my_job", job_dict, "paasta-dev") result = tron_tools.format_tron_job_dict(job_config) assert mock_get_action_config.call_args_list == [ mock.call(job_config, "normal", job_dict["actions"]["normal"]), mock.call(job_config, "cleanup", job_dict["cleanup_action"]), ] assert mock_format_action.call_count == 2 assert result == { "node": "batch_server", "schedule": "daily 12:10:00", "max_runtime": "2h", "actions": { mock_get_action_config.return_value.get_action_name.return_value: mock_format_action.return_value }, "cleanup_action": mock_format_action.return_value, "monitoring": {"team": "noop"}, }
def test_get_action_config_load_deployments_false(self, mock_load_deployments): action_dict = {"command": "echo first"} job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "service": "my_service", "deploy_group": "prod", "max_runtime": "2h", "actions": {"normal": action_dict}, "monitoring": {"team": "noop"}, } soa_dir = "/other_dir" cluster = "paasta-dev" job_config = tron_tools.TronJobConfig( "my_job", job_dict, cluster, load_deployments=False, soa_dir=soa_dir ) mock_load_deployments.side_effect = NoDeploymentsAvailable action_config = job_config._get_action_config("normal", action_dict) assert mock_load_deployments.call_count == 0 assert action_config == tron_tools.TronActionConfig( service="my_service", cluster=cluster, instance=tron_tools.compose_instance("my_job", "normal"), config_dict={ "command": "echo first", "service": "my_service", "deploy_group": "prod", "monitoring": {"team": "noop"}, }, branch_dict=None, soa_dir=soa_dir, )
def test_load_tron_service_config_interprets_correctly(mock_load_tron_yaml): mock_load_tron_yaml.return_value = { 'extra': 'data', 'jobs': [{ 'job1': { 'actions': [{ 'action1': {} }], }, }], } actual1, actual2 = tron_tools.load_tron_service_config( service='service', cluster='test-cluster', load_deployments=False, soa_dir='fake', ) expected1 = [ tron_tools.TronJobConfig( service='service', cluster='test-cluster', config_dict={'job1': { 'actions': [{ 'action1': {} }] }}, load_deployments=False, soa_dir='fake', ), ] expected2 = {'extra': 'data'} assert actual1 == expected1 assert actual2 == expected2
def test_format_tron_job_dict_with_cleanup_action( self, mock_format_action, mock_get_action_config, ): job_dict = { 'name': 'my_job', 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'my_service', 'deploy_group': 'prod', 'max_runtime': '2h', 'actions': [{ 'name': 'normal', 'command': 'echo first', }], 'cleanup_action': { 'command': 'rm *', }, } job_config = tron_tools.TronJobConfig(job_dict, 'paasta-dev') result = tron_tools.format_tron_job_dict(job_config) assert mock_get_action_config.call_count == 2 assert mock_format_action.call_count == 2 assert result == { 'name': 'my_job', 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'max_runtime': '2h', 'actions': [mock_format_action.return_value], 'cleanup_action': mock_format_action.return_value, }
def test_get_monitoring(self, tronfig_monitoring): job_dict = {'monitoring': tronfig_monitoring} job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake_cluster') assert job_config.get_monitoring() == { 'team': ('tronfig_team' if 'team' in tronfig_monitoring else 'default_team'), }
def test_get_monitoring(self, tronfig_monitoring): job_dict = {"monitoring": tronfig_monitoring} job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake_cluster") assert job_config.get_monitoring() == { "team": ("tronfig_team" if "team" in tronfig_monitoring else "default_team") }
def test_validate_monitoring(self): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "monitoring": {"team": "noop", "page": True}, "actions": {"first": {"command": "echo first"}}, } job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert len(errors) == 0
def test_create_complete_config( self, mock_yaml_dump, mock_format_master_config, mock_format_job, mock_tron_service_config, mock_tron_system_config, mock_system_config, service, ): job_config = tron_tools.TronJobConfig({}) other_config = { 'my_config_value': [1, 2], } mock_format_master_config.return_value = other_config mock_tron_service_config.return_value = ( [job_config], other_config, ) soa_dir = '/testing/services' assert tron_tools.create_complete_config( service, soa_dir) == mock_yaml_dump.return_value mock_tron_service_config.assert_called_once_with( service, mock_tron_system_config.return_value.get_cluster_name.return_value, True, soa_dir, ) if service == MASTER_NAMESPACE: mock_format_master_config.assert_called_once_with( other_config, mock_system_config.return_value.get_volumes.return_value, mock_system_config.return_value.get_dockercfg_location. return_value, ) else: assert mock_format_master_config.call_count == 0 mock_format_job.assert_called_once_with( job_config, mock_system_config.return_value.get_cluster_fqdn_format. return_value, mock_tron_system_config.return_value.get_default_paasta_cluster. return_value, ) complete_config = other_config.copy() complete_config.update({ 'jobs': [mock_format_job.return_value], }) mock_yaml_dump.assert_called_once_with( complete_config, Dumper=mock.ANY, default_flow_style=mock.ANY, )
def test_create_complete_config( self, mock_yaml_dump, mock_format_master_config, mock_format_job, mock_tron_service_config, mock_tron_system_config, mock_system_config, service, ): job_config = tron_tools.TronJobConfig('my_job', {}, 'fake-cluster') other_config = { 'my_config_value': [1, 2], } mock_format_master_config.return_value = other_config mock_tron_service_config.return_value = ( [job_config], other_config, ) soa_dir = '/testing/services' cluster = 'fake-cluster' assert tron_tools.create_complete_config( service=service, cluster=cluster, soa_dir=soa_dir, ) == mock_yaml_dump.return_value mock_tron_service_config.assert_called_once_with( service=service, cluster=cluster, load_deployments=True, soa_dir=soa_dir, ) if service == MASTER_NAMESPACE: mock_format_master_config.assert_called_once_with( other_config, mock_system_config.return_value.get_volumes.return_value, mock_system_config.return_value.get_dockercfg_location. return_value, ) else: assert mock_format_master_config.call_count == 0 mock_format_job.assert_called_once_with(job_config, ) complete_config = other_config.copy() complete_config.update({ 'jobs': { 'my_job': mock_format_job.return_value, }, }) mock_yaml_dump.assert_called_once_with( complete_config, Dumper=mock.ANY, default_flow_style=mock.ANY, )
def test_validate_monitoring_without_team(self): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "monitoring": {"page": True}, "actions": {"first": {"command": "echo first"}}, } job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert errors == [] assert job_config.get_monitoring()["team"] == "default_team"
def test_format_tron_job_dict( self, mock_format_action, mock_get_action_config, action_list, ): action_name = 'normal' action_dict = { 'command': 'echo first', } if action_list: action_dict['name'] = 'normal' actions = [action_dict] else: actions = {action_name: action_dict} job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'my_service', 'deploy_group': 'prod', 'max_runtime': '2h', 'actions': actions, 'expected_runtime': '1h', 'monitoring': { 'team': 'noop' }, } soa_dir = '/other_dir' cluster = 'paasta-dev' job_config = tron_tools.TronJobConfig('my_job', job_dict, cluster, soa_dir=soa_dir) result = tron_tools.format_tron_job_dict(job_config) mock_get_action_config.assert_called_once_with(job_config, action_name, action_dict) mock_format_action.assert_called_once_with( mock_get_action_config.return_value) assert result == { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'max_runtime': '2h', 'actions': { mock_get_action_config.return_value.get_action_name.return_value: mock_format_action.return_value, }, 'expected_runtime': '1h', 'monitoring': { 'team': 'noop' }, }
def test_validate_monitoring_with_invalid_team(self): job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "monitoring": {"team": "invalid_team", "page": True}, "actions": {"first": {"command": "echo first"}}, } job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster") errors = job_config.validate() assert errors == [ "Invalid team name: invalid_team. Do you mean one of these: ['valid_team']" ]
def test_get_action_config_load_deployments_false( self, mock_load_deployments, ): action_dict = { 'command': 'echo first', } job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'my_service', 'deploy_group': 'prod', 'max_runtime': '2h', 'actions': { 'normal': action_dict }, 'monitoring': { 'team': 'noop' }, } soa_dir = '/other_dir' cluster = 'paasta-dev' job_config = tron_tools.TronJobConfig( 'my_job', job_dict, cluster, load_deployments=False, soa_dir=soa_dir, ) mock_load_deployments.side_effect = NoDeploymentsAvailable action_config = job_config._get_action_config('normal', action_dict) assert mock_load_deployments.call_count == 0 assert action_config == tron_tools.TronActionConfig( service='my_service', cluster=cluster, instance=tron_tools.compose_instance('my_job', 'normal'), config_dict={ 'command': 'echo first', 'service': 'my_service', 'deploy_group': 'prod', 'monitoring': { 'team': 'noop' }, }, branch_dict=None, soa_dir=soa_dir, )
def test_validate_monitoring_with_invalid_team(self): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'monitoring': { 'team': 'invalid_team', 'page': True, }, 'actions': { 'first': { 'command': 'echo first', }, }, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') errors = job_config.validate() assert errors == ["Invalid team name: invalid_team. Do you mean one of these: ['valid_team']"]
def test_validate_monitoring_without_team(self): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'monitoring': { 'page': True, }, 'actions': { 'first': { 'command': 'echo first', }, }, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') errors = job_config.validate() assert errors == [] assert job_config.get_monitoring()['team'] == 'default_team'
def test_validate_monitoring(self): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'monitoring': { 'team': 'noop', 'page': True, }, 'actions': { 'first': { 'command': 'echo first', }, }, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') errors = job_config.validate() assert len(errors) == 0
def test_validate_monitoring_without_team(self, mock_teams): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'monitoring': { 'page': True, }, 'actions': { 'first': { 'command': 'echo first', }, }, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') errors = job_config.validate() assert errors == ['Team name is required for monitoring']
def test_format_tron_job_dict_with_cleanup_action( self, mock_format_action, mock_get_action_config, ): job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'my_service', 'deploy_group': 'prod', 'max_runtime': '2h', 'actions': { 'normal': { 'command': 'echo first', }, }, 'cleanup_action': { 'command': 'rm *', }, 'monitoring': { 'team': 'noop' }, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'paasta-dev') result = tron_tools.format_tron_job_dict(job_config) assert mock_get_action_config.call_args_list == [ mock.call(job_config, 'normal', job_dict['actions']['normal']), mock.call(job_config, 'cleanup', job_dict['cleanup_action']), ] assert mock_format_action.call_count == 2 assert result == { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'max_runtime': '2h', 'actions': { mock_get_action_config.return_value.get_action_name.return_value: mock_format_action.return_value, }, 'cleanup_action': mock_format_action.return_value, 'monitoring': { 'team': 'noop' }, }
def test_format_tron_job_dict(self, mock_format_action, mock_get_action_config): action_name = "normal" action_dict = {"command": "echo first"} actions = {action_name: action_dict} job_dict = { "node": "batch_server", "schedule": "daily 12:10:00", "service": "my_service", "deploy_group": "prod", "max_runtime": "2h", "actions": actions, "expected_runtime": "1h", "monitoring": { "team": "noop" }, } soa_dir = "/other_dir" cluster = "paasta-dev" job_config = tron_tools.TronJobConfig("my_job", job_dict, cluster, soa_dir=soa_dir) result = tron_tools.format_tron_job_dict(job_config) mock_get_action_config.assert_called_once_with(job_config, action_name, action_dict) mock_format_action.assert_called_once_with( mock_get_action_config.return_value) assert result == { "node": "batch_server", "schedule": "daily 12:10:00", "max_runtime": "2h", "actions": { mock_get_action_config.return_value.get_action_name.return_value: mock_format_action.return_value }, "expected_runtime": "1h", "monitoring": { "team": "noop" }, }
def test_create_complete_config( self, mock_yaml_dump, mock_format_job, mock_tron_service_config, mock_tron_system_config, mock_system_config, ): job_config = tron_tools.TronJobConfig({}) other_config = { 'my_config_value': [1, 2], } mock_tron_service_config.return_value = ( [job_config], other_config, ) service = 'my_app' soa_dir = '/testing/services' assert tron_tools.create_complete_config( service, soa_dir) == mock_yaml_dump.return_value mock_tron_service_config.assert_called_once_with( service, mock_tron_system_config.return_value.get_cluster_name.return_value, soa_dir, ) mock_format_job.assert_called_once_with( job_config, mock_system_config.return_value.get_cluster_fqdn_format. return_value, mock_tron_system_config.return_value.get_default_paasta_cluster. return_value, ) complete_config = other_config.copy() complete_config.update({ 'jobs': [mock_format_job.return_value], }) mock_yaml_dump.assert_called_once_with( complete_config, Dumper=mock.ANY, default_flow_style=mock.ANY, )
def test_get_action_config_no_deployment( self, mock_load_deployments, ): action_dict = { 'command': 'echo first', } job_dict = { 'node': 'batch_server', 'schedule': 'daily 12:10:00', 'service': 'my_service', 'deploy_group': 'prod', 'max_runtime': '2h', 'actions': {'normal': action_dict}, } job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster') mock_load_deployments.side_effect = NoDeploymentsAvailable with pytest.raises(tron_tools.InvalidTronConfig): job_config._get_action_config('normal', action_dict)