def test_validate_all_actions(self):
job_dict = {
'name':
'my_job',
'node':
'batch_server',
'schedule':
'daily 12:10:00',
'service':
'testservice',
'actions': [
{
'name': 'first',
'command': 'echo first',
'cpus': 'bad string',
},
{
'name': 'second',
'command': 'echo second',
'mem': 'not a number',
},
],
'cleanup_action': {
'command': 'rm *',
'cpus': 'also bad',
},
}
job_config = tron_tools.TronJobConfig(job_dict, 'fake-cluster')
errors = job_config.validate()
assert len(errors) == 3
def test_validate_action_valid_deploy_group(self, mock_pipeline_config):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'deploy_group': 'deploy_group_1',
'monitoring': {
'team': 'noop',
'page': True,
},
'actions': {
'first': {
'command': 'echo first',
'deploy_group': 'deploy_group_2',
},
},
}
mock_pipeline_config.return_value = [{
'step': 'deploy_group_1'
}, {
'step': 'deploy_group_2'
}]
job_config = tron_tools.TronJobConfig('my_job', job_dict,
'fake-cluster')
errors = job_config.validate()
assert len(errors) == 0
def test_load_tron_service_config(self, mock_load_tron_yaml):
mock_load_tron_yaml.return_value = {
"_template": {
"actions": {
"action1": {}
}
},
"job1": {
"actions": {
"action1": {}
}
},
}
job_configs = tron_tools.load_tron_service_config(
service="service",
cluster="test-cluster",
load_deployments=False,
soa_dir="fake",
)
assert job_configs == [
tron_tools.TronJobConfig(
name="job1",
service="service",
cluster="test-cluster",
config_dict={"actions": {
"action1": {}
}},
load_deployments=False,
soa_dir="fake",
)
]
mock_load_tron_yaml.assert_called_once_with(service="service",
cluster="test-cluster",
soa_dir="fake")
def test_validate_all_actions(self):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": "testservice",
"actions": {
"first": {
"command": "echo first",
"cpus": "bad string"
},
"second": {
"command": "echo second",
"mem": "not a number"
},
},
"cleanup_action": {
"command": "rm *",
"cpus": "also bad"
},
"monitoring": {
"team": "noop"
},
}
job_config = tron_tools.TronJobConfig("my_job", job_dict,
"fake-cluster")
errors = job_config.validate()
assert len(errors) == 3
def test_load_tron_service_config_jobs_list(self, mock_load_yaml):
job_1 = {'name': 'job_1'}
config_dict = {
'jobs': [job_1],
}
mock_load_yaml.return_value = config_dict
soa_dir = '/other/services'
job_configs, extra_config = tron_tools.load_tron_service_config(
service='foo',
cluster='dev',
load_deployments=True,
soa_dir=soa_dir,
)
assert extra_config == {}
assert job_configs == [
tron_tools.TronJobConfig(
name='job_1',
config_dict=job_1,
cluster='dev',
load_deployments=True,
service='foo',
soa_dir=soa_dir,
),
]
def test_load_tron_service_config(self, mock_load_tron_yaml):
mock_load_tron_yaml.return_value = {
'_template': {'actions': {'action1': {}}},
'extra': 'data',
'jobs': {
'job1': {
'actions': {'action1': {}},
},
},
}
job_configs, extra_config = tron_tools.load_tron_service_config(
service='service', cluster='test-cluster', load_deployments=False, soa_dir='fake',
)
assert job_configs == [
tron_tools.TronJobConfig(
name='job1',
service='service',
cluster='test-cluster',
config_dict={'actions': {'action1': {}}},
load_deployments=False,
soa_dir='fake',
),
]
assert extra_config == {'extra': 'data'} # template filtered out
mock_load_tron_yaml.assert_called_once_with(
service='service',
cluster='test-cluster',
soa_dir='fake',
)
def test_validate_all_actions(self, mock_get_pipeline_deploy_groups):
job_dict = {
"node": "paasta",
"deploy_group": "test",
"schedule": "daily 12:10:00",
"service": "testservice",
"actions": {
"first": {
"command": "echo first",
"cpus": "bad string"
},
"second": {
"command": "echo second",
"mem": "not a number"
},
},
"cleanup_action": {
"command": "rm *",
"cpus": "also bad"
},
"monitoring": {
"team": "noop"
},
}
mock_get_pipeline_deploy_groups.return_value = ["test"]
job_config = tron_tools.TronJobConfig("my_job", job_dict,
"fake-cluster")
errors = job_config.validate()
assert len(errors) == 3
def test_create_complete_config(
self,
mock_yaml_dump,
mock_format_job,
mock_tron_service_config,
mock_tron_system_config,
mock_system_config,
service,
):
job_config = tron_tools.TronJobConfig("my_job", {}, "fake-cluster")
mock_tron_service_config.return_value = [job_config]
soa_dir = "/testing/services"
cluster = "fake-cluster"
assert (tron_tools.create_complete_config(
service=service, cluster=cluster,
soa_dir=soa_dir) == mock_yaml_dump.return_value)
mock_tron_service_config.assert_called_once_with(service=service,
cluster=cluster,
load_deployments=True,
soa_dir=soa_dir)
mock_format_job.assert_called_once_with(job_config)
complete_config = {"jobs": {"my_job": mock_format_job.return_value}}
mock_yaml_dump.assert_called_once_with(complete_config,
Dumper=mock.ANY,
default_flow_style=mock.ANY)
def test_validate_action_valid_deploy_group(
self, mock_get_pipeline_deploy_groups):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"deploy_group": "deploy_group_1",
"monitoring": {
"team": "noop",
"page": True
},
"actions": {
"first": {
"command": "echo first",
"deploy_group": "deploy_group_2"
}
},
}
mock_get_pipeline_deploy_groups.return_value = [
"deploy_group_1",
"deploy_group_2",
]
job_config = tron_tools.TronJobConfig("my_job", job_dict,
"fake-cluster")
errors = job_config.validate()
assert len(errors) == 0
def test_format_tron_job_dict_with_cleanup_action(
self, mock_format_action, mock_get_action_config
):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": "my_service",
"deploy_group": "prod",
"max_runtime": "2h",
"actions": {"normal": {"command": "echo first"}},
"cleanup_action": {"command": "rm *"},
"monitoring": {"team": "noop"},
}
job_config = tron_tools.TronJobConfig("my_job", job_dict, "paasta-dev")
result = tron_tools.format_tron_job_dict(job_config)
assert mock_get_action_config.call_args_list == [
mock.call(job_config, "normal", job_dict["actions"]["normal"]),
mock.call(job_config, "cleanup", job_dict["cleanup_action"]),
]
assert mock_format_action.call_count == 2
assert result == {
"node": "batch_server",
"schedule": "daily 12:10:00",
"max_runtime": "2h",
"actions": {
mock_get_action_config.return_value.get_action_name.return_value: mock_format_action.return_value
},
"cleanup_action": mock_format_action.return_value,
"monitoring": {"team": "noop"},
}
def test_get_action_config_load_deployments_false(self, mock_load_deployments):
action_dict = {"command": "echo first"}
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": "my_service",
"deploy_group": "prod",
"max_runtime": "2h",
"actions": {"normal": action_dict},
"monitoring": {"team": "noop"},
}
soa_dir = "/other_dir"
cluster = "paasta-dev"
job_config = tron_tools.TronJobConfig(
"my_job", job_dict, cluster, load_deployments=False, soa_dir=soa_dir
)
mock_load_deployments.side_effect = NoDeploymentsAvailable
action_config = job_config._get_action_config("normal", action_dict)
assert mock_load_deployments.call_count == 0
assert action_config == tron_tools.TronActionConfig(
service="my_service",
cluster=cluster,
instance=tron_tools.compose_instance("my_job", "normal"),
config_dict={
"command": "echo first",
"service": "my_service",
"deploy_group": "prod",
"monitoring": {"team": "noop"},
},
branch_dict=None,
soa_dir=soa_dir,
)
def test_load_tron_service_config_interprets_correctly(mock_load_tron_yaml):
mock_load_tron_yaml.return_value = {
'extra': 'data',
'jobs': [{
'job1': {
'actions': [{
'action1': {}
}],
},
}],
}
actual1, actual2 = tron_tools.load_tron_service_config(
service='service',
cluster='test-cluster',
load_deployments=False,
soa_dir='fake',
)
expected1 = [
tron_tools.TronJobConfig(
service='service',
cluster='test-cluster',
config_dict={'job1': {
'actions': [{
'action1': {}
}]
}},
load_deployments=False,
soa_dir='fake',
),
]
expected2 = {'extra': 'data'}
assert actual1 == expected1
assert actual2 == expected2
def test_format_tron_job_dict_with_cleanup_action(
self,
mock_format_action,
mock_get_action_config,
):
job_dict = {
'name': 'my_job',
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': [{
'name': 'normal',
'command': 'echo first',
}],
'cleanup_action': {
'command': 'rm *',
},
}
job_config = tron_tools.TronJobConfig(job_dict, 'paasta-dev')
result = tron_tools.format_tron_job_dict(job_config)
assert mock_get_action_config.call_count == 2
assert mock_format_action.call_count == 2
assert result == {
'name': 'my_job',
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'max_runtime': '2h',
'actions': [mock_format_action.return_value],
'cleanup_action': mock_format_action.return_value,
}
def test_get_monitoring(self, tronfig_monitoring):
job_dict = {'monitoring': tronfig_monitoring}
job_config = tron_tools.TronJobConfig('my_job', job_dict,
'fake_cluster')
assert job_config.get_monitoring() == {
'team': ('tronfig_team'
if 'team' in tronfig_monitoring else 'default_team'),
}
def test_get_monitoring(self, tronfig_monitoring):
job_dict = {"monitoring": tronfig_monitoring}
job_config = tron_tools.TronJobConfig("my_job", job_dict,
"fake_cluster")
assert job_config.get_monitoring() == {
"team": ("tronfig_team"
if "team" in tronfig_monitoring else "default_team")
}
def test_validate_monitoring(self):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"monitoring": {"team": "noop", "page": True},
"actions": {"first": {"command": "echo first"}},
}
job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster")
errors = job_config.validate()
assert len(errors) == 0
def test_create_complete_config(
self,
mock_yaml_dump,
mock_format_master_config,
mock_format_job,
mock_tron_service_config,
mock_tron_system_config,
mock_system_config,
service,
):
job_config = tron_tools.TronJobConfig({})
other_config = {
'my_config_value': [1, 2],
}
mock_format_master_config.return_value = other_config
mock_tron_service_config.return_value = (
[job_config],
other_config,
)
soa_dir = '/testing/services'
assert tron_tools.create_complete_config(
service, soa_dir) == mock_yaml_dump.return_value
mock_tron_service_config.assert_called_once_with(
service,
mock_tron_system_config.return_value.get_cluster_name.return_value,
True,
soa_dir,
)
if service == MASTER_NAMESPACE:
mock_format_master_config.assert_called_once_with(
other_config,
mock_system_config.return_value.get_volumes.return_value,
mock_system_config.return_value.get_dockercfg_location.
return_value,
)
else:
assert mock_format_master_config.call_count == 0
mock_format_job.assert_called_once_with(
job_config,
mock_system_config.return_value.get_cluster_fqdn_format.
return_value,
mock_tron_system_config.return_value.get_default_paasta_cluster.
return_value,
)
complete_config = other_config.copy()
complete_config.update({
'jobs': [mock_format_job.return_value],
})
mock_yaml_dump.assert_called_once_with(
complete_config,
Dumper=mock.ANY,
default_flow_style=mock.ANY,
)
def test_create_complete_config(
self,
mock_yaml_dump,
mock_format_master_config,
mock_format_job,
mock_tron_service_config,
mock_tron_system_config,
mock_system_config,
service,
):
job_config = tron_tools.TronJobConfig('my_job', {}, 'fake-cluster')
other_config = {
'my_config_value': [1, 2],
}
mock_format_master_config.return_value = other_config
mock_tron_service_config.return_value = (
[job_config],
other_config,
)
soa_dir = '/testing/services'
cluster = 'fake-cluster'
assert tron_tools.create_complete_config(
service=service,
cluster=cluster,
soa_dir=soa_dir,
) == mock_yaml_dump.return_value
mock_tron_service_config.assert_called_once_with(
service=service,
cluster=cluster,
load_deployments=True,
soa_dir=soa_dir,
)
if service == MASTER_NAMESPACE:
mock_format_master_config.assert_called_once_with(
other_config,
mock_system_config.return_value.get_volumes.return_value,
mock_system_config.return_value.get_dockercfg_location.
return_value,
)
else:
assert mock_format_master_config.call_count == 0
mock_format_job.assert_called_once_with(job_config, )
complete_config = other_config.copy()
complete_config.update({
'jobs': {
'my_job': mock_format_job.return_value,
},
})
mock_yaml_dump.assert_called_once_with(
complete_config,
Dumper=mock.ANY,
default_flow_style=mock.ANY,
)
def test_validate_monitoring_without_team(self):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"monitoring": {"page": True},
"actions": {"first": {"command": "echo first"}},
}
job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster")
errors = job_config.validate()
assert errors == []
assert job_config.get_monitoring()["team"] == "default_team"
def test_format_tron_job_dict(
self,
mock_format_action,
mock_get_action_config,
action_list,
):
action_name = 'normal'
action_dict = {
'command': 'echo first',
}
if action_list:
action_dict['name'] = 'normal'
actions = [action_dict]
else:
actions = {action_name: action_dict}
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': actions,
'expected_runtime': '1h',
'monitoring': {
'team': 'noop'
},
}
soa_dir = '/other_dir'
cluster = 'paasta-dev'
job_config = tron_tools.TronJobConfig('my_job',
job_dict,
cluster,
soa_dir=soa_dir)
result = tron_tools.format_tron_job_dict(job_config)
mock_get_action_config.assert_called_once_with(job_config, action_name,
action_dict)
mock_format_action.assert_called_once_with(
mock_get_action_config.return_value)
assert result == {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'max_runtime': '2h',
'actions': {
mock_get_action_config.return_value.get_action_name.return_value:
mock_format_action.return_value,
},
'expected_runtime': '1h',
'monitoring': {
'team': 'noop'
},
}
def test_validate_monitoring_with_invalid_team(self):
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"monitoring": {"team": "invalid_team", "page": True},
"actions": {"first": {"command": "echo first"}},
}
job_config = tron_tools.TronJobConfig("my_job", job_dict, "fake-cluster")
errors = job_config.validate()
assert errors == [
"Invalid team name: invalid_team. Do you mean one of these: ['valid_team']"
]
def test_get_action_config_load_deployments_false(
self,
mock_load_deployments,
):
action_dict = {
'command': 'echo first',
}
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': {
'normal': action_dict
},
'monitoring': {
'team': 'noop'
},
}
soa_dir = '/other_dir'
cluster = 'paasta-dev'
job_config = tron_tools.TronJobConfig(
'my_job',
job_dict,
cluster,
load_deployments=False,
soa_dir=soa_dir,
)
mock_load_deployments.side_effect = NoDeploymentsAvailable
action_config = job_config._get_action_config('normal', action_dict)
assert mock_load_deployments.call_count == 0
assert action_config == tron_tools.TronActionConfig(
service='my_service',
cluster=cluster,
instance=tron_tools.compose_instance('my_job', 'normal'),
config_dict={
'command': 'echo first',
'service': 'my_service',
'deploy_group': 'prod',
'monitoring': {
'team': 'noop'
},
},
branch_dict=None,
soa_dir=soa_dir,
)
def test_validate_monitoring_with_invalid_team(self):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'monitoring': {
'team': 'invalid_team',
'page': True,
},
'actions': {
'first': {
'command': 'echo first',
},
},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster')
errors = job_config.validate()
assert errors == ["Invalid team name: invalid_team. Do you mean one of these: ['valid_team']"]
def test_validate_monitoring_without_team(self):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'monitoring': {
'page': True,
},
'actions': {
'first': {
'command': 'echo first',
},
},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster')
errors = job_config.validate()
assert errors == []
assert job_config.get_monitoring()['team'] == 'default_team'
def test_validate_monitoring(self):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'monitoring': {
'team': 'noop',
'page': True,
},
'actions': {
'first': {
'command': 'echo first',
},
},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster')
errors = job_config.validate()
assert len(errors) == 0
def test_validate_monitoring_without_team(self, mock_teams):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'monitoring': {
'page': True,
},
'actions': {
'first': {
'command': 'echo first',
},
},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict,
'fake-cluster')
errors = job_config.validate()
assert errors == ['Team name is required for monitoring']
def test_format_tron_job_dict_with_cleanup_action(
self,
mock_format_action,
mock_get_action_config,
):
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': {
'normal': {
'command': 'echo first',
},
},
'cleanup_action': {
'command': 'rm *',
},
'monitoring': {
'team': 'noop'
},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict, 'paasta-dev')
result = tron_tools.format_tron_job_dict(job_config)
assert mock_get_action_config.call_args_list == [
mock.call(job_config, 'normal', job_dict['actions']['normal']),
mock.call(job_config, 'cleanup', job_dict['cleanup_action']),
]
assert mock_format_action.call_count == 2
assert result == {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'max_runtime': '2h',
'actions': {
mock_get_action_config.return_value.get_action_name.return_value:
mock_format_action.return_value,
},
'cleanup_action': mock_format_action.return_value,
'monitoring': {
'team': 'noop'
},
}
def test_format_tron_job_dict(self, mock_format_action,
mock_get_action_config):
action_name = "normal"
action_dict = {"command": "echo first"}
actions = {action_name: action_dict}
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": "my_service",
"deploy_group": "prod",
"max_runtime": "2h",
"actions": actions,
"expected_runtime": "1h",
"monitoring": {
"team": "noop"
},
}
soa_dir = "/other_dir"
cluster = "paasta-dev"
job_config = tron_tools.TronJobConfig("my_job",
job_dict,
cluster,
soa_dir=soa_dir)
result = tron_tools.format_tron_job_dict(job_config)
mock_get_action_config.assert_called_once_with(job_config, action_name,
action_dict)
mock_format_action.assert_called_once_with(
mock_get_action_config.return_value)
assert result == {
"node": "batch_server",
"schedule": "daily 12:10:00",
"max_runtime": "2h",
"actions": {
mock_get_action_config.return_value.get_action_name.return_value:
mock_format_action.return_value
},
"expected_runtime": "1h",
"monitoring": {
"team": "noop"
},
}
def test_create_complete_config(
self,
mock_yaml_dump,
mock_format_job,
mock_tron_service_config,
mock_tron_system_config,
mock_system_config,
):
job_config = tron_tools.TronJobConfig({})
other_config = {
'my_config_value': [1, 2],
}
mock_tron_service_config.return_value = (
[job_config],
other_config,
)
service = 'my_app'
soa_dir = '/testing/services'
assert tron_tools.create_complete_config(
service, soa_dir) == mock_yaml_dump.return_value
mock_tron_service_config.assert_called_once_with(
service,
mock_tron_system_config.return_value.get_cluster_name.return_value,
soa_dir,
)
mock_format_job.assert_called_once_with(
job_config,
mock_system_config.return_value.get_cluster_fqdn_format.
return_value,
mock_tron_system_config.return_value.get_default_paasta_cluster.
return_value,
)
complete_config = other_config.copy()
complete_config.update({
'jobs': [mock_format_job.return_value],
})
mock_yaml_dump.assert_called_once_with(
complete_config,
Dumper=mock.ANY,
default_flow_style=mock.ANY,
)
def test_get_action_config_no_deployment(
self,
mock_load_deployments,
):
action_dict = {
'command': 'echo first',
}
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': {'normal': action_dict},
}
job_config = tron_tools.TronJobConfig('my_job', job_dict, 'fake-cluster')
mock_load_deployments.side_effect = NoDeploymentsAvailable
with pytest.raises(tron_tools.InvalidTronConfig):
job_config._get_action_config('normal', action_dict)
