def test_format_tron_action_dict_default_executor(self):
action_dict = {
"command": "echo something",
"requires": ["required_action"],
"retries": 2,
"expected_runtime": "30m",
}
branch_dict = {
"docker_image": "my_service:paasta-123abcde",
"git_sha": "aabbcc44",
"desired_state": "start",
"force_bounce": None,
}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "do_something"),
config_dict=action_dict,
branch_dict=branch_dict,
cluster="test-cluster",
)
with mock.patch.object(
action_config,
"get_docker_registry",
return_value="docker-registry.com:400"), mock.patch(
"paasta_tools.utils.load_system_paasta_config",
autospec=True):
result = tron_tools.format_tron_action_dict(action_config)
assert result["executor"] == "mesos"
def spark_action_config(self):
action_dict = {
"name":
"print",
"command":
"spark-submit something",
"aws_credentials_yaml":
"/some/yaml/path",
"executor":
"spark",
"spark_args": {
"spark.eventLog.enabled": "false"
},
"spark_paasta_cluster":
"fake-spark-cluster",
"spark_paasta_pool":
"fake-spark-pool",
"extra_volumes": [{
"containerPath": "/nail/tmp",
"hostPath": "/nail/tmp",
"mode": "RW"
}],
}
return tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("cool_job", "print"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={"docker_image": ""},
)
def test_format_tron_action_dict_default_executor(self):
action_dict = {
"command": "echo something",
"requires": ["required_action"],
"retries": 2,
"expected_runtime": "30m",
}
branch_dict = {
"docker_image": "my_service:paasta-123abcde",
"git_sha": "aabbcc44",
"desired_state": "start",
"force_bounce": None,
}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "do_something"),
config_dict=action_dict,
branch_dict=branch_dict,
cluster="test-cluster",
)
result = tron_tools.format_tron_action_dict(action_config)
assert result == {
"command": "echo something",
"requires": ["required_action"],
"retries": 2,
"expected_runtime": "30m",
}
def test_format_tron_action_dict_default_executor(self):
action_dict = {
'name': 'do_something',
'command': 'echo something',
'requires': ['required_action'],
'retries': 2,
'expected_runtime': '30m',
}
branch_dict = {
'docker_image': 'my_service:paasta-123abcde',
'git_sha': 'aabbcc44',
'desired_state': 'start',
'force_bounce': None,
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
config_dict=action_dict,
branch_dict=branch_dict,
)
result = tron_tools.format_tron_action_dict(action_config,
'{cluster:s}.com')
assert result == {
'name': 'do_something',
'command': 'echo something',
'requires': ['required_action'],
'retries': 2,
'expected_runtime': '30m',
}
def test_get_action_config_load_deployments_false(self, mock_load_deployments):
action_dict = {"command": "echo first"}
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": "my_service",
"deploy_group": "prod",
"max_runtime": "2h",
"actions": {"normal": action_dict},
"monitoring": {"team": "noop"},
}
soa_dir = "/other_dir"
cluster = "paasta-dev"
job_config = tron_tools.TronJobConfig(
"my_job", job_dict, cluster, load_deployments=False, soa_dir=soa_dir
)
mock_load_deployments.side_effect = NoDeploymentsAvailable
action_config = job_config._get_action_config("normal", action_dict)
assert mock_load_deployments.call_count == 0
assert action_config == tron_tools.TronActionConfig(
service="my_service",
cluster=cluster,
instance=tron_tools.compose_instance("my_job", "normal"),
config_dict={
"command": "echo first",
"service": "my_service",
"deploy_group": "prod",
"monitoring": {"team": "noop"},
},
branch_dict=None,
soa_dir=soa_dir,
)
def action_config(self):
action_dict = {"name": "print", "command": "echo something"}
return tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("cool_job", "print"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={"docker_image": "foo:latest"},
)
def test_get_executor_default(self):
action_dict = {"name": "do_something", "command": "echo something"}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "do_something"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_executor() is None
def test_get_action_name(self):
action_dict = {"name": "sleep", "command": "sleep 10"}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "sleep"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_action_name() == "sleep"
def test_get_job_name(self):
action_dict = {"name": "print", "command": "echo something"}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("cool_job", "print"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_job_name() == "cool_job"
def test_get_executor_default(self):
action_dict = {
'name': 'do_something',
'command': 'echo something',
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_executor() is None
def test_get_action_name(self):
action_dict = {
'name': 'sleep',
'command': 'sleep 10',
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'sleep'),
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_action_name() == 'sleep'
def test_get_job_name(self):
action_dict = {
'name': 'print',
'command': 'echo something',
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('cool_job', 'print'),
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_job_name() == 'cool_job'
def test_format_tron_action_dict_paasta_no_branch_dict(self):
action_dict = {
"command": "echo something",
"requires": ["required_action"],
"retries": 2,
"service": "my_service",
"deploy_group": "prod",
"executor": "paasta",
"cpus": 2,
"mem": 1200,
"disk": 42,
"pool": "special_pool",
"env": {"SHELL": "/bin/bash"},
"extra_volumes": [
{"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"}
],
}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "do_something"),
config_dict=action_dict,
branch_dict=None,
cluster="paasta-dev",
)
with mock.patch(
"paasta_tools.utils.InstanceConfig.use_docker_disk_quota",
autospec=True,
return_value=False,
):
result = tron_tools.format_tron_action_dict(action_config)
assert result == {
"command": "echo something",
"requires": ["required_action"],
"retries": 2,
"docker_image": "",
"executor": "mesos",
"cpus": 2,
"mem": 1200,
"disk": 42,
"env": mock.ANY,
"extra_volumes": [
{"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"}
],
"docker_parameters": mock.ANY,
"constraints": [
{"attribute": "pool", "operator": "LIKE", "value": "special_pool"}
],
}
assert result["env"]["SHELL"] == "/bin/bash"
assert isinstance(result["docker_parameters"], list)
def action_config(self):
action_dict = {
"name": "print",
"command": "spark-submit something",
"aws_credentials_yaml": "/some/yaml/path",
}
return tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("cool_job", "print"),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={"docker_image": "foo:latest"},
)
def test_get_cluster(self):
action_dict = {
'name': 'do_something',
'command': 'echo something',
'cluster': 'dev-oregon',
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_cluster() == 'dev-oregon'
def test_format_tron_action_dict_paasta_no_branch_dict(self):
action_dict = {
'command': 'echo something',
'requires': ['required_action'],
'retries': 2,
'service': 'my_service',
'deploy_group': 'prod',
'executor': 'paasta',
'cpus': 2,
'mem': 1200,
'pool': 'special_pool',
'env': {'SHELL': '/bin/bash'},
'extra_volumes': [
{'containerPath': '/nail/tmp', 'hostPath': '/nail/tmp', 'mode': 'RW'},
],
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
config_dict=action_dict,
branch_dict=None,
cluster="paasta-dev",
)
result = tron_tools.format_tron_action_dict(action_config)
assert result == {
'command': 'echo something',
'requires': ['required_action'],
'retries': 2,
'docker_image': '',
'executor': 'mesos',
'cpus': 2,
'mem': 1200,
'env': mock.ANY,
'extra_volumes': [{
'container_path': '/nail/tmp',
'host_path': '/nail/tmp',
'mode': 'RW',
}],
'docker_parameters': mock.ANY,
'constraints': [
{
'attribute': 'pool',
'operator': 'LIKE',
'value': 'special_pool',
},
],
}
assert result['env']['SHELL'] == '/bin/bash'
assert isinstance(result['docker_parameters'], list)
def test_get_executor_paasta(self):
action_dict = {
'name': 'do_something',
'command': 'echo something',
'executor': 'paasta',
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
cluster="fake-cluster",
config_dict=action_dict,
branch_dict={},
)
assert action_config.get_executor() == 'mesos'
def test_get_action_config_load_deployments_false(
self,
mock_load_deployments,
):
action_dict = {
'command': 'echo first',
}
job_dict = {
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': 'my_service',
'deploy_group': 'prod',
'max_runtime': '2h',
'actions': {
'normal': action_dict
},
'monitoring': {
'team': 'noop'
},
}
soa_dir = '/other_dir'
cluster = 'paasta-dev'
job_config = tron_tools.TronJobConfig(
'my_job',
job_dict,
cluster,
load_deployments=False,
soa_dir=soa_dir,
)
mock_load_deployments.side_effect = NoDeploymentsAvailable
action_config = job_config._get_action_config('normal', action_dict)
assert mock_load_deployments.call_count == 0
assert action_config == tron_tools.TronActionConfig(
service='my_service',
cluster=cluster,
instance=tron_tools.compose_instance('my_job', 'normal'),
config_dict={
'command': 'echo first',
'service': 'my_service',
'deploy_group': 'prod',
'monitoring': {
'team': 'noop'
},
},
branch_dict=None,
soa_dir=soa_dir,
)
def test_format_tron_action_dict_paasta(self):
action_dict = {
'name':
'do_something',
'command':
'echo something',
'requires': ['required_action'],
'retries':
2,
'cluster':
'paasta-dev',
'service':
'my_service',
'deploy_group':
'prod',
'executor':
'paasta',
'cpus':
2,
'mem':
1200,
'pool':
'special_pool',
'env': {
'SHELL': '/bin/bash'
},
'extra_volumes': [
{
'containerPath': '/nail/tmp',
'hostPath': '/nail/tmp',
'mode': 'RW'
},
],
}
branch_dict = {
'docker_image': 'my_service:paasta-123abcde',
'git_sha': 'aabbcc44',
'desired_state': 'start',
'force_bounce': None,
}
action_config = tron_tools.TronActionConfig(
service='my_service',
instance=tron_tools.compose_instance('my_job', 'do_something'),
config_dict=action_dict,
branch_dict=branch_dict,
)
with mock.patch.object(
action_config,
'get_docker_registry',
return_value='docker-registry.com:400',
):
result = tron_tools.format_tron_action_dict(
action_config, '{cluster:s}.com')
assert result == {
'name':
'do_something',
'command':
'echo something',
'requires': ['required_action'],
'retries':
2,
'mesos_address':
'paasta-dev.com',
'docker_image':
mock.ANY,
'executor':
'mesos',
'cpus':
2,
'mem':
1200,
'env':
mock.ANY,
'extra_volumes': [{
'container_path': '/nail/tmp',
'host_path': '/nail/tmp',
'mode': 'RW',
}],
'docker_parameters':
mock.ANY,
'constraints': [
{
'attribute': 'pool',
'operator': 'LIKE',
'value': 'special_pool',
},
],
}
expected_docker = '{}/{}'.format('docker-registry.com:400',
branch_dict['docker_image'])
assert result['docker_image'] == expected_docker
assert result['env']['SHELL'] == '/bin/bash'
assert isinstance(result['docker_parameters'], list)
def test_get_action_config(
self,
mock_load_deployments,
action_service,
action_deploy,
action_cluster,
):
"""Check resulting action config with various overrides from the action."""
action_dict = {
'name': 'normal',
'command': 'echo first',
}
if action_service:
action_dict['service'] = action_service
if action_deploy:
action_dict['deploy_group'] = action_deploy
if action_cluster:
action_dict['cluster'] = action_cluster
job_service = 'my_service'
job_deploy = 'prod'
default_cluster = 'paasta-dev'
expected_service = action_service or job_service
expected_deploy = action_deploy or job_deploy
expected_cluster = action_cluster or default_cluster
job_dict = {
'name': 'my_job',
'node': 'batch_server',
'schedule': 'daily 12:10:00',
'service': job_service,
'deploy_group': job_deploy,
'max_runtime': '2h',
'actions': [action_dict],
}
soa_dir = '/other_dir'
job_config = tron_tools.TronJobConfig(job_dict, soa_dir=soa_dir)
action_config = job_config._get_action_config(action_dict,
default_cluster)
mock_load_deployments.assert_called_once_with(expected_service,
soa_dir)
mock_deployments_json = mock_load_deployments.return_value
mock_deployments_json.get_docker_image_for_deploy_group.assert_called_once_with(
expected_deploy)
mock_deployments_json.get_git_sha_for_deploy_group.assert_called_once_with(
expected_deploy)
expected_branch_dict = {
'docker_image':
mock_deployments_json.get_docker_image_for_deploy_group.
return_value,
'git_sha':
mock_deployments_json.get_git_sha_for_deploy_group.return_value,
'desired_state':
'start',
'force_bounce':
None,
}
assert action_config == tron_tools.TronActionConfig(
service=expected_service,
instance=tron_tools.compose_instance('my_job', 'normal'),
config_dict={
'name': 'normal',
'command': 'echo first',
'cluster': expected_cluster,
'service': expected_service,
'deploy_group': expected_deploy,
},
branch_dict=expected_branch_dict,
soa_dir=soa_dir,
)
def test_format_tron_action_dict_paasta(self):
action_dict = {
"command":
"echo something",
"requires": ["required_action"],
"retries":
2,
"retries_delay":
"5m",
"service":
"my_service",
"deploy_group":
"prod",
"executor":
"paasta",
"cpus":
2,
"mem":
1200,
"disk":
42,
"pool":
"special_pool",
"env": {
"SHELL": "/bin/bash"
},
"extra_volumes": [{
"containerPath": "/nail/tmp",
"hostPath": "/nail/tmp",
"mode": "RW"
}],
"trigger_downstreams":
True,
"triggered_by": ["foo.bar.{shortdate}"],
"trigger_timeout":
"5m",
}
branch_dict = {
"docker_image": "my_service:paasta-123abcde",
"git_sha": "aabbcc44",
"desired_state": "start",
"force_bounce": None,
}
action_config = tron_tools.TronActionConfig(
service="my_service",
instance=tron_tools.compose_instance("my_job", "do_something"),
config_dict=action_dict,
branch_dict=branch_dict,
cluster="test-cluster",
)
with mock.patch.object(
action_config,
"get_docker_registry",
return_value="docker-registry.com:400"), mock.patch(
"paasta_tools.utils.InstanceConfig.use_docker_disk_quota",
autospec=True,
return_value=False,
):
result = tron_tools.format_tron_action_dict(action_config)
assert result == {
"command":
"echo something",
"requires": ["required_action"],
"retries":
2,
"retries_delay":
"5m",
"docker_image":
mock.ANY,
"executor":
"mesos",
"cpus":
2,
"mem":
1200,
"disk":
42,
"env":
mock.ANY,
"extra_volumes": [{
"container_path": "/nail/tmp",
"host_path": "/nail/tmp",
"mode": "RW"
}],
"docker_parameters":
mock.ANY,
"constraints": [{
"attribute": "pool",
"operator": "LIKE",
"value": "special_pool"
}],
"trigger_downstreams":
True,
"triggered_by": ["foo.bar.{shortdate}"],
"trigger_timeout":
"5m",
}
expected_docker = "{}/{}".format("docker-registry.com:400",
branch_dict["docker_image"])
assert result["docker_image"] == expected_docker
assert result["env"]["SHELL"] == "/bin/bash"
assert isinstance(result["docker_parameters"], list)
def test_get_action_config(self, mock_load_deployments, action_service,
action_deploy):
"""Check resulting action config with various overrides from the action."""
action_dict = {"command": "echo first"}
if action_service:
action_dict["service"] = action_service
if action_deploy:
action_dict["deploy_group"] = action_deploy
job_service = "my_service"
job_deploy = "prod"
expected_service = action_service or job_service
expected_deploy = action_deploy or job_deploy
expected_cluster = "paasta-dev"
job_dict = {
"node": "batch_server",
"schedule": "daily 12:10:00",
"service": job_service,
"deploy_group": job_deploy,
"max_runtime": "2h",
"actions": {
"normal": action_dict
},
"monitoring": {
"team": "noop"
},
}
soa_dir = "/other_dir"
job_config = tron_tools.TronJobConfig("my_job",
job_dict,
expected_cluster,
soa_dir=soa_dir)
action_config = job_config._get_action_config("normal",
action_dict=action_dict)
mock_load_deployments.assert_called_once_with(expected_service,
soa_dir)
mock_deployments_json = mock_load_deployments.return_value
mock_deployments_json.get_docker_image_for_deploy_group.assert_called_once_with(
expected_deploy)
mock_deployments_json.get_git_sha_for_deploy_group.assert_called_once_with(
expected_deploy)
expected_branch_dict = {
"docker_image":
mock_deployments_json.get_docker_image_for_deploy_group.
return_value,
"git_sha":
mock_deployments_json.get_git_sha_for_deploy_group.return_value,
"desired_state":
"start",
"force_bounce":
None,
}
expected_input_action_config = {
"command": "echo first",
"service": expected_service,
"deploy_group": expected_deploy,
"monitoring": {
"team": "noop"
},
}
assert action_config == tron_tools.TronActionConfig(
service=expected_service,
instance=tron_tools.compose_instance("my_job", "normal"),
config_dict=expected_input_action_config,
branch_dict=expected_branch_dict,
soa_dir=soa_dir,
cluster=expected_cluster,
)