def make_tron_config(
action_runner=None,
output_stream_dir='/tmp',
command_context=None,
ssh_options=None,
time_zone=pytz.timezone("EST"),
state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE,
nodes=None,
node_pools=None,
jobs=None,
mesos_options=None,
):
return schema.TronConfig(
action_runner=action_runner or FrozenDict(),
output_stream_dir=output_stream_dir,
command_context=command_context
or FrozenDict(batch_dir='/tron/batch/test/foo',
python='/usr/bin/python'),
ssh_options=ssh_options or make_ssh_options(),
time_zone=time_zone,
state_persistence=state_persistence,
nodes=nodes or make_nodes(),
node_pools=node_pools or make_node_pools(),
jobs=jobs or make_master_jobs(),
mesos_options=mesos_options or make_mesos_options(),
)
def test_attributes(self):
expected = make_named_tron_config(jobs=FrozenDict({
'test_job':
make_job(
name="test_job",
namespace='test_namespace',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None,
),
expected_runtime=datetime.timedelta(1),
)
}))
test_config = validate_fragment(
'test_namespace',
dict(jobs=[
dict(
name="test_job",
namespace='test_namespace',
node="node0",
schedule="interval 20s",
actions=[dict(name="action", command="command")],
cleanup_action=dict(command="command"),
)
]))
assert_equal(test_config, expected)
def make_job(**kwargs):
kwargs.setdefault('namespace', 'MASTER')
kwargs.setdefault('name', f"{kwargs['namespace']}.job_name")
kwargs.setdefault('node', 'node0')
kwargs.setdefault('enabled', True)
kwargs.setdefault('monitoring', {})
kwargs.setdefault(
'schedule',
schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
))
kwargs.setdefault('actions', FrozenDict({'action': make_action()}))
kwargs.setdefault('queueing', True)
kwargs.setdefault('run_limit', 50)
kwargs.setdefault('all_nodes', False)
kwargs.setdefault('cleanup_action', make_cleanup_action())
kwargs.setdefault('max_runtime')
kwargs.setdefault('allow_overlap', False)
kwargs.setdefault('time_zone', None)
kwargs.setdefault('expected_runtime', datetime.timedelta(0, 3600))
return schema.ConfigJob(**kwargs)
def make_node_pools():
return FrozenDict({
'NodePool':
schema.ConfigNodePool(
nodes=('node0', 'node1'),
name='NodePool',
),
})
def validator(value, config_context):
if isinstance(value, dict):
value = [{'name': name, **config} for name, config in value.items()]
msg = "Duplicate name %%s at %s" % config_context.path
name_dict = UniqueNameDict(msg)
for item in valid(value, config_context):
name_dict[item.name] = item
return FrozenDict(**name_dict)
def make_nodes():
return FrozenDict({
'node0':
schema.ConfigNode(
name='node0',
username='******',
hostname='node0',
port=22,
),
'node1':
schema.ConfigNode(
name='node1',
username='******',
hostname='node1',
port=22,
),
})
def valid_command_context(context, config_context):
# context can be any dict.
return FrozenDict(**valid_dict(context or {}, config_context))
def validator(value, config_context):
msg = "Duplicate name %%s at %s" % config_context.path
name_dict = UniqueNameDict(msg)
for item in valid(value, config_context):
name_dict[item.name] = item
return FrozenDict(**name_dict)
def test_valid_jobs_and_services_success(self):
test_config = BASE_CONFIG + textwrap.dedent("""
jobs:
-
name: "test_job0"
node: node0
schedule: "interval 20s"
actions:
-
name: "action0_0"
command: "test_command0.0"
cleanup_action:
command: "test_command0.1"
services:
-
name: "test_service0"
node: node0
command: "service_command0"
count: 2
pid_file: "/var/run/%(name)s-%(instance_number)s.pid"
monitor_interval: 20
""")
expected_jobs = {
'MASTER.test_job0':
schema.ConfigJob(name='MASTER.test_job0',
namespace='MASTER',
node='node0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None),
actions=FrozenDict({
'action0_0':
schema.ConfigAction(name='action0_0',
command='test_command0.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=schema.ConfigCleanupAction(
command='test_command0.1',
name='cleanup',
node=None),
enabled=True,
allow_overlap=False,
max_runtime=None)
}
expected_services = {
'MASTER.test_service0':
schema.ConfigService(
name='MASTER.test_service0',
namespace='MASTER',
node='node0',
pid_file='/var/run/%(name)s-%(instance_number)s.pid',
command='service_command0',
monitor_interval=20,
monitor_retries=3,
restart_delay=None,
count=2)
}
config = manager.from_string(test_config)
context = config_utils.ConfigContext('config', ['node0'], None,
MASTER_NAMESPACE)
config_parse.validate_jobs_and_services(config, context)
assert_equal(expected_jobs, config['jobs'])
assert_equal(expected_services, config['services'])
def test_attributes(self):
expected = schema.NamedTronConfig(
jobs=FrozenDict(
{
'test_job0':
schema.ConfigJob(name='test_job0',
namespace='test_namespace',
node='node0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None,
),
actions=FrozenDict({
'action0_0':
schema.ConfigAction(
name='action0_0',
command='test_command0.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=schema.ConfigCleanupAction(
name='cleanup',
command='test_command0.1',
node=None),
enabled=True,
max_runtime=None,
allow_overlap=False),
'test_job1':
schema.ConfigJob(
name='test_job1',
namespace='test_namespace',
node='node0',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set([1, 3, 5]),
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions=FrozenDict(
{
'action1_1':
schema.ConfigAction(name='action1_1',
command='test_command1.1',
requires=('action1_0', ),
node=None),
'action1_0':
schema.ConfigAction(
name='action1_0',
command='test_command1.0 %(some_var)s',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=True),
'test_job2':
schema.ConfigJob(
name='test_job2',
namespace='test_namespace',
node='node1',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
),
actions=FrozenDict(
{
'action2_0':
schema.ConfigAction(name='action2_0',
command='test_command2.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'test_job3':
schema.ConfigJob(
name='test_job3',
namespace='test_namespace',
node='node1',
schedule=ConfigConstantScheduler(),
enabled=True,
actions=FrozenDict(
{
'action3_1':
schema.ConfigAction(name='action3_1',
command='test_command3.1',
requires=(),
node=None),
'action3_0':
schema.ConfigAction(name='action3_0',
command='test_command3.0',
requires=(),
node=None),
'action3_2':
schema.ConfigAction(name='action3_2',
command='test_command3.2',
requires=('action3_0',
'action3_1'),
node='node0')
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'test_job4':
schema.ConfigJob(
name='test_job4',
namespace='test_namespace',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=0,
minute=0,
second=0,
original="00:00:00 ",
jitter=None,
),
actions=FrozenDict({
'action4_0':
schema.ConfigAction(name='action4_0',
command='test_command4.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=True,
cleanup_action=None,
enabled=False,
max_runtime=None,
allow_overlap=False)
}),
services=FrozenDict({
'service0':
schema.ConfigService(
namespace='test_namespace',
name='service0',
node='NodePool',
pid_file='/var/run/%(name)s-%(instance_number)s.pid',
command='service_command0',
monitor_interval=20,
monitor_retries=3,
restart_delay=None,
count=2)
}))
test_config = validate_fragment('test_namespace',
yaml.load(self.config))
assert_equal(test_config.jobs['test_job0'], expected.jobs['test_job0'])
assert_equal(test_config.jobs['test_job1'], expected.jobs['test_job1'])
assert_equal(test_config.jobs['test_job2'], expected.jobs['test_job2'])
assert_equal(test_config.jobs['test_job3'], expected.jobs['test_job3'])
assert_equal(test_config.jobs['test_job4'], expected.jobs['test_job4'])
assert_equal(test_config.jobs, expected.jobs)
assert_equal(test_config.services, expected.services)
assert_equal(test_config, expected)
assert_equal(test_config.jobs['test_job4'].enabled, False)
def test_attributes(self):
expected = schema.TronConfig(
action_runner=FrozenDict(),
output_stream_dir='/tmp',
command_context=FrozenDict({
'python': '/usr/bin/python',
'batch_dir': '/tron/batch/test/foo'
}),
ssh_options=schema.ConfigSSHOptions(
agent=False,
identities=('tests/test_id_rsa', ),
known_hosts_file=None,
connect_timeout=30,
idle_connection_timeout=3600,
jitter_min_load=4,
jitter_max_delay=20,
jitter_load_factor=1,
),
notification_options=None,
time_zone=pytz.timezone("EST"),
state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE,
nodes=FrozenDict({
'node0':
schema.ConfigNode(name='node0',
username=os.environ['USER'],
hostname='node0',
port=22),
'node1':
schema.ConfigNode(name='node1',
username=os.environ['USER'],
hostname='node1',
port=22)
}),
node_pools=FrozenDict({
'nodePool':
schema.ConfigNodePool(nodes=('node0', 'node1'),
name='nodePool')
}),
jobs=FrozenDict({
'MASTER.test_job0':
schema.ConfigJob(name='MASTER.test_job0',
namespace='MASTER',
node='node0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None),
actions=FrozenDict({
'action0_0':
schema.ConfigAction(
name='action0_0',
command='test_command0.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=schema.ConfigCleanupAction(
name='cleanup',
command='test_command0.1',
node=None),
enabled=True,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job1':
schema.ConfigJob(
name='MASTER.test_job1',
namespace='MASTER',
node='node0',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set([1, 3, 5]),
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions=FrozenDict({
'action1_1':
schema.ConfigAction(name='action1_1',
command='test_command1.1',
requires=('action1_0', ),
node=None),
'action1_0':
schema.ConfigAction(name='action1_0',
command='test_command1.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=True),
'MASTER.test_job2':
schema.ConfigJob(name='MASTER.test_job2',
namespace='MASTER',
node='node1',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
),
actions=FrozenDict({
'action2_0':
schema.ConfigAction(
name='action2_0',
command='test_command2.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job3':
schema.ConfigJob(
name='MASTER.test_job3',
namespace='MASTER',
node='node1',
schedule=ConfigConstantScheduler(),
enabled=True,
actions=FrozenDict({
'action3_1':
schema.ConfigAction(name='action3_1',
command='test_command3.1',
requires=(),
node=None),
'action3_0':
schema.ConfigAction(name='action3_0',
command='test_command3.0',
requires=(),
node=None),
'action3_2':
schema.ConfigAction(name='action3_2',
command='test_command3.2',
requires=('action3_0',
'action3_1'),
node='node0')
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job4':
schema.ConfigJob(name='MASTER.test_job4',
namespace='MASTER',
node='nodePool',
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=0,
minute=0,
second=0,
original='00:00:00 ',
jitter=None,
),
actions=FrozenDict({
'action4_0':
schema.ConfigAction(
name='action4_0',
command='test_command4.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=True,
cleanup_action=None,
enabled=False,
max_runtime=None,
allow_overlap=False)
}),
services=FrozenDict({
'MASTER.service0':
schema.ConfigService(
name='MASTER.service0',
namespace='MASTER',
node='nodePool',
pid_file='/var/run/%(name)s-%(instance_number)s.pid',
command='service_command0',
monitor_interval=20,
monitor_retries=3,
restart_delay=None,
count=2)
}))
test_config = valid_config_from_yaml(self.config)
assert_equal(test_config.command_context, expected.command_context)
assert_equal(test_config.ssh_options, expected.ssh_options)
assert_equal(test_config.notification_options,
expected.notification_options)
assert_equal(test_config.time_zone, expected.time_zone)
assert_equal(test_config.nodes, expected.nodes)
assert_equal(test_config.node_pools, expected.node_pools)
assert_equal(test_config.jobs['MASTER.test_job0'],
expected.jobs['MASTER.test_job0'])
assert_equal(test_config.jobs['MASTER.test_job1'],
expected.jobs['MASTER.test_job1'])
assert_equal(test_config.jobs['MASTER.test_job2'],
expected.jobs['MASTER.test_job2'])
assert_equal(test_config.jobs['MASTER.test_job3'],
expected.jobs['MASTER.test_job3'])
assert_equal(test_config.jobs['MASTER.test_job4'],
expected.jobs['MASTER.test_job4'])
assert_equal(test_config.jobs, expected.jobs)
assert_equal(test_config.services, expected.services)
assert_equal(test_config, expected)
assert_equal(test_config.jobs['MASTER.test_job4'].enabled, False)
def test_valid_jobs_success(self):
test_config = dict(jobs=[
dict(name="test_job0",
node='node0',
schedule="interval 20s",
expected_runtime="20m",
actions=[
dict(name="action",
command="command",
expected_runtime="20m"),
dict(
name="action_mesos",
command="command",
executor='mesos',
cpus=4,
mem=300,
constraints=[
dict(attribute='pool',
operator='LIKE',
value='default')
],
docker_image='my_container:latest',
docker_parameters=[
dict(key='label', value='labelA'),
dict(key='label', value='labelB')
],
env=dict(USER='******'),
extra_volumes=[
dict(container_path='/tmp',
host_path='/home/tmp',
mode='RO')
],
),
dict(
name="test_trigger_attrs",
command="foo",
triggered_by=["foo.bar"],
trigger_downstreams=True,
),
],
cleanup_action=dict(command="command"))
],
**BASE_CONFIG)
expected_jobs = FrozenDict({
'MASTER.test_job0':
make_job(
name='MASTER.test_job0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None,
),
actions=FrozenDict({
'action':
make_action(expected_runtime=datetime.timedelta(0,
1200), ),
'action_mesos':
make_action(
name='action_mesos',
executor='mesos',
cpus=4.0,
mem=300.0,
constraints=(schema.ConfigConstraint(
attribute='pool',
operator='LIKE',
value='default',
), ),
docker_image='my_container:latest',
docker_parameters=(
schema.ConfigParameter(
key='label',
value='labelA',
),
schema.ConfigParameter(
key='label',
value='labelB',
),
),
env={'USER': '******'},
extra_volumes=(schema.ConfigVolume(
container_path='/tmp',
host_path='/home/tmp',
mode='RO',
), ),
expected_runtime=datetime.timedelta(hours=24),
),
'test_trigger_attrs':
make_action(
name="test_trigger_attrs",
command="foo",
triggered_by=("foo.bar", ),
trigger_downstreams=True,
),
}),
expected_runtime=datetime.timedelta(0, 1200),
),
})
context = config_utils.ConfigContext(
'config',
['node0'],
None,
MASTER_NAMESPACE,
)
config_parse.validate_jobs(test_config, context)
assert_equal(expected_jobs, test_config['jobs'])
def make_command_context():
return FrozenDict({
'python': '/usr/bin/python',
'batch_dir': '/tron/batch/test/foo',
})
def make_master_jobs():
return FrozenDict({
'MASTER.test_job0':
make_job(name='MASTER.test_job0',
schedule=schedule_parse.ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None,
),
expected_runtime=datetime.timedelta(1)),
'MASTER.test_job1':
make_job(
name='MASTER.test_job1',
schedule=schedule_parse.ConfigDailyScheduler(
days={1, 3, 5},
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions=FrozenDict({
'action':
make_action(requires=('action1', ),
expected_runtime=datetime.timedelta(0, 7200)),
'action1':
make_action(name='action1',
expected_runtime=datetime.timedelta(0, 7200)),
}),
time_zone=pytz.timezone("Pacific/Auckland"),
expected_runtime=datetime.timedelta(1),
cleanup_action=None,
allow_overlap=True,
),
'MASTER.test_job2':
make_job(
name='MASTER.test_job2',
node='node1',
actions=FrozenDict({
'action2_0':
make_action(
name='action2_0',
command='test_command2.0',
)
}),
time_zone=pytz.timezone("Pacific/Auckland"),
expected_runtime=datetime.timedelta(1),
cleanup_action=None,
),
'MASTER.test_job_actions_dict':
make_job(
name='MASTER.test_job_actions_dict',
node='node1',
schedule=ConfigConstantScheduler(),
actions=FrozenDict({
'action':
make_action(),
'action1':
make_action(name='action1'),
'action2':
make_action(
name='action2',
requires=('action', 'action1'),
node='node0',
),
}),
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
'MASTER.test_job4':
make_job(
name='MASTER.test_job4',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
original="00:00:00 ",
hour=0,
minute=0,
second=0,
days=set(),
jitter=None,
),
all_nodes=True,
enabled=False,
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
'MASTER.test_job_mesos':
make_job(
name='MASTER.test_job_mesos',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
original="00:00:00 ",
hour=0,
minute=0,
second=0,
days=set(),
jitter=None,
),
actions=FrozenDict({
'action_mesos':
make_action(
name='action_mesos',
command='test_command_mesos',
executor='mesos',
cpus=0.1,
mem=100,
docker_image='container:latest',
),
}),
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
})
