def make_job(**kwargs):
kwargs.setdefault('namespace', 'MASTER')
kwargs.setdefault('name', f"{kwargs['namespace']}.job_name")
kwargs.setdefault('node', 'node0')
kwargs.setdefault('enabled', True)
kwargs.setdefault('monitoring', {})
kwargs.setdefault(
'schedule',
schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
))
kwargs.setdefault('actions', {'action': make_action()})
kwargs.setdefault('queueing', True)
kwargs.setdefault('run_limit', 50)
kwargs.setdefault('all_nodes', False)
kwargs.setdefault('cleanup_action', make_cleanup_action())
kwargs.setdefault('max_runtime')
kwargs.setdefault('allow_overlap', False)
kwargs.setdefault('time_zone', None)
kwargs.setdefault('expected_runtime', datetime.timedelta(0, 3600))
return schema.ConfigJob(**kwargs)
def make_master_jobs():
return {
'MASTER.test_job0':
make_job(name='MASTER.test_job0',
schedule=make_mock_schedule(),
expected_runtime=datetime.timedelta(1)),
'MASTER.test_job1':
make_job(
name='MASTER.test_job1',
schedule=schedule_parse.ConfigDailyScheduler(
days={1, 3, 5},
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions={
'action':
make_action(requires=('action1', ),
expected_runtime=datetime.timedelta(0, 7200)),
'action1':
make_action(name='action1',
expected_runtime=datetime.timedelta(0, 7200)),
},
time_zone=pytz.timezone("Pacific/Auckland"),
expected_runtime=datetime.timedelta(1),
cleanup_action=None,
allow_overlap=True,
),
'MASTER.test_job2':
make_job(
name='MASTER.test_job2',
node='node1',
actions={
'action2_0':
make_action(
name='action2_0',
command='test_command2.0',
)
},
time_zone=pytz.timezone("Pacific/Auckland"),
expected_runtime=datetime.timedelta(1),
cleanup_action=None,
),
'MASTER.test_job_actions_dict':
make_job(
name='MASTER.test_job_actions_dict',
node='node1',
schedule=make_mock_schedule(),
actions={
'action':
make_action(),
'action1':
make_action(name='action1'),
'action2':
make_action(
name='action2',
requires=('action', 'action1'),
node='node0',
),
},
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
'MASTER.test_job4':
make_job(
name='MASTER.test_job4',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
original="00:00:00 ",
hour=0,
minute=0,
second=0,
days=set(),
jitter=None,
),
all_nodes=True,
enabled=False,
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
'MASTER.test_job_mesos':
make_job(
name='MASTER.test_job_mesos',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
original="00:00:00 ",
hour=0,
minute=0,
second=0,
days=set(),
jitter=None,
),
actions={
'action_mesos':
make_action(
name='action_mesos',
command='test_command_mesos',
executor=schema.ExecutorTypes.mesos.value,
cpus=0.1,
mem=100,
disk=600,
docker_image='container:latest',
),
},
cleanup_action=None,
expected_runtime=datetime.timedelta(1),
),
}
def test_attributes(self):
expected = schema.NamedTronConfig(
jobs=FrozenDict(
{
'test_job0':
schema.ConfigJob(name='test_job0',
namespace='test_namespace',
node='node0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None,
),
actions=FrozenDict({
'action0_0':
schema.ConfigAction(
name='action0_0',
command='test_command0.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=schema.ConfigCleanupAction(
name='cleanup',
command='test_command0.1',
node=None),
enabled=True,
max_runtime=None,
allow_overlap=False),
'test_job1':
schema.ConfigJob(
name='test_job1',
namespace='test_namespace',
node='node0',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set([1, 3, 5]),
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions=FrozenDict(
{
'action1_1':
schema.ConfigAction(name='action1_1',
command='test_command1.1',
requires=('action1_0', ),
node=None),
'action1_0':
schema.ConfigAction(
name='action1_0',
command='test_command1.0 %(some_var)s',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=True),
'test_job2':
schema.ConfigJob(
name='test_job2',
namespace='test_namespace',
node='node1',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
),
actions=FrozenDict(
{
'action2_0':
schema.ConfigAction(name='action2_0',
command='test_command2.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'test_job3':
schema.ConfigJob(
name='test_job3',
namespace='test_namespace',
node='node1',
schedule=ConfigConstantScheduler(),
enabled=True,
actions=FrozenDict(
{
'action3_1':
schema.ConfigAction(name='action3_1',
command='test_command3.1',
requires=(),
node=None),
'action3_0':
schema.ConfigAction(name='action3_0',
command='test_command3.0',
requires=(),
node=None),
'action3_2':
schema.ConfigAction(name='action3_2',
command='test_command3.2',
requires=('action3_0',
'action3_1'),
node='node0')
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'test_job4':
schema.ConfigJob(
name='test_job4',
namespace='test_namespace',
node='NodePool',
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=0,
minute=0,
second=0,
original="00:00:00 ",
jitter=None,
),
actions=FrozenDict({
'action4_0':
schema.ConfigAction(name='action4_0',
command='test_command4.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=True,
cleanup_action=None,
enabled=False,
max_runtime=None,
allow_overlap=False)
}),
services=FrozenDict({
'service0':
schema.ConfigService(
namespace='test_namespace',
name='service0',
node='NodePool',
pid_file='/var/run/%(name)s-%(instance_number)s.pid',
command='service_command0',
monitor_interval=20,
monitor_retries=3,
restart_delay=None,
count=2)
}))
test_config = validate_fragment('test_namespace',
yaml.load(self.config))
assert_equal(test_config.jobs['test_job0'], expected.jobs['test_job0'])
assert_equal(test_config.jobs['test_job1'], expected.jobs['test_job1'])
assert_equal(test_config.jobs['test_job2'], expected.jobs['test_job2'])
assert_equal(test_config.jobs['test_job3'], expected.jobs['test_job3'])
assert_equal(test_config.jobs['test_job4'], expected.jobs['test_job4'])
assert_equal(test_config.jobs, expected.jobs)
assert_equal(test_config.services, expected.services)
assert_equal(test_config, expected)
assert_equal(test_config.jobs['test_job4'].enabled, False)
def assert_parse(self, config, expected):
config = self.validate(config)
expected = schedule_parse.ConfigDailyScheduler(*expected, jitter=None)
assert_equal(config, expected)
def test_attributes(self):
expected = schema.TronConfig(
action_runner=FrozenDict(),
output_stream_dir='/tmp',
command_context=FrozenDict({
'python': '/usr/bin/python',
'batch_dir': '/tron/batch/test/foo'
}),
ssh_options=schema.ConfigSSHOptions(
agent=False,
identities=('tests/test_id_rsa', ),
known_hosts_file=None,
connect_timeout=30,
idle_connection_timeout=3600,
jitter_min_load=4,
jitter_max_delay=20,
jitter_load_factor=1,
),
notification_options=None,
time_zone=pytz.timezone("EST"),
state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE,
nodes=FrozenDict({
'node0':
schema.ConfigNode(name='node0',
username=os.environ['USER'],
hostname='node0',
port=22),
'node1':
schema.ConfigNode(name='node1',
username=os.environ['USER'],
hostname='node1',
port=22)
}),
node_pools=FrozenDict({
'nodePool':
schema.ConfigNodePool(nodes=('node0', 'node1'),
name='nodePool')
}),
jobs=FrozenDict({
'MASTER.test_job0':
schema.ConfigJob(name='MASTER.test_job0',
namespace='MASTER',
node='node0',
schedule=ConfigIntervalScheduler(
timedelta=datetime.timedelta(0, 20),
jitter=None),
actions=FrozenDict({
'action0_0':
schema.ConfigAction(
name='action0_0',
command='test_command0.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=schema.ConfigCleanupAction(
name='cleanup',
command='test_command0.1',
node=None),
enabled=True,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job1':
schema.ConfigJob(
name='MASTER.test_job1',
namespace='MASTER',
node='node0',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set([1, 3, 5]),
hour=0,
minute=30,
second=0,
original="00:30:00 MWF",
jitter=None,
),
actions=FrozenDict({
'action1_1':
schema.ConfigAction(name='action1_1',
command='test_command1.1',
requires=('action1_0', ),
node=None),
'action1_0':
schema.ConfigAction(name='action1_0',
command='test_command1.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=True),
'MASTER.test_job2':
schema.ConfigJob(name='MASTER.test_job2',
namespace='MASTER',
node='node1',
enabled=True,
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=16,
minute=30,
second=0,
original="16:30:00 ",
jitter=None,
),
actions=FrozenDict({
'action2_0':
schema.ConfigAction(
name='action2_0',
command='test_command2.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job3':
schema.ConfigJob(
name='MASTER.test_job3',
namespace='MASTER',
node='node1',
schedule=ConfigConstantScheduler(),
enabled=True,
actions=FrozenDict({
'action3_1':
schema.ConfigAction(name='action3_1',
command='test_command3.1',
requires=(),
node=None),
'action3_0':
schema.ConfigAction(name='action3_0',
command='test_command3.0',
requires=(),
node=None),
'action3_2':
schema.ConfigAction(name='action3_2',
command='test_command3.2',
requires=('action3_0',
'action3_1'),
node='node0')
}),
queueing=True,
run_limit=50,
all_nodes=False,
cleanup_action=None,
max_runtime=None,
allow_overlap=False),
'MASTER.test_job4':
schema.ConfigJob(name='MASTER.test_job4',
namespace='MASTER',
node='nodePool',
schedule=schedule_parse.ConfigDailyScheduler(
days=set(),
hour=0,
minute=0,
second=0,
original='00:00:00 ',
jitter=None,
),
actions=FrozenDict({
'action4_0':
schema.ConfigAction(
name='action4_0',
command='test_command4.0',
requires=(),
node=None)
}),
queueing=True,
run_limit=50,
all_nodes=True,
cleanup_action=None,
enabled=False,
max_runtime=None,
allow_overlap=False)
}),
services=FrozenDict({
'MASTER.service0':
schema.ConfigService(
name='MASTER.service0',
namespace='MASTER',
node='nodePool',
pid_file='/var/run/%(name)s-%(instance_number)s.pid',
command='service_command0',
monitor_interval=20,
monitor_retries=3,
restart_delay=None,
count=2)
}))
test_config = valid_config_from_yaml(self.config)
assert_equal(test_config.command_context, expected.command_context)
assert_equal(test_config.ssh_options, expected.ssh_options)
assert_equal(test_config.notification_options,
expected.notification_options)
assert_equal(test_config.time_zone, expected.time_zone)
assert_equal(test_config.nodes, expected.nodes)
assert_equal(test_config.node_pools, expected.node_pools)
assert_equal(test_config.jobs['MASTER.test_job0'],
expected.jobs['MASTER.test_job0'])
assert_equal(test_config.jobs['MASTER.test_job1'],
expected.jobs['MASTER.test_job1'])
assert_equal(test_config.jobs['MASTER.test_job2'],
expected.jobs['MASTER.test_job2'])
assert_equal(test_config.jobs['MASTER.test_job3'],
expected.jobs['MASTER.test_job3'])
assert_equal(test_config.jobs['MASTER.test_job4'],
expected.jobs['MASTER.test_job4'])
assert_equal(test_config.jobs, expected.jobs)
assert_equal(test_config.services, expected.services)
assert_equal(test_config, expected)
assert_equal(test_config.jobs['MASTER.test_job4'].enabled, False)
