def make_tron_config( action_runner=None, output_stream_dir='/tmp', command_context=None, ssh_options=None, time_zone=pytz.timezone("EST"), state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE, nodes=None, node_pools=None, jobs=None, mesos_options=None, ): return schema.TronConfig( action_runner=action_runner or FrozenDict(), output_stream_dir=output_stream_dir, command_context=command_context or FrozenDict(batch_dir='/tron/batch/test/foo', python='/usr/bin/python'), ssh_options=ssh_options or make_ssh_options(), time_zone=time_zone, state_persistence=state_persistence, nodes=nodes or make_nodes(), node_pools=node_pools or make_node_pools(), jobs=jobs or make_master_jobs(), mesos_options=mesos_options or make_mesos_options(), )
def test_attributes(self): expected = make_named_tron_config(jobs=FrozenDict({ 'test_job': make_job( name="test_job", namespace='test_namespace', schedule=ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None, ), expected_runtime=datetime.timedelta(1), ) })) test_config = validate_fragment( 'test_namespace', dict(jobs=[ dict( name="test_job", namespace='test_namespace', node="node0", schedule="interval 20s", actions=[dict(name="action", command="command")], cleanup_action=dict(command="command"), ) ])) assert_equal(test_config, expected)
def make_job(**kwargs): kwargs.setdefault('namespace', 'MASTER') kwargs.setdefault('name', f"{kwargs['namespace']}.job_name") kwargs.setdefault('node', 'node0') kwargs.setdefault('enabled', True) kwargs.setdefault('monitoring', {}) kwargs.setdefault( 'schedule', schedule_parse.ConfigDailyScheduler( days=set(), hour=16, minute=30, second=0, original="16:30:00 ", jitter=None, )) kwargs.setdefault('actions', FrozenDict({'action': make_action()})) kwargs.setdefault('queueing', True) kwargs.setdefault('run_limit', 50) kwargs.setdefault('all_nodes', False) kwargs.setdefault('cleanup_action', make_cleanup_action()) kwargs.setdefault('max_runtime') kwargs.setdefault('allow_overlap', False) kwargs.setdefault('time_zone', None) kwargs.setdefault('expected_runtime', datetime.timedelta(0, 3600)) return schema.ConfigJob(**kwargs)
def make_node_pools(): return FrozenDict({ 'NodePool': schema.ConfigNodePool( nodes=('node0', 'node1'), name='NodePool', ), })
def validator(value, config_context): if isinstance(value, dict): value = [{'name': name, **config} for name, config in value.items()] msg = "Duplicate name %%s at %s" % config_context.path name_dict = UniqueNameDict(msg) for item in valid(value, config_context): name_dict[item.name] = item return FrozenDict(**name_dict)
def make_nodes(): return FrozenDict({ 'node0': schema.ConfigNode( name='node0', username='******', hostname='node0', port=22, ), 'node1': schema.ConfigNode( name='node1', username='******', hostname='node1', port=22, ), })
def valid_command_context(context, config_context): # context can be any dict. return FrozenDict(**valid_dict(context or {}, config_context))
def validator(value, config_context): msg = "Duplicate name %%s at %s" % config_context.path name_dict = UniqueNameDict(msg) for item in valid(value, config_context): name_dict[item.name] = item return FrozenDict(**name_dict)
def test_valid_jobs_and_services_success(self): test_config = BASE_CONFIG + textwrap.dedent(""" jobs: - name: "test_job0" node: node0 schedule: "interval 20s" actions: - name: "action0_0" command: "test_command0.0" cleanup_action: command: "test_command0.1" services: - name: "test_service0" node: node0 command: "service_command0" count: 2 pid_file: "/var/run/%(name)s-%(instance_number)s.pid" monitor_interval: 20 """) expected_jobs = { 'MASTER.test_job0': schema.ConfigJob(name='MASTER.test_job0', namespace='MASTER', node='node0', schedule=ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None), actions=FrozenDict({ 'action0_0': schema.ConfigAction(name='action0_0', command='test_command0.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=schema.ConfigCleanupAction( command='test_command0.1', name='cleanup', node=None), enabled=True, allow_overlap=False, max_runtime=None) } expected_services = { 'MASTER.test_service0': schema.ConfigService( name='MASTER.test_service0', namespace='MASTER', node='node0', pid_file='/var/run/%(name)s-%(instance_number)s.pid', command='service_command0', monitor_interval=20, monitor_retries=3, restart_delay=None, count=2) } config = manager.from_string(test_config) context = config_utils.ConfigContext('config', ['node0'], None, MASTER_NAMESPACE) config_parse.validate_jobs_and_services(config, context) assert_equal(expected_jobs, config['jobs']) assert_equal(expected_services, config['services'])
def test_attributes(self): expected = schema.NamedTronConfig( jobs=FrozenDict( { 'test_job0': schema.ConfigJob(name='test_job0', namespace='test_namespace', node='node0', schedule=ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None, ), actions=FrozenDict({ 'action0_0': schema.ConfigAction( name='action0_0', command='test_command0.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=schema.ConfigCleanupAction( name='cleanup', command='test_command0.1', node=None), enabled=True, max_runtime=None, allow_overlap=False), 'test_job1': schema.ConfigJob( name='test_job1', namespace='test_namespace', node='node0', enabled=True, schedule=schedule_parse.ConfigDailyScheduler( days=set([1, 3, 5]), hour=0, minute=30, second=0, original="00:30:00 MWF", jitter=None, ), actions=FrozenDict( { 'action1_1': schema.ConfigAction(name='action1_1', command='test_command1.1', requires=('action1_0', ), node=None), 'action1_0': schema.ConfigAction( name='action1_0', command='test_command1.0 %(some_var)s', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=True), 'test_job2': schema.ConfigJob( name='test_job2', namespace='test_namespace', node='node1', enabled=True, schedule=schedule_parse.ConfigDailyScheduler( days=set(), hour=16, minute=30, second=0, original="16:30:00 ", jitter=None, ), actions=FrozenDict( { 'action2_0': schema.ConfigAction(name='action2_0', command='test_command2.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=False), 'test_job3': schema.ConfigJob( name='test_job3', namespace='test_namespace', node='node1', schedule=ConfigConstantScheduler(), enabled=True, actions=FrozenDict( { 'action3_1': schema.ConfigAction(name='action3_1', command='test_command3.1', requires=(), node=None), 'action3_0': schema.ConfigAction(name='action3_0', command='test_command3.0', requires=(), node=None), 'action3_2': schema.ConfigAction(name='action3_2', command='test_command3.2', requires=('action3_0', 'action3_1'), node='node0') }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=False), 'test_job4': schema.ConfigJob( name='test_job4', namespace='test_namespace', node='NodePool', schedule=schedule_parse.ConfigDailyScheduler( days=set(), hour=0, minute=0, second=0, original="00:00:00 ", jitter=None, ), actions=FrozenDict({ 'action4_0': schema.ConfigAction(name='action4_0', command='test_command4.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=True, cleanup_action=None, enabled=False, max_runtime=None, allow_overlap=False) }), services=FrozenDict({ 'service0': schema.ConfigService( namespace='test_namespace', name='service0', node='NodePool', pid_file='/var/run/%(name)s-%(instance_number)s.pid', command='service_command0', monitor_interval=20, monitor_retries=3, restart_delay=None, count=2) })) test_config = validate_fragment('test_namespace', yaml.load(self.config)) assert_equal(test_config.jobs['test_job0'], expected.jobs['test_job0']) assert_equal(test_config.jobs['test_job1'], expected.jobs['test_job1']) assert_equal(test_config.jobs['test_job2'], expected.jobs['test_job2']) assert_equal(test_config.jobs['test_job3'], expected.jobs['test_job3']) assert_equal(test_config.jobs['test_job4'], expected.jobs['test_job4']) assert_equal(test_config.jobs, expected.jobs) assert_equal(test_config.services, expected.services) assert_equal(test_config, expected) assert_equal(test_config.jobs['test_job4'].enabled, False)
def test_attributes(self): expected = schema.TronConfig( action_runner=FrozenDict(), output_stream_dir='/tmp', command_context=FrozenDict({ 'python': '/usr/bin/python', 'batch_dir': '/tron/batch/test/foo' }), ssh_options=schema.ConfigSSHOptions( agent=False, identities=('tests/test_id_rsa', ), known_hosts_file=None, connect_timeout=30, idle_connection_timeout=3600, jitter_min_load=4, jitter_max_delay=20, jitter_load_factor=1, ), notification_options=None, time_zone=pytz.timezone("EST"), state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE, nodes=FrozenDict({ 'node0': schema.ConfigNode(name='node0', username=os.environ['USER'], hostname='node0', port=22), 'node1': schema.ConfigNode(name='node1', username=os.environ['USER'], hostname='node1', port=22) }), node_pools=FrozenDict({ 'nodePool': schema.ConfigNodePool(nodes=('node0', 'node1'), name='nodePool') }), jobs=FrozenDict({ 'MASTER.test_job0': schema.ConfigJob(name='MASTER.test_job0', namespace='MASTER', node='node0', schedule=ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None), actions=FrozenDict({ 'action0_0': schema.ConfigAction( name='action0_0', command='test_command0.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=schema.ConfigCleanupAction( name='cleanup', command='test_command0.1', node=None), enabled=True, max_runtime=None, allow_overlap=False), 'MASTER.test_job1': schema.ConfigJob( name='MASTER.test_job1', namespace='MASTER', node='node0', enabled=True, schedule=schedule_parse.ConfigDailyScheduler( days=set([1, 3, 5]), hour=0, minute=30, second=0, original="00:30:00 MWF", jitter=None, ), actions=FrozenDict({ 'action1_1': schema.ConfigAction(name='action1_1', command='test_command1.1', requires=('action1_0', ), node=None), 'action1_0': schema.ConfigAction(name='action1_0', command='test_command1.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=True), 'MASTER.test_job2': schema.ConfigJob(name='MASTER.test_job2', namespace='MASTER', node='node1', enabled=True, schedule=schedule_parse.ConfigDailyScheduler( days=set(), hour=16, minute=30, second=0, original="16:30:00 ", jitter=None, ), actions=FrozenDict({ 'action2_0': schema.ConfigAction( name='action2_0', command='test_command2.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=False), 'MASTER.test_job3': schema.ConfigJob( name='MASTER.test_job3', namespace='MASTER', node='node1', schedule=ConfigConstantScheduler(), enabled=True, actions=FrozenDict({ 'action3_1': schema.ConfigAction(name='action3_1', command='test_command3.1', requires=(), node=None), 'action3_0': schema.ConfigAction(name='action3_0', command='test_command3.0', requires=(), node=None), 'action3_2': schema.ConfigAction(name='action3_2', command='test_command3.2', requires=('action3_0', 'action3_1'), node='node0') }), queueing=True, run_limit=50, all_nodes=False, cleanup_action=None, max_runtime=None, allow_overlap=False), 'MASTER.test_job4': schema.ConfigJob(name='MASTER.test_job4', namespace='MASTER', node='nodePool', schedule=schedule_parse.ConfigDailyScheduler( days=set(), hour=0, minute=0, second=0, original='00:00:00 ', jitter=None, ), actions=FrozenDict({ 'action4_0': schema.ConfigAction( name='action4_0', command='test_command4.0', requires=(), node=None) }), queueing=True, run_limit=50, all_nodes=True, cleanup_action=None, enabled=False, max_runtime=None, allow_overlap=False) }), services=FrozenDict({ 'MASTER.service0': schema.ConfigService( name='MASTER.service0', namespace='MASTER', node='nodePool', pid_file='/var/run/%(name)s-%(instance_number)s.pid', command='service_command0', monitor_interval=20, monitor_retries=3, restart_delay=None, count=2) })) test_config = valid_config_from_yaml(self.config) assert_equal(test_config.command_context, expected.command_context) assert_equal(test_config.ssh_options, expected.ssh_options) assert_equal(test_config.notification_options, expected.notification_options) assert_equal(test_config.time_zone, expected.time_zone) assert_equal(test_config.nodes, expected.nodes) assert_equal(test_config.node_pools, expected.node_pools) assert_equal(test_config.jobs['MASTER.test_job0'], expected.jobs['MASTER.test_job0']) assert_equal(test_config.jobs['MASTER.test_job1'], expected.jobs['MASTER.test_job1']) assert_equal(test_config.jobs['MASTER.test_job2'], expected.jobs['MASTER.test_job2']) assert_equal(test_config.jobs['MASTER.test_job3'], expected.jobs['MASTER.test_job3']) assert_equal(test_config.jobs['MASTER.test_job4'], expected.jobs['MASTER.test_job4']) assert_equal(test_config.jobs, expected.jobs) assert_equal(test_config.services, expected.services) assert_equal(test_config, expected) assert_equal(test_config.jobs['MASTER.test_job4'].enabled, False)
def test_valid_jobs_success(self): test_config = dict(jobs=[ dict(name="test_job0", node='node0', schedule="interval 20s", expected_runtime="20m", actions=[ dict(name="action", command="command", expected_runtime="20m"), dict( name="action_mesos", command="command", executor='mesos', cpus=4, mem=300, constraints=[ dict(attribute='pool', operator='LIKE', value='default') ], docker_image='my_container:latest', docker_parameters=[ dict(key='label', value='labelA'), dict(key='label', value='labelB') ], env=dict(USER='******'), extra_volumes=[ dict(container_path='/tmp', host_path='/home/tmp', mode='RO') ], ), dict( name="test_trigger_attrs", command="foo", triggered_by=["foo.bar"], trigger_downstreams=True, ), ], cleanup_action=dict(command="command")) ], **BASE_CONFIG) expected_jobs = FrozenDict({ 'MASTER.test_job0': make_job( name='MASTER.test_job0', schedule=ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None, ), actions=FrozenDict({ 'action': make_action(expected_runtime=datetime.timedelta(0, 1200), ), 'action_mesos': make_action( name='action_mesos', executor='mesos', cpus=4.0, mem=300.0, constraints=(schema.ConfigConstraint( attribute='pool', operator='LIKE', value='default', ), ), docker_image='my_container:latest', docker_parameters=( schema.ConfigParameter( key='label', value='labelA', ), schema.ConfigParameter( key='label', value='labelB', ), ), env={'USER': '******'}, extra_volumes=(schema.ConfigVolume( container_path='/tmp', host_path='/home/tmp', mode='RO', ), ), expected_runtime=datetime.timedelta(hours=24), ), 'test_trigger_attrs': make_action( name="test_trigger_attrs", command="foo", triggered_by=("foo.bar", ), trigger_downstreams=True, ), }), expected_runtime=datetime.timedelta(0, 1200), ), }) context = config_utils.ConfigContext( 'config', ['node0'], None, MASTER_NAMESPACE, ) config_parse.validate_jobs(test_config, context) assert_equal(expected_jobs, test_config['jobs'])
def make_command_context(): return FrozenDict({ 'python': '/usr/bin/python', 'batch_dir': '/tron/batch/test/foo', })
def make_master_jobs(): return FrozenDict({ 'MASTER.test_job0': make_job(name='MASTER.test_job0', schedule=schedule_parse.ConfigIntervalScheduler( timedelta=datetime.timedelta(0, 20), jitter=None, ), expected_runtime=datetime.timedelta(1)), 'MASTER.test_job1': make_job( name='MASTER.test_job1', schedule=schedule_parse.ConfigDailyScheduler( days={1, 3, 5}, hour=0, minute=30, second=0, original="00:30:00 MWF", jitter=None, ), actions=FrozenDict({ 'action': make_action(requires=('action1', ), expected_runtime=datetime.timedelta(0, 7200)), 'action1': make_action(name='action1', expected_runtime=datetime.timedelta(0, 7200)), }), time_zone=pytz.timezone("Pacific/Auckland"), expected_runtime=datetime.timedelta(1), cleanup_action=None, allow_overlap=True, ), 'MASTER.test_job2': make_job( name='MASTER.test_job2', node='node1', actions=FrozenDict({ 'action2_0': make_action( name='action2_0', command='test_command2.0', ) }), time_zone=pytz.timezone("Pacific/Auckland"), expected_runtime=datetime.timedelta(1), cleanup_action=None, ), 'MASTER.test_job_actions_dict': make_job( name='MASTER.test_job_actions_dict', node='node1', schedule=ConfigConstantScheduler(), actions=FrozenDict({ 'action': make_action(), 'action1': make_action(name='action1'), 'action2': make_action( name='action2', requires=('action', 'action1'), node='node0', ), }), cleanup_action=None, expected_runtime=datetime.timedelta(1), ), 'MASTER.test_job4': make_job( name='MASTER.test_job4', node='NodePool', schedule=schedule_parse.ConfigDailyScheduler( original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None, ), all_nodes=True, enabled=False, cleanup_action=None, expected_runtime=datetime.timedelta(1), ), 'MASTER.test_job_mesos': make_job( name='MASTER.test_job_mesos', node='NodePool', schedule=schedule_parse.ConfigDailyScheduler( original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None, ), actions=FrozenDict({ 'action_mesos': make_action( name='action_mesos', command='test_command_mesos', executor='mesos', cpus=0.1, mem=100, docker_image='container:latest', ), }), cleanup_action=None, expected_runtime=datetime.timedelta(1), ), })