def test_cron_policy_alias(): cron_schedule = '*/10 * * * *' CRON_HELLO_WORLD = HELLO_WORLD(cron_schedule=cron_schedule) tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.KILL_EXISTING tti = convert_pystachio_to_thrift( CRON_HELLO_WORLD(cron_policy='RUN_OVERLAP')) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP tti = convert_pystachio_to_thrift( CRON_HELLO_WORLD(cron_collision_policy='RUN_OVERLAP')) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP with pytest.raises(ValueError): tti = convert_pystachio_to_thrift( CRON_HELLO_WORLD(cron_policy='RUN_OVERLAP', cron_collision_policy='RUN_OVERLAP')) with pytest.raises(ValueError): tti = convert_pystachio_to_thrift( CRON_HELLO_WORLD(cron_collision_policy='GARBAGE'))
def test_cron_collision_policy(): cron_schedule = '*/10 * * * *' CRON_HELLO_WORLD = HELLO_WORLD(cron_schedule=cron_schedule) tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.KILL_EXISTING tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD(cron_collision_policy='RUN_OVERLAP')) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP with pytest.raises(ValueError): tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD(cron_collision_policy='GARBAGE'))
def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD) expected_key = JobKey( role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser()) assert job.cronSchedule is None assert tti.job == expected_key assert tti.jobName == 'hello_world' assert tti.isService is False assert tti.numCpus == 0.1 assert tti.ramMb == 64 assert tti.diskMb == 64 assert tti.requestedPorts == set() assert tti.production is False assert tti.priority == 0 assert tti.maxTaskFailures == 1 assert tti.constraints == set() assert tti.metadata == set() assert tti.environment == HELLO_WORLD.environment().get() assert tti.tier is None
def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD) expected_key = JobKey(role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser()) assert job.cronSchedule is None assert tti.job == expected_key assert tti.jobName == 'hello_world' assert tti.isService is False assert tti.numCpus == 0.1 assert tti.ramMb == 64 assert tti.diskMb == 64 assert tti.requestedPorts == set() assert tti.production is False assert tti.priority == 0 assert tti.maxTaskFailures == 1 assert tti.constraints == set() assert tti.metadata == set() assert tti.environment == HELLO_WORLD.environment().get() assert tti.tier is None
def test_config_with_custom_executor_config(): job = convert_pystachio_to_thrift( HELLO_WORLD(executor_config=ExecutorConfig( name="CustomExecutor", data="{test:'payload'}"))) assert str(job.taskConfig.executorConfig.name) == "CustomExecutor" assert str(job.taskConfig.executorConfig.data) == "{test:'payload'}"
def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD, ports=frozenset(['health'])) expected_key = JobKey( role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(user=getpass.getuser()) assert job.cronSchedule is None assert tti.job == expected_key assert tti.isService is False assert tti.numCpus == 0.1 assert tti.ramMb == 64 assert tti.diskMb == 64 assert tti.requestedPorts == frozenset(['health']) assert tti.production is False assert tti.priority == 0 assert tti.maxTaskFailures == 1 assert tti.constraints == set() assert tti.metadata == set() assert tti.tier is None assert Resource(numCpus=0.1) in list(tti.resources) assert Resource(ramMb=64) in list(tti.resources) assert Resource(diskMb=64) in list(tti.resources) assert Resource(namedPort='health') in list(tti.resources)
def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD, ports=frozenset(['health'])) expected_key = JobKey( role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(user=getpass.getuser()) assert job.cronSchedule is None assert tti.job == expected_key assert tti.isService is False assert tti.numCpus == 0.1 assert tti.ramMb == 64 assert tti.diskMb == 64 assert tti.requestedPorts == frozenset(['health']) assert tti.production is False assert tti.priority == 0 assert tti.maxTaskFailures == 1 assert tti.constraints == set() assert tti.metadata == set() assert tti.tier is None assert Resource(numCpus=0.1) in list(tti.resources) assert Resource(ramMb=64) in list(tti.resources) assert Resource(diskMb=64) in list(tti.resources) assert Resource(namedPort='health') in list(tti.resources) assert Resource(numGpus=2) in list(tti.resources)
def test_config_with_explicit_thermos_executor_config(): job = convert_pystachio_to_thrift( HELLO_WORLD(executor_config=ExecutorConfig(name=AURORA_EXECUTOR_NAME))) assert str(job.taskConfig.executorConfig.name) == AURORA_EXECUTOR_NAME assert json.loads( job.taskConfig.executorConfig.data) == HELLO_WORLD_EXECUTOR_DATA
def test_docker_with_parameters(): helloworld = HELLO_WORLD( container=Container( docker=Docker(image='test_image', parameters=[Parameter(name='foo', value='bar')]) ) ) job = convert_pystachio_to_thrift(helloworld) assert job.taskConfig.container.docker.image == 'test_image'
def test_metadata_in_config(): job = convert_pystachio_to_thrift(HELLO_WORLD, metadata=[('alpha', 1)]) assert job.instanceCount == 1 tti = job.taskConfig assert len(tti.metadata) == 1 pi = iter(tti.metadata).next() assert pi.key == 'alpha' assert pi.value == '1'
def test_config_with_docker_image(): image_name = 'some-image' image_tag = 'some-tag' job = convert_pystachio_to_thrift( HELLO_WORLD(container=Mesos(image=DockerImage(name=image_name, tag=image_tag)))) assert job.taskConfig.container.mesos.image.appc is None assert job.taskConfig.container.mesos.image.docker.name == image_name assert job.taskConfig.container.mesos.image.docker.tag == image_tag
def test_config_with_appc_image(): image_name = 'some-image' image_id = 'some-image-id' job = convert_pystachio_to_thrift( HELLO_WORLD(container=Mesos(image=AppcImage(name=image_name, image_id=image_id)))) assert job.taskConfig.container.mesos.image.docker is None assert job.taskConfig.container.mesos.image.appc.name == image_name assert job.taskConfig.container.mesos.image.appc.imageId == image_id
def test_count_sla_policy(): hwc = HELLO_WORLD( sla_policy=PystachioCountSlaPolicy(count=10, duration_secs=1800)) job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.slaPolicy.percentageSlaPolicy is None assert job.taskConfig.slaPolicy.coordinatorSlaPolicy is None assert job.taskConfig.slaPolicy.countSlaPolicy == CountSlaPolicy( count=10, durationSecs=1800)
def test_coordinator_sla_policy_status_key(): hwc = HELLO_WORLD(sla_policy=PystachioCoordinatorSlaPolicy( coordinator_url='some-url', status_key='key')) job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.slaPolicy.percentageSlaPolicy is None assert job.taskConfig.slaPolicy.countSlaPolicy is None assert job.taskConfig.slaPolicy.coordinatorSlaPolicy == CoordinatorSlaPolicy( coordinatorUrl='some-url', statusKey='key')
def test_packages_in_config(): job = convert_pystachio_to_thrift(HELLO_WORLD, packages = [('alpha', 'beta', 1)]) assert job.instanceCount == 1 tti = job.taskConfig assert len(tti.packages) == 1 pi = iter(tti.packages).next() assert pi.role == 'alpha' assert pi.name == 'beta' assert pi.version == 1
def test_config_with_duplicate_metadata(): expected_metadata_tuples = frozenset([("city", "LA")]) job = convert_pystachio_to_thrift( HELLO_WORLD(metadata=[ Metadata(key=key, value=value) for key, value in expected_metadata_tuples]), metadata=[('city', "LA")]) tti = job.taskConfig metadata_tuples = frozenset((key_value.key, key_value.value) for key_value in tti.metadata) assert metadata_tuples == expected_metadata_tuples
def test_count_sla_policy(): hwc = HELLO_WORLD( sla_policy=PystachioCountSlaPolicy(count=10, duration_secs=1800) ) job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.slaPolicy.percentageSlaPolicy is None assert job.taskConfig.slaPolicy.coordinatorSlaPolicy is None assert job.taskConfig.slaPolicy.countSlaPolicy == CountSlaPolicy( count=10, durationSecs=1800)
def test_coordinator_sla_policy_status_key(): hwc = HELLO_WORLD( sla_policy=PystachioCoordinatorSlaPolicy(coordinator_url='some-url', status_key='key') ) job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.slaPolicy.percentageSlaPolicy is None assert job.taskConfig.slaPolicy.countSlaPolicy is None assert job.taskConfig.slaPolicy.coordinatorSlaPolicy == CoordinatorSlaPolicy( coordinatorUrl='some-url', statusKey='key' )
def test_disable_partition_policy(): hwc = HELLO_WORLD( production=True, priority=200, service=True, cron_collision_policy='RUN_OVERLAP', partition_policy=PystachioPartitionPolicy(reschedule=False), constraints={ 'dedicated': 'root', 'cpu': 'x86_64' }, environment='prod') job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.partitionPolicy == PartitionPolicy(False, 0)
def test_cron_policy_alias(): cron_schedule = "*/10 * * * *" CRON_HELLO_WORLD = HELLO_WORLD(cron_schedule=cron_schedule) tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.KILL_EXISTING tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD(cron_policy="RUN_OVERLAP")) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD(cron_collision_policy="RUN_OVERLAP")) assert tti.cronSchedule == cron_schedule assert tti.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP with pytest.raises(ValueError): tti = convert_pystachio_to_thrift( CRON_HELLO_WORLD(cron_policy="RUN_OVERLAP", cron_collision_policy="RUN_OVERLAP") ) with pytest.raises(ValueError): tti = convert_pystachio_to_thrift(CRON_HELLO_WORLD(cron_collision_policy="GARBAGE"))
def test_disable_partition_policy(): hwc = HELLO_WORLD( production=True, priority=200, service=True, cron_collision_policy='RUN_OVERLAP', partition_policy=PystachioPartitionPolicy(reschedule=False), constraints={ 'dedicated': 'root', 'cpu': 'x86_64' }, environment='prod' ) job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.partitionPolicy == PartitionPolicy(False, 0)
def test_unbound_references(): def job_command(cmdline): return AuroraConfig(HELLO_WORLD(task=SimpleTask("hello_world", cmdline))).raw() # bindingless and bad => good bindings should work convert_pystachio_to_thrift(job_command("echo hello world")) convert_pystachio_to_thrift(job_command("echo {{mesos.user}}").bind(mesos={"user": "******"})) # unbound with pytest.raises(InvalidConfig): convert_pystachio_to_thrift(job_command("echo {{mesos.user}}"))
def test_unbound_references(): def job_command(cmdline): return AuroraConfig(HELLO_WORLD(task=SimpleTask('hello_world', cmdline))).raw() # bindingless and bad => good bindings should work convert_pystachio_to_thrift(job_command('echo hello world')) convert_pystachio_to_thrift(job_command('echo {{mesos.user}}') .bind(mesos={'user': '******'})) # unbound with pytest.raises(InvalidConfig): convert_pystachio_to_thrift(job_command('echo {{mesos.user}}'))
def test_config_with_volumes(): image_name = 'some-image' image_tag = 'some-tag' host_path = '/etc/secrets/role/' container_path = '/etc/secrets/' volume = Volume(host_path=host_path, container_path=container_path, mode=Mode('RO')) container = Mesos(image=DockerImage(name=image_name, tag=image_tag), volumes=[volume]) job = convert_pystachio_to_thrift(HELLO_WORLD(container=container)) assert len(job.taskConfig.container.mesos.volumes) == 1 thrift_volume = job.taskConfig.container.mesos.volumes[0] assert thrift_volume.hostPath == host_path assert thrift_volume.containerPath == container_path assert thrift_volume.mode == ThriftMode.RO
def test_config_with_options(): hwc = HELLO_WORLD( production=True, priority=200, service=True, cron_policy="RUN_OVERLAP", constraints={"dedicated": "root", "cpu": "x86_64"}, environment="prod", ) job = convert_pystachio_to_thrift(hwc) assert job.instanceCount == 1 tti = job.taskConfig assert tti.production assert tti.priority == 200 assert tti.isService assert job.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP assert len(tti.constraints) == 2 assert tti.environment == "prod" assert job.key.environment == "prod"
def test_config_with_options(): hwc = HELLO_WORLD(production=True, priority=200, service=True, cron_collision_policy='RUN_OVERLAP', constraints={ 'dedicated': 'root', 'cpu': 'x86_64' }, environment='prod') job = convert_pystachio_to_thrift(hwc) assert job.instanceCount == 1 tti = job.taskConfig assert tti.production assert tti.priority == 200 assert tti.isService assert job.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP assert len(tti.constraints) == 2 assert job.key.environment == 'prod'
def test_config_with_options(): hwc = HELLO_WORLD( production=True, priority=200, service=True, cron_collision_policy='RUN_OVERLAP', constraints={ 'dedicated': 'root', 'cpu': 'x86_64' }, environment='prod' ) job = convert_pystachio_to_thrift(hwc) assert job.instanceCount == 1 tti = job.taskConfig assert tti.production assert tti.priority == 200 assert tti.isService assert job.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP assert len(tti.constraints) == 2 assert job.key.environment == 'prod'
def test_config_with_bad_resources(): MB = 1048576 hwtask = HELLO_WORLD.task() convert_pystachio_to_thrift(HELLO_WORLD) good_resources = [Resources(cpu=1.0, ram=1 * MB, disk=1 * MB)] bad_resources = [ Resources(cpu=0, ram=1 * MB, disk=1 * MB), Resources(cpu=1, ram=0 * MB, disk=1 * MB), Resources(cpu=1, ram=1 * MB, disk=0 * MB), Resources(cpu=1, ram=1 * MB - 1, disk=1 * MB), Resources(cpu=1, ram=1 * MB, disk=1 * MB - 1), ] for resource in good_resources: convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource))) for resource in bad_resources: with pytest.raises(ValueError): convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
def test_config_with_bad_resources(): MB = 1048576 hwtask = HELLO_WORLD.task() convert_pystachio_to_thrift(HELLO_WORLD) good_resources = [ Resources(cpu=1.0, ram=1 * MB, disk=1 * MB) ] bad_resources = [ Resources(cpu=0, ram=1 * MB, disk=1 * MB), Resources(cpu=1, ram=0 * MB, disk=1 * MB), Resources(cpu=1, ram=1 * MB, disk=0 * MB), Resources(cpu=1, ram=1 * MB - 1, disk=1 * MB), Resources(cpu=1, ram=1 * MB, disk=1 * MB - 1) ] for resource in good_resources: convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource))) for resource in bad_resources: with pytest.raises(ValueError): convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
def test_no_sla_policy(): hwc = HELLO_WORLD() job = convert_pystachio_to_thrift(hwc) assert job.taskConfig.slaPolicy is None
def test_config_with_tier(): config = HELLO_WORLD(tier='devel') job = convert_pystachio_to_thrift(config) assert job.taskConfig.tier == 'devel'
def test_config_with_explicit_thermos_executor_config(): job = convert_pystachio_to_thrift( HELLO_WORLD(executor_config=ExecutorConfig(name=AURORA_EXECUTOR_NAME))) assert str(job.taskConfig.executorConfig.name) == AURORA_EXECUTOR_NAME assert json.loads(job.taskConfig.executorConfig.data) == HELLO_WORLD_EXECUTOR_DATA