class MesosJob(Struct):
name = Default(String, '{{task.name}}')
role = Required(String)
contact = String
cluster = Required(String)
environment = Required(String)
instances = Default(Integer, 1)
task = Required(Task)
announce = Announcer
tier = String
cron_schedule = String
cron_collision_policy = Default(String, "KILL_EXISTING")
update_config = Default(UpdateConfig, UpdateConfig())
constraints = Map(String, String)
service = Default(Boolean, False)
max_task_failures = Default(Integer, 1)
production = Default(Boolean, False)
priority = Default(Integer, 0)
health_check_config = Default(HealthCheckConfig, HealthCheckConfig())
# TODO(wickman) Make Default(Any, LifecycleConfig()) once pystachio #17 is addressed.
lifecycle = Default(LifecycleConfig, DefaultLifecycleConfig)
task_links = Map(String, String) # Unsupported. See AURORA-739
enable_hooks = Default(
Boolean,
False) # enable client API hooks; from env python-list 'hooks'
# Specifying a `Container` with a `docker` property for Docker jobs is deprecated, instead just
# specify the value of the container property to be a `Docker` container directly.
container = Choice([Container, Docker, Mesos])
def test_choice_type():
IntStr = Choice("IntStrFloat", (Integer, String))
one = IntStr(123)
two = IntStr("123")
three = IntStr("abc")
assert one.unwrap() == Integer(123)
assert two.unwrap() == Integer(123)
assert three.unwrap() == String("abc")
def test_choice_triple():
Triple = Choice((Integer, Float, String))
one = Triple(123)
two = Triple(123.456)
three = Triple("123.abc")
assert one.check().ok()
assert two.check().ok()
assert three.check().ok()
def test_choice_error():
IntFloat = Choice((Integer, Float))
one = IntFloat(123)
two = IntFloat(123.456)
three = IntFloat("123.abc")
assert one.check().ok()
assert two.check().ok()
assert not three.check().ok()
def test_hashing():
IntStr = Choice("IntStrFloat", (Integer, String))
map = {IntStr(123): 'foo', IntStr("123"): 'bar', IntStr("abc"): 'baz'}
assert IntStr(123) in map
assert IntStr("123") in map
assert IntStr("abc") in map
assert IntStr(456) not in map
assert IntStr("456") not in map
assert IntStr("def") not in map
def test_choice_primlist():
"""Test that choices with a list value work correctly."""
C = Choice([String, List(Integer)])
c = C([1, 2, 3])
assert c.check().ok()
c = C("hello")
assert c.check().ok()
c = C([1, 2, "{{x}}"])
assert not c.check().ok()
assert c.bind(x=3).check().ok()
class UpdateConfig(Struct):
batch_size = Default(Integer, 1)
watch_secs = Default(Integer, 45)
max_per_shard_failures = Default(Integer, 0)
max_total_failures = Default(Integer, 0)
rollback_on_failure = Default(Boolean, True)
wait_for_batch_completion = Default(Boolean, False)
pulse_interval_secs = Integer
sla_aware = Default(Boolean, False)
update_strategy = Choice([QueueUpdateStrategy,
BatchUpdateStrategy,
VariableBatchUpdateStrategy])
def test_get_choice_in_struct():
class Foo(Struct):
foo = Required(String)
class Bar(Struct):
bar = Required(String)
Item = Choice("Item", (Foo, Bar))
class Qux(Struct):
item = Choice([String, List(Item)])
b = Qux(item=[Foo(foo="fubar")])
assert b.get() == frozendict({'item': (frozendict({'foo': u'fubar'}), )})
def test_repr():
class Dumb(Struct):
one = String
class ChoiceDefaultStruct(Struct):
a = Default(Choice("IntOrDumb", [Dumb, Integer]), 28)
b = Integer
class OtherStruct(Struct):
first = ChoiceDefaultStruct
second = String
C = Choice([String, List(Integer)])
testvalone = C("hello")
testvaltwo = C([1, 2, 3])
assert repr(testvalone) == "Choice_String_IntegerList('hello')"
assert repr(testvaltwo) == "Choice_String_IntegerList([1, 2, 3])"
def test_choice_interpolation():
IntFloat = Choice((Integer, Float))
one = IntFloat('{{abc}}')
two = IntFloat('{{a}}{{b}}')
one_int = one.bind(abc=34)
assert isinstance(one_int.interpolate()[0], Integer)
assert one_int.check().ok()
one_fl = one.bind(abc=123.354)
assert isinstance(one_fl.interpolate()[0], Float)
assert one_fl.check().ok()
one_str = one.bind(abc="def")
assert not one_str.check().ok()
assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')]
two_one = two.bind(a=12, b=23)
assert two_one.check().ok()
assert two_one.unwrap() == Integer(1223)
two_two = two.bind(a=12, b=".34")
assert two_two.check().ok()
assert two_two.unwrap() == Float(12.34)
def to_thrift_update_settings(self, instances=None):
"""Converts UpdateConfig into thrift JobUpdateSettings object.
Arguments:
instances - optional list of instances to update.
"""
if self.update_strategy is Empty:
update_strategy = Choice([
PystachioQueueUpdateStrategy, PystachioBatchUpdateStrategy,
PystachioVariableBatchUpdateStrategy
])
if self.wait_for_batch_completion:
self.update_strategy = update_strategy(
PystachioBatchUpdateStrategy(batch_size=self.batch_size))
else:
self.update_strategy = update_strategy(
PystachioQueueUpdateStrategy(batch_size=self.batch_size))
else:
unwrapped = self.update_strategy.unwrap()
if isinstance(unwrapped, PystachioQueueUpdateStrategy):
self.batch_size = self.update_strategy.groupSize
elif isinstance(unwrapped, PystachioBatchUpdateStrategy):
self.batch_size = self.update_strategy.groupSize
self.wait_for_batch_completion = True
elif isinstance(unwrapped, PystachioBatchUpdateStrategy):
self.batch_size = self.update_strategy.groupSizes[0]
return JobUpdateSettings(
updateGroupSize=self.batch_size,
maxPerInstanceFailures=self.max_per_instance_failures,
maxFailedInstances=self.max_total_failures,
minWaitInInstanceRunningMs=self.watch_secs * 1000,
rollbackOnFailure=self.rollback_on_failure,
waitForBatchCompletion=self.wait_for_batch_completion,
updateOnlyTheseInstances=self.instances_to_ranges(instances)
if instances else None,
updateStrategy=create_update_strategy_config(self.update_strategy),
blockIfNoPulsesAfterMs=(self.pulse_interval_secs * 1000
if self.pulse_interval_secs else None),
slaAware=self.sla_aware)
class Mesos(Struct):
image = Choice([AppcImage, DockerImage])
class Mesos(Struct): image = Choice([AppcImage, DockerImage]) volumes = Default(List(Volume), [])
class TestUpdaterUtil(unittest.TestCase):
EXPECTED_JOB_UPDATE_SETTINGS = JobUpdateSettings(
blockIfNoPulsesAfterMs=None,
updateOnlyTheseInstances=None,
slaAware=False,
maxPerInstanceFailures=0,
waitForBatchCompletion=False,
rollbackOnFailure=True,
minWaitInInstanceRunningMs=45000,
updateGroupSize=1,
maxFailedInstances=0)
UPDATE_STRATEGIES = Choice([
PystachioQueueUpdateStrategy, PystachioBatchUpdateStrategy,
PystachioVariableBatchUpdateStrategy
])
def test_multiple_ranges(self):
"""Test multiple ranges."""
ranges = [
repr(e)
for e in UpdaterConfig.instances_to_ranges([1, 2, 3, 5, 7, 8])
]
assert 3 == len(ranges), "Wrong number of ranges:%s" % len(ranges)
assert repr(Range(first=1, last=3)) in ranges, "Missing range [1,3]"
assert repr(Range(first=5, last=5)) in ranges, "Missing range [5,5]"
assert repr(Range(first=7, last=8)) in ranges, "Missing range [7,8]"
def test_one_element(self):
"""Test one ID in the list."""
ranges = [repr(e) for e in UpdaterConfig.instances_to_ranges([1])]
assert 1 == len(ranges), "Wrong number of ranges:%s" % len(ranges)
assert repr(Range(first=1, last=1)) in ranges, "Missing range [1,1]"
def test_none_list(self):
"""Test None list produces None result."""
assert UpdaterConfig.instances_to_ranges(
None) is None, "Result must be None."
def test_empty_list(self):
"""Test empty list produces None result."""
assert UpdaterConfig.instances_to_ranges(
[]) is None, "Result must be None."
def test_pulse_interval_secs(self):
config = UpdaterConfig(
UpdateConfig(batch_size=1,
watch_secs=1,
max_per_shard_failures=1,
max_total_failures=1,
pulse_interval_secs=60))
assert 60000 == config.to_thrift_update_settings(
).blockIfNoPulsesAfterMs
def test_pulse_interval_unset(self):
config = UpdaterConfig(
UpdateConfig(batch_size=1,
watch_secs=1,
max_per_shard_failures=1,
max_total_failures=1))
assert config.to_thrift_update_settings(
).blockIfNoPulsesAfterMs is None
def test_pulse_interval_too_low(self):
threshold = UpdaterConfig.MIN_PULSE_INTERVAL_SECONDS
with raises(ValueError) as e:
UpdaterConfig(
UpdateConfig(batch_size=1,
watch_secs=1,
max_per_shard_failures=1,
max_total_failures=1,
pulse_interval_secs=threshold - 1))
assert 'Pulse interval seconds must be at least %s seconds.' % threshold in e.value.message
def test_to_thrift_update_settings_strategy(self):
"""Test to_thrift produces an expected thrift update settings configuration
from a Pystachio update object.
"""
config = UpdaterConfig(
UpdateConfig(update_strategy=self.UPDATE_STRATEGIES(
PystachioVariableBatchUpdateStrategy(
batch_sizes=[1, 2, 3, 4], autopause_after_batch=True))))
thrift_update_config = config.to_thrift_update_settings()
update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS)
update_settings.updateStrategy = JobUpdateStrategy(
batchStrategy=None,
queueStrategy=None,
varBatchStrategy=VariableBatchJobUpdateStrategy(
groupSizes=(1, 2, 3, 4), autopauseAfterBatch=True))
assert thrift_update_config == update_settings
def test_to_thrift_update_settings_no_strategy_queue(self):
"""Test to_thrift produces an expected thrift update settings configuration
from a Pystachio update object that doesn't include an update strategy.
The configuration in this test should be converted to a
QueueJobUpdateStrategy.
"""
config = UpdaterConfig(UpdateConfig())
thrift_update_config = config.to_thrift_update_settings()
update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS)
update_settings.updateStrategy = JobUpdateStrategy(
batchStrategy=None,
queueStrategy=QueueJobUpdateStrategy(groupSize=1),
varBatchStrategy=None)
assert thrift_update_config == update_settings
def test_to_thrift_update_settings_no_strategy_batch(self):
"""Test to_thrift produces an expected thrift update settings configuration
from a Pystachio update object that doesn't include an update strategy.
The configuration in this test should be converted to a
BatchJobUpdateStrategy.
"""
config = UpdaterConfig(UpdateConfig(wait_for_batch_completion=True))
thrift_update_config = config.to_thrift_update_settings()
update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS)
update_settings.updateStrategy = JobUpdateStrategy(
batchStrategy=BatchJobUpdateStrategy(groupSize=1,
autopauseAfterBatch=False),
queueStrategy=None,
varBatchStrategy=None)
update_settings.waitForBatchCompletion = True
assert thrift_update_config == update_settings
def test_wait_for_batch_completion_and_update_strategy(self):
"""Test setting wait_for_batch_completion along with an update strategy.
This combination should result in a fast fail.
"""
with raises(ValueError) as e:
UpdaterConfig(
UpdateConfig(wait_for_batch_completion=True,
update_strategy=self.UPDATE_STRATEGIES(
PystachioBatchUpdateStrategy(batch_size=3))))
assert ('Ambiguous update configuration. Cannot combine '
'wait_batch_completion with an '
'explicit update strategy.' in e.value.message)
def test_batch_size_and_update_strategy(self):
"""Test setting a batch size along with an update strategy.
This combination should result in a fast fail.
"""
with raises(ValueError) as e:
UpdaterConfig(
UpdateConfig(batch_size=2,
update_strategy=self.UPDATE_STRATEGIES(
PystachioBatchUpdateStrategy(batch_size=3))))
assert ('Ambiguous update configuration. Cannot combine '
'update strategy with batch size. Please set batch'
'size inside of update strategy instead.' in e.value.message)
class SOne(Struct):
a = Choice((Integer, Float))
b = String
class Yuck(Struct):
one = Choice([Foo, Integer])
two = String
def test_choice_string_enum():
TestEnum = Enum("TestEnum", ("A", "B", "C"))
TestChoice = Choice("TestChoice", (TestEnum, String))
v = TestChoice("A")
assert isinstance(v.interpolate()[0], TestEnum)
assert isinstance(TestChoice("Q").interpolate()[0], String)
class Qux(Struct):
item = Choice([String, List(Item)])
class Bar(Struct):
a = Choice("StringOrFoo", [Integer, String, Foo])
b = String
class ChoiceDefaultStruct(Struct):
a = Default(Choice("IntOrDumb", [Dumb, Integer]), 28)
b = Integer
