class MesosJob(Struct): name = Default(String, '{{task.name}}') role = Required(String) contact = String cluster = Required(String) environment = Required(String) instances = Default(Integer, 1) task = Required(Task) announce = Announcer tier = String cron_schedule = String cron_collision_policy = Default(String, "KILL_EXISTING") update_config = Default(UpdateConfig, UpdateConfig()) constraints = Map(String, String) service = Default(Boolean, False) max_task_failures = Default(Integer, 1) production = Default(Boolean, False) priority = Default(Integer, 0) health_check_config = Default(HealthCheckConfig, HealthCheckConfig()) # TODO(wickman) Make Default(Any, LifecycleConfig()) once pystachio #17 is addressed. lifecycle = Default(LifecycleConfig, DefaultLifecycleConfig) task_links = Map(String, String) # Unsupported. See AURORA-739 enable_hooks = Default( Boolean, False) # enable client API hooks; from env python-list 'hooks' # Specifying a `Container` with a `docker` property for Docker jobs is deprecated, instead just # specify the value of the container property to be a `Docker` container directly. container = Choice([Container, Docker, Mesos])
def test_choice_type(): IntStr = Choice("IntStrFloat", (Integer, String)) one = IntStr(123) two = IntStr("123") three = IntStr("abc") assert one.unwrap() == Integer(123) assert two.unwrap() == Integer(123) assert three.unwrap() == String("abc")
def test_choice_triple(): Triple = Choice((Integer, Float, String)) one = Triple(123) two = Triple(123.456) three = Triple("123.abc") assert one.check().ok() assert two.check().ok() assert three.check().ok()
def test_choice_error(): IntFloat = Choice((Integer, Float)) one = IntFloat(123) two = IntFloat(123.456) three = IntFloat("123.abc") assert one.check().ok() assert two.check().ok() assert not three.check().ok()
def test_hashing(): IntStr = Choice("IntStrFloat", (Integer, String)) map = {IntStr(123): 'foo', IntStr("123"): 'bar', IntStr("abc"): 'baz'} assert IntStr(123) in map assert IntStr("123") in map assert IntStr("abc") in map assert IntStr(456) not in map assert IntStr("456") not in map assert IntStr("def") not in map
def test_choice_primlist(): """Test that choices with a list value work correctly.""" C = Choice([String, List(Integer)]) c = C([1, 2, 3]) assert c.check().ok() c = C("hello") assert c.check().ok() c = C([1, 2, "{{x}}"]) assert not c.check().ok() assert c.bind(x=3).check().ok()
class UpdateConfig(Struct): batch_size = Default(Integer, 1) watch_secs = Default(Integer, 45) max_per_shard_failures = Default(Integer, 0) max_total_failures = Default(Integer, 0) rollback_on_failure = Default(Boolean, True) wait_for_batch_completion = Default(Boolean, False) pulse_interval_secs = Integer sla_aware = Default(Boolean, False) update_strategy = Choice([QueueUpdateStrategy, BatchUpdateStrategy, VariableBatchUpdateStrategy])
def test_get_choice_in_struct(): class Foo(Struct): foo = Required(String) class Bar(Struct): bar = Required(String) Item = Choice("Item", (Foo, Bar)) class Qux(Struct): item = Choice([String, List(Item)]) b = Qux(item=[Foo(foo="fubar")]) assert b.get() == frozendict({'item': (frozendict({'foo': u'fubar'}), )})
def test_repr(): class Dumb(Struct): one = String class ChoiceDefaultStruct(Struct): a = Default(Choice("IntOrDumb", [Dumb, Integer]), 28) b = Integer class OtherStruct(Struct): first = ChoiceDefaultStruct second = String C = Choice([String, List(Integer)]) testvalone = C("hello") testvaltwo = C([1, 2, 3]) assert repr(testvalone) == "Choice_String_IntegerList('hello')" assert repr(testvaltwo) == "Choice_String_IntegerList([1, 2, 3])"
def test_choice_interpolation(): IntFloat = Choice((Integer, Float)) one = IntFloat('{{abc}}') two = IntFloat('{{a}}{{b}}') one_int = one.bind(abc=34) assert isinstance(one_int.interpolate()[0], Integer) assert one_int.check().ok() one_fl = one.bind(abc=123.354) assert isinstance(one_fl.interpolate()[0], Float) assert one_fl.check().ok() one_str = one.bind(abc="def") assert not one_str.check().ok() assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')] two_one = two.bind(a=12, b=23) assert two_one.check().ok() assert two_one.unwrap() == Integer(1223) two_two = two.bind(a=12, b=".34") assert two_two.check().ok() assert two_two.unwrap() == Float(12.34)
def to_thrift_update_settings(self, instances=None): """Converts UpdateConfig into thrift JobUpdateSettings object. Arguments: instances - optional list of instances to update. """ if self.update_strategy is Empty: update_strategy = Choice([ PystachioQueueUpdateStrategy, PystachioBatchUpdateStrategy, PystachioVariableBatchUpdateStrategy ]) if self.wait_for_batch_completion: self.update_strategy = update_strategy( PystachioBatchUpdateStrategy(batch_size=self.batch_size)) else: self.update_strategy = update_strategy( PystachioQueueUpdateStrategy(batch_size=self.batch_size)) else: unwrapped = self.update_strategy.unwrap() if isinstance(unwrapped, PystachioQueueUpdateStrategy): self.batch_size = self.update_strategy.groupSize elif isinstance(unwrapped, PystachioBatchUpdateStrategy): self.batch_size = self.update_strategy.groupSize self.wait_for_batch_completion = True elif isinstance(unwrapped, PystachioBatchUpdateStrategy): self.batch_size = self.update_strategy.groupSizes[0] return JobUpdateSettings( updateGroupSize=self.batch_size, maxPerInstanceFailures=self.max_per_instance_failures, maxFailedInstances=self.max_total_failures, minWaitInInstanceRunningMs=self.watch_secs * 1000, rollbackOnFailure=self.rollback_on_failure, waitForBatchCompletion=self.wait_for_batch_completion, updateOnlyTheseInstances=self.instances_to_ranges(instances) if instances else None, updateStrategy=create_update_strategy_config(self.update_strategy), blockIfNoPulsesAfterMs=(self.pulse_interval_secs * 1000 if self.pulse_interval_secs else None), slaAware=self.sla_aware)
class Mesos(Struct): image = Choice([AppcImage, DockerImage])
class Mesos(Struct): image = Choice([AppcImage, DockerImage]) volumes = Default(List(Volume), [])
class TestUpdaterUtil(unittest.TestCase): EXPECTED_JOB_UPDATE_SETTINGS = JobUpdateSettings( blockIfNoPulsesAfterMs=None, updateOnlyTheseInstances=None, slaAware=False, maxPerInstanceFailures=0, waitForBatchCompletion=False, rollbackOnFailure=True, minWaitInInstanceRunningMs=45000, updateGroupSize=1, maxFailedInstances=0) UPDATE_STRATEGIES = Choice([ PystachioQueueUpdateStrategy, PystachioBatchUpdateStrategy, PystachioVariableBatchUpdateStrategy ]) def test_multiple_ranges(self): """Test multiple ranges.""" ranges = [ repr(e) for e in UpdaterConfig.instances_to_ranges([1, 2, 3, 5, 7, 8]) ] assert 3 == len(ranges), "Wrong number of ranges:%s" % len(ranges) assert repr(Range(first=1, last=3)) in ranges, "Missing range [1,3]" assert repr(Range(first=5, last=5)) in ranges, "Missing range [5,5]" assert repr(Range(first=7, last=8)) in ranges, "Missing range [7,8]" def test_one_element(self): """Test one ID in the list.""" ranges = [repr(e) for e in UpdaterConfig.instances_to_ranges([1])] assert 1 == len(ranges), "Wrong number of ranges:%s" % len(ranges) assert repr(Range(first=1, last=1)) in ranges, "Missing range [1,1]" def test_none_list(self): """Test None list produces None result.""" assert UpdaterConfig.instances_to_ranges( None) is None, "Result must be None." def test_empty_list(self): """Test empty list produces None result.""" assert UpdaterConfig.instances_to_ranges( []) is None, "Result must be None." def test_pulse_interval_secs(self): config = UpdaterConfig( UpdateConfig(batch_size=1, watch_secs=1, max_per_shard_failures=1, max_total_failures=1, pulse_interval_secs=60)) assert 60000 == config.to_thrift_update_settings( ).blockIfNoPulsesAfterMs def test_pulse_interval_unset(self): config = UpdaterConfig( UpdateConfig(batch_size=1, watch_secs=1, max_per_shard_failures=1, max_total_failures=1)) assert config.to_thrift_update_settings( ).blockIfNoPulsesAfterMs is None def test_pulse_interval_too_low(self): threshold = UpdaterConfig.MIN_PULSE_INTERVAL_SECONDS with raises(ValueError) as e: UpdaterConfig( UpdateConfig(batch_size=1, watch_secs=1, max_per_shard_failures=1, max_total_failures=1, pulse_interval_secs=threshold - 1)) assert 'Pulse interval seconds must be at least %s seconds.' % threshold in e.value.message def test_to_thrift_update_settings_strategy(self): """Test to_thrift produces an expected thrift update settings configuration from a Pystachio update object. """ config = UpdaterConfig( UpdateConfig(update_strategy=self.UPDATE_STRATEGIES( PystachioVariableBatchUpdateStrategy( batch_sizes=[1, 2, 3, 4], autopause_after_batch=True)))) thrift_update_config = config.to_thrift_update_settings() update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS) update_settings.updateStrategy = JobUpdateStrategy( batchStrategy=None, queueStrategy=None, varBatchStrategy=VariableBatchJobUpdateStrategy( groupSizes=(1, 2, 3, 4), autopauseAfterBatch=True)) assert thrift_update_config == update_settings def test_to_thrift_update_settings_no_strategy_queue(self): """Test to_thrift produces an expected thrift update settings configuration from a Pystachio update object that doesn't include an update strategy. The configuration in this test should be converted to a QueueJobUpdateStrategy. """ config = UpdaterConfig(UpdateConfig()) thrift_update_config = config.to_thrift_update_settings() update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS) update_settings.updateStrategy = JobUpdateStrategy( batchStrategy=None, queueStrategy=QueueJobUpdateStrategy(groupSize=1), varBatchStrategy=None) assert thrift_update_config == update_settings def test_to_thrift_update_settings_no_strategy_batch(self): """Test to_thrift produces an expected thrift update settings configuration from a Pystachio update object that doesn't include an update strategy. The configuration in this test should be converted to a BatchJobUpdateStrategy. """ config = UpdaterConfig(UpdateConfig(wait_for_batch_completion=True)) thrift_update_config = config.to_thrift_update_settings() update_settings = copy.deepcopy(self.EXPECTED_JOB_UPDATE_SETTINGS) update_settings.updateStrategy = JobUpdateStrategy( batchStrategy=BatchJobUpdateStrategy(groupSize=1, autopauseAfterBatch=False), queueStrategy=None, varBatchStrategy=None) update_settings.waitForBatchCompletion = True assert thrift_update_config == update_settings def test_wait_for_batch_completion_and_update_strategy(self): """Test setting wait_for_batch_completion along with an update strategy. This combination should result in a fast fail. """ with raises(ValueError) as e: UpdaterConfig( UpdateConfig(wait_for_batch_completion=True, update_strategy=self.UPDATE_STRATEGIES( PystachioBatchUpdateStrategy(batch_size=3)))) assert ('Ambiguous update configuration. Cannot combine ' 'wait_batch_completion with an ' 'explicit update strategy.' in e.value.message) def test_batch_size_and_update_strategy(self): """Test setting a batch size along with an update strategy. This combination should result in a fast fail. """ with raises(ValueError) as e: UpdaterConfig( UpdateConfig(batch_size=2, update_strategy=self.UPDATE_STRATEGIES( PystachioBatchUpdateStrategy(batch_size=3)))) assert ('Ambiguous update configuration. Cannot combine ' 'update strategy with batch size. Please set batch' 'size inside of update strategy instead.' in e.value.message)
class SOne(Struct): a = Choice((Integer, Float)) b = String
class Yuck(Struct): one = Choice([Foo, Integer]) two = String
def test_choice_string_enum(): TestEnum = Enum("TestEnum", ("A", "B", "C")) TestChoice = Choice("TestChoice", (TestEnum, String)) v = TestChoice("A") assert isinstance(v.interpolate()[0], TestEnum) assert isinstance(TestChoice("Q").interpolate()[0], String)
class Qux(Struct): item = Choice([String, List(Item)])
class Bar(Struct): a = Choice("StringOrFoo", [Integer, String, Foo]) b = String
class ChoiceDefaultStruct(Struct): a = Default(Choice("IntOrDumb", [Dumb, Integer]), 28) b = Integer