def standard_bindings(cls, job):
# Rewrite now-deprecated bindings into their proper form.
return job.bind({
Ref.from_address('mesos.role'): '{{role}}',
Ref.from_address('mesos.cluster'): '{{cluster}}',
Ref.from_address('thermos.user'): '{{role}}',
})
def task_links(self):
# {{mesos.instance}} --> %shard_id%
# {{thermos.ports[foo]}} --> %port:foo%
task_links = self._job.task_links()
if task_links is Empty:
return task_links
_, uninterp = task_links.interpolate()
substitutions = {Ref.from_address('mesos.instance'): '%shard_id%'}
port_scope = Ref.from_address('thermos.ports')
for ref in uninterp:
subscope = port_scope.scoped_to(ref)
if subscope:
substitutions[ref] = '%%port:%s%%' % subscope.action().value
return task_links.bind(substitutions)
def task_links(self):
# {{mesos.instance}} --> %shard_id%
# {{thermos.ports[foo]}} --> %port:foo%
task_links = self._job.task_links()
if task_links is Empty:
return task_links
_, uninterp = task_links.interpolate()
substitutions = {
Ref.from_address('mesos.instance'): '%shard_id%'
}
port_scope = Ref.from_address('thermos.ports')
for ref in uninterp:
subscope = port_scope.scoped_to(ref)
if subscope:
substitutions[ref] = '%%port:%s%%' % subscope.action().value
return task_links.bind(substitutions)
def mesos_task_instance_from_assigned_task(assigned_task):
"""Deserialize MesosTaskInstance from an AssignedTask thrift."""
thermos_task = assigned_task.task.executorConfig.data
if not thermos_task:
raise ValueError('Task did not have a thermos config!')
try:
json_blob = json.loads(thermos_task)
except (TypeError, ValueError) as e:
raise ValueError('Could not deserialize thermos config: %s' % e)
# As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
# or we can be sending a MesosJob. So handle both possible cases. Once everyone
# is using MesosJob, then we can begin to leverage additional information that
# becomes available such as cluster.
if 'instance' in json_blob:
return MesosTaskInstance.json_loads(thermos_task)
# This is a MesosJob
mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
unbound_refs = []
for ref in refs:
# If the ref is {{thermos.task_id}} or a subscope of
# {{thermos.ports}}, it currently gets bound by the Thermos Runner,
# so we must leave them unbound.
#
# {{thermos.user}} is a legacy binding which we can safely ignore.
#
# TODO(wickman) These should be rewritten by the mesos client to use
# %%style%% replacements in order to allow us to better type-check configs
# client-side.
if ref == Ref.from_address('thermos.task_id'):
continue
if Ref.subscope(Ref.from_address('thermos.ports'), ref):
continue
if ref == Ref.from_address('thermos.user'):
continue
else:
unbound_refs.append(ref)
if len(unbound_refs) != 0:
raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, unbound_refs)))
return mti
def test_choice_interpolation():
IntFloat = Choice((Integer, Float))
one = IntFloat('{{abc}}')
two = IntFloat('{{a}}{{b}}')
one_int = one.bind(abc=34)
assert isinstance(one_int.interpolate()[0], Integer)
assert one_int.check().ok()
one_fl = one.bind(abc=123.354)
assert isinstance(one_fl.interpolate()[0], Float)
assert one_fl.check().ok()
one_str = one.bind(abc="def")
assert not one_str.check().ok()
assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')]
two_one = two.bind(a=12, b=23)
assert two_one.check().ok()
assert two_one.unwrap() == Integer(1223)
two_two = two.bind(a=12, b=".34")
assert two_two.check().ok()
assert two_two.unwrap() == Float(12.34)
def test_choice_interpolation():
IntFloat = Choice((Integer, Float))
one = IntFloat('{{abc}}')
two = IntFloat('{{a}}{{b}}')
one_int = one.bind(abc=34)
assert isinstance(one_int.interpolate()[0], Integer)
assert one_int.check().ok()
one_fl = one.bind(abc=123.354)
assert isinstance(one_fl.interpolate()[0], Float)
assert one_fl.check().ok()
one_str = one.bind(abc="def")
assert not one_str.check().ok()
assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')]
two_one = two.bind(a=12, b=23)
assert two_one.check().ok()
assert two_one.unwrap() == Integer(1223)
two_two = two.bind(a=12, b=".34")
assert two_two.check().ok()
assert two_two.unwrap() == Float(12.34)
def add_binding_callback(option, opt, value, parser):
if not getattr(parser.values, option_name, None):
setattr(parser.values, option_name, [])
if len(value.split('=')) != 2:
raise ParseError('Binding must be of the form NAME=VALUE')
name, value = value.split('=')
try:
ref = Ref.from_address(name)
except Ref.InvalidRefError as e:
raise ParseError('Could not parse ref %s: %s' % (name, e))
getattr(parser.values, option_name).append({ref: value})
def add_binding_callback(option, opt, value, parser):
if not getattr(parser.values, option_name, None):
setattr(parser.values, option_name, [])
if len(value.split('=')) != 2:
raise ParseError('Binding must be of the form NAME=VALUE')
name, value = value.split('=')
try:
ref = Ref.from_address(name)
except Ref.InvalidRefError as e:
raise ParseError('Could not parse ref %s: %s' % (name, e))
getattr(parser.values, option_name).append({ref: value})
def bind(self, config, match, env, binding_dict):
cluster = CLUSTERS[config.cluster()]
image = match
ref_str = 'docker.image[%s][%s]' % image[2:4]
ref = Ref.from_address(ref_str)
if ref_str in binding_dict:
(image_data, image_struct) = binding_dict[ref_str]
else:
image_data = '%s:%s' % (image[2], image[3])
image_struct = DockerRegistryClient.resolve(cluster, image[2], image[3])
binding_dict[ref_str] = (image_data, image_struct)
config.bind({ref: image_struct})
def extract(cls, obj):
port_scope = Ref.from_address('thermos.ports')
_, uninterp = obj.interpolate()
ports = []
for ref in uninterp:
subscope = port_scope.scoped_to(ref)
if subscope is not None:
if not subscope.is_index():
raise cls.InvalidPorts(
'Bad port specification "%s" (should be of form "thermos.ports[name]"' % ref.address())
ports.append(subscope.action().value)
return ports
def extract(cls, obj):
port_scope = Ref.from_address('thermos.ports')
_, uninterp = obj.interpolate()
ports = []
for ref in uninterp:
subscope = port_scope.scoped_to(ref)
if subscope is not None:
if not subscope.is_index():
raise cls.InvalidPorts(
'Bad port specification "%s" (should be of form "thermos.ports[name]"'
% ref.address())
ports.append(subscope.action().value)
return ports
def bind(self, config, match, env, binding_dict):
cluster = CLUSTERS[config.cluster()]
image = match
ref_str = 'docker.image[%s][%s]' % image[2:4]
ref = Ref.from_address(ref_str)
if ref_str in binding_dict:
(image_data, image_struct) = binding_dict[ref_str]
else:
image_data = '%s:%s' % (image[2], image[3])
image_struct = DockerRegistryClient.resolve(
cluster, image[2], image[3])
binding_dict[ref_str] = (image_data, image_struct)
config.bind({ref: image_struct})
def binding_parser(binding):
"""Pystachio takes bindings in the form of a list of dictionaries. Each pystachio binding
becomes another dictionary in the list. So we need to convert the bindings specified by
the user from a list of "name=value" formatted strings to a list of the dictionaries
expected by pystachio.
"""
binding_parts = binding.split("=")
if len(binding_parts) != 2:
raise ValueError('Binding parameter must be formatted name=value')
try:
ref = Ref.from_address(binding_parts[0])
except Ref.InvalidRefError as e:
raise ValueError("Could not parse binding parameter %s: %s" % (binding, e))
return {ref: binding_parts[1]}
def binding_parser(binding):
"""Pystachio takes bindings in the form of a list of dictionaries. Each pystachio binding
becomes another dictionary in the list. So we need to convert the bindings specified by
the user from a list of "name=value" formatted strings to a list of the dictionaries
expected by pystachio.
"""
binding_parts = binding.split("=", 1)
if len(binding_parts) < 2:
raise ValueError('Binding parameter must be formatted name=value')
try:
ref = Ref.from_address(binding_parts[0])
except Ref.InvalidRefError as e:
raise ValueError("Could not parse binding parameter %s: %s" % (binding, e))
return {ref: binding_parts[1]}
def bind(self, config, match, env, binding_dict):
cluster = CLUSTERS[config.cluster()]
name, version = match[1:3]
ref_str = 'sacker[%s][%s]' % (name, version)
ref = Ref.from_address(ref_str)
if ref_str in binding_dict:
s3_struct = binding_dict[ref_str]
else:
s3_struct = get_sacker_binding(cluster, name, version)
binding_dict[ref_str] = s3_struct
config.bind({ref: s3_struct})
config.add_metadata(
key='sacker',
value='%s/%s sha:%s' % (name, s3_struct.version(), s3_struct.sha())
)
def test_choice_in_struct():
class SOne(Struct):
a = Choice((Integer, Float))
b = String
one = SOne(a=12, b="abc")
assert one.check().ok()
assert one.interpolate()[0].a().unwrap() == Integer(12)
two = SOne(a="1{{q}}2", b="hi there")
assert not two.check().ok()
refs = two.interpolate()[1]
assert refs == [Ref.from_address('q')]
two_int = two.bind(q="34")
assert two_int.check().ok()
assert two_int.a().unwrap() == Integer(1342)
two_fl = two.bind(q="3.4")
assert two_fl.check().ok()
assert two_fl.a().unwrap() == Float(13.42)
two_str = two.bind(q="abc")
assert not two_str.check().ok()
def test_choice_in_struct():
class SOne(Struct):
a = Choice((Integer, Float))
b = String
one = SOne(a=12, b="abc")
assert one.check().ok()
assert one.interpolate()[0].a().unwrap() == Integer(12)
two = SOne(a="1{{q}}2", b="hi there")
assert not two.check().ok()
refs = two.interpolate()[1]
assert refs == [Ref.from_address('q')]
two_int = two.bind(q="34")
assert two_int.check().ok()
assert two_int.a().unwrap() == Integer(1342)
two_fl = two.bind(q="3.4")
assert two_fl.check().ok()
assert two_fl.a().unwrap() == Float(13.42)
two_str = two.bind(q="abc")
assert not two_str.check().ok()
def filter_aliased_fields(job):
return job(**dict((key, Empty) for key in ALIASED_FIELDS))
def assert_valid_field(field, identifier):
VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
if not isinstance(identifier, Compatibility.string):
raise InvalidConfig("%s must be a string" % field)
if not VALID_IDENTIFIER.match(identifier):
raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
return identifier
MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
MESOS_HOSTNAME_REF = Ref.from_address('mesos.hostname')
THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment', fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
task_raw = job.task()
def filter_aliased_fields(job):
return job(**dict((key, Empty) for key in ALIASED_FIELDS))
def assert_valid_field(field, identifier):
VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
if not isinstance(identifier, Compatibility.string):
raise InvalidConfig("%s must be a string" % field)
if not VALID_IDENTIFIER.match(identifier):
raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
return identifier
MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
MESOS_HOSTNAME_REF = Ref.from_address('mesos.hostname')
THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment',
fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
def filter_aliased_fields(job):
return job(**dict((key, Empty) for key in ALIASED_FIELDS))
def assert_valid_field(field, identifier):
VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
if not isinstance(identifier, Compatibility.string):
raise InvalidConfig("%s must be a string" % field)
if not VALID_IDENTIFIER.match(identifier):
raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
return identifier
MESOS_INSTANCE_REF = Ref.from_address("mesos.instance")
MESOS_HOSTNAME_REF = Ref.from_address("mesos.hostname")
THERMOS_PORT_SCOPE_REF = Ref.from_address("thermos.ports")
THERMOS_TASK_ID_REF = Ref.from_address("thermos.task_id")
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field("role", fully_interpolated(job.role())),
environment=assert_valid_field("environment", fully_interpolated(job.environment())),
name=assert_valid_field("name", fully_interpolated(job.name())),
)
def uncached_bind(self, config, match, env, binding_dict):
self.uncached_binds += 1
ref = Ref.from_address('%s[%s]' % match)
binding = {ref: 'C(%s)' % match[1]}
config.bind(binding)
return binding
def bind(self, config, match, env, binding_dict):
# TODO(wickman) You should be able to take a match tuple + matcher
# object and return the ref.
self.binds += 1
ref = Ref.from_address('%s[%s]' % match)
config.bind({ref: 'U(%s)' % match[1]})
def bind(self, config, match, env, binding_dict):
# TODO(wickman) You should be able to take a match tuple + matcher
# object and return the ref.
self.binds += 1
ref = Ref.from_address('%s[%s]' % match)
config.bind({ref: 'U(%s)' % match[1]})
def uncached_bind(self, config, match, env, binding_dict):
self.uncached_binds += 1
ref = Ref.from_address('%s[%s]' % match)
binding = {ref: 'C(%s)' % match[1]}
config.bind(binding)
return binding