Exemplo n.º 1
0
 def standard_bindings(cls, job):
   # Rewrite now-deprecated bindings into their proper form.
   return job.bind({
     Ref.from_address('mesos.role'): '{{role}}',
     Ref.from_address('mesos.cluster'): '{{cluster}}',
     Ref.from_address('thermos.user'): '{{role}}',
   })
Exemplo n.º 2
0
 def task_links(self):
     # {{mesos.instance}} --> %shard_id%
     # {{thermos.ports[foo]}} --> %port:foo%
     task_links = self._job.task_links()
     if task_links is Empty:
         return task_links
     _, uninterp = task_links.interpolate()
     substitutions = {Ref.from_address('mesos.instance'): '%shard_id%'}
     port_scope = Ref.from_address('thermos.ports')
     for ref in uninterp:
         subscope = port_scope.scoped_to(ref)
         if subscope:
             substitutions[ref] = '%%port:%s%%' % subscope.action().value
     return task_links.bind(substitutions)
Exemplo n.º 3
0
 def task_links(self):
   # {{mesos.instance}} --> %shard_id%
   # {{thermos.ports[foo]}} --> %port:foo%
   task_links = self._job.task_links()
   if task_links is Empty:
     return task_links
   _, uninterp = task_links.interpolate()
   substitutions = {
     Ref.from_address('mesos.instance'): '%shard_id%'
   }
   port_scope = Ref.from_address('thermos.ports')
   for ref in uninterp:
     subscope = port_scope.scoped_to(ref)
     if subscope:
       substitutions[ref] = '%%port:%s%%' % subscope.action().value
   return task_links.bind(substitutions)
Exemplo n.º 4
0
def mesos_task_instance_from_assigned_task(assigned_task):
  """Deserialize MesosTaskInstance from an AssignedTask thrift."""
  thermos_task = assigned_task.task.executorConfig.data

  if not thermos_task:
    raise ValueError('Task did not have a thermos config!')

  try:
    json_blob = json.loads(thermos_task)
  except (TypeError, ValueError) as e:
    raise ValueError('Could not deserialize thermos config: %s' % e)

  # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
  # or we can be sending a MesosJob.  So handle both possible cases.  Once everyone
  # is using MesosJob, then we can begin to leverage additional information that
  # becomes available such as cluster.
  if 'instance' in json_blob:
    return MesosTaskInstance.json_loads(thermos_task)

  # This is a MesosJob
  mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
  unbound_refs = []
  for ref in refs:
    # If the ref is {{thermos.task_id}} or a subscope of
    # {{thermos.ports}}, it currently gets bound by the Thermos Runner,
    # so we must leave them unbound.
    #
    # {{thermos.user}} is a legacy binding which we can safely ignore.
    #
    # TODO(wickman) These should be rewritten by the mesos client to use
    # %%style%% replacements in order to allow us to better type-check configs
    # client-side.
    if ref == Ref.from_address('thermos.task_id'):
      continue
    if Ref.subscope(Ref.from_address('thermos.ports'), ref):
      continue
    if ref == Ref.from_address('thermos.user'):
      continue
    else:
      unbound_refs.append(ref)

  if len(unbound_refs) != 0:
    raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, unbound_refs)))

  return mti
Exemplo n.º 5
0
def test_choice_interpolation():
  IntFloat = Choice((Integer, Float))
  one = IntFloat('{{abc}}')
  two = IntFloat('{{a}}{{b}}')
  one_int = one.bind(abc=34)
  assert isinstance(one_int.interpolate()[0], Integer)
  assert one_int.check().ok()
  one_fl = one.bind(abc=123.354)
  assert isinstance(one_fl.interpolate()[0], Float)
  assert one_fl.check().ok()
  one_str = one.bind(abc="def")
  assert not one_str.check().ok()
  assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')]
  two_one =  two.bind(a=12, b=23)
  assert two_one.check().ok()
  assert two_one.unwrap() == Integer(1223)
  two_two = two.bind(a=12, b=".34")
  assert two_two.check().ok()
  assert two_two.unwrap() == Float(12.34)
Exemplo n.º 6
0
def test_choice_interpolation():
  IntFloat = Choice((Integer, Float))
  one = IntFloat('{{abc}}')
  two = IntFloat('{{a}}{{b}}')
  one_int = one.bind(abc=34)
  assert isinstance(one_int.interpolate()[0], Integer)
  assert one_int.check().ok()
  one_fl = one.bind(abc=123.354)
  assert isinstance(one_fl.interpolate()[0], Float)
  assert one_fl.check().ok()
  one_str = one.bind(abc="def")
  assert not one_str.check().ok()
  assert two.interpolate()[1] == [Ref.from_address('a'), Ref.from_address('b')]
  two_one =  two.bind(a=12, b=23)
  assert two_one.check().ok()
  assert two_one.unwrap() == Integer(1223)
  two_two = two.bind(a=12, b=".34")
  assert two_two.check().ok()
  assert two_two.unwrap() == Float(12.34)
Exemplo n.º 7
0
 def add_binding_callback(option, opt, value, parser):
   if not getattr(parser.values, option_name, None):
     setattr(parser.values, option_name, [])
   if len(value.split('=')) != 2:
     raise ParseError('Binding must be of the form NAME=VALUE')
   name, value = value.split('=')
   try:
     ref = Ref.from_address(name)
   except Ref.InvalidRefError as e:
     raise ParseError('Could not parse ref %s: %s' % (name, e))
   getattr(parser.values, option_name).append({ref: value})
Exemplo n.º 8
0
 def add_binding_callback(option, opt, value, parser):
     if not getattr(parser.values, option_name, None):
         setattr(parser.values, option_name, [])
     if len(value.split('=')) != 2:
         raise ParseError('Binding must be of the form NAME=VALUE')
     name, value = value.split('=')
     try:
         ref = Ref.from_address(name)
     except Ref.InvalidRefError as e:
         raise ParseError('Could not parse ref %s: %s' % (name, e))
     getattr(parser.values, option_name).append({ref: value})
Exemplo n.º 9
0
 def bind(self, config, match, env, binding_dict):
   cluster = CLUSTERS[config.cluster()]
   image = match
   ref_str = 'docker.image[%s][%s]' % image[2:4]
   ref = Ref.from_address(ref_str)
   if ref_str in binding_dict:
     (image_data, image_struct) = binding_dict[ref_str]
   else:
     image_data = '%s:%s' % (image[2], image[3])
     image_struct = DockerRegistryClient.resolve(cluster, image[2], image[3])
   binding_dict[ref_str] = (image_data, image_struct)
   config.bind({ref: image_struct})
Exemplo n.º 10
0
 def extract(cls, obj):
   port_scope = Ref.from_address('thermos.ports')
   _, uninterp = obj.interpolate()
   ports = []
   for ref in uninterp:
     subscope = port_scope.scoped_to(ref)
     if subscope is not None:
       if not subscope.is_index():
         raise cls.InvalidPorts(
           'Bad port specification "%s" (should be of form "thermos.ports[name]"' % ref.address())
       ports.append(subscope.action().value)
   return ports
Exemplo n.º 11
0
 def extract(cls, obj):
     port_scope = Ref.from_address('thermos.ports')
     _, uninterp = obj.interpolate()
     ports = []
     for ref in uninterp:
         subscope = port_scope.scoped_to(ref)
         if subscope is not None:
             if not subscope.is_index():
                 raise cls.InvalidPorts(
                     'Bad port specification "%s" (should be of form "thermos.ports[name]"'
                     % ref.address())
             ports.append(subscope.action().value)
     return ports
Exemplo n.º 12
0
 def bind(self, config, match, env, binding_dict):
     cluster = CLUSTERS[config.cluster()]
     image = match
     ref_str = 'docker.image[%s][%s]' % image[2:4]
     ref = Ref.from_address(ref_str)
     if ref_str in binding_dict:
         (image_data, image_struct) = binding_dict[ref_str]
     else:
         image_data = '%s:%s' % (image[2], image[3])
         image_struct = DockerRegistryClient.resolve(
             cluster, image[2], image[3])
     binding_dict[ref_str] = (image_data, image_struct)
     config.bind({ref: image_struct})
Exemplo n.º 13
0
def binding_parser(binding):
  """Pystachio takes bindings in the form of a list of dictionaries. Each pystachio binding
  becomes another dictionary in the list. So we need to convert the bindings specified by
  the user from a list of "name=value" formatted strings to a list of the dictionaries
  expected by pystachio.
  """
  binding_parts = binding.split("=")
  if len(binding_parts) != 2:
    raise ValueError('Binding parameter must be formatted name=value')
  try:
    ref = Ref.from_address(binding_parts[0])
  except Ref.InvalidRefError as e:
    raise ValueError("Could not parse binding parameter %s: %s" % (binding, e))
  return {ref: binding_parts[1]}
Exemplo n.º 14
0
def binding_parser(binding):
  """Pystachio takes bindings in the form of a list of dictionaries. Each pystachio binding
  becomes another dictionary in the list. So we need to convert the bindings specified by
  the user from a list of "name=value" formatted strings to a list of the dictionaries
  expected by pystachio.
  """
  binding_parts = binding.split("=", 1)
  if len(binding_parts) < 2:
    raise ValueError('Binding parameter must be formatted name=value')
  try:
    ref = Ref.from_address(binding_parts[0])
  except Ref.InvalidRefError as e:
    raise ValueError("Could not parse binding parameter %s: %s" % (binding, e))
  return {ref: binding_parts[1]}
Exemplo n.º 15
0
 def bind(self, config, match, env, binding_dict):
   cluster = CLUSTERS[config.cluster()]
   name, version = match[1:3]
   ref_str = 'sacker[%s][%s]' % (name, version)
   ref = Ref.from_address(ref_str)
   if ref_str in binding_dict:
     s3_struct = binding_dict[ref_str]
   else:
     s3_struct = get_sacker_binding(cluster, name, version)
   binding_dict[ref_str] = s3_struct
   config.bind({ref: s3_struct})
   config.add_metadata(
       key='sacker',
       value='%s/%s sha:%s' % (name, s3_struct.version(), s3_struct.sha())
   )
Exemplo n.º 16
0
def test_choice_in_struct():
  class SOne(Struct):
    a = Choice((Integer, Float))
    b = String

  one = SOne(a=12, b="abc")
  assert one.check().ok()
  assert one.interpolate()[0].a().unwrap() == Integer(12)

  two = SOne(a="1{{q}}2", b="hi there")
  assert not two.check().ok()
  refs = two.interpolate()[1]
  assert refs == [Ref.from_address('q')]

  two_int = two.bind(q="34")
  assert two_int.check().ok()
  assert two_int.a().unwrap() == Integer(1342)

  two_fl = two.bind(q="3.4")
  assert two_fl.check().ok()
  assert two_fl.a().unwrap() == Float(13.42)

  two_str = two.bind(q="abc")
  assert not two_str.check().ok()
Exemplo n.º 17
0
def test_choice_in_struct():
    class SOne(Struct):
        a = Choice((Integer, Float))
        b = String

    one = SOne(a=12, b="abc")
    assert one.check().ok()
    assert one.interpolate()[0].a().unwrap() == Integer(12)

    two = SOne(a="1{{q}}2", b="hi there")
    assert not two.check().ok()
    refs = two.interpolate()[1]
    assert refs == [Ref.from_address('q')]

    two_int = two.bind(q="34")
    assert two_int.check().ok()
    assert two_int.a().unwrap() == Integer(1342)

    two_fl = two.bind(q="3.4")
    assert two_fl.check().ok()
    assert two_fl.a().unwrap() == Float(13.42)

    two_str = two.bind(q="abc")
    assert not two_str.check().ok()
Exemplo n.º 18
0

def filter_aliased_fields(job):
  return job(**dict((key, Empty) for key in ALIASED_FIELDS))


def assert_valid_field(field, identifier):
  VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
  if not isinstance(identifier, Compatibility.string):
    raise InvalidConfig("%s must be a string" % field)
  if not VALID_IDENTIFIER.match(identifier):
    raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
  return identifier


MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
MESOS_HOSTNAME_REF = Ref.from_address('mesos.hostname')
THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')


def convert(job, metadata=frozenset(), ports=frozenset()):
  """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

  owner = Identity(user=getpass.getuser())
  key = JobKey(
    role=assert_valid_field('role', fully_interpolated(job.role())),
    environment=assert_valid_field('environment', fully_interpolated(job.environment())),
    name=assert_valid_field('name', fully_interpolated(job.name())))

  task_raw = job.task()
Exemplo n.º 19
0
def convert(job, metadata=frozenset(), ports=frozenset()):
    """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

    owner = Identity(user=getpass.getuser())
    key = JobKey(
        role=assert_valid_field('role', fully_interpolated(job.role())),
        environment=assert_valid_field('environment',
                                       fully_interpolated(job.environment())),
        name=assert_valid_field('name', fully_interpolated(job.name())))

    task_raw = job.task()

    MB = 1024 * 1024
    task = TaskConfig()

    def not_empty_or(item, default):
        return default if item is Empty else fully_interpolated(item)

    # job components
    task.production = fully_interpolated(job.production(), bool)
    task.isService = select_service_bit(job)
    task.maxTaskFailures = fully_interpolated(job.max_task_failures())
    task.priority = fully_interpolated(job.priority())
    task.contactEmail = not_empty_or(job.contact(), None)
    task.tier = not_empty_or(job.tier(), None)

    if job.has_partition_policy():
        task.partitionPolicy = PartitionPolicy(
            fully_interpolated(job.partition_policy().reschedule()),
            fully_interpolated(job.partition_policy().delay_secs()))

    # Add metadata to a task, to display in the scheduler UI.
    metadata_set = frozenset()
    if job.has_metadata():
        customized_metadata = job.metadata()
        metadata_set |= frozenset(
            (str(fully_interpolated(key_value_metadata.key())),
             str(fully_interpolated(key_value_metadata.value())))
            for key_value_metadata in customized_metadata)
    metadata_set |= frozenset(
        (str(key), str(value)) for key, value in metadata)
    task.metadata = frozenset(
        Metadata(key=key, value=value) for key, value in metadata_set)

    # task components
    if not task_raw.has_resources():
        raise InvalidConfig('Task must specify resources!')

    if (fully_interpolated(task_raw.resources().ram()) == 0
            or fully_interpolated(task_raw.resources().disk()) == 0):
        raise InvalidConfig(
            'Must specify ram and disk resources, got ram:%r disk:%r' %
            (fully_interpolated(task_raw.resources().ram()),
             fully_interpolated(task_raw.resources().disk())))

    numCpus = fully_interpolated(task_raw.resources().cpu())
    ramMb = fully_interpolated(task_raw.resources().ram()) / MB
    diskMb = fully_interpolated(task_raw.resources().disk()) / MB
    if numCpus <= 0 or ramMb <= 0 or diskMb <= 0:
        raise InvalidConfig(
            'Task has invalid resources.  cpu/ramMb/diskMb must all be positive: '
            'cpu:%r ramMb:%r diskMb:%r' % (numCpus, ramMb, diskMb))
    numGpus = fully_interpolated(task_raw.resources().gpu())

    task.resources = frozenset([
        Resource(numCpus=numCpus),
        Resource(ramMb=ramMb),
        Resource(diskMb=diskMb)
    ] + [Resource(namedPort=p)
         for p in ports] + ([Resource(numGpus=numGpus)] if numGpus else []))

    task.job = key
    task.owner = owner
    task.taskLinks = {}  # See AURORA-739
    task.constraints = constraints_to_thrift(
        not_empty_or(job.constraints(), {}))
    task.container = create_container_config(job.container())

    underlying, refs = job.interpolate()

    # need to fake an instance id for the sake of schema checking
    underlying_checked = underlying.bind(mesos={
        'instance': 31337,
        'hostname': ''
    })
    try:
        ThermosTaskValidator.assert_valid_task(underlying_checked.task())
    except ThermosTaskValidator.InvalidTaskError as e:
        raise InvalidConfig('Task is invalid: %s' % e)
    if not underlying_checked.check().ok():
        raise InvalidConfig('Job not fully specified: %s' %
                            underlying.check().message())

    unbound = []
    for ref in refs:
        if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF,
                   MESOS_HOSTNAME_REF) or (Ref.subscope(
                       THERMOS_PORT_SCOPE_REF, ref)):
            continue
        unbound.append(ref)

    if unbound:
        raise InvalidConfig('Config contains unbound variables: %s' %
                            ' '.join(map(str, unbound)))

    # set the executor that will be used by the Mesos task. Thermos is the default
    executor = job.executor_config()
    if fully_interpolated(executor.name()) == AURORA_EXECUTOR_NAME:
        task.executorConfig = ExecutorConfig(
            name=AURORA_EXECUTOR_NAME,
            data=filter_aliased_fields(underlying).json_dumps())
    else:
        task.executorConfig = ExecutorConfig(
            name=fully_interpolated(executor.name()),
            data=fully_interpolated(executor.data()))

    return JobConfiguration(
        key=key,
        owner=owner,
        cronSchedule=not_empty_or(job.cron_schedule(), None),
        cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()),
        taskConfig=task,
        instanceCount=fully_interpolated(job.instances()))
Exemplo n.º 20
0

def filter_aliased_fields(job):
    return job(**dict((key, Empty) for key in ALIASED_FIELDS))


def assert_valid_field(field, identifier):
    VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
    if not isinstance(identifier, Compatibility.string):
        raise InvalidConfig("%s must be a string" % field)
    if not VALID_IDENTIFIER.match(identifier):
        raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
    return identifier


MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
MESOS_HOSTNAME_REF = Ref.from_address('mesos.hostname')
THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')


def convert(job, metadata=frozenset(), ports=frozenset()):
    """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

    owner = Identity(user=getpass.getuser())
    key = JobKey(
        role=assert_valid_field('role', fully_interpolated(job.role())),
        environment=assert_valid_field('environment',
                                       fully_interpolated(job.environment())),
        name=assert_valid_field('name', fully_interpolated(job.name())))
Exemplo n.º 21
0

def filter_aliased_fields(job):
    return job(**dict((key, Empty) for key in ALIASED_FIELDS))


def assert_valid_field(field, identifier):
    VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
    if not isinstance(identifier, Compatibility.string):
        raise InvalidConfig("%s must be a string" % field)
    if not VALID_IDENTIFIER.match(identifier):
        raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
    return identifier


MESOS_INSTANCE_REF = Ref.from_address("mesos.instance")
MESOS_HOSTNAME_REF = Ref.from_address("mesos.hostname")
THERMOS_PORT_SCOPE_REF = Ref.from_address("thermos.ports")
THERMOS_TASK_ID_REF = Ref.from_address("thermos.task_id")


def convert(job, metadata=frozenset(), ports=frozenset()):
    """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

    owner = Identity(user=getpass.getuser())
    key = JobKey(
        role=assert_valid_field("role", fully_interpolated(job.role())),
        environment=assert_valid_field("environment", fully_interpolated(job.environment())),
        name=assert_valid_field("name", fully_interpolated(job.name())),
    )
Exemplo n.º 22
0
 def uncached_bind(self, config, match, env, binding_dict):
     self.uncached_binds += 1
     ref = Ref.from_address('%s[%s]' % match)
     binding = {ref: 'C(%s)' % match[1]}
     config.bind(binding)
     return binding
Exemplo n.º 23
0
 def bind(self, config, match, env, binding_dict):
     # TODO(wickman) You should be able to take a match tuple + matcher
     # object and return the ref.
     self.binds += 1
     ref = Ref.from_address('%s[%s]' % match)
     config.bind({ref: 'U(%s)' % match[1]})
Exemplo n.º 24
0
def convert(job, packages=frozenset(), ports=frozenset()):
    """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

    owner = Identity(role=fully_interpolated(job.role()),
                     user=getpass.getuser())
    key = JobKey(
        role=assert_valid_field('role', fully_interpolated(job.role())),
        environment=assert_valid_field('environment',
                                       fully_interpolated(job.environment())),
        name=assert_valid_field('name', fully_interpolated(job.name())))

    task_raw = job.task()

    MB = 1024 * 1024
    task = TaskConfig()

    def not_empty_or(item, default):
        return default if item is Empty else fully_interpolated(item)

    # job components
    task.jobName = fully_interpolated(job.name())
    task.environment = fully_interpolated(job.environment())
    task.production = fully_interpolated(job.production(), bool)
    task.isService = select_service_bit(job)
    task.maxTaskFailures = fully_interpolated(job.max_task_failures())
    task.priority = fully_interpolated(job.priority())
    task.contactEmail = not_empty_or(job.contact(), None)

    # Add package tuples to a task, to display in the scheduler UI.
    task.packages = frozenset(
        Package(role=str(role), name=str(package_name), version=int(version))
        for role, package_name, version in packages)

    # task components
    if not task_raw.has_resources():
        raise InvalidConfig('Task must specify resources!')

    if (fully_interpolated(task_raw.resources().ram()) == 0
            or fully_interpolated(task_raw.resources().disk()) == 0):
        raise InvalidConfig(
            'Must specify ram and disk resources, got ram:%r disk:%r' %
            (fully_interpolated(task_raw.resources().ram()),
             fully_interpolated(task_raw.resources().disk())))

    task.numCpus = fully_interpolated(task_raw.resources().cpu())
    task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
    task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
    if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
        raise InvalidConfig(
            'Task has invalid resources.  cpu/ramMb/diskMb must all be positive: '
            'cpu:%r ramMb:%r diskMb:%r' %
            (task.numCpus, task.ramMb, task.diskMb))

    task.owner = owner
    task.requestedPorts = ports
    task.taskLinks = not_empty_or(job.task_links(), {})
    task.constraints = constraints_to_thrift(
        not_empty_or(job.constraints(), {}))

    underlying, refs = job.interpolate()

    # need to fake an instance id for the sake of schema checking
    underlying_checked = underlying.bind(mesos={'instance': 31337})
    try:
        ThermosTaskValidator.assert_valid_task(underlying_checked.task())
    except ThermosTaskValidator.InvalidTaskError as e:
        raise InvalidConfig('Task is invalid: %s' % e)
    if not underlying_checked.check().ok():
        raise InvalidConfig('Job not fully specified: %s' %
                            underlying.check().message())

    unbound = []
    for ref in refs:
        if ref == THERMOS_TASK_ID_REF or ref == MESOS_INSTANCE_REF or (
                Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
            continue
        unbound.append(ref)

    if unbound:
        raise InvalidConfig('Config contains unbound variables: %s' %
                            ' '.join(map(str, unbound)))

    cron_schedule = not_empty_or(job.cron_schedule(), '')
    cron_policy = select_cron_policy(job.cron_policy(),
                                     job.cron_collision_policy())

    task.executorConfig = ExecutorConfig(
        name=AURORA_EXECUTOR_NAME,
        data=filter_aliased_fields(underlying).json_dumps())

    return JobConfiguration(key=key,
                            owner=owner,
                            cronSchedule=cron_schedule,
                            cronCollisionPolicy=cron_policy,
                            taskConfig=task,
                            instanceCount=fully_interpolated(job.instances()))
Exemplo n.º 25
0
def convert(job, metadata=frozenset(), ports=frozenset()):
  """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""

  owner = Identity(user=getpass.getuser())
  key = JobKey(
    role=assert_valid_field('role', fully_interpolated(job.role())),
    environment=assert_valid_field('environment', fully_interpolated(job.environment())),
    name=assert_valid_field('name', fully_interpolated(job.name())))

  task_raw = job.task()

  MB = 1024 * 1024
  task = TaskConfig()

  def not_empty_or(item, default):
    return default if item is Empty else fully_interpolated(item)

  # job components
  task.production = fully_interpolated(job.production(), bool)
  task.isService = select_service_bit(job)
  task.maxTaskFailures = fully_interpolated(job.max_task_failures())
  task.priority = fully_interpolated(job.priority())
  task.contactEmail = not_empty_or(job.contact(), None)
  task.tier = not_empty_or(job.tier(), None)

  # Add metadata to a task, to display in the scheduler UI.
  task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata)

  # task components
  if not task_raw.has_resources():
    raise InvalidConfig('Task must specify resources!')

  if (fully_interpolated(task_raw.resources().ram()) == 0
      or fully_interpolated(task_raw.resources().disk()) == 0):
    raise InvalidConfig('Must specify ram and disk resources, got ram:%r disk:%r' % (
      fully_interpolated(task_raw.resources().ram()),
      fully_interpolated(task_raw.resources().disk())))

  task.numCpus = fully_interpolated(task_raw.resources().cpu())
  task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
  task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
  if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
    raise InvalidConfig('Task has invalid resources.  cpu/ramMb/diskMb must all be positive: '
        'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb))

  task.job = key
  task.owner = owner
  task.requestedPorts = ports
  task.taskLinks = {}  # See AURORA-739
  task.constraints = constraints_to_thrift(not_empty_or(job.constraints(), {}))
  task.container = create_container_config(job.container())

  underlying, refs = job.interpolate()

  # need to fake an instance id for the sake of schema checking
  underlying_checked = underlying.bind(mesos={'instance': 31337, 'hostname': ''})
  try:
    ThermosTaskValidator.assert_valid_task(underlying_checked.task())
  except ThermosTaskValidator.InvalidTaskError as e:
    raise InvalidConfig('Task is invalid: %s' % e)
  if not underlying_checked.check().ok():
    raise InvalidConfig('Job not fully specified: %s' % underlying.check().message())

  unbound = []
  for ref in refs:
    if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF, MESOS_HOSTNAME_REF) or (
        Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
      continue
    unbound.append(ref)

  if unbound:
    raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound)))

  task.executorConfig = ExecutorConfig(
      name=AURORA_EXECUTOR_NAME,
      data=filter_aliased_fields(underlying).json_dumps())

  return JobConfiguration(
      key=key,
      owner=owner,
      cronSchedule=not_empty_or(job.cron_schedule(), None),
      cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()),
      taskConfig=task,
      instanceCount=fully_interpolated(job.instances()))
 def uncached_bind(self, config, match, env, binding_dict):
   self.uncached_binds += 1
   ref = Ref.from_address('%s[%s]' % match)
   binding = {ref: 'C(%s)' % match[1]}
   config.bind(binding)
   return binding
 def bind(self, config, match, env, binding_dict):
   # TODO(wickman) You should be able to take a match tuple + matcher
   # object and return the ref.
   self.binds += 1
   ref = Ref.from_address('%s[%s]' % match)
   config.bind({ref: 'U(%s)' % match[1]})