def launchTask(self, driver, task):
   """Queue a new garbage collection run, and drop any currently-enqueued runs."""
   if self._slave_id is None:
     self._slave_id = task.slave_id.value
   task_id = task.task_id.value
   self.log('launchTask() got task_id: %s' % task_id)
   if task_id == self._task_id:
     self.log('=> GC with task_id %s currently running - ignoring' % task_id)
     return
   elif task_id in self._gc_task_queue:
     self.log('=> Already have task_id %s queued - ignoring' % task_id)
     return
   try:
     art = thrift_deserialize(AdjustRetainedTasks(), task.data)
   except Exception as err:
     self.log('Error deserializing task: %s' % err)
     self.send_update(
         self._driver, task_id, mesos_pb2.TASK_FAILED, 'Deserialization of GC task failed')
     return
   try:
     prev_task_id, _ = self._gc_task_queue.popitem(0)
   except KeyError:  # no enqueued GC tasks - reset counter
     self._dropped_tasks.write(0)
   else:
     self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
     self._dropped_tasks.increment()
     self.log('=> Updating scheduler')
     self.send_update(self._driver, prev_task_id, mesos_pb2.TASK_FINISHED,
                      'Garbage collection skipped - GC executor received another task')
   self.log('=> Adding %s to GC queue' % task_id)
   self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
Exemple #2
0
def assigned_task_from_mesos_task(task):
    """Deserialize AssignedTask from a launchTask task protocol buffer."""
    try:
        assigned_task = thrift_deserialize(AssignedTask(), task.data)
    except (EOFError, TException) as e:
        raise TaskInfoError('Could not deserialize task! %s' % e)
    return assigned_task
def assigned_task_from_mesos_task(task):
  """Deserialize AssignedTask from a launchTask task protocol buffer."""
  try:
    assigned_task = thrift_deserialize(AssignedTask(), task.data)
  except (EOFError, TException) as e:
    raise ValueError('Could not deserialize task! %s' % e)
  return assigned_task
 def launchTask(self, driver, task):
   """Queue a new garbage collection run, and drop any currently-enqueued runs."""
   if self._slave_id is None:
     self._slave_id = task.slave_id.value
   task_id = task.task_id.value
   self.log('launchTask() got task_id: %s' % task_id)
   if task_id == self._task_id:
     self.log('=> GC with task_id %s currently running - ignoring' % task_id)
     return
   elif task_id in self._gc_task_queue:
     self.log('=> Already have task_id %s queued - ignoring' % task_id)
     return
   try:
     art = thrift_deserialize(AdjustRetainedTasks(), task.data)
   except Exception as err:
     self.log('Error deserializing task: %s' % err)
     self.send_update(
         self._driver, task_id, mesos_pb.TASK_FAILED, 'Deserialization of GC task failed')
     return
   try:
     prev_task_id, _ = self._gc_task_queue.popitem(0)
   except KeyError: # no enqueued GC tasks - reset counter
     self._dropped_tasks.write(0)
   else:
     self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
     self._dropped_tasks.increment()
     self.log('=> Updating scheduler')
     self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
                      'Garbage collection skipped - GC executor received another task')
   self.log('=> Adding %s to GC queue' % task_id)
   self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
Exemple #5
0
    def run(self):
        self._run_count += 1
        atexit.register(self.cleanup)

        if self.script_filename:
            os.unlink(self.script_filename)

        with temporary_file(cleanup=False) as fp:
            self.script_filename = fp.name
            fp.write(
                self.RUN_JOB_SCRIPT % {
                    'filename': self.job_filename,
                    'sandbox': self.sandbox,
                    'root': self.tempdir,
                    'task_id': self.task_id,
                    'state_filename': self.state_filename,
                    'portmap': repr(self.portmap),
                    'success_rate': self.success_rate,
                    'random_seed': self.random_seed + self._run_count,
                })

        with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
            self.po = subprocess.Popen([sys.executable, self.script_filename],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            try:
                so, se = self.po.communicate()
            except OSError as e:
                if e.errno == errno.ECHILD:
                    so = se = 'Killed'
                else:
                    raise

        rc = self.po.returncode
        if rc != 0:
            if os.path.exists(self.job_filename):
                config = open(self.job_filename).read()
            else:
                config = 'Nonexistent!'
            if 'THERMOS_DEBUG' in os.environ:
                print(
                    "Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n"
                    % (config, so, se))

        try:
            with open(self.state_filename, 'r') as fp:
                self.state = thrift_deserialize(RunnerState(), fp.read())
        except Exception as e:
            if 'THERMOS_DEBUG' in os.environ:
                print('Failed to load Runner state: %s' % e, file=sys.stderr)
            self.state = RunnerState()

        try:
            self.reconstructed_state = CheckpointDispatcher.from_file(
                self.pathspec.getpath('runner_checkpoint'))
        except:
            self.reconstructed_state = None
        self.initialized = True
        return rc
Exemple #6
0
  def run(self):
    self._run_count += 1
    atexit.register(self.cleanup)

    if self.script_filename:
      os.unlink(self.script_filename)

    with temporary_file(cleanup=False) as fp:
      self.script_filename = fp.name
      fp.write(self.RUN_JOB_SCRIPT % {
        'filename': self.job_filename,
        'sandbox': self.sandbox,
        'root': self.tempdir,
        'task_id': self.task_id,
        'state_filename': self.state_filename,
        'success_rate': self.success_rate,
        'random_seed': self.random_seed + self._run_count,
        'extra_task_runner_args': self.extra_task_runner_args,
      })

    with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
      self.po = subprocess.Popen([sys.executable, self.script_filename],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      try:
        so, se = self.po.communicate()
      except OSError as e:
        if e.errno == errno.ECHILD:
          so = se = 'Killed'
        else:
          raise

    rc = self.po.returncode
    if rc != 0:
      if os.path.exists(self.job_filename):
        with open(self.job_filename) as fp:
          config = fp.read()
      else:
        config = 'Nonexistent!'
      if 'THERMOS_DEBUG' in os.environ:
        print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (
            config, so, se))

    try:
      with open(self.state_filename, 'r') as fp:
        self.state = thrift_deserialize(RunnerState(), fp.read())
    except Exception as e:
      if 'THERMOS_DEBUG' in os.environ:
        print('Failed to load Runner state: %s' % e, file=sys.stderr)
      self.state = RunnerState()

    try:
      self.reconstructed_state = CheckpointDispatcher.from_file(
          self.pathspec.getpath('runner_checkpoint'))
    except Exception as e:
      print('Failed to replay checkpoint: %s' % e, file=sys.stderr)
      self.reconstructed_state = None
    self.initialized = True
    return rc
Exemple #7
0
 def unpack_thrift(cls, blob):
   if not isinstance(blob, ThriftServiceInstance):
     blob = thrift_deserialize(ThriftServiceInstance(), blob)
   additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
     for name, value in blob.additionalEndpoints.items())
   return cls(
     service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
     additional_endpoints=additional_endpoints,
     status=Status.unpack_thrift(blob.status))
Exemple #8
0
 def unpack_thrift(cls, blob):
   if not isinstance(blob, ThriftServiceInstance):
     blob = thrift_deserialize(ThriftServiceInstance(), blob)
   additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
     for name, value in blob.additionalEndpoints.items())
   return cls(
     service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
     additional_endpoints=additional_endpoints,
     status=Status.from_thrift(blob.status),
     shard=blob.shard)
Exemple #9
0
 def unpack_thrift(cls, blob, member_id=None):
   if not isinstance(blob, ThriftServiceInstance):
     blob = thrift_deserialize(ThriftServiceInstance(), blob)
   additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
     for name, value in blob.additionalEndpoints.items())
   member_id = cls.__check_int(member_id)
   return cls(
     service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
     additional_endpoints=additional_endpoints,
     status=Status.from_thrift(blob.status),
     shard=blob.shard,
     member_id=member_id)
 def sendFrameworkMessage(self, message):
   self.messages.append(thrift_deserialize(SchedulerMessage(), message))
Exemple #11
0
 def sendFrameworkMessage(self, message):  # noqa
     self.messages.append(thrift_deserialize(SchedulerMessage(), message))
Exemple #12
0
def deserialize(thrift_object, buf: bytes, protocol_cls=TBinaryProtocol):
    return thrift_deserialize(thrift_object, buf,
                              TProtocolFactory(protocol_cls))