def launchTask(self, driver, task):
"""Queue a new garbage collection run, and drop any currently-enqueued runs."""
if self._slave_id is None:
self._slave_id = task.slave_id.value
task_id = task.task_id.value
self.log('launchTask() got task_id: %s' % task_id)
if task_id == self._task_id:
self.log('=> GC with task_id %s currently running - ignoring' % task_id)
return
elif task_id in self._gc_task_queue:
self.log('=> Already have task_id %s queued - ignoring' % task_id)
return
try:
art = thrift_deserialize(AdjustRetainedTasks(), task.data)
except Exception as err:
self.log('Error deserializing task: %s' % err)
self.send_update(
self._driver, task_id, mesos_pb2.TASK_FAILED, 'Deserialization of GC task failed')
return
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
except KeyError: # no enqueued GC tasks - reset counter
self._dropped_tasks.write(0)
else:
self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
self._dropped_tasks.increment()
self.log('=> Updating scheduler')
self.send_update(self._driver, prev_task_id, mesos_pb2.TASK_FINISHED,
'Garbage collection skipped - GC executor received another task')
self.log('=> Adding %s to GC queue' % task_id)
self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
def assigned_task_from_mesos_task(task):
"""Deserialize AssignedTask from a launchTask task protocol buffer."""
try:
assigned_task = thrift_deserialize(AssignedTask(), task.data)
except (EOFError, TException) as e:
raise TaskInfoError('Could not deserialize task! %s' % e)
return assigned_task
def assigned_task_from_mesos_task(task):
"""Deserialize AssignedTask from a launchTask task protocol buffer."""
try:
assigned_task = thrift_deserialize(AssignedTask(), task.data)
except (EOFError, TException) as e:
raise ValueError('Could not deserialize task! %s' % e)
return assigned_task
def launchTask(self, driver, task):
"""Queue a new garbage collection run, and drop any currently-enqueued runs."""
if self._slave_id is None:
self._slave_id = task.slave_id.value
task_id = task.task_id.value
self.log('launchTask() got task_id: %s' % task_id)
if task_id == self._task_id:
self.log('=> GC with task_id %s currently running - ignoring' % task_id)
return
elif task_id in self._gc_task_queue:
self.log('=> Already have task_id %s queued - ignoring' % task_id)
return
try:
art = thrift_deserialize(AdjustRetainedTasks(), task.data)
except Exception as err:
self.log('Error deserializing task: %s' % err)
self.send_update(
self._driver, task_id, mesos_pb.TASK_FAILED, 'Deserialization of GC task failed')
return
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
except KeyError: # no enqueued GC tasks - reset counter
self._dropped_tasks.write(0)
else:
self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
self._dropped_tasks.increment()
self.log('=> Updating scheduler')
self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
'Garbage collection skipped - GC executor received another task')
self.log('=> Adding %s to GC queue' % task_id)
self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
def run(self):
self._run_count += 1
atexit.register(self.cleanup)
if self.script_filename:
os.unlink(self.script_filename)
with temporary_file(cleanup=False) as fp:
self.script_filename = fp.name
fp.write(
self.RUN_JOB_SCRIPT % {
'filename': self.job_filename,
'sandbox': self.sandbox,
'root': self.tempdir,
'task_id': self.task_id,
'state_filename': self.state_filename,
'portmap': repr(self.portmap),
'success_rate': self.success_rate,
'random_seed': self.random_seed + self._run_count,
})
with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
self.po = subprocess.Popen([sys.executable, self.script_filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
so, se = self.po.communicate()
except OSError as e:
if e.errno == errno.ECHILD:
so = se = 'Killed'
else:
raise
rc = self.po.returncode
if rc != 0:
if os.path.exists(self.job_filename):
config = open(self.job_filename).read()
else:
config = 'Nonexistent!'
if 'THERMOS_DEBUG' in os.environ:
print(
"Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n"
% (config, so, se))
try:
with open(self.state_filename, 'r') as fp:
self.state = thrift_deserialize(RunnerState(), fp.read())
except Exception as e:
if 'THERMOS_DEBUG' in os.environ:
print('Failed to load Runner state: %s' % e, file=sys.stderr)
self.state = RunnerState()
try:
self.reconstructed_state = CheckpointDispatcher.from_file(
self.pathspec.getpath('runner_checkpoint'))
except:
self.reconstructed_state = None
self.initialized = True
return rc
def run(self):
self._run_count += 1
atexit.register(self.cleanup)
if self.script_filename:
os.unlink(self.script_filename)
with temporary_file(cleanup=False) as fp:
self.script_filename = fp.name
fp.write(self.RUN_JOB_SCRIPT % {
'filename': self.job_filename,
'sandbox': self.sandbox,
'root': self.tempdir,
'task_id': self.task_id,
'state_filename': self.state_filename,
'success_rate': self.success_rate,
'random_seed': self.random_seed + self._run_count,
'extra_task_runner_args': self.extra_task_runner_args,
})
with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
self.po = subprocess.Popen([sys.executable, self.script_filename],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
so, se = self.po.communicate()
except OSError as e:
if e.errno == errno.ECHILD:
so = se = 'Killed'
else:
raise
rc = self.po.returncode
if rc != 0:
if os.path.exists(self.job_filename):
with open(self.job_filename) as fp:
config = fp.read()
else:
config = 'Nonexistent!'
if 'THERMOS_DEBUG' in os.environ:
print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (
config, so, se))
try:
with open(self.state_filename, 'r') as fp:
self.state = thrift_deserialize(RunnerState(), fp.read())
except Exception as e:
if 'THERMOS_DEBUG' in os.environ:
print('Failed to load Runner state: %s' % e, file=sys.stderr)
self.state = RunnerState()
try:
self.reconstructed_state = CheckpointDispatcher.from_file(
self.pathspec.getpath('runner_checkpoint'))
except Exception as e:
print('Failed to replay checkpoint: %s' % e, file=sys.stderr)
self.reconstructed_state = None
self.initialized = True
return rc
def unpack_thrift(cls, blob):
if not isinstance(blob, ThriftServiceInstance):
blob = thrift_deserialize(ThriftServiceInstance(), blob)
additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
for name, value in blob.additionalEndpoints.items())
return cls(
service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
additional_endpoints=additional_endpoints,
status=Status.unpack_thrift(blob.status))
def unpack_thrift(cls, blob):
if not isinstance(blob, ThriftServiceInstance):
blob = thrift_deserialize(ThriftServiceInstance(), blob)
additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
for name, value in blob.additionalEndpoints.items())
return cls(
service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
additional_endpoints=additional_endpoints,
status=Status.from_thrift(blob.status),
shard=blob.shard)
def unpack_thrift(cls, blob, member_id=None):
if not isinstance(blob, ThriftServiceInstance):
blob = thrift_deserialize(ThriftServiceInstance(), blob)
additional_endpoints = dict((name, Endpoint.unpack_thrift(value))
for name, value in blob.additionalEndpoints.items())
member_id = cls.__check_int(member_id)
return cls(
service_endpoint=Endpoint.unpack_thrift(blob.serviceEndpoint),
additional_endpoints=additional_endpoints,
status=Status.from_thrift(blob.status),
shard=blob.shard,
member_id=member_id)
def sendFrameworkMessage(self, message): self.messages.append(thrift_deserialize(SchedulerMessage(), message))
def sendFrameworkMessage(self, message): # noqa
self.messages.append(thrift_deserialize(SchedulerMessage(), message))
def deserialize(thrift_object, buf: bytes, protocol_cls=TBinaryProtocol):
return thrift_deserialize(thrift_object, buf,
TProtocolFactory(protocol_cls))