def _drain_hosts(self, drainable_hosts):
""""Drains tasks from the specified hosts.
This will move active tasks on these hosts to the DRAINING state, causing them to be
rescheduled elsewhere.
:param drainable_hosts: Hosts that are in maintenance mode and ready to be drained
:type drainable_hosts: gen.apache.aurora.ttypes.Hosts
:rtype: set of host names failed to drain
"""
check_and_log_response(self._client.drain_hosts(drainable_hosts))
drainable_hostnames = [hostname for hostname in drainable_hosts.hostNames]
total_wait = self.STATUS_POLL_INTERVAL
not_drained_hostnames = set(drainable_hostnames)
while not self._wait_event.is_set() and not_drained_hostnames:
log.info("Waiting for hosts to be in DRAINED: %s" % not_drained_hostnames)
self._wait_event.wait(self.STATUS_POLL_INTERVAL.as_(Time.SECONDS))
statuses = self.check_status(list(not_drained_hostnames))
not_drained_hostnames = set(h[0] for h in statuses if h[1] != "DRAINED")
total_wait += self.STATUS_POLL_INTERVAL
if not_drained_hostnames and total_wait > self.MAX_STATUS_WAIT:
log.warning(
"Failed to move all hosts into DRAINED within %s:\n%s"
% (
self.MAX_STATUS_WAIT,
"\n".join("\tHost:%s\tStatus:%s" % h for h in sorted(statuses) if h[1] != "DRAINED"),
)
)
break
return not_drained_hostnames
def _apply_states(self):
"""
os.stat() the corresponding checkpoint stream of this task and determine if there are new ckpt
records. Attempt to read those records and update the high watermark for that stream.
Returns True if new states were applied, False otherwise.
"""
ckpt_offset = None
try:
ckpt_offset = os.stat(self._runner_ckpt).st_size
updated = False
if self._ckpt_head < ckpt_offset:
with open(self._runner_ckpt, "r") as fp:
fp.seek(self._ckpt_head)
rr = ThriftRecordReader(fp, RunnerCkpt)
while True:
runner_update = rr.try_read()
if not runner_update:
break
try:
self._dispatcher.dispatch(self._runnerstate, runner_update)
except CheckpointDispatcher.InvalidSequenceNumber as e:
log.error("Checkpoint stream is corrupt: %s" % e)
break
new_ckpt_head = fp.tell()
updated = self._ckpt_head != new_ckpt_head
self._ckpt_head = new_ckpt_head
return updated
except OSError as e:
if e.errno == errno.ENOENT:
# The log doesn't yet exist, will retry later.
log.warning("Could not read from checkpoint %s" % self._runner_ckpt)
return False
else:
raise
def _fast_forward_stream(self, process_name):
log.debug('Fast forwarding %s stream to seq=%s' %
(process_name, self._watermarks[process_name]))
assert self._processes.get(process_name) is not None
fp = self._processes[process_name]
rr = ThriftRecordReader(fp, RunnerCkpt)
current_watermark = -1
records = 0
while current_watermark < self._watermarks[process_name]:
last_pos = fp.tell()
record = rr.try_read()
if record is None:
break
new_watermark = record.process_status.seq
if new_watermark > self._watermarks[process_name]:
log.debug(
'Over-seeked %s [watermark = %s, high watermark = %s], rewinding.'
% (process_name, new_watermark,
self._watermarks[process_name]))
fp.seek(last_pos)
break
current_watermark = new_watermark
records += 1
if current_watermark < self._watermarks[process_name]:
log.warning(
'Only able to fast forward to %s@sequence=%s, high watermark is %s'
% (process_name, current_watermark,
self._watermarks[process_name]))
if records:
log.debug('Fast forwarded %s %s record(s) to seq=%s.' %
(process_name, records, current_watermark))
def method_wrapper(*args):
with self._lock:
start = time.time()
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
# Only automatically append a SessionKey if this is not part of the read-only API.
auth_args = () if hasattr(ReadOnlyScheduler.Iface, method_name) else (self.session_key(),)
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
resp = method(*(args + auth_args))
if resp is not None and resp.responseCode == ResponseCode.ERROR_TRANSIENT:
raise self.TransientError(", ".join(
[m.message for m in resp.details] if resp.details else []))
if resp.serverInfo.thriftAPIVersion != THRIFT_API_VERSION:
raise self.APIVersionError("Client Version: %s, Server Version: %s" %
(THRIFT_API_VERSION, resp.serverInfo.thriftAPIVersion))
return resp
except (TTransport.TTransportException, self.TimeoutError, self.TransientError) as e:
if not self._terminating.is_set():
log.warning('Connection error with scheduler: %s, reconnecting...' % e)
self.invalidate()
self._terminating.wait(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating.is_set():
raise self.ThriftInternalError("Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating.is_set():
raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
method_name, self.cluster.name))
def on_finish(service_instance):
try:
self._members[member_id] = ServiceInstance.unpack(service_instance)
except Exception as e:
log.warning('Failed to deserialize endpoint: %s' % e)
return
self._on_join(self._members[member_id])
def _drain_hosts(self, drainable_hosts):
""""Drains tasks from the specified hosts.
This will move active tasks on these hosts to the DRAINING state, causing them to be
rescheduled elsewhere.
:param drainable_hosts: Hosts that are in maintenance mode and ready to be drained
:type drainable_hosts: gen.apache.aurora.ttypes.Hosts
"""
check_and_log_response(self._client.drain_hosts(drainable_hosts))
not_ready_hostnames = [
hostname for hostname in drainable_hosts.hostNames
]
while not_ready_hostnames:
resp = self._client.maintenance_status(
Hosts(set(not_ready_hostnames)))
if not resp.result.maintenanceStatusResult.statuses:
not_ready_hostnames = None
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.DRAINED:
log.warning(
'%s is currently in status %s' %
(host_status.host,
MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
else:
not_ready_hostnames.remove(host_status.host)
def setup_child_subreaping():
"""
This uses the prctl(2) syscall to set the `PR_SET_CHILD_SUBREAPER` flag. This
means if any children processes need to be reparented, they will be reparented
to this process.
More documentation here: http://man7.org/linux/man-pages/man2/prctl.2.html
and here: https://lwn.net/Articles/474787/
Callers should reap terminal children to prevent zombies.
"""
log.debug("Calling prctl(2) with PR_SET_CHILD_SUBREAPER")
# This constant is taken from prctl.h
PR_SET_CHILD_SUBREAPER = 36
try:
library_name = ctypes.util.find_library('c')
if library_name is None:
log.warning("libc is not found. Unable to call prctl!")
log.warning("Children subreaping is disabled!")
return
libc = ctypes.CDLL(library_name, use_errno=True)
# If we are on a system where prctl doesn't exist, this will throw an
# attribute error.
ret = libc.prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0)
if ret != 0:
errno = ctypes.get_errno()
raise OSError(errno, os.strerror(errno))
except Exception as e:
log.error("Unable to call prctl %s" % e)
log.error("Children subreaping is disabled!")
def allocate_port(self, name, port=None):
if port is not None:
if name in self._ports and self._ports[name] != port:
raise EphemeralPortAllocator.PortConflict(
'Port binding %s=>%s conflicts with current binding %s=>%s' % (
name, port, name, self._ports[name]))
else:
self._ports[name] = port
return port
if name in self._ports:
return self._ports[name]
while True:
rand_port = random.randint(*EphemeralPortAllocator.SOCKET_RANGE)
# if this ever needs to be performant, make a peer set.
if rand_port in self._ports.values():
continue
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', rand_port))
s.close()
self._ports[name] = rand_port
break
except OSError as e:
if e.errno == errno.EADDRINUSE:
log.warning('Could not bind port: %s' % e)
time.sleep(0.2)
continue
else:
raise
return self._ports[name]
def _complete_maintenance(self, drained_hosts):
"""End the maintenance status for a give set of hosts."""
check_and_log_response(self._client.end_maintenance(drained_hosts))
resp = self._client.maintenance_status(drained_hosts)
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.NONE:
log.warning('%s is DRAINING or in DRAINED' % host_status.host)
def _drain_hosts(self, drainable_hosts, clock=time):
""""Drains tasks from the specified hosts.
This will move active tasks on these hosts to the DRAINING state, causing them to be
rescheduled elsewhere.
:param drainable_hosts: Hosts that are in maintenance mode and ready to be drained
:type drainable_hosts: gen.apache.aurora.ttypes.Hosts
:param clock: time module for testing
:type clock: time
"""
check_and_log_response(self._client.drain_hosts(drainable_hosts))
not_ready_hostnames = [hostname for hostname in drainable_hosts.hostNames]
while not_ready_hostnames:
log.info("Sleeping for %s." % self.START_MAINTENANCE_DELAY)
clock.sleep(self.START_MAINTENANCE_DELAY.as_(Time.SECONDS))
resp = self._client.maintenance_status(Hosts(set(not_ready_hostnames)))
if not resp.result.maintenanceStatusResult.statuses:
not_ready_hostnames = None
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.DRAINED:
log.warning('%s is currently in status %s' %
(host_status.host, MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
else:
not_ready_hostnames.remove(host_status.host)
def _construct_scheduler(self):
"""
Populates:
self._scheduler
self._client
"""
self._scheduler = SchedulerClient.get(self.cluster,
verbose=self.verbose)
assert self._scheduler, "Could not find scheduler (cluster = %s)" % self.cluster.name
start = time.time()
while (time.time() - start) < self.CONNECT_MAXIMUM_WAIT.as_(
Time.SECONDS):
try:
self._client = self._scheduler.get_thrift_client()
break
except SchedulerClient.CouldNotConnect as e:
log.warning('Could not connect to scheduler: %s' % e)
if not self._client:
raise self.TimeoutError(
'Timed out trying to connect to scheduler at %s' %
self.cluster.name)
server_version = self._client.getVersion().result.getVersionResult
if server_version != CURRENT_API_VERSION:
raise self.APIVersionError(
"Client Version: %s, Server Version: %s" %
(CURRENT_API_VERSION, server_version))
def _apply_states(self):
"""
os.stat() the corresponding checkpoint stream of this task and determine if there are new ckpt
records. Attempt to read those records and update the high watermark for that stream.
Returns True if new states were applied, False otherwise.
"""
ckpt_offset = None
try:
ckpt_offset = os.stat(self._runner_ckpt).st_size
updated = False
if self._ckpt_head < ckpt_offset:
with open(self._runner_ckpt, 'r') as fp:
fp.seek(self._ckpt_head)
rr = ThriftRecordReader(fp, RunnerCkpt)
while True:
runner_update = rr.try_read()
if not runner_update:
break
try:
self._dispatcher.dispatch(self._runnerstate, runner_update)
except CheckpointDispatcher.InvalidSequenceNumber as e:
log.error('Checkpoint stream is corrupt: %s' % e)
break
new_ckpt_head = fp.tell()
updated = self._ckpt_head != new_ckpt_head
self._ckpt_head = new_ckpt_head
return updated
except OSError as e:
if e.errno == errno.ENOENT:
# The log doesn't yet exist, will retry later.
log.warning('Could not read from checkpoint %s' % self._runner_ckpt)
return False
else:
raise
def _complete_maintenance(self, drained_hosts):
"""End the maintenance status for a give set of hosts."""
check_and_log_response(self._client.end_maintenance(drained_hosts))
resp = self._client.maintenance_status(drained_hosts)
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.NONE:
log.warning('%s is DRAINING or in DRAINED' % host_status.host)
def get_completion(result):
try:
children = result.get()
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, do_monitor)
return
except ke.NoNodeError:
wait_exists()
return
except ke.KazooException as e:
log.warning('Unexpected get_completion result: (%s)%s' % (type(e), e))
return
children = [child for child in children if self.znode_owned(child)]
_, new = self._update_children(children)
for child in new:
def devnull(*args, **kw): pass
self.info(child, callback=devnull)
monitor_queue = self._monitor_queue[:]
self._monitor_queue = []
members = set(Membership(self.znode_to_id(child)) for child in children)
for membership, capture in monitor_queue:
if set(membership) != members:
capture.set(members)
else:
self._monitor_queue.append((membership, capture))
def method_wrapper(*args):
with self._lock:
start = time.time()
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
auth_args = () if method_name in self.UNAUTHENTICATED_RPCS else (self.session_key(),)
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
return method(*(args + auth_args))
except (TTransport.TTransportException, self.TimeoutError) as e:
if not self._terminating:
log.warning('Connection error with scheduler: %s, reconnecting...' % e)
self.invalidate()
self._terminating.wait(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating:
raise self.ThriftInternalError("Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating:
raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
method_name, self.cluster.name))
def method_wrapper(*args):
with self._lock:
start = time.time()
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
resp = method(*args)
if resp is not None and resp.responseCode == ResponseCode.ERROR_TRANSIENT:
raise self.TransientError(", ".join(
[m.message for m in resp.details] if resp.details else []))
return resp
except TRequestsTransport.AuthError as e:
log.error(self.scheduler_client().get_failed_auth_message())
raise self.AuthError(e)
except (TTransport.TTransportException, self.TimeoutError, self.TransientError) as e:
if not self._terminating.is_set():
log.warning('Connection error with scheduler: %s, reconnecting...' % e)
self.invalidate()
self._terminating.wait(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating.is_set():
raise self.ThriftInternalError("Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating.is_set():
raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
method_name, self.cluster.name))
def __call__(self, endpoint, use_post_method=False, expected_response=None, expected_response_code=None):
"""Returns a (boolean, string|None) tuple of (call success, failure reason)"""
try:
response, response_code = self.query(endpoint, "" if use_post_method else None)
response = response.strip().lower()
if expected_response and response != expected_response.lower():
reason = 'Response differs from expected response (expected "%s", got "%s")'
def shorten(string):
return (
string
if len(string) < self.FAILURE_REASON_LENGTH
else "%s..." % string[: self.FAILURE_REASON_LENGTH - 3]
)
log.warning(reason % (expected_response, response))
return (False, reason % (shorten(str(expected_response)), shorten(str(response))))
elif expected_response_code and response_code != expected_response_code:
reason = "Response code differs from expected response (expected %i, got %i)"
log.warning(reason % (expected_response_code, response_code))
return (False, reason % (expected_response_code, response_code))
else:
return (True, None)
except self.QueryError as e:
return (False, str(e))
def method_wrapper(*args):
with self._lock:
start = time.time()
# TODO(wfarner): The while loop causes failed unit tests to spin for the retry
# period (currently 10 minutes). Figure out a better approach.
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
# Only automatically append a SessionKey if this is not part of the read-only API.
auth_args = () if hasattr(ReadOnlyScheduler.Iface, method_name) else (self.session_key(),)
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
return method(*(args + auth_args))
except (TTransport.TTransportException, self.TimeoutError) as e:
if not self._terminating:
log.warning('Connection error with scheduler: %s, reconnecting...' % e)
self.invalidate()
self._terminating.wait(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating:
raise self.ThriftInternalError("Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating:
raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
method_name, self.cluster.name))
def _drain_hosts(self, drainable_hosts):
""""Drains tasks from the specified hosts.
This will move active tasks on these hosts to the DRAINING state, causing them to be
rescheduled elsewhere.
:param drainable_hosts: Hosts that are in maintenance mode and ready to be drained
:type drainable_hosts: gen.apache.aurora.ttypes.Hosts
:rtype: set of host names failed to drain
"""
check_and_log_response(self._client.drain_hosts(drainable_hosts))
drainable_hostnames = [hostname for hostname in drainable_hosts.hostNames]
total_wait = self.STATUS_POLL_INTERVAL
not_drained_hostnames = set(drainable_hostnames)
while not self._wait_event.is_set() and not_drained_hostnames:
log.info('Waiting for hosts to be in DRAINED: %s' % not_drained_hostnames)
self._wait_event.wait(self.STATUS_POLL_INTERVAL.as_(Time.SECONDS))
statuses = self.check_status(list(not_drained_hostnames))
not_drained_hostnames = set(h[0] for h in statuses if h[1] != 'DRAINED')
total_wait += self.STATUS_POLL_INTERVAL
if not_drained_hostnames and total_wait > self.MAX_STATUS_WAIT:
log.warning('Failed to move all hosts into DRAINED within %s:\n%s' %
(self.MAX_STATUS_WAIT,
'\n'.join("\tHost:%s\tStatus:%s" % h for h in sorted(statuses) if h[1] != 'DRAINED')))
break
return not_drained_hostnames
def acreate_completion(result):
try:
# TODO(wickman) Kazoo has a bug:
# https://github.com/python-zk/kazoo/issues/106
# https://github.com/python-zk/kazoo/pull/107
# Remove this one 1.3 is cut.
path = self._zk.unchroot(result.get())
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, do_join)
return
except ke.KazooException as e:
log.warning('Unexpected Kazoo result in join: (%s)%s' % (type(e), e))
membership = Membership.error()
else:
created_id = self.znode_to_id(path)
membership = Membership(created_id)
with self._member_lock:
result_future = self._members.get(membership, Future())
result_future.set_result(blob)
self._members[membership] = result_future
if expire_callback:
self._once(KazooState.CONNECTED, expire_notifier)
do_exists(path)
membership_capture.set(membership)
def __call__(self,
endpoint,
use_post_method=False,
expected_response=None,
expected_response_code=None):
"""Returns a (boolean, string|None) tuple of (call success, failure reason)"""
try:
response, response_code = self.query(
endpoint, '' if use_post_method else None)
response = response.strip().lower()
if expected_response and response != expected_response.lower():
reason = 'Response differs from expected response (expected "%s", got "%s")'
def shorten(string):
return (string if len(string) < self.FAILURE_REASON_LENGTH
else "%s..." %
string[:self.FAILURE_REASON_LENGTH - 3])
log.warning(reason % (expected_response, response))
return (
False, reason %
(shorten(str(expected_response)), shorten(str(response))))
elif expected_response_code and response_code != expected_response_code:
reason = 'Response code differs from expected response (expected %i, got %i)'
log.warning(reason % (expected_response_code, response_code))
return (False,
reason % (expected_response_code, response_code))
else:
return (True, None)
except self.QueryError as e:
return (False, str(e))
def _fast_forward_stream(self, process_name):
log.debug('Fast forwarding %s stream to seq=%s' % (process_name,
self._watermarks[process_name]))
assert self._processes.get(process_name) is not None
fp = self._processes[process_name]
rr = ThriftRecordReader(fp, RunnerCkpt)
current_watermark = -1
records = 0
while current_watermark < self._watermarks[process_name]:
last_pos = fp.tell()
record = rr.try_read()
if record is None:
break
new_watermark = record.process_status.seq
if new_watermark > self._watermarks[process_name]:
log.debug('Over-seeked %s [watermark = %s, high watermark = %s], rewinding.' % (
process_name, new_watermark, self._watermarks[process_name]))
fp.seek(last_pos)
break
current_watermark = new_watermark
records += 1
if current_watermark < self._watermarks[process_name]:
log.warning('Only able to fast forward to %s@sequence=%s, high watermark is %s' % (
process_name, current_watermark, self._watermarks[process_name]))
if records:
log.debug('Fast forwarded %s %s record(s) to seq=%s.' % (process_name, records,
current_watermark))
def get_completion(result):
try:
children = result.get()
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, do_monitor)
return
except ke.NoNodeError:
wait_exists()
return
except ke.KazooException as e:
log.warning('Unexpected get_completion result: (%s)%s' %
(type(e), e))
return
children = [child for child in children if self.znode_owned(child)]
_, new = self._update_children(children)
for child in new:
def devnull(*args, **kw):
pass
self.info(child, callback=devnull)
monitor_queue = self._monitor_queue[:]
self._monitor_queue = []
members = set(
Membership(self.znode_to_id(child)) for child in children)
for membership, capture in monitor_queue:
if set(membership) != members:
capture.set(members)
else:
self._monitor_queue.append((membership, capture))
def sample(self):
""" Collate and aggregate ProcessSamples for process and children
Returns None: result is stored in self.value
"""
try:
last_sample, last_stamp = self._sample, self._stamp
if self._process is None:
self._process = Process(self._pid)
parent = self._process
parent_sample = process_to_sample(parent)
new_samples = dict((proc.pid, process_to_sample(proc)) for proc in parent.get_children(recursive=True))
new_samples[self._pid] = parent_sample
except PsutilError as e:
log.warning("Error during process sampling: %s" % e)
self._sample = ProcessSample.empty()
self._rate = 0.0
else:
last_stamp = self._stamp
self._stamp = time()
# for most stats, calculate simple sum to aggregate
self._sample = sum(new_samples.values(), ProcessSample.empty())
# cpu consumption is more complicated
# We require at least 2 generations of a process before we can calculate rate, so for all
# current processes that were not running in the previous sample, compare to an empty sample
if self._sampled_tree and last_stamp:
new = new_samples.values()
old = [self._sampled_tree.get(pid, ProcessSample.empty()) for pid in new_samples.keys()]
new_user_sys = sum(map(attrgetter("user"), new)) + sum(map(attrgetter("system"), new))
old_user_sys = sum(map(attrgetter("user"), old)) + sum(map(attrgetter("system"), old))
self._rate = (new_user_sys - old_user_sys) / (self._stamp - last_stamp)
log.debug("Calculated rate for pid=%s and children: %s" % (self._process.pid, self._rate))
self._sampled_tree = new_samples
def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI):
"""
Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple
if one can be found given the args, potentially querying the scheduler with the returned client.
Calls die() with an appropriate error message otherwise.
Arguments:
args: args from app command invocation.
options: options from app command invocation. must have env and cluster attributes.
client_factory: a callable (cluster) -> AuroraClientAPI.
"""
if not len(args) > 0:
die('job path is required')
try:
job_key = AuroraJobKey.from_path(args[0])
client = client_factory(job_key.cluster)
config_file = args[1] if len(args) > 1 else None # the config for hooks
return client, job_key, config_file
except AuroraJobKey.Error:
log.warning("Failed to parse job path, falling back to compatibility mode")
role = args[0] if len(args) > 0 else None
name = args[1] if len(args) > 1 else None
env = None
config_file = None # deprecated form does not support hooks functionality
cluster = options.cluster
if not cluster:
die('cluster is required')
client = client_factory(cluster)
return client, cls._disambiguate_or_die(client, role, env, name), config_file
def _construct_scheduler(self):
"""
Populates:
self._scheduler_client
self._client
"""
self._scheduler_client = SchedulerClient.get(self.cluster, verbose=self.verbose)
assert self._scheduler_client, "Could not find scheduler (cluster = %s)" % self.cluster.name
start = time.time()
while (time.time() - start) < self.CONNECT_MAXIMUM_WAIT.as_(Time.SECONDS):
try:
# this can wind up generating any kind of error, because it turns into
# a call to a dynamically set authentication module.
self._client = self._scheduler_client.get_thrift_client()
break
except SchedulerClient.CouldNotConnect as e:
log.warning('Could not connect to scheduler: %s' % e)
except Exception as e:
# turn any auth module exception into an auth error.
log.debug('Warning: got an unknown exception during authentication:')
log.debug(traceback.format_exc())
raise self.AuthenticationError('Error connecting to scheduler: %s' % e)
if not self._client:
raise self.TimeoutError('Timed out trying to connect to scheduler at %s' % self.cluster.name)
server_version = self._client.getVersion().result.getVersionResult
if server_version != CURRENT_API_VERSION:
raise self.APIVersionError("Client Version: %s, Server Version: %s" %
(CURRENT_API_VERSION, server_version))
def setup_child_subreaping():
"""
This uses the prctl(2) syscall to set the `PR_SET_CHILD_SUBREAPER` flag. This
means if any children processes need to be reparented, they will be reparented
to this process.
More documentation here: http://man7.org/linux/man-pages/man2/prctl.2.html
and here: https://lwn.net/Articles/474787/
Callers should reap terminal children to prevent zombies.
"""
log.debug("Calling prctl(2) with PR_SET_CHILD_SUBREAPER")
# This constant is taken from prctl.h
PR_SET_CHILD_SUBREAPER = 36
try:
library_name = ctypes.util.find_library('c')
if library_name is None:
log.warning("libc is not found. Unable to call prctl!")
log.warning("Children subreaping is disabled!")
return
libc = ctypes.CDLL(library_name, use_errno=True)
# If we are on a system where prctl doesn't exist, this will throw an
# attribute error.
ret = libc.prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0)
if ret != 0:
errno = ctypes.get_errno()
raise OSError(errno, os.strerror(errno))
except Exception as e:
log.error("Unable to call prctl %s" % e)
log.error("Children subreaping is disabled!")
def _construct_scheduler(self):
"""
Populates:
self._scheduler_client
self._client
"""
self._scheduler_client = SchedulerClient.get(self.cluster,
verbose=self.verbose,
**self._kwargs)
assert self._scheduler_client, "Could not find scheduler (cluster = %s)" % self.cluster.name
start = time.time()
while (time.time() - start) < self.CONNECT_MAXIMUM_WAIT.as_(
Time.SECONDS):
try:
# this can wind up generating any kind of error, because it turns into
# a call to a dynamically set authentication module.
self._client = self._scheduler_client.get_thrift_client()
break
except SchedulerClient.CouldNotConnect as e:
log.warning('Could not connect to scheduler: %s' % e)
except Exception as e:
# turn any auth module exception into an auth error.
log.debug(
'Warning: got an unknown exception during authentication:')
log.debug(traceback.format_exc())
raise self.AuthError('Error connecting to scheduler: %s' % e)
if not self._client:
raise self.TimeoutError(
'Timed out trying to connect to scheduler at %s' %
self.cluster.name)
def _drain_hosts(self, drainable_hosts):
""""Drains tasks from the specified hosts.
This will move active tasks on these hosts to the DRAINING state, causing them to be
rescheduled elsewhere.
:param drainable_hosts: Hosts that are in maintenance mode and ready to be drained
:type drainable_hosts: gen.apache.aurora.ttypes.Hosts
:rtype: set of host names failed to drain
"""
check_and_log_response(self._client.drain_hosts(drainable_hosts))
drainable_hostnames = [hostname for hostname in drainable_hosts.hostNames]
total_wait = self.STATUS_POLL_INTERVAL
not_drained_hostnames = set(drainable_hostnames)
while not self._wait_event.is_set() and not_drained_hostnames:
self._wait_event.wait(self.STATUS_POLL_INTERVAL.as_(Time.SECONDS))
not_drained_hostnames = self.check_if_drained(drainable_hostnames)
total_wait += self.STATUS_POLL_INTERVAL
if not_drained_hostnames and total_wait > self.MAX_STATUS_WAIT:
log.warning('Failed to move all hosts into DRAINED within %s' % self.MAX_STATUS_WAIT)
break
return not_drained_hostnames
def method_wrapper(*args):
with self._lock:
start = time.time()
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
resp = method(*args)
if resp is not None and resp.responseCode == ResponseCode.ERROR_TRANSIENT:
raise self.TransientError(", ".join(
[m.message for m in resp.details] if resp.details else []))
return resp
except TRequestsTransport.AuthError as e:
log.error(self.scheduler_client().get_failed_auth_message())
raise self.AuthError(e)
except (TTransport.TTransportException, self.TimeoutError, self.TransientError) as e:
if not self._terminating.is_set():
log.warning('Connection error with scheduler: %s, reconnecting...' % e)
self.invalidate()
self._terminating.wait(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating.is_set():
raise self.ThriftInternalError("Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating.is_set():
raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
method_name, self.cluster.name))
def run(self):
"""Thread entrypoint. Loop indefinitely, polling collectors at self._collection_interval and
collating samples."""
log.debug('Commencing resource monitoring for task "%s"' % self._task_id)
next_process_collection = 0
next_disk_collection = 0
while not self._kill_signal.is_set():
now = time.time()
if now > next_process_collection:
next_process_collection = now + self._process_collection_interval
actives = set(self._get_active_processes())
current = set(self._process_collectors)
for process in current - actives:
self._process_collectors.pop(process)
for process in actives - current:
self._process_collectors[process] = self._process_collector_factory(process.pid)
for process, collector in self._process_collectors.items():
collector.sample()
if now > next_disk_collection:
next_disk_collection = now + self._disk_collection_interval
if not self._disk_collector:
sandbox = self._task_monitor.get_sandbox()
if sandbox:
self._disk_collector = self._disk_collector_class(sandbox)
if self._disk_collector:
self._disk_collector.sample()
else:
log.debug('No sandbox detected yet for %s' % self._task_id)
try:
aggregated_procs = sum(map(attrgetter('procs'), self._process_collectors.values()))
aggregated_sample = sum(map(attrgetter('value'), self._process_collectors.values()),
ProcessSample.empty())
disk_value = self._disk_collector.value if self._disk_collector else 0
self._history.add(now, self.ResourceResult(aggregated_procs, aggregated_sample, disk_value))
except ValueError as err:
log.warning("Error recording resource sample: %s" % err)
# Sleep until any of the following conditions are met:
# - it's time for the next disk collection
# - it's time for the next process collection
# - the result from the last disk collection is available via the DiskCollector
# - the TaskResourceMonitor has been killed via self._kill_signal
now = time.time()
next_collection = min(next_process_collection - now, next_disk_collection - now)
if self._disk_collector:
waiter = EventMuxer(self._kill_signal, self._disk_collector.completed_event)
else:
waiter = self._kill_signal
waiter.wait(timeout=max(0, next_collection))
log.debug('Stopping resource monitoring for task "%s"' % self._task_id)
def run(self):
"""Thread entrypoint. Loop indefinitely, polling collectors at self._collection_interval and
collating samples."""
log.debug('Commencing resource monitoring for task "%s"' % self._task_id)
next_process_collection = 0
next_disk_collection = 0
while not self._kill_signal.is_set():
now = time.time()
if now > next_process_collection:
next_process_collection = now + self._process_collection_interval
actives = set(self._get_active_processes())
current = set(self._process_collectors)
for process in current - actives:
self._process_collectors.pop(process)
for process in actives - current:
self._process_collectors[process] = self._process_collector_factory(process.pid)
for process, collector in self._process_collectors.items():
collector.sample()
if now > next_disk_collection:
next_disk_collection = now + self._disk_collection_interval
if not self._disk_collector:
sandbox = self._task_monitor.get_sandbox()
if sandbox:
self._disk_collector = self._disk_collector_class(sandbox)
if self._disk_collector:
self._disk_collector.sample()
else:
log.debug('No sandbox detected yet for %s' % self._task_id)
try:
aggregated_procs = sum(map(attrgetter('procs'), self._process_collectors.values()))
aggregated_sample = sum(map(attrgetter('value'), self._process_collectors.values()),
ProcessSample.empty())
disk_value = self._disk_collector.value if self._disk_collector else 0
self._history.add(now, self.ResourceResult(aggregated_procs, aggregated_sample, disk_value))
except ValueError as err:
log.warning("Error recording resource sample: %s" % err)
# Sleep until any of the following conditions are met:
# - it's time for the next disk collection
# - it's time for the next process collection
# - the result from the last disk collection is available via the DiskCollector
# - the TaskResourceMonitor has been killed via self._kill_signal
now = time.time()
next_collection = min(next_process_collection - now, next_disk_collection - now)
if self._disk_collector:
waiter = EventMuxer(self._kill_signal, self._disk_collector.completed_event)
else:
waiter = self._kill_signal
waiter.wait(timeout=max(0, next_collection))
log.debug('Stopping resource monitoring for task "%s"' % self._task_id)
def __iter__(self):
"""Iterate over the services (ServiceInstance objects) in this ServerSet."""
for member in self._group.list():
try:
yield ServiceInstance.unpack(self._group.info(member))
except Exception as e:
log.warning('Failed to deserialize endpoint: %s' % e)
continue
def run(self):
self.runner._run_plan(self.runner._finalizing_plan)
log.debug("TaskRunnerStage[FINALIZING]: Finalization remaining: %s" % self.runner._finalization_remaining())
if self.runner.deadlocked(self.runner._finalizing_plan):
log.warning("Finalizing plan deadlocked.")
return None
if self.runner._finalization_remaining() > 0 and not self.runner._finalizing_plan.is_complete():
return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
def __iter__(self):
"""Iterate over the services (ServiceInstance objects) in this ServerSet."""
for member in self._group.list():
try:
yield ServiceInstance.unpack(self._group.info(member))
except Exception as e:
log.warning('Failed to deserialize endpoint: %s' % e)
continue
def perform_maintenance(self, hostnames, grouping_function=DEFAULT_GROUPING,
percentage=None, duration=None, output_file=None):
"""Put hosts into maintenance mode and drain them.
Walk through the process of putting hosts into maintenance and draining them of tasks. The hosts
will remain in maintenance mode upon completion.
:param hostnames: A list of hostnames to operate upon
:type hostnames: list of strings
:param grouping_function: How to split up the hostname into groups
:type grouping_function: function
:param percentage: SLA percentage to use
:type percentage: float
:param duration: SLA duration to use
:type duration: twitter.common.quantity.Time
:param output_file: file to write hosts that were not drained due to failed SLA check
:type output_file: string
:rtype: set of host names that were successfully drained
"""
hostnames = self.start_maintenance(hostnames)
not_drained_hostnames = set()
for hosts in self.iter_batches(hostnames, grouping_function):
log.info('Beginning SLA check for %s' % hosts.hostNames)
unsafe_hostnames = self._check_sla(
list(hosts.hostNames),
grouping_function,
percentage,
duration)
if unsafe_hostnames:
log.warning('Some hosts did not pass SLA check and will not be drained! '
'Skipping hosts: %s' % unsafe_hostnames)
not_drained_hostnames |= unsafe_hostnames
drainable_hostnames = hosts.hostNames - unsafe_hostnames
if not drainable_hostnames:
continue
hosts = Hosts(drainable_hostnames)
else:
log.info('All hosts passed SLA check.')
self._drain_hosts(hosts)
if not_drained_hostnames:
output = '\n'.join(list(not_drained_hostnames))
log.info('The following hosts did not pass SLA check and were not drained:')
print(output)
if output_file:
try:
with open(output_file, 'w') as fp:
fp.write(output)
fp.write('\n')
log.info('Written unsafe host names into: %s' % output_file)
except IOError as e:
log.error('Failed to write into the output file: %s' % e)
return set(hostnames) - not_drained_hostnames
def run(self):
self.runner._run_plan(self.runner._finalizing_plan)
log.debug('TaskRunnerStage[FINALIZING]: Finalization remaining: %s' %
self.runner._finalization_remaining())
if self.runner.deadlocked(self.runner._finalizing_plan):
log.warning('Finalizing plan deadlocked.')
return None
if self.runner._finalization_remaining() > 0 and not self.runner._finalizing_plan.is_complete():
return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
def get_completion(_, rc, children):
if rc in self._zk.COMPLETION_RETRY:
do_monitor()
return
if rc != zookeeper.OK:
log.warning('Unexpected get_completion return code: %s' % ZooKeeper.ReturnCode(rc))
promise.set(set([Membership.error()]))
return
self._update_children(children)
set_different(promise, membership, self._members)
def run(self):
"""Thread entrypoint. Loop indefinitely, polling collectors at self._collection_interval and
collating samples."""
log.debug('Commencing resource monitoring for task "%s"' % self._task_id)
next_process_collection = 0
next_disk_collection = 0
while not self._kill_signal.is_set():
now = time.time()
if now > next_process_collection:
next_process_collection = now + self._process_collection_interval
actives = set(self._get_active_processes())
current = set(self._process_collectors)
for process in current - actives:
log.debug('Process "%s" (pid %s) no longer active, removing from monitored processes' %
(process.process, process.pid))
self._process_collectors.pop(process)
for process in actives - current:
log.debug('Adding process "%s" (pid %s) to resource monitoring' %
(process.process, process.pid))
self._process_collectors[process] = self._process_collector_factory(process.pid)
for process, collector in self._process_collectors.items():
log.debug('Collecting sample for process "%s" (pid %s) and children' %
(process.process, process.pid))
collector.sample()
if now > next_disk_collection:
next_disk_collection = now + self._disk_collection_interval
log.debug('Collecting disk sample for %s' % self._sandbox)
self._disk_collector.sample()
try:
aggregated_procs = sum(map(attrgetter('procs'), self._process_collectors.values()))
aggregated_sample = sum(map(attrgetter('value'), self._process_collectors.values()),
ProcessSample.empty())
self._history.add(now, self.ResourceResult(aggregated_procs, aggregated_sample,
self._disk_collector.value))
log.debug("Recorded resource sample at %s" % now)
except ValueError as err:
log.warning("Error recording resource sample: %s" % err)
# Sleep until any of the following conditions are met:
# - it's time for the next disk collection
# - it's time for the next process collection
# - the result from the last disk collection is available via the DiskCollector
# - the TaskResourceMonitor has been killed via self._kill_signal
now = time.time()
next_collection = min(next_process_collection - now, next_disk_collection - now)
EventMuxer(self._kill_signal, self._disk_collector.completed_event
).wait(timeout=max(0, next_collection))
log.debug('Stopping resource monitoring for task "%s"' % self._task_id)
def _reconnect(self):
"""Reconnect to ZK and update endpoints once complete."""
for _ in range(self._retries):
try:
self._zk.restart()
self._start()
break
except ZooKeeper.ConnectionTimeout:
log.warning('Connection establishment to %r timed out, retrying.' % self._zk)
else:
raise ServerSetClient.ReconnectFailed('Re-establishment of connection to ZK servers failed')
def _reconnect(self):
"""Reconnect to ZK and update endpoints once complete."""
for _ in range(self._retries):
try:
self._zk.restart()
self._start()
break
except ZooKeeper.ConnectionTimeout:
log.warning("Connection establishment to %r timed out, retrying." % self._zk)
else:
raise ServerSetClient.ReconnectFailed("Re-establishment of connection to ZK servers failed")
def get_completion(_, rc, children):
if rc in self._zk.COMPLETION_RETRY:
do_monitor()
return
if rc != zookeeper.OK:
log.warning('Unexpected get_completion return code: %s' %
ZooKeeper.ReturnCode(rc))
promise.set(set([Membership.error()]))
return
self._update_children(children)
set_different(promise, membership, self._members)
def _complete_maintenance(self, drained_hosts):
"""End the maintenance status for a given set of hosts.
:param drained_hosts: Hosts that are drained and finished being operated upon
:type drained_hosts: gen.apache.aurora.ttypes.Hosts
"""
check_and_log_response(self._client.end_maintenance(drained_hosts))
resp = self._client.maintenance_status(drained_hosts)
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.NONE:
log.warning('%s is DRAINING or in DRAINED' % host_status.host)
def _complete_maintenance(self, drained_hosts):
"""End the maintenance status for a given set of hosts.
:param drained_hosts: Hosts that are drained and finished being operated upon
:type drained_hosts: gen.apache.aurora.ttypes.Hosts
"""
check_and_log_response(self._client.end_maintenance(drained_hosts))
resp = self._client.maintenance_status(drained_hosts)
for host_status in resp.result.maintenanceStatusResult.statuses:
if host_status.mode != MaintenanceMode.NONE:
log.warning('%s is DRAINING or in DRAINED' % host_status.host)
def _maybe_update_failure_count(self, is_healthy, reason):
if not is_healthy:
log.warning('Health check failure: %s' % reason)
self.current_consecutive_failures += 1
if self.current_consecutive_failures > self.max_consecutive_failures:
log.warning('Reached consecutive failure limit.')
self.healthy = False
self.reason = reason
else:
if self.current_consecutive_failures > 0:
log.debug('Reset consecutive failures counter.')
self.current_consecutive_failures = 0
def _maybe_update_failure_count(self, is_healthy, reason):
if not is_healthy:
log.warning('Health check failure: %s' % reason)
self.current_consecutive_failures += 1
if self.current_consecutive_failures > self.max_consecutive_failures:
log.warning('Reached consecutive failure limit.')
self.healthy = False
self.reason = reason
else:
if self.current_consecutive_failures > 0:
log.debug('Reset consecutive failures counter.')
self.current_consecutive_failures = 0
def resolve_ports(mesos_task, portmap):
"""Given a MesosTaskInstance and the portmap of resolved ports from the scheduler,
create a fully resolved map of port name => port number for the thermos
runner and discovery manager."""
task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
task_portmap.update(portmap)
task_portmap = PortResolver.resolve(task_portmap)
for name, port in task_portmap.items():
if not isinstance(port, int):
log.warning('Task has unmapped port: %s => %s' % (name, port))
return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))
def get_completion(_, rc, children):
if rc in self._zk.COMPLETION_RETRY:
do_monitor()
return
if rc == zookeeper.NONODE:
wait_exists()
return
if rc != zookeeper.OK:
log.warning('Unexpected get_completion return code: %s' % ReturnCode(rc))
capture.set(set([Membership.error()]))
return
self._update_children(children)
set_different(capture, membership, self._members)
def resolve_ports(mesos_task, portmap):
"""Given a MesosTaskInstance and the portmap of resolved ports from the scheduler,
create a fully resolved map of port name => port number for the thermos
runner and discovery manager."""
task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
task_portmap.update(portmap)
task_portmap = PortResolver.resolve(task_portmap)
for name, port in task_portmap.items():
if not isinstance(port, int):
log.warning('Task has unmapped port: %s => %s' % (name, port))
return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))
def get_completion(_, rc, children):
if rc in self._zk.COMPLETION_RETRY:
do_monitor()
return
if rc == zookeeper.NONODE:
wait_exists()
return
if rc != zookeeper.OK:
log.warning('Unexpected get_completion return code: %s' %
ReturnCode(rc))
capture.set(set([Membership.error()]))
return
self._update_children(children)
set_different(capture, membership, self._members)
def _kill(self):
processes = TaskRunnerHelper.scan_tree(self._state)
for process, pid_tuple in processes.items():
current_run = self._current_process_run(process)
coordinator_pid, pid, tree = pid_tuple
if TaskRunnerHelper.is_process_terminal(current_run.state):
if coordinator_pid or pid or tree:
log.warning(
'Terminal process (%s) still has running pids:' %
process)
log.warning(' coordinator_pid: %s' % coordinator_pid)
log.warning(' pid: %s' % pid)
log.warning(' tree: %s' % tree)
TaskRunnerHelper.kill_process(self.state, process)
else:
if coordinator_pid or pid or tree:
log.info('Transitioning %s to KILLED' % process)
self._set_process_status(process,
ProcessState.KILLED,
stop_time=self._clock.time(),
return_code=-1)
else:
log.info('Transitioning %s to LOST' % process)
if current_run.state != ProcessState.WAITING:
self._set_process_status(process, ProcessState.LOST)
def _do_health_check(self):
if self._should_enforce_deadline():
# This is needed otherwise it is possible to flap between
# successful health-checks and failed health-checks, never
# really satisfying the criteria for either healthy or unhealthy.
log.warning(
'Exhausted attempts before satisfying liveness criteria.')
self.healthy = False
self.reason = 'Not enough successful health checks in time.'
return self.healthy, self.reason
is_healthy, reason = self._perform_check_if_not_disabled()
if self.attempts <= self.max_attempts_to_running:
self.attempts += 1
self._maybe_update_health_check_count(is_healthy, reason)
return is_healthy, reason
def adelete_completion(result):
try:
success = result.get()
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, do_cancel)
return
except ke.NoNodeError:
success = True
except ke.KazooException as e:
log.warning('Unexpected Kazoo result in cancel: (%s)%s' %
(type(e), e))
success = False
future = self._members.pop(member.id, Future())
future.set_result(Membership.error())
capture.set(success)
def get_completion(result):
try:
children = result.get()
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, do_monitor)
return
except ke.NoNodeError:
wait_exists()
return
except ke.KazooException as e:
log.warning('Unexpected get_completion result: (%s)%s' %
(type(e), e))
capture.set(set([Membership.error()]))
return
self._update_children(children)
set_different(capture, membership, self._members)
def exists_completion(result):
try:
stat = result.get()
except self.DISCONNECT_EXCEPTIONS:
self._once(KazooState.CONNECTED, wait_exists)
return
except ke.NoNodeError:
wait_exists()
return
except ke.KazooException as e:
log.warning('Unexpected exists_completion result: (%s)%s' %
(type(e), e))
return
if stat:
do_monitor()
def method_wrapper(*args):
with self._lock:
start = time.time()
while not self._terminating.is_set() and (
time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(
Time.SECONDS):
# Only automatically append a SessionKey if this is not part of the read-only API.
auth_args = () if hasattr(
ReadOnlyScheduler.Iface,
method_name) else (self.session_key(), )
try:
method = getattr(self.client(), method_name)
if not callable(method):
return method
resp = method(*(args + auth_args))
if resp is not None and resp.responseCode == ResponseCode.ERROR_TRANSIENT:
raise self.TransientError(
", ".join([m.message for m in resp.
details] if resp.details else []))
if resp.serverInfo.thriftAPIVersion != THRIFT_API_VERSION:
raise self.APIVersionError(
"Client Version: %s, Server Version: %s" %
(THRIFT_API_VERSION,
resp.serverInfo.thriftAPIVersion))
return resp
except (TTransport.TTransportException, self.TimeoutError,
self.TransientError) as e:
if not self._terminating.is_set():
log.warning(
'Connection error with scheduler: %s, reconnecting...'
% e)
self.invalidate()
self._terminating.wait(
self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
except Exception as e:
# Take any error that occurs during the RPC call, and transform it
# into something clients can handle.
if not self._terminating.is_set():
raise self.ThriftInternalError(
"Error during thrift call %s to %s: %s" %
(method_name, self.cluster.name, e))
if not self._terminating.is_set():
raise self.TimeoutError(
'Timed out attempting to issue %s to %s' %
(method_name, self.cluster.name))
def process_to_sample(process):
""" Given a psutil.Process, return a current ProcessSample """
try:
# the nonblocking get_cpu_percent call is stateful on a particular Process object, and hence
# >2 consecutive calls are required before it will return a non-zero value
rate = process.cpu_percent(0.0) / 100.0
cpu_times = process.cpu_times()
user, system = cpu_times.user, cpu_times.system
memory_info = process.memory_info()
rss, vms = memory_info.rss, memory_info.vms
nice = process.nice()
status = process.status()
threads = process.num_threads()
return ProcessSample(rate, user, system, rss, vms, nice, status, threads)
except (AccessDenied, NoSuchProcess) as e:
log.warning('Error during process sampling [pid=%s]: %s' % (process.pid, e))
return ProcessSample.empty()
def reap_children(cls):
pids = set()
while True:
try:
pid, status, rusage = os.wait3(os.WNOHANG)
if pid == 0:
break
pids.add(pid)
log.debug('Detected terminated process: pid=%s, status=%s, rusage=%s' % (
pid, status, rusage))
except OSError as e:
if e.errno != errno.ECHILD:
log.warning('Unexpected error when calling waitpid: %s' % e)
break
return pids
