def _nssm_run(option, service_name, *args):
args = [NSSM_PATH, option, service_name] + list(args)
logger.debug('NSSM args: {}'.format(args))
p = Popen(args, stdout=PIPE, stderr=STDOUT)
p.wait()
returncode = p.returncode
output = _get_clear_output(p.stdout.read())
return returncode, output
def manage_watchers(self):
if self.managing_watchers_future is not None:
logger.debug('manage_watchers is already running...')
return
try:
self.managing_watchers_future = self.arbiter.manage_watchers()
self.loop.add_future(self.managing_watchers_future,
self._manage_watchers_cb)
except ConflictError:
logger.debug('manage_watchers is conflicting with another command')
def reap_processes(self):
"""Reap all the processes for this watcher.
"""
if self.is_stopped():
logger.debug('Do not reap processes as the watcher is stopped')
return
# reap_process changes our dict, look through the copy of keys
for pid in list(self.processes.keys()):
self.reap_process(pid)
def execute(self, arbiter, props):
if 'name' in props:
watcher = self._get_watcher(arbiter, props['name'])
processes = watcher.get_active_processes()
status = [(p.pid, p.status) for p in processes]
logger.debug('here is the status of the processes %s' % status)
return {"pids": [p.pid for p in processes]}
else:
watchers = sorted(arbiter._watchers_names)
return {"watchers": [name for name in watchers]}
def start(self):
logger.debug('Starting controller')
self.initialize()
if self.check_delay > 0:
# The specific case (check_delay < 0)
# so with no period callback to manage_watchers
# is probably "unit tests only"
self.caller = ioloop.PeriodicCallback(self.manage_watchers,
self.check_delay, self.loop)
self.caller.start()
self.started = True
def rm_watcher(self, name, nostop=False):
"""Deletes a watcher.
Options:
- **name**: name of the watcher to delete
"""
logger.debug('Deleting %r watcher', name)
# remove the watcher from the list
watcher = self._watchers_names.pop(name.lower())
del self.watchers[self.watchers.index(watcher)]
if not nostop:
# stop the watcher
yield watcher._stop()
def config(cls, *args):
logger.debug('Configuring ant-agent service parameters')
returncode, output = _nssm_run('set', cls.name, 'AppDirectory',
ROOT_DIR)
if returncode != 0:
if '\x07c\x9a' in output:
raise CallError(cls.not_exist_msg)
elif '\xd2b\xdd' in output:
raise CallError(cls.access_err_msg)
raise CallError(output)
returncode, output = _nssm_run('set', cls.name, 'AppParameters',
*([CIRCLED_PATH] + list(args)))
if returncode != 0:
raise CallError(output)
logger.debug('ant-agent service parameters configured')
def _log(self, *args, **kw):
if os.environ.get('DEBUG') is None:
return func(self, *args, **kw)
from circle import logger
cls = self.__class__.__name__
global INDENTATION_LEVEL
func_name = func.func_name if hasattr(func, 'func_name')\
else func.__name__
logger.debug(' ' * INDENTATION_LEVEL + '"{}.{}" starts'.format(
cls, func_name))
INDENTATION_LEVEL += 1
try:
return func(self, *args, **kw)
finally:
INDENTATION_LEVEL -= 1
logger.debug(' ' * INDENTATION_LEVEL + '"{}.{}" ends'.format(
cls, func_name))
def _start(self):
"""Start."""
logger.info('Starting watcher: "{}"'.format(self.name))
if not self.is_stopped():
if len(self.processes) < self.numprocesses:
self.reap_processes()
yield self.spawn_processes()
return
found_wids = bool(self._found_wids)
self._status = "starting"
if self.stdout_stream and hasattr(self.stdout_stream, 'open'):
self.stdout_stream.open()
if self.stderr_stream and hasattr(self.stderr_stream, 'open'):
self.stderr_stream.open()
self._create_redirectors()
self.reap_processes()
yield self.spawn_processes()
# If not self.processes, the before_spawn or after_spawn hooks have
# probably prevented startup so give up
if not self.processes:
logger.debug('Aborting startup')
# stop streams too since we are bailing on this watcher completely
yield self._stop()
return
self._status = "active"
if found_wids:
logger.info('Watcher "{}" already running'.format(self.name))
else:
logger.info('Watcher "{}" started'.format(self.name))
hook_status = yield self.run_hooks('post_start')
if not hook_status:
yield self._stop()
def _do_rollover(self):
if self._file:
self._file.close()
self._file = None
current_time = int(time_.time())
dst_now = time_.localtime(current_time)[-1]
t = self._rollover_at - self._interval
if self._utc:
time_touple = time_.gmtime(t)
else:
time_touple = time_.localtime(t)
dst_then = time_touple[-1]
if dst_now != dst_then:
if dst_now:
addend = 3600
else:
addend = -3600
time_touple = time_.localtime(t + addend)
dfn = self._filename + '.' + time_.strftime(self._suffix, time_touple)
if os.path.exists(dfn):
os.remove(dfn)
if os.path.exists(self._filename):
os.rename(self._filename, dfn)
logger.debug('Log rotating %s -> %s' % (self._filename, dfn))
if self._backup_count > 0:
for f in self._get_files_to_delete():
os.remove(f)
self._file = self._open()
new_rollover_at = self._compute_rollover(current_time)
while new_rollover_at <= current_time:
new_rollover_at = new_rollover_at + self._interval
self._rollover_at = new_rollover_at
def _do_rollover(self):
"""
Do a rollover, as described in __init__().
"""
if self._file:
self._file.close()
self._file = None
if self._backup_count > 0:
for i in range(self._backup_count - 1, 0, -1):
sfn = '{}.{}'.format(self._filename, i)
dfn = '{}.{}'.format(self._filename, i + 1)
if os.path.exists(sfn):
logger.debug('Log rotating {} -> {}'.format(sfn, dfn))
if os.path.exists(dfn):
os.remove(dfn)
os.rename(sfn, dfn)
dfn = self._filename + '.1'
if os.path.exists(dfn):
os.remove(dfn)
os.rename(self._filename, dfn)
logger.debug('Log rotating {} -> {}'.format(self._filename, dfn))
self._file = self._open()
def _stop(self, close_output_streams=False):
if self.is_stopped():
return
self._status = "stopping"
logger.debug('Stopping the watcher: "{}"'.format(self.name))
logger.debug('Gracefully stopping processes [{}] for {}s'.format(
self.name, self.graceful_timeout))
yield self.kill_processes()
self.reap_processes()
# stop redirectors
if self.stream_redirector:
self.stream_redirector.stop()
self.stream_redirector = None
if close_output_streams:
if self.stdout_stream and hasattr(self.stdout_stream, 'close'):
self.stdout_stream.close()
if self.stderr_stream and hasattr(self.stderr_stream, 'close'):
self.stderr_stream.close()
self._status = "stopped"
logger.info('Watcher: "{}" stopped'.format(self.name))
yield self.run_hooks('post_stop')
def dispatch(self, msg):
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(None,
msg,
'json invalid',
errno=errors.INVALID_JSON)
mid = json_msg.get('id')
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
try:
cmd = self.controller.commands[cmd_name.lower()]
except KeyError:
error_ = 'unknown command: {!r}'.format(cmd_name)
return self.send_error(mid,
msg,
error_,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.controller.arbiter, properties)
if isinstance(resp, Future):
if properties.get('waiting', False):
cb = functools.partial(self._dispatch_callback_future, msg,
mid, cmd_name, True)
resp.add_done_callback(cb)
else:
cb = functools.partial(self._dispatch_callback_future, msg,
mid, cmd_name, False)
resp.add_done_callback(cb)
self._dispatch_callback(msg, mid, cmd_name, None)
else:
self._dispatch_callback(msg, mid, cmd_name, resp)
except MessageError as e:
return self.send_error(mid,
msg,
str(e),
errno=errors.MESSAGE_ERROR)
except ConflictError as e:
if self.controller.managing_watchers_future is not None:
logger.debug('The command conflicts with running '
'manage_watchers, re-executing it at '
'the end')
cb = functools.partial(self.dispatch, msg)
self.controller.loop.add_future(
self.controller.managing_watchers_future, cb)
return
# conflicts between two commands, sending error...
return self.send_error(mid,
msg,
str(e),
errno=errors.COMMAND_ERROR)
except OSError as e:
return self.send_error(mid, msg, str(e), errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = 'Command {!r}: {}'.format(msg, value)
logger.debug('Error: command {!r}: {}\n\n{}'.format(
msg, value, tb))
return self.send_error(mid,
msg,
reason,
tb,
errno=errors.COMMAND_ERROR)
def send_signal(self, sig):
"""Sends a signal **sig** to the process."""
logger.debug('sending signal {} to {}'.format(sig, self.pid))
return self._worker.send_signal(sig)
def format_args(self):
""" It's possible to use environment variables and some other variables
that are available in this context, when spawning the processes.
"""
logger.debug('cmd: ' + to_string(self.cmd))
logger.debug('args: ' + str(self.args))
current_env = ObjectDict(self.env.copy())
format_kwargs = {
'wid': self.wid,
'shell': self.shell,
'args': self.args,
'env': current_env,
'working_dir': self.working_dir,
'uid': self.uid,
'gid': self.gid,
'rlimits': self.rlimits,
'executable': self.executable,
'use_fds': self.use_fds
}
if self.watcher is not None:
for option in self.watcher.optnames:
if option not in format_kwargs\
and hasattr(self.watcher, option):
format_kwargs[option] = getattr(self.watcher, option)
cmd = replace_gnu_args(self.cmd, **format_kwargs)
if '$WID' in cmd or (self.args and '$WID' in self.args):
msg = "Using $WID in the command is deprecated. You should use "\
"the python string format instead. In your case, this "\
"means replacing the $WID in your command by $(WID)."
warnings.warn(msg, DeprecationWarning)
self.cmd = cmd.replace('$WID', str(self.wid))
if self.args is not None:
if isinstance(self.args, basestring):
args = shlex.split(
to_string(replace_gnu_args(self.args, **format_kwargs)))
else:
args = [
to_string(replace_gnu_args(arg, **format_kwargs))
for arg in self.args
]
args = shlex.split(to_string(cmd), posix=not IS_WINDOWS) + args
else:
args = shlex.split(to_string(cmd), posix=not IS_WINDOWS)
if self.shell:
# subprocess.Popen(shell=True) implies that 1st arg is the
# requested command, remaining args are applied to sh.
args = [' '.join(quote(arg) for arg in args)]
shell_args = format_kwargs.get('shell_args', None)
if shell_args and IS_WINDOWS:
logger.warn("shell_args won't apply for "
"windows platforms: {}".format(shell_args))
elif isinstance(shell_args, basestring):
args += shlex.split(
to_string(replace_gnu_args(shell_args, **format_kwargs)))
elif shell_args:
args += [
to_string(replace_gnu_args(arg, **format_kwargs))
for arg in shell_args
]
elif format_kwargs.get('shell_args', False):
logger.warn("shell_args is defined but won't be used "
"in this context: {}".format(
format_kwargs['shell_args']))
logger.debug('Process args: {}'.format(args))
return args
def reap_process(self, pid, status=None):
"""ensure that the process is killed (and not a zombie)"""
if pid not in self.processes:
return
process = self.processes.pop(pid)
timeout = 0.001
while status is None:
if IS_WINDOWS:
try:
# On Windows we can't use waitpid as it's blocking,
# so we use psutils' wait
status = process.wait(timeout=timeout)
except TimeoutExpired:
continue
else:
try:
_, status = os.waitpid(pid, os.WNOHANG)
except OSError as e:
if e.errno == errno.EAGAIN:
time.sleep(timeout)
continue
elif e.errno == errno.ECHILD:
status = None
else:
raise
if status is None:
# nothing to do here, we do not have any child
# process running
# but we still need to send the "reap" signal.
#
# This can happen if poll() or wait() were called on
# the underlying process.
logger.debug('Reaping already dead process {} [{}]'.format(
pid, self.name))
process.stop()
return
# get return code
if hasattr(os, 'WIFSIGNALED'):
exit_code = 0
if os.WIFSIGNALED(status):
# The Python Popen object returns <-signal> in it's returncode
# property if the process exited on a signal, so emulate that
# behavior here so that pubsub clients watching for reap can
# distinguish between an exit with a non-zero exit code and
# a signal'd exit. This is also consistent with the notify
# event reap message above that uses the returncode function
# (that ends up calling Popen.returncode)
exit_code = -os.WTERMSIG(status)
# process exited using exit(2) system call; return the
# integer exit(2) system call has been called with
elif os.WIFEXITED(status):
exit_code = os.WEXITSTATUS(status)
else:
# should never happen
raise RuntimeError("Unknown process exit status")
else:
# On Windows we don't have such distinction
exit_code = status
# if the process is dead or a zombie try to definitely stop it.
if process.status in (DEAD_OR_ZOMBIE, UNEXISTING):
process.stop()
logger.debug('Reaping process {} [{}]'.format(pid, self.name))
return exit_code