def _reload(self, graceful=True, sequential=False):
""" reload
"""
if not graceful and sequential:
logger.warn("with graceful=False, sequential=True is ignored")
if self.prereload_fn is not None:
self.prereload_fn(self)
if not graceful:
yield self._restart()
return
if self.is_stopped():
yield self._start()
elif self.send_hup:
for process in self.processes.values():
logger.info("SENDING HUP to %s" % process.pid)
process.send_signal(signal.SIGHUP)
else:
if sequential:
active_processes = self.get_active_processes()
for process in active_processes:
yield self.kill_process(process)
self.reap_process(process.pid)
self.spawn_process()
yield gen.sleep(self.warmup_delay)
else:
for i in range(self.numprocesses):
self.spawn_process()
yield self.manage_processes()
logger.info('%s reloaded', self.name)
def stop(cls):
logger.info('Stopping ant-agent service')
returncode, output = _sc_run('stop', cls.name)
if returncode != 0:
raise CallError(output)
_wait_util(cls, 'stopped')
logger.info('ant-agent stopped')
def start(cls, *args):
logger.info('Starting ant-agent service')
# CircleWinService.config(*args)
returncode, output = _sc_run('start', cls.name)
if returncode != 0:
raise CallError(output)
_wait_util(cls, 'running')
logger.info('ant-agent started')
def _stop(self):
logger.info('Arbiter exiting')
self._stopping = True
yield self._stop_watchers(close_output_streams=True)
if self._provided_loop:
cb = self.stop_controller_and_close_sockets
self.loop.add_callback(cb)
else:
# stop_controller_and_close_sockets will be
# called in the end of start() method
self.loop.add_callback(self.loop.stop)
def _register(self):
logger.info('Registering signals...')
for sig in self.SIGNALS:
self._old[sig] = signal.getsignal(sig)
signal.signal(sig, self.signal)
# Don't let SIGQUIT and SIGUSR1 disturb active requests
# by interrupting system calls
if hasattr(signal, 'siginterrupt'): # python >= 2.6
signal.siginterrupt(signal.SIGQUIT, False)
signal.siginterrupt(signal.SIGUSR1, False)
def install(cls, *args):
logger.info('Installing ant-upgrade service')
returncode, output = _nssm_run('install', cls.name, UPGRADE_START)
if returncode == 0 and 'Administrator access' in output:
raise CallError(output)
elif returncode == 5:
raise AlreadyExist(cls.exist_msg)
elif returncode != 0:
raise CallError(output)
logger.info('ant-upgrade installed')
def remove(cls):
"""If you want to remove the service, please make sure stop it first.
Otherwise you will get error message: "Error deleting". """
logger.info('Removing ant-upgrade service')
returncode, output = _nssm_run('remove', cls.name, 'confirm')
if returncode == 3:
raise NotExistError(cls.not_exist_msg)
elif returncode == 4:
raise CallError(cls.remove_err_msg)
elif returncode != 0:
raise CallError(output)
logger.info('ant-upgrade removed')
def signal(self, sig, frame=None):
signame = self.SIG_NAMES.get(sig)
logger.info('Got signal SIG_{}'.format(signame.upper()))
if signame is not None:
try:
handler = getattr(self, "handle_%s" % signame)
handler()
except AttributeError:
pass
except Exception as e:
tb = traceback.format_exc()
logger.error("error: %s [%s]" % (e, tb))
sys.exit(1)
def win_daemonize():
logger.info('Starting deamon mode. '
'The AntCircled service will be started.')
args = sys.argv[1:]
if '--daemon' in args:
args.remove('--daemon')
try:
if not CircleWinService.exists():
CircleWinService.install(*args)
CircleWinService.start(*args)
sys.exit(0)
except (AlreadyExist, NotExistError, CallError) as e:
logger.error(e)
sys.exit(1)
def install(cls, *args):
logger.info('Installing ant-agent service')
if not os.path.exists(CIRCLED_PATH):
raise CallError('{} not exists.'.format(CIRCLED_PATH))
returncode, output = _nssm_run('install', cls.name, BIN_START)
if returncode == 0 and 'Administrator access' in output:
raise CallError(output)
elif returncode == 5:
raise AlreadyExist(cls.exist_msg)
elif returncode != 0:
raise CallError(output)
logger.info('ant-agent installed')
def kill_process(self, process, stop_signal=None, graceful_timeout=None):
"""Kill process (stop_signal, graceful_timeout then SIGKILL)
"""
if stop_signal is None:
stop_signal = self.stop_signal
if graceful_timeout is None:
graceful_timeout = self.graceful_timeout
if process.stopping:
raise gen.Return(False)
try:
logger.info('Kill process "{}" (pid {})'.format(
self.name, process.pid))
if self.stop_children:
self.send_signal_process(process, stop_signal)
else:
self.send_signal(process.pid, stop_signal)
except NoSuchProcess:
raise gen.Return(False)
process.stopping = True
waited = 0
while waited < graceful_timeout:
if not process.is_alive():
break
yield gen.sleep(0.1)
waited += 0.1
if waited >= graceful_timeout:
# On Windows we can't send a SIGKILL signal, but the
# process.stop function will terminate the process
# later anyway
if hasattr(signal, 'SIGKILL'):
# We are not smart anymore
self.send_signal_process(process,
signal.SIGKILL,
recursive=True)
if self.stream_redirector:
self.stream_redirector.remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def run_hooks(self, hook_name):
hooks = self.config.get('hooks', None)
if not hooks:
raise gen.Return(True)
hook = hooks.get(hook_name, None)
if not hook:
logger.info('Watcher: "{}" have no {} hooks'.format(
self.name, hook_name))
raise gen.Return(True)
self.running_hook = True
logger.info('Watcher: "{}" run {} hooks'.format(self.name, hook_name))
status, output = yield async_get_status_output(hook,
cwd=self.working_dir)
self.running_hook = False
if status:
logger.error(output)
raise gen.Return(False)
raise gen.Return(True)
def _stop(self, close_output_streams=False):
if self.is_stopped():
return
self._status = "stopping"
logger.debug('Stopping the watcher: "{}"'.format(self.name))
logger.debug('Gracefully stopping processes [{}] for {}s'.format(
self.name, self.graceful_timeout))
yield self.kill_processes()
self.reap_processes()
# stop redirectors
if self.stream_redirector:
self.stream_redirector.stop()
self.stream_redirector = None
if close_output_streams:
if self.stdout_stream and hasattr(self.stdout_stream, 'close'):
self.stdout_stream.close()
if self.stderr_stream and hasattr(self.stderr_stream, 'close'):
self.stderr_stream.close()
self._status = "stopped"
logger.info('Watcher: "{}" stopped'.format(self.name))
yield self.run_hooks('post_stop')
def start(self, callback=None):
"""Starts all the watchers.
If the ioloop has been provided during __init__() call,
starts all watchers as a standard coroutine
If the ioloop hasn't been provided during __init__() call (default),
starts all watchers and the eventloop (and blocks here). In this mode
the method MUST NOT yield anything because it's called as a standard
method.
:param callback: Callback called after all the watchers have been
started, when the loop hasn't been provided.
:type function:
"""
logger.info('Starting circle arbiter on pid {}'.format(self.pid))
self.initialize()
# start controller
self.ctrl.start()
self._restarting = False
try:
# initialize processes
logger.info('Starting watchers')
if self._provided_loop:
yield self.start_watchers()
else:
# start_watchers will be called just after the start_io_loop()
if not callback:
def callback(x):
pass
self.loop.add_future(self.start_watchers(), callback)
logger.info('Arbiter now waiting for commands')
self._running = True
if not self._provided_loop:
# If an event loop is not provided, block at this line
self.start_io_loop()
finally:
if not self._provided_loop:
# If an event loop is not provided, do some cleaning
self.stop_controller_and_close_sockets()
raise gen.Return(self._restarting)
def _start(self):
"""Start."""
logger.info('Starting watcher: "{}"'.format(self.name))
if not self.is_stopped():
if len(self.processes) < self.numprocesses:
self.reap_processes()
yield self.spawn_processes()
return
found_wids = bool(self._found_wids)
self._status = "starting"
if self.stdout_stream and hasattr(self.stdout_stream, 'open'):
self.stdout_stream.open()
if self.stderr_stream and hasattr(self.stderr_stream, 'open'):
self.stderr_stream.open()
self._create_redirectors()
self.reap_processes()
yield self.spawn_processes()
# If not self.processes, the before_spawn or after_spawn hooks have
# probably prevented startup so give up
if not self.processes:
logger.debug('Aborting startup')
# stop streams too since we are bailing on this watcher completely
yield self._stop()
return
self._status = "active"
if found_wids:
logger.info('Watcher "{}" already running'.format(self.name))
else:
logger.info('Watcher "{}" started'.format(self.name))
hook_status = yield self.run_hooks('post_start')
if not hook_status:
yield self._stop()
def spawn_process(self, recovery_wid=None):
"""Spawn process.
Return True if ok, False if the watcher must be stopped
"""
if self.is_stopped():
return True
cmd = util.replace_gnu_args(self.cmd, env=self.env)
nb_tries = 0
# start the redirector now so we can catch any startup errors
if self.stream_redirector:
self.stream_redirector.start()
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_stream is not None
pipe_stderr = self.stderr_stream is not None
# noinspection PyPep8Naming
ProcessClass = self._process_class
try:
process = ProcessClass(
self.name,
recovery_wid or self._nextwid,
cmd,
args=self.args,
working_dir=self.working_dir,
shell=self.shell,
uid=self.uid,
gid=self.gid,
env=self.env,
rlimits=self.rlimits,
executable=self.executable,
# todo, watcher remove 'use_fds'
use_fds=False,
watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdin=self.close_child_stdin,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if self.stream_redirector:
self.stream_redirector.add_redirections(process)
self.processes[process.pid] = process
logger.info('Running process "{}" (pid {})'.format(
self.name, process.pid))
# catch ValueError as well, as a misconfigured rlimit setting could
# lead to bad infinite retries here
except (OSError, ValueError) as e:
logger.warn('Error in %s: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
return process.started
return False