def _add_periodic_task(cls, task):
"""Add a periodic task to the list of periodic tasks.
The task should already be decorated by @periodic_task.
:return: whether task was actually enabled
"""
name = task._periodic_name
if task._periodic_spacing < 0:
LOG.info(_LI('Skipping periodic task %(task)s because '
'its interval is negative'),
{'task': name})
return False
if not task._periodic_enabled:
LOG.info(_LI('Skipping periodic task %(task)s because '
'it is disabled'),
{'task': name})
return False
# A periodic spacing of zero indicates that this task should
# be run on the default interval to avoid running too
# frequently.
if task._periodic_spacing == 0:
task._periodic_spacing = DEFAULT_INTERVAL
cls._periodic_tasks.append((name, task))
cls._periodic_spacing[name] = task._periodic_spacing
return True
def _wait_child(self):
try:
# Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_LW('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def get_all_user_all_weibo_info(self, *args, **kwargs):
LOG.info(_LI('Looping exec function get_all_user_all_weibo_info'))
if self.is_all_and_runing:
return
else:
self.is_all_and_runing = True
if not CONF.is_all:
return
users = db_api.db_userdata_get_all()
tg = kwargs.pop('tg', None)
for user in users:
homepage = user.homepage
nickname = user.screen_name
LOG.info(_LI('get %(nickname)s weibo %(homepage)s info to db'),
{'nickname': nickname,
'homepage': homepage})
kwargs['homepage'] = homepage
kwargs['nickname'] = nickname
try:
if not tg:
self.get_all_page_one_user_weibo_info(**kwargs)
else:
tg.add_thread(self.get_all_page_one_user_weibo_info,
*args,
**kwargs)
except Exception:
self.is_all_and_runing = False
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_LI('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
self.launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(
self.launcher)
if not _is_sighup_and_daemon(signo):
self.launcher.wait()
break
self.launcher.restart()
os._exit(status)
LOG.debug('Started child %d', pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def update_one_weibo(self, url, name):
try:
self.simulogin.eventlet_one_url(url, name)
except exception.ResetLoginError:
LOG.debug(_LI('reset login %(nickname)s weibo'),
{'nickname': name})
self.reset_login_weibo
eventlet.greenthread.sleep(10)
except Exception:
eventlet.greenthread.sleep(20)
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read(1)
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
if self.launcher:
self.launcher.stop()
sys.exit(1)
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
# systemd.notify_once()
if self.conf.log_options:
LOG.debug('Full set of CONF:')
self.conf.log_opt_values(LOG, logging.DEBUG)
try:
while True:
self.handle_signal()
self._respawn_children()
# No signal means that stop was called. Don't clean up here.
if not self.sigcaught:
return
signame = self.signal_handler.signals_to_name[self.sigcaught]
LOG.info(_LI('Caught %s, stopping children'), signame)
if not _is_sighup_and_daemon(self.sigcaught):
break
self.conf.reload_config_files()
for service in set(
[wrap.service for wrap in self.children.values()]):
service.reset()
for pid in self.children:
os.kill(pid, signal.SIGTERM)
self.running = True
self.sigcaught = None
except eventlet.greenlet.GreenletExit:
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
# if we are here it means that we try to do gracefull shutdown.
# add alarm watching that graceful_shutdown_timeout is not exceeded
if (self.conf.graceful_shutdown_timeout and
self.signal_handler.is_signal_supported('SIGALRM')):
signal.alarm(self.conf.graceful_shutdown_timeout)
self.stop()
def init_host(self, tg, **kwargs):
LOG.info(_LI('Willing init host function.......'))
if CONF.is_all:
pid = os.fork()
if pid == 0:
child_started = False
while True:
enable_spawn = kwargs.get('enable_spawn', True)
if enable_spawn:
eventlet.spawn(self.get_all_user_all_weibo_info,
**kwargs)
child_started = True
else:
kwargs['tg'] = tg
self.get_all_user_all_weibo_info(**kwargs)
child_started = True
if not child_started:
break
os._exit(2)
LOG.debug(_LI('Started child %d' % pid))
def inner(self, *args, **kw):
f.ntime = ntime
n = 1
while n <= f.ntime:
LOG.debug(_LI('Execing %s time fucntin %s' % (n, f.__name__)))
if n == f.ntime:
return f(self, *args, **kw)
try:
return f(self, *args, **kw)
except Exception:
n = n + 1
eventlet.greenthread.sleep(60)
def remove_external_lock_file(name, lock_file_prefix=None, lock_path=None,
semaphores=None):
"""Remove an external lock file when it's not used anymore
This will be helpful when we have a lot of lock files
"""
with internal_lock(name, semaphores=semaphores):
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
try:
os.remove(lock_file_path)
except OSError:
LOG.info(_LI('Failed to remove file %(file)s'),
{'file': lock_file_path})
def launch_service(self, service, workers=1):
"""Launch a service with a given number of workers.
:param service: a service to launch, must be an instance of
:class:`oslo_service.service.ServiceBase`
:param workers: a number of processes in which a service
will be running
"""
_check_service_base(service)
wrap = ServiceWrapper(service, workers)
LOG.info(_LI('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
def update_webo_info(self, **kwargs):
LOG.info(_LI('Looping exec function update_webo_info'))
udata = self.simulogin.get_db_userdata_all()
initial_delay = 0
for u in udata:
nickname = u.get('screen_name', None)
if nickname in self.exist_udata_name:
continue
self.exist_udata_name.append(nickname)
url = u.get('homepage', None)
if 'is_all' not in url:
url = url + '?is_all=1'
# 设置微博每 10 分钟更新 一次
tg = kwargs.get('tg', None)
if not tg:
return
LOG.info(_LI('update %(nickname)s weibo info to db'),
{'nickname': nickname})
tg.add_timer(CONF.weiboinfo_interval,
self.update_one_weibo,
initial_delay * 10,
url, nickname)
initial_delay = initial_delay + 1
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = self.signal_handler.signals_to_name[exc.signo]
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
return status, signo
def stop(self):
"""Terminate child processes and wait on each."""
self.running = False
LOG.debug("Stop services.")
for service in set(
[wrap.service for wrap in self.children.values()]):
service.stop()
LOG.debug("Killing children.")
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
# if self.conf.log_options:
# LOG.debug('Full set of CONF:')
# self.conf.log_opt_values(LOG, logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = SignalHandler().signals_to_name[exc.signo]
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
return status, signo
def update_userdata_info(self, **kwargs):
LOG.info(_LI('Looping exec function update_userdata_info'))
self.udata.get_all_names_json()
self.udata.save_all_users()
def _fast_exit(self, signo, frame):
LOG.info(_LI('Caught SIGINT signal, instantaneous exiting'))
os._exit(1)
def download_all_img(self, **kwargs):
LOG.info(_LI('Looping exec function download_all_img'))
self.imagedl.download()
def _on_alarm_exit(self, signo, frame):
LOG.info(_LI('Graceful shutdown timeout exceeded, '
'instantaneous exiting'))
os._exit(1)
