def _add_periodic_task(cls, task): """Add a periodic task to the list of periodic tasks. The task should already be decorated by @periodic_task. :return: whether task was actually enabled """ name = task._periodic_name if task._periodic_spacing < 0: LOG.info(_LI('Skipping periodic task %(task)s because ' 'its interval is negative'), {'task': name}) return False if not task._periodic_enabled: LOG.info(_LI('Skipping periodic task %(task)s because ' 'it is disabled'), {'task': name}) return False # A periodic spacing of zero indicates that this task should # be run on the default interval to avoid running too # frequently. if task._periodic_spacing == 0: task._periodic_spacing = DEFAULT_INTERVAL cls._periodic_tasks.append((name, task)) cls._periodic_spacing[name] = task._periodic_spacing return True
def _wait_child(self): try: # Don't block if no child processes have exited pid, status = os.waitpid(0, os.WNOHANG) if not pid: return None except OSError as exc: if exc.errno not in (errno.EINTR, errno.ECHILD): raise return None if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'), dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) LOG.info(_LI('Child %(pid)s exited with status %(code)d'), dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_LW('pid %d not in child list'), pid) return None wrap = self.children.pop(pid) wrap.children.remove(pid) return wrap
def get_all_user_all_weibo_info(self, *args, **kwargs): LOG.info(_LI('Looping exec function get_all_user_all_weibo_info')) if self.is_all_and_runing: return else: self.is_all_and_runing = True if not CONF.is_all: return users = db_api.db_userdata_get_all() tg = kwargs.pop('tg', None) for user in users: homepage = user.homepage nickname = user.screen_name LOG.info(_LI('get %(nickname)s weibo %(homepage)s info to db'), {'nickname': nickname, 'homepage': homepage}) kwargs['homepage'] = homepage kwargs['nickname'] = nickname try: if not tg: self.get_all_page_one_user_weibo_info(**kwargs) else: tg.add_thread(self.get_all_page_one_user_weibo_info, *args, **kwargs) except Exception: self.is_all_and_runing = False
def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: # Limit ourselves to one process a second (over the period of # number of workers * 1 second). This will allow workers to # start up quickly but ensure we don't fork off children that # die instantly too quickly. if time.time() - wrap.forktimes[0] < wrap.workers: LOG.info(_LI('Forking too fast, sleeping')) time.sleep(1) wrap.forktimes.pop(0) wrap.forktimes.append(time.time()) pid = os.fork() if pid == 0: self.launcher = self._child_process(wrap.service) while True: self._child_process_handle_signal() status, signo = self._child_wait_for_exit_or_signal( self.launcher) if not _is_sighup_and_daemon(signo): self.launcher.wait() break self.launcher.restart() os._exit(status) LOG.debug('Started child %d', pid) wrap.children.add(pid) self.children[pid] = wrap return pid
def update_one_weibo(self, url, name): try: self.simulogin.eventlet_one_url(url, name) except exception.ResetLoginError: LOG.debug(_LI('reset login %(nickname)s weibo'), {'nickname': name}) self.reset_login_weibo eventlet.greenthread.sleep(10) except Exception: eventlet.greenthread.sleep(20)
def _pipe_watcher(self): # This will block until the write end is closed when the parent # dies unexpectedly self.readpipe.read(1) LOG.info(_LI('Parent process has died unexpectedly, exiting')) if self.launcher: self.launcher.stop() sys.exit(1)
def wait(self): """Loop waiting on children to die and respawning as necessary.""" # systemd.notify_once() if self.conf.log_options: LOG.debug('Full set of CONF:') self.conf.log_opt_values(LOG, logging.DEBUG) try: while True: self.handle_signal() self._respawn_children() # No signal means that stop was called. Don't clean up here. if not self.sigcaught: return signame = self.signal_handler.signals_to_name[self.sigcaught] LOG.info(_LI('Caught %s, stopping children'), signame) if not _is_sighup_and_daemon(self.sigcaught): break self.conf.reload_config_files() for service in set( [wrap.service for wrap in self.children.values()]): service.reset() for pid in self.children: os.kill(pid, signal.SIGTERM) self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: LOG.info(_LI("Wait called after thread killed. Cleaning up.")) # if we are here it means that we try to do gracefull shutdown. # add alarm watching that graceful_shutdown_timeout is not exceeded if (self.conf.graceful_shutdown_timeout and self.signal_handler.is_signal_supported('SIGALRM')): signal.alarm(self.conf.graceful_shutdown_timeout) self.stop()
def init_host(self, tg, **kwargs): LOG.info(_LI('Willing init host function.......')) if CONF.is_all: pid = os.fork() if pid == 0: child_started = False while True: enable_spawn = kwargs.get('enable_spawn', True) if enable_spawn: eventlet.spawn(self.get_all_user_all_weibo_info, **kwargs) child_started = True else: kwargs['tg'] = tg self.get_all_user_all_weibo_info(**kwargs) child_started = True if not child_started: break os._exit(2) LOG.debug(_LI('Started child %d' % pid))
def inner(self, *args, **kw): f.ntime = ntime n = 1 while n <= f.ntime: LOG.debug(_LI('Execing %s time fucntin %s' % (n, f.__name__))) if n == f.ntime: return f(self, *args, **kw) try: return f(self, *args, **kw) except Exception: n = n + 1 eventlet.greenthread.sleep(60)
def remove_external_lock_file(name, lock_file_prefix=None, lock_path=None, semaphores=None): """Remove an external lock file when it's not used anymore This will be helpful when we have a lot of lock files """ with internal_lock(name, semaphores=semaphores): lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) try: os.remove(lock_file_path) except OSError: LOG.info(_LI('Failed to remove file %(file)s'), {'file': lock_file_path})
def launch_service(self, service, workers=1): """Launch a service with a given number of workers. :param service: a service to launch, must be an instance of :class:`oslo_service.service.ServiceBase` :param workers: a number of processes in which a service will be running """ _check_service_base(service) wrap = ServiceWrapper(service, workers) LOG.info(_LI('Starting %d workers'), wrap.workers) while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap)
def update_webo_info(self, **kwargs): LOG.info(_LI('Looping exec function update_webo_info')) udata = self.simulogin.get_db_userdata_all() initial_delay = 0 for u in udata: nickname = u.get('screen_name', None) if nickname in self.exist_udata_name: continue self.exist_udata_name.append(nickname) url = u.get('homepage', None) if 'is_all' not in url: url = url + '?is_all=1' # 设置微博每 10 分钟更新 一次 tg = kwargs.get('tg', None) if not tg: return LOG.info(_LI('update %(nickname)s weibo info to db'), {'nickname': nickname}) tg.add_timer(CONF.weiboinfo_interval, self.update_one_weibo, initial_delay * 10, url, nickname) initial_delay = initial_delay + 1
def _child_wait_for_exit_or_signal(self, launcher): status = 0 signo = 0 # NOTE(johannes): All exceptions are caught to ensure this # doesn't fallback into the loop spawning children. It would # be bad for a child to spawn more children. try: launcher.wait() except SignalExit as exc: signame = self.signal_handler.signals_to_name[exc.signo] LOG.info(_LI('Child caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code except BaseException: LOG.exception(_LE('Unhandled exception')) status = 2 return status, signo
def stop(self): """Terminate child processes and wait on each.""" self.running = False LOG.debug("Stop services.") for service in set( [wrap.service for wrap in self.children.values()]): service.stop() LOG.debug("Killing children.") for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_LI('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 # if self.conf.log_options: # LOG.debug('Full set of CONF:') # self.conf.log_opt_values(LOG, logging.DEBUG) try: if ready_callback: ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = SignalHandler().signals_to_name[exc.signo] LOG.info(_LI('Caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code finally: self.stop() return status, signo
def update_userdata_info(self, **kwargs): LOG.info(_LI('Looping exec function update_userdata_info')) self.udata.get_all_names_json() self.udata.save_all_users()
def _fast_exit(self, signo, frame): LOG.info(_LI('Caught SIGINT signal, instantaneous exiting')) os._exit(1)
def download_all_img(self, **kwargs): LOG.info(_LI('Looping exec function download_all_img')) self.imagedl.download()
def _on_alarm_exit(self, signo, frame): LOG.info(_LI('Graceful shutdown timeout exceeded, ' 'instantaneous exiting')) os._exit(1)