def spawn_worker(self): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENER, self.app, self.timeout/2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: self.WORKERS[pid] = worker return pid # Process Child worker_pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker_pid) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except: self.log.exception("Exception in worker process:\n%s", traceback.format_exc()) if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker_pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: pass
def spawn_worker(self): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: self.WORKERS[pid] = worker return pid # Process Child worker_pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker_pid) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except: self.log.exception("Exception in worker process:\n%s", traceback.format_exc()) if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker_pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: pass
def run_gunicorn_worker(): if not einhorn.is_worker(): print >> sys.stderr, "This process does not appear to be running under Einhorn." sys.exit(1) app = PasterApplication() util._setproctitle("worker [%s]" % app.cfg.proc_name) worker = EinhornSyncWorker(app.cfg, app) worker.init_process()
def spawn_worker(self): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) if self.use_cuda: iter_cnt = 0 while iter_cnt < len(self.gpu_ids) and self.gpu_count[ self.gpu_id] >= self.gpu_limit: self.gpu_id = next(self.gpu_cycle) os.environ["CUDA_VISIBLE_DEVICES"] = self.gpu_id self.gpu_count[self.gpu_id] += 1 worker.gpu_id = self.gpu_id pid = os.fork() if pid != 0: worker.pid = pid self.WORKERS[pid] = worker return pid # Do not inherit the temporary files of other workers for sibling in self.WORKERS.values(): sibling.tmp.close() # Process Child worker.pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker.pid) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except AppImportError as e: self.log.debug("Exception while loading the application", exc_info=True) print("%s" % e, file=sys.stderr) sys.stderr.flush() sys.exit(self.APP_LOAD_ERROR) except Exception: self.log.exception("Exception in worker process") if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker.pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except Exception: self.log.warning("Exception during worker exit:\n%s", traceback.format_exc())
def reload(self): old_address = self.cfg.address # reset old environment for k in self.cfg.env: if k in self.cfg.env_orig: # reset the key to the value it had before # we launched gunicorn os.environ[k] = self.cfg.env_orig[k] else: # delete the value set by gunicorn try: del os.environ[k] except KeyError: pass # reload conf self.app.reload() self.setup(self.app) # reopen log files self.log.reopen_files() # do we need to change listener ? if old_address != self.cfg.address: # close all listeners for l in self.LISTENERS: l.close() # init new listeners self.LISTENERS = sock.create_sockets(self.cfg, self.log) listeners_str = ",".join([str(l) for l in self.LISTENERS]) self.log.info("Listening at: %s", listeners_str) # do some actions on reload self.cfg.on_reload(self) # unlink pidfile if self.pidfile is not None: self.pidfile.unlink() # create new pidfile if self.cfg.pidfile is not None: self.pidfile = Pidfile(self.cfg.pidfile) self.pidfile.create(self.pid) # set new proc_name util._setproctitle("master [%s]" % self.proc_name) # spawn new workers for _ in range(self.cfg.workers): self.spawn_worker() # manage workers self.manage_workers()
def run(self): "Main master loop." self.start() util._setproctitle("master [%s]" % self.proc_name) "Starting new customer process - dataset checker " self.checker_alive = False self.checker_pipe = os.pipe() for p in self.checker_pipe: util.set_non_blocking(p) util.close_on_exec(p) self.manage_checker() try: self.manage_workers() while True: self.maybe_promote_master() sig = self.SIG_QUEUE.pop(0) if self.SIG_QUEUE else None if sig is None: self.sleep() self.murder_workers() self.manage_workers() self.manage_checker(update_workers_pids=True) continue if sig not in self.SIG_NAMES: self.log.info("Ignoring unknown signal: %s", sig) continue signame = self.SIG_NAMES.get(sig) handler = getattr(self, "handle_%s" % signame, None) if not handler: self.log.error("Unhandled signal: %s", signame) continue self.log.info("Handling signal: %s", signame) handler() self.wakeup() except StopIteration: self.halt() except KeyboardInterrupt: self.halt() except HaltServer as inst: self.halt(reason=inst.reason, exit_status=inst.exit_status) except SystemExit: raise except Exception: self.log.info("Unhandled exception in main loop", exc_info=True) self.stop(False) if self.pidfile is not None: self.pidfile.unlink() sys.exit(-1)
def run(self): "Main master loop." self.start() util._setproctitle("master [%s]" % self.proc_name) try: self.manage_workers() #master 的事件循环 接收不同指令 或者 检查worker的健康状态 while True: # 没有信号的时候 循环一直进行 不停的进行 None 分支的代码 # 判断信号队列有么有值就可以响应信号了,为什么还要用管道来唤醒进程呢? # 猜测应该是减少while的循环频率,但是为了即时响应信号,用select + pipe的读写来即时获取到信号的到来 #self.log.info("master main loop, sigs: %s"%self.SIG_QUEUE) sig = self.SIG_QUEUE.pop(0) if len(self.SIG_QUEUE) else None if sig is None: #self.log.info("master sleep before") #信号读写在这里发生 self.sleep() #self.log.info("master sleep after") self.murder_workers() self.manage_workers() continue if sig not in self.SIG_NAMES: self.log.info("Ignoring unknown signal: %s", sig) continue signame = self.SIG_NAMES.get(sig) handler = getattr(self, "handle_%s" % signame, None) if not handler: self.log.error("Unhandled signal: %s", signame) continue self.log.info("Handling signal: %s", signame) #如果是一个可处理的信号 handler() self.wakeup() except StopIteration: self.halt() except KeyboardInterrupt: self.halt() except HaltServer as inst: self.halt(reason=inst.reason, exit_status=inst.exit_status) except SystemExit: raise except Exception: self.log.info("Unhandled exception in main loop", exc_info=True) self.stop(False) if self.pidfile is not None: self.pidfile.unlink() sys.exit(-1)
def spawn_worker(self): self.worker_age += 1 worker_id = -1 for i in range(self.num_workers): if i not in self.WORKER_IDS.values(): worker_id = i break worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: self.WORKERS[pid] = worker self.WORKER_IDS[pid] = worker_id return pid # Process Child worker_pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker_pid) self.cfg.post_fork(self, worker) if self.num_workers > 1 and worker_id != -1: os.environ['GUNICORN_WORKER_INSTANCE'] = str(worker_id) worker.init_process() sys.exit(0) except SystemExit: raise except AppImportError as e: self.log.debug("Exception while loading the application: \n%s", traceback.format_exc()) print("%s" % e, file=sys.stderr) sys.stderr.flush() sys.exit(self.APP_LOAD_ERROR) except: self.log.exception("Exception in worker process:\n%s", traceback.format_exc()) if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker_pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: pass
def maybe_promote_master(self): if self.master_pid == 0: return if self.master_pid != os.getppid(): self.log.info("Master has been promoted.") # reset master infos self.master_name = "Master" self.master_pid = 0 self.proc_name = self.cfg.proc_name del os.environ['GUNICORN_PID'] # rename the pidfile if self.pidfile is not None: self.pidfile.rename(self.cfg.pidfile) # reset proctitle util._setproctitle("master [%s]" % self.proc_name)
def run(self): "Main master loop." self.start() util._setproctitle("master [%s]" % self.proc_name) try: self.manage_workers() while True: self.maybe_promote_master() sig = self.SIG_QUEUE.pop(0) if len(self.SIG_QUEUE) else None if sig is None: self.sleep() self.murder_workers() self.manage_workers() continue if sig not in self.SIG_NAMES: self.log.info("Ignoring unknown signal: %s", sig) continue signame = self.SIG_NAMES.get(sig) handler = getattr(self, "handle_%s" % signame, None) if not handler: self.log.error("Unhandled signal: %s", signame) continue self.log.info("Handling signal: %s", signame) handler() self.wakeup() except StopIteration: self.halt() except KeyboardInterrupt: self.halt() except HaltServer as inst: self.halt(reason=inst.reason, exit_status=inst.exit_status) except SystemExit: raise except Exception: self.log.info("Unhandled exception in main loop", exc_info=True) self.stop(False) if self.pidfile is not None: self.pidfile.unlink() sys.exit(-1)
def spawn_worker(self): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: worker.pid = pid self.WORKERS[pid] = worker return pid # Do not inherit the temporary files of other workers for sibling in self.WORKERS.values(): sibling.tmp.close() # Process Child worker.pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker.pid) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except AppImportError as e: self.log.debug("Exception while loading the application", exc_info=True) print("%s" % e, file=sys.stderr) sys.stderr.flush() sys.exit(self.APP_LOAD_ERROR) except: self.log.exception("Exception in worker process") if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker.pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: self.log.warning("Exception during worker exit:\n%s", traceback.format_exc())
def run(self): "Main master loop." self.start() util._setproctitle("master [%s]" % self.proc_name) self.manage_workers() while True: try: self.reap_workers() sig = self.SIG_QUEUE.pop(0) if len(self.SIG_QUEUE) else None if sig is None: self.sleep() self.murder_workers() self.manage_workers() continue if sig not in self.SIG_NAMES: self.log.info("Ignoring unknown signal: %s" % sig) continue signame = self.SIG_NAMES.get(sig) handler = getattr(self, "handle_%s" % signame, None) if not handler: self.log.error("Unhandled signal: %s" % signame) continue self.log.info("Handling signal: %s" % signame) handler() self.wakeup() except StopIteration: break except KeyboardInterrupt: break except Exception: self.log.info("Unhandled exception in main loop:\n%s" % traceback.format_exc()) self.stop(False) if self.pidfile: del self.pidfile sys.exit(-1) self.stop() self.log.info("Shutting down: %s" % self.master_name) if self.pidfile: del self.pidfile sys.exit(0)
def spawn_worker(self): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) worker.index = self.next_worker_index() self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: worker.pid = pid self.WORKERS[pid] = worker return pid # Process Child worker.pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.info("Booting worker with pid: %s", worker.pid) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except AppImportError as e: self.log.debug("Exception while loading the application", exc_info=True) print("%s" % e, file=sys.stderr) sys.stderr.flush() sys.exit(self.APP_LOAD_ERROR) except: self.log.exception("Exception in worker process"), if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)", worker.pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: self.log.warning("Exception during worker exit:\n%s", traceback.format_exc())
def spawn_workers(self): """\ Spawn new workers as needed. This is where a worker process leaves the main loop of the master process. """ for i in range(self.num_workers - len(self.WORKERS.keys())): self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENER, self.app, self.timeout/2.0, self.cfg) self.cfg.pre_fork(self, worker) pid = os.fork() if pid != 0: self.WORKERS[pid] = worker continue # Process Child worker_pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self.log.debug("Booting worker: %s (age: %s)" % ( worker_pid, self.worker_age)) self.cfg.post_fork(self, worker) worker.init_process() sys.exit(0) except SystemExit: raise except: self.log.exception("Exception in worker process:") if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: self.log.info("Worker exiting (pid: %s)" % worker_pid) try: worker.tmp.close() os.unlink(worker.tmpname) except: pass
def reload(self): old_address = self.cfg.address # reload conf self.app.reload() self.setup(self.app) # reopen log files self.log.reopen_files() # do we need to change listener ? if old_address != self.cfg.address: # close all listeners [l.close for l in self.LISTENERS] # init new listeners self.LISTENERS = create_sockets(self.cfg, self.log) self.log.info("Listening at: %s", ",".join(str(self.LISTENERS))) # do some actions on reload self.cfg.on_reload(self) # unlink pidfile if self.pidfile is not None: self.pidfile.unlink() # create new pidfile if self.cfg.pidfile is not None: self.pidfile = Pidfile(self.cfg.pidfile) self.pidfile.create(self.pid) # set new proc_name util._setproctitle("master [%s]" % self.proc_name) # spawn new workers for i in range(self.cfg.workers): self.spawn_worker() # manage workers self.manage_workers()
def maybe_promote_master(self): # self._log('promote master master_pid=%s' % self.master_pid) if self.master_pid == 0: return # os.getppid: Return the parent’s process id. On Unix # the id returned is the one of the init process (1) # todo: 不太理解 self._log('promote master mid=%s ppid=%s' % (self.master_pid, os.getppid())) if self.master_pid == os.getppid(): return self._log("Master has been promoted.") # reset master infos self.master_name = "Master" self.master_pid = 0 self.proc_name = self.cfg.proc_name del os.environ['GUNICORN_PID'] # rename the pidfile if self.pidfile is not None: self.pidfile.rename(self.cfg.pidfile) # reset proc title util._setproctitle("master [%s]" % self.proc_name)
def run(self): """Main master loop. """ self._log('run') self.start() util._setproctitle("master [%s]" % self.proc_name) abc = 'RUN>' try: # 看是否需要 增/减 worker self.manage_workers() while True: self.maybe_promote_master() # 周期性检测信号 sig = self.SIG_QUEUE.pop(0) if self.SIG_QUEUE else None if sig is None: # 处理 PIPE 中的消息(数据) self.sleep() # 终止unused/idle worker self.murder_workers() # 看是否需要 增/减 worker self.manage_workers() continue self._log('run got sig=%s' % sig) if sig not in self.SIG_NAMES: self.log.info("Ignoring unknown signal: %s", sig) continue signame = self.SIG_NAMES.get(sig) handler = getattr(self, "handle_%s" % signame, None) if not handler: self.log.error("Unhandled signal: %s", signame) continue self._log("Handling signal: %s" % signame) handler() # todo: 为啥此处要 wakeup? self._log('run will wakeup') self.wakeup(abc='RUN>') except StopIteration: self._log('run except1: StopIteration') self.halt(abc=abc) except KeyboardInterrupt: self._log('run except1: KeyboardInterrupt') self.halt(abc=abc) except HaltServer as e: self._log('run except2: %s' % e) self.halt(reason=e.reason, exit_status=e.exit_status, abc=abc) except SystemExit: self._log('run except3: SystemExit') raise except Exception as e: self._log('run except4: %s' % e) self.log.info("Unhandled exception in main loop", exc_info=True) # 此处直接可以调用 halt 方法的吧? self.stop(False, abc=abc) if self.pidfile is not None: self.pidfile.unlink() sys.exit(-1)
def spawn_worker(self, abc=''): self._log('%s spawn_worker' % abc) self.worker_age += 1 worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS, self.app, self.timeout / 2.0, self.cfg, self.log) self.cfg.pre_fork(self, worker) # hook before fork self._log('spawn_worker before fork') pid = os.fork() self._log('spawn_worker after fork') if pid != 0: # 父进程中 worker.pid = pid self.WORKERS[pid] = worker return pid # 在子进程中 # Do not inherit the temporary files of other workers for sibling in self.WORKERS.values(): sibling.tmp.close() # Process Child worker.pid = os.getpid() try: util._setproctitle("worker [%s]" % self.proc_name) self._log("Booting worker with pid: %s" % worker.pid) self.cfg.post_fork(self, worker) worker._log('spawn_worker before worker init_process') # 现在是在子进程中, 下面会hang住 worker.init_process() # 如: 直到Ctrl-C worker退出, 才会执行到这里 worker._log('spawn_worker after worker init_process') sys.exit(0) except SystemExit: # 比如: worker.init_process中因为修改代码而reload时, 将会运行到这儿 worker._log('spawn_worker except SystemExit') raise except AppImportError as e: self.log.debug("Exception while loading the application", exc_info=True) print("%s" % e, file=sys.stderr) sys.stderr.flush() sys.exit(self.APP_LOAD_ERROR) except Exception as e: worker._log('spawn_worker except %s' % e) self.log.exception("Exception in worker process") if not worker.booted: sys.exit(self.WORKER_BOOT_ERROR) sys.exit(-1) finally: # worker.init_process中因为修改代码而reload时, 将会运行到这儿 worker._log('spawn_worker finally worker exiting') self.log.info("Worker exiting (pid: %s)", worker.pid) try: worker.tmp.close() self.cfg.worker_exit(self, worker) except: self.log.warning("Exception during worker exit:\n%s", traceback.format_exc())