def __init__(self, slave): """spawn worker threads""" self.slave = slave self.lock = Lock() self.pb = PostBox() self.bytesSynced=0 for i in range(int(gconf.sync_jobs)): t = Thread(target=self.syncjob) t.start()
def service_loop(self): """fire up worker threads, get messages and dispatch among them""" for i in range(self.wnum): t = Thread(target=self.worker) t.start() try: while True: self.q.put(recv(self.inf)) except EOFError: logging.info("terminating on reaching EOF.")
def init_keep_alive(cls): """start the keep-alive thread """ timo = int(gconf.timeout or 0) if timo > 0: def keep_alive(): while True: vi, gap = cls.keepalive_payload_hook(timo, timo * 0.5) cls.slave.server.keep_alive(vi) time.sleep(gap) t = Thread(target=keep_alive) t.start()
def crawl_loop(self): """start the keep-alive thread and iterate .crawl""" timo = int(gconf.timeout or 0) if timo > 0: def keep_alive(): while True: vi, gap = self.keepalive_payload_hook(timo, timo * 0.5) self.slave.server.keep_alive(vi) time.sleep(gap) t = Thread(target=keep_alive) t.start() self.lastreport['time'] = time.time() while not self.terminate: self.crawl()
def multiplex(self, wspx, suuid, slave_vol, slave_host, master, slavenodes): argv = [os.path.basename(sys.executable), sys.argv[0]] cpids = set() agents = set() ta = [] for wx in wspx: def wmon(w): cpid, _ = self.monitor(w, argv, cpids, agents, slave_vol, slave_host, master, suuid, slavenodes) time.sleep(1) self.lock.acquire() for cpid in cpids: errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH]) for apid in agents: errno_wrap(os.kill, [apid, signal.SIGKILL], [ESRCH]) self.lock.release() finalize(exval=1) t = Thread(target=wmon, args=[wx]) t.start() ta.append(t) # monitor status was being updated in each monitor thread. It # should not be done as it can cause deadlock for a worker start. # set_monitor_status uses flock to synchronize multple instances # updating the file. Since each monitor thread forks worker and # agent, these processes can hold the reference to fd of status # file causing deadlock to workers which starts later as flock # will not be release until all references to same fd is closed. # It will also cause fd leaks. self.lock.acquire() set_monitor_status(gconf.get("state-file"), self.ST_STARTED) self.lock.release() for t in ta: t.join()
def __init__(self, i, o): self.inf, self.out = ioparse(i, o) self.jtab = {} t = Thread(target=self.listen) t.start()