def __init__(self, slave):
"""spawn worker threads"""
self.slave = slave
self.lock = Lock()
self.pb = PostBox()
self.bytesSynced=0
for i in range(int(gconf.sync_jobs)):
t = Thread(target=self.syncjob)
t.start()
def service_loop(self):
"""fire up worker threads, get messages and dispatch among them"""
for i in range(self.wnum):
t = Thread(target=self.worker)
t.start()
try:
while True:
self.q.put(recv(self.inf))
except EOFError:
logging.info("terminating on reaching EOF.")
def init_keep_alive(cls):
"""start the keep-alive thread """
timo = int(gconf.timeout or 0)
if timo > 0:
def keep_alive():
while True:
vi, gap = cls.keepalive_payload_hook(timo, timo * 0.5)
cls.slave.server.keep_alive(vi)
time.sleep(gap)
t = Thread(target=keep_alive)
t.start()
def crawl_loop(self):
"""start the keep-alive thread and iterate .crawl"""
timo = int(gconf.timeout or 0)
if timo > 0:
def keep_alive():
while True:
vi, gap = self.keepalive_payload_hook(timo, timo * 0.5)
self.slave.server.keep_alive(vi)
time.sleep(gap)
t = Thread(target=keep_alive)
t.start()
self.lastreport['time'] = time.time()
while not self.terminate:
self.crawl()
def multiplex(self, wspx, suuid, slave_vol, slave_host, master,
slavenodes):
argv = [os.path.basename(sys.executable), sys.argv[0]]
cpids = set()
agents = set()
ta = []
for wx in wspx:
def wmon(w):
cpid, _ = self.monitor(w, argv, cpids, agents, slave_vol,
slave_host, master, suuid, slavenodes)
time.sleep(1)
self.lock.acquire()
for cpid in cpids:
errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
for apid in agents:
errno_wrap(os.kill, [apid, signal.SIGKILL], [ESRCH])
self.lock.release()
finalize(exval=1)
t = Thread(target=wmon, args=[wx])
t.start()
ta.append(t)
# monitor status was being updated in each monitor thread. It
# should not be done as it can cause deadlock for a worker start.
# set_monitor_status uses flock to synchronize multple instances
# updating the file. Since each monitor thread forks worker and
# agent, these processes can hold the reference to fd of status
# file causing deadlock to workers which starts later as flock
# will not be release until all references to same fd is closed.
# It will also cause fd leaks.
self.lock.acquire()
set_monitor_status(gconf.get("state-file"), self.ST_STARTED)
self.lock.release()
for t in ta:
t.join()
def __init__(self, i, o):
self.inf, self.out = ioparse(i, o)
self.jtab = {}
t = Thread(target=self.listen)
t.start()
