def serve(self):
logger = logging.getLogger()
processes = []
try:
while True:
self.logger.debug('Serve Forever')
tasks = []
task = []
while not self.queue.empty():
job = self.queue.get_nowait()
task.append(job)
if len(task) == self.jobsPerWorker or self.queue.empty():
tasks.append(task)
task = []
self.logger.debug('Loaded Tasks')
while len(tasks) > 0:
while len(processes) == self.numWorker:
newProcesses = []
for p in processes:
if not p.is_alive():
p.join()
else:
newProcesses.append(p)
processes = newProcesses
sleep(1)
self.logger.debug('{} Free Worker'.format(self.numWorker -
len(processes)))
for i in range(
min(self.numWorker - len(processes), len(tasks))):
task = tasks.pop(0)
p = Process(target=self.worker, args=(task, ))
p.start()
self.logger.debug('Start Fresh Worker')
processes.append(p)
sleep(5)
except (Exception, SystemExit, KeyboardInterrupt) as e:
logger.exception(e)
finally:
for p in processes:
p.terminate()
p.join()
logger.fatal('Spider does not serve anymore')
# 进程间通信是通过Queue、Pipes等实现的。
def write(q):
print('Process to write: %s' % os.getpid())
for value in ['A', 'B', 'C']:
print('Put %s to queue...' % value)
q.put(value)
time.sleep(random.random())
def read(q):
print('Process to read: %s' % os.getpid())
while True:
value = q.get(True)
print('Get %s from queue.' % value)
if __name__ == '__main__':
# 父进程创建Queue，并传给各个子进程：
q = Queue()
pw = Process(target=write, args=(q, ))
pr = Process(target=read, args=(q, ))
pw.start()
pr.start()
# 等待pw结束:
pw.join()
# pr进程里是死循环，无法等待其结束，只能强行终止:
pr.terminate()
