def process_mq_cleaning(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
# This is only used by thrift, each process has its own EntityExtractor instance
singletons[EntityExtractor.__name__] = EntityExtractor(conf)
singletons[EEWorkerPool.__name__] = EEWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
worker_pool = singletons[EEWorkerPool.__name__]
worker_pool.start_all()
heartbeat_thread = HeartbeatThread("entity_extractor", conf)
heartbeat_thread.start()
try:
handler = EntityExtractorHandler(conf, worker_pool)
processor = EntityExtractorService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport, tfactory,
pfactory)
server.setNumThreads(1)
server.serve()
except Exception, e:
log.error(str(e))
log.error(traceback.format_exc())
def main(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
try:
pid_file = make_pid_file(sys.path[0])
try:
converter = ConvertProccessor(conf)
selecter = SelectProcessor(conf)
worker_pool = CrawlerMergeWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
singletons[CrawlerMergeWorkerPool.__name__] = worker_pool
worker_pool.start_all()
running = True
heartbeat_thread = HeartbeatThread("crawl_merge", conf)
heartbeat_thread.start()
handler = CrawlerMergeHandler(conf, converter, selecter)
processor = CrawlerMergeService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server'].get('port'))
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(conf['server']['server_thread_num'])
server.serve()
except Exception, e:
conf.get('log').error(traceback.format_exc())
except:
pass
finally:
remove_pid_file(sys.path[0])
if running:
worker_pool.stop_all()
def main(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
try:
pid_file = make_pid_file(sys.path[0])
try:
datasaver = DataSaver(conf)
singletons[DataSaver.__name__] = datasaver
worker_pool = DatasaverWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
singletons[DatasaverWorkerPool.__name__] = worker_pool
worker_pool.start_all()
running = True
heartbeat_thread = HeartbeatThread("datasaver", conf)
heartbeat_thread.start()
handler = DataSaverHandler()
processor = DataSaverService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(conf['server']['server_thread_num'])
log.debug("start main thrift")
server.serve()
except Exception, e:
log.debug(traceback.format_exc())
if running:
worker_pool.stop_all()
def main(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
handler = None
try:
make_pid_file(sys.path[0])
try:
handler = DownloadHandler(conf)
#多进程启动
worker_pool = DownloaderWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
singletons[DownloaderWorkerPool.__name__] = worker_pool
worker_pool.start_all()
running = True
#心跳开始
heartbeat_thread = HeartbeatThread("downloader", conf)
heartbeat_thread.start()
#thrift接口
processor = DownloadService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server'].get('port'))
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(conf['server'].get('server_thread_num'))
server.serve()
except Exception as e:
import traceback
conf.get('log').error(e.message)
if running:
worker_pool.stop_all()
if handler:
# 必须在server阻塞完成之后调用hander的stop方法去回收phantomjs
handler.stop()
except:
pass
finally:
remove_pid_file(sys.path[0])
os.kill(os.getpid(), signal.SIGKILL)
class SchedulerServer(object):
def __init__(self, conf, scheduler):
self.conf = conf
self.log = conf['log']
self.scheduler = scheduler
self.process_thread_num = conf['server']['process_thread_num']
thread_locals = {'processor': (SchedulerProccessor, (conf['log'], self.scheduler)),
'profiler': (profiler_creator, ())}
self.process_pool = ThreadPool(self.process_thread_num, thread_locals)
self.input_thread = InputThread(conf['beanstalk_conf'], conf['log'], self.process_pool)
self.heartbeat_thread = HeartbeatThread('scheduler', self.conf)
def start(self):
self.input_thread.start()
self.heartbeat_thread.start()
self.log.info("start_server\tSchedulerServer!")
def stop(self, message):
self.input_thread.stop()
self.heartbeat_thread.stop()
self.log.info("stop_server\tstatus:%s!" % message)
exit(1)
class DownloaderServer(object):
def __init__(self, conf):
self.log = conf.get('log')
thread_locals = {
'processor': (DownloaderProccessor, (self.log, conf)),
'profiler': (profiler_creator, ())
}
self.process_pool = ThreadPool(
conf['local_server'].get('process_thread_num'), thread_locals)
self.input_thread = InputThread(conf.get('beanstalk_conf'), self.log,
self.process_pool)
self.heartbeat_thread = HeartbeatThread('download', conf)
self.log.info('初始化线程信息完成..')
def start(self):
self.input_thread.start()
self.heartbeat_thread.start()
self.log.info("start DownloadServer!")
def stop(self, message):
self.log.info("stop DownloadServer %s!" % message)
self.input_thread.stop()
def main(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
pid_file = make_pid_file(sys.path[0])
try:
# This is only used by thrift, each process has its own EntityExtractor instance
singletons[SingleSourceMerger.__name__] = SingleSourceMerger(conf)
singletons[SMWorkerPool.__name__] = SMWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
singletons[RecordLockManager.__name__] = RecordLockManager(
conf,
prefix='single-src',
backlog_path=os.path.join(crawler_basic_path, 'ss_backlog'))
worker_pool = singletons[SMWorkerPool.__name__]
worker_pool.start_all()
conf['log'] = log
heartbeat_thread = HeartbeatThread("single_src_merge", conf)
heartbeat_thread.start()
try:
handler = SMThriftHandler(conf, worker_pool)
processor = SingleSourceMergeService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(1)
server.serve()
except Exception, e:
log.error(str(e))
log.error(traceback.format_exc())
if running:
worker_pool.stop_all()