class InputThread(threading.Thread):
def __init__(self, beanstalk_conf, log=None, process_pool=None):
threading.Thread.__init__(self)
self.daemon = True
self.running = True
self.beanstalk = PyBeanstalk(beanstalk_conf['host'], beanstalk_conf['port'])
self.out_beanstalk = PyBeanstalk(beanstalk_conf['host'], beanstalk_conf['port'])
self.input_tube = beanstalk_conf['input_tube']
self.output_tube = beanstalk_conf['output_tube']
self.log = log
if not self.log:
self.log = LogHandler("i_input_thread")
self.process_pool = process_pool
self.wlock = threading.Lock()
def stop(self):
self.log.warning("stop input_thread")
self.running = False
proccesor = None
try:
while True:
if self.process_pool.get_task_num() == 0:
if self.process_pool.thread_local_constructors.has_key('processor'):
processor = self.process_pool.thread_local_constructors['processor'][1][1]
self.log.warning("prepare call scheduler_processor to stop scheduler")
processor.save_status()
break
else:
self.log.info("wait tasks be consumed over, wait 5s")
time.sleep(5)
self.beanstalk.__del__() # 关闭连接不再接受数据
except Exception, e:
self.log.error("stop input_thread fail:%s" % e.message)
class InputThread(threading.Thread):
def __init__(self, beanstalk_conf, log=None, process_pool=None):
threading.Thread.__init__(self)
self.daemon = True
self.running = True
assert beanstalk_conf is not None
assert log is not None
assert process_pool is not None
self.beanstalk = PyBeanstalk(beanstalk_conf['host'], beanstalk_conf['port'])
self.out_beanstalk = PyBeanstalk(beanstalk_conf['host'], beanstalk_conf['port'])
self.input_tube = beanstalk_conf['input_tube']
self.output_tube = beanstalk_conf['output_tube']
self.log = log
if not self.log:
self.log = LogHandler("i_input_thread")
self.process_pool = process_pool
self.t_lock = threading.Lock()
def stop(self):
self.log.warning("stop input_thread")
self.running = False
try:
while True:
if self.process_pool.get_task_num() <= 0:
# if 'processor' in self.process_pool.thread_local_constructors:
# processor = self.process_pool.thread_local_constructors['processor'][1][1]
# self.log.warning("prepare call scheduler_processor to stop scheduler")
# processor.save_status()
break
else:
self.log.info("wait tasks be consumed over, wait 5s")
time.sleep(5)
self.beanstalk.__del__() # 关闭连接不再接受数据
except Exception as e:
self.log.error("stop input_thread fail")
self.log.exception(e)
def run(self):
job_num = 0
while self.running and self.input_tube:
try:
job = self.beanstalk.reserve(self.input_tube, 30)
if job is not None:
job_num += 1
body = job.body
job.delete()
self.process_pool.queue_task(self.__on_task_start, (body,), self.__on_task_finished)
task_num = self.process_pool.get_task_num()
if task_num >= 50:
self.log.info("place_processor\ttasks:%d" % task_num)
time.sleep(2)
else:
self.log.info("not msg from:%s" % self.input_tube)
except SocketError as e:
time.sleep(30)
self.log.error('beanstalk\tconnect\tfail\tstart\treconnect')
self.log.exception(e)
try:
self.beanstalk.reconnect()
self.out_beanstalk.reconnect()
self.log.error('beanstalk\treconnect\tsuccess')
except Exception as e:
self.log.error('beanstalk\treconnect\tfail')
self.log.exception(e)
except Exception as e:
self.log.error("not msg from:%s\tresult:" % self.input_tube)
self.log.exception(e)
@staticmethod
def __on_task_start(task, **thread_locals):
result = None
if 'profiler' in thread_locals:
thread_locals['profiler'].begin()
if 'processor' in thread_locals:
result = thread_locals['processor'].do_task(task)
return result
def __on_task_finished(self, (result), **thread_locals):
self.t_lock.acquire()
proccesor = None
if 'processor' in thread_locals:
proccesor = thread_locals['processor']
if 'profiler' in thread_locals:
thread_locals['profiler'].end()
if result and isinstance(result, basestring):
self.__output_msg(result, proccesor)
elif isinstance(result, list):
for message in result:
self.__output_msg(message, proccesor)
self.t_lock.release()
# signal.signal(signal.SIGINT, signal_process)
# signal.signal(signal.SIGQUIT, signal_process)
# signal.signal(signal.SIGUSR1, lambda a, b: profiling_signal_handler("downloader", a, b))
file_path = 'downloader_smart_test.toml'
opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help'])
for name, value in opt:
if name == "-f":
file_path = value
elif name in ("-h", "--help"):
usaget()
sys.exit()
else:
assert False, "unhandled option"
if file_path is None or file_path.strip() == '':
raise Exception('配置文件路径错误..')
with open(file_path, 'rb') as fp:
config = pytoml.load(fp)
logger_name = config["local_server"].get('name') + str(
config["local_server"].get('port'))
logger = LogHandler(logger_name)
config['log'] = logger
logger.info(logger_name)
logger.info('开始启动服务....')
main(config)
except getopt.GetoptError:
raise Exception(
'参数读取错误...argv = {argv}'.format(argv=" ".join(sys.argv)))