def main(): """ 主函数 """ conf_file = '' # 用户指定一个配置文件 if len(sys.argv) > 1: conf_file = sys.argv[1] init_config(conf_file) server = get_configs('server') alias = safe_get_host('server', 'alias') active_config = server.get('active_config', 'false') # 动态注册task for module in server['modules'].split(): try: importlib.import_module(module) except ImportError: raise Exception('Not imported the %s module' % module) # 连结服务器 ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db'])) # 开启一个命令线程 command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif get_configs('queues'): # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} queue_config = ztq_core.get_queue_config() for queue_name, sleeps in get_configs('queues').items(): queue_config[queue_name] = {'title': queue_name} # for ztq_app job_threads[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} init_job_threads(job_threads) loggers = get_configs('log') initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()
def run(self): """ 阻塞方式找到任务,并自动调用""" # 如果上次有任务在运行还没结束,重新执行 jobs = ztq_core.get_job_state(safe_get_host('server', 'alias')) if self.name in jobs: self.start_job(jobs[self.name]) # 队列批处理模式 # batch_size: 批处理的阀值,达到这个阀值,就执行一次batch_func # batch_func: # 1, 执行一批batch_size 大小的任务后,后续自动执行这个方法方法 # 2, 执行一批小于batch_size 大小的任务后,再得不到任务,后续自动执行这个方法 batch_config = CONFIG.get("batch_queue", {}).get(self.queue_name, {}) batch_size = batch_config.get('batch_size', None) or -1 batch_func = batch_config.get('batch_func', None) or (lambda *args, **kw: -1) run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT # 循环执行任务 while not self._stop: try: task = ztq_core.pop_task( self.queue_name, timeout=queue_tiemout, from_right=self.from_right ) except ztq_core.ConnectionError: print 'ERROR: Not connected the server\n' task = None time.sleep(3) if task is None: # 没有后续任务了。执行batch_func if run_job_index > 0: run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT batch_func() continue self.start_job(task) if batch_size > 0: if run_job_index >= batch_size - 1: # 完成了一批任务。执行batch_func run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT batch_func() else: run_job_index += 1 queue_tiemout = -1 if self.sleep_time: time.sleep(self.sleep_time)
def run(self): """ 阻塞方式找到任务,并自动调用""" # 如果上次有任务在运行还没结束,重新执行 jobs = ztq_core.get_job_state(safe_get_host('server', 'alias')) if self.name in jobs: self.start_job(jobs[self.name]) # 队列批处理模式 # batch_size: 批处理的阀值,达到这个阀值,就执行一次batch_func # batch_func: # 1, 执行一批batch_size 大小的任务后,后续自动执行这个方法方法 # 2, 执行一批小于batch_size 大小的任务后,再得不到任务,后续自动执行这个方法 batch_config = CONFIG.get("batch_queue", {}).get(self.queue_name, {}) batch_size = batch_config.get('batch_size', None) or -1 batch_func = batch_config.get('batch_func', None) or (lambda *args, **kw: -1) run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT # 循环执行任务 while not self._stop: try: task = ztq_core.pop_task(self.queue_name, timeout=queue_tiemout, from_right=self.from_right) except ztq_core.ConnectionError: print 'ERROR: Not connected the server\n' task = None time.sleep(3) if task is None: # 没有后续任务了。执行batch_func if run_job_index > 0: run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT batch_func() continue self.start_job(task) if batch_size > 0: if run_job_index >= batch_size - 1: # 完成了一批任务。执行batch_func run_job_index = 0 queue_tiemout = QUEUE_TIMEOUT batch_func() else: run_job_index += 1 queue_tiemout = -1 if self.sleep_time: time.sleep(self.sleep_time)
def start_job(self, task): self.start_job_time = int(time.time()) task['runtime'].update({ 'worker': safe_get_host('server', 'alias'), 'thread': self.getName(), 'start': self.start_job_time, }) # 记录当前在做什么 task['process'] = {'ident': self.ident} thread_context.job = task try: self.run_task = ztq_core.task_registry[task['func']] # started report report_job(comment='start the job') try: self.run_task(*task['args'], **task['kw']) except TypeError: # keyword must string is a bug in python if sys.version[:5] < '2.6.5': raise Exception, ("We not supported %s version of python," "Please update it to 2.6.5 or later." % sys.version[:5]) else: raise if task.get('callback', None): callback_args = task.get('callback_args', ()) callback_kw = task.get('callback_kw', {}) ztq_core.push_task(task['callback'], *callback_args, **callback_kw) except Exception, e: reason = traceback.format_exception(*sys.exc_info()) # 将错误信息记录到服务器 return_code = str(e.args[0]) if len(e.args) > 1 else 300 task['runtime']['return'] = return_code task['runtime']['reason'] = reason[-11:] task['runtime']['end'] = int(time.time()) ztq_core.push_runtime_error(self.queue_name, task) # 错误回调 if task.get('fcallback', None): callback_args = task.get('fcallback_args', ()) callback_kw = task.get('fcallback_kw', {}) callback_kw['return_code'] = return_code callback_kw['return_msg'] = reason[-1] ztq_core.push_task(task['fcallback'], *callback_args, **callback_kw) # 在终端打印错误信息 #reason.insert(0, str(datetime.datetime.today()) + '\n') logger.error(''.join(reason))
def init(self): """ 开机初始化工作 """ reboot = False worker_state = ztq_core.get_worker_state() if self.worker_name in worker_state: # 重启,读取服务器配置信息 reboot = True # 记录系统日志 system_log = ztq_core.get_system_log_queue() system_log.push(dict( host=safe_get_host('server', 'alias'), alias=self.worker_name, type=reboot and 'reboot' or 'power', timestamp=self.login_time,)) # 报告机器状态 worker_state[self.worker_name] = report(self.login_time)
def start_job(self, task): self.start_job_time = int(time.time()) task['runtime'].update({'worker': safe_get_host('server', 'alias'), 'thread': self.getName(), 'start': self.start_job_time, }) # 记录当前在做什么 task['process'] = {'ident':self.ident} thread_context.job = task try: self.run_task = ztq_core.task_registry[task['func']] # started report report_job(comment='start the job') try: self.run_task(*task['args'], **task['kw']) except TypeError: # keyword must string is a bug in python if sys.version[:5] < '2.6.5': raise Exception,("We not supported %s version of python," "Please update it to 2.6.5 or later."%sys.version[:5]) else: raise if task.get('callback', None): callback_args = task.get('callback_args', ()) callback_kw = task.get('callback_kw', {}) ztq_core.push_task(task['callback'], *callback_args, **callback_kw) except Exception, e: reason = traceback.format_exception(*sys.exc_info()) # 将错误信息记录到服务器 return_code = str(e.args[0]) if len(e.args) > 1 else 300 task['runtime']['return'] = return_code task['runtime']['reason'] = reason[-11:] task['runtime']['end'] = int( time.time() ) ztq_core.push_runtime_error(self.queue_name, task) # 错误回调 if task.get('fcallback', None): callback_args = task.get('fcallback_args', ()) callback_kw = task.get('fcallback_kw', {}) callback_kw['return_code'] = return_code callback_kw['return_msg'] = reason[-1] ztq_core.push_task(task['fcallback'], *callback_args, **callback_kw) # 在终端打印错误信息 #reason.insert(0, str(datetime.datetime.today()) + '\n') logger.error(''.join(reason))
def init(self): """ 开机初始化工作 """ reboot = False worker_state = ztq_core.get_worker_state() if self.worker_name in worker_state: # 重启,读取服务器配置信息 reboot = True # 记录系统日志 system_log = ztq_core.get_system_log_queue() system_log.push( dict( host=safe_get_host('server', 'alias'), alias=self.worker_name, type=reboot and 'reboot' or 'power', timestamp=self.login_time, )) # 报告机器状态 worker_state[self.worker_name] = report(self.login_time)
def report(start_time): """ 转换器向服务器报告状态 """ cpu_style = get_cpu_style() cpu_percent = get_cpu_usage() mem_percent, mem_total = get_mem_usage() ip = safe_get_host('server', 'alias') traceback_dict = {} for thread_id, frame in sys._current_frames().items(): traceback_dict[thread_id] = traceback.format_stack(frame) # 向服务器报告 return dict( ip=ip, cpu_style=cpu_style, cpu_percent=cpu_percent, mem_total=mem_total, mem_percent=mem_percent, started=start_time, timestamp=int(time.time()), traceback=traceback_dict, )
def init_job_threads(config_dict): # 启动工作线程 set_job_threads(config_dict) # 保存信息 config = {} for queue_name, values in config_dict.items(): sleeps = [str(x['interval']) for x in values] config[queue_name] = ','.join(sleeps) set_config(config, 'queues') # 将一些信息补全,让监视界面认为这个worker已经启动 alias = safe_get_host('server', 'alias') # 初始化 dispatcher_config = ztq_core.get_dispatcher_config() if not dispatcher_config: dispatcher_config = {'queue_weight': {}, 'worker_weight': {}} # set dispatcher config worker_weight = dispatcher_config['worker_weight'] if not worker_weight.get(alias, 0): worker_weight[alias] = 5 ztq_core.set_dispatcher_config(dispatcher_config) # set worker config worker_config = ztq_core.get_worker_config() worker_config[alias] = config_dict
def init_job_threads(config_dict): # 启动工作线程 set_job_threads(config_dict) # 保存信息 config = {} for queue_name, values in config_dict.items(): sleeps = [str(x['interval']) for x in values] config[queue_name] = ','.join(sleeps) set_config(config, 'queues') # 将一些信息补全,让监视界面认为这个worker已经启动 alias = safe_get_host('server', 'alias') # 初始化 dispatcher_config = ztq_core.get_dispatcher_config() if not dispatcher_config: dispatcher_config = {'queue_weight':{},'worker_weight':{}} # set dispatcher config worker_weight = dispatcher_config['worker_weight'] if not worker_weight.get(alias, 0): worker_weight[alias] = 5 ztq_core.set_dispatcher_config(dispatcher_config) # set worker config worker_config = ztq_core.get_worker_config() worker_config[alias] = config_dict
def __init__(self, worker_name=''): super(CommandThread, self).__init__() self.login_time = int(time.time()) self.worker_name = worker_name or safe_get_host('server', 'alias')
def main(): """ 主函数 """ conf_file = '' # 用户指定一个配置文件 if len(sys.argv) > 1: conf_file = sys.argv[1] init_config(conf_file) server = get_configs('server') alias = safe_get_host('server', 'alias') active_config = server.get('active_config', 'false') # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: raise Exception('Not imported the %s module' % module) # 连结服务器 ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db'])) # 开启一个命令线程 command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n' % os.getpid()) worker_state = ztq_core.get_worker_state() if active_config.lower( ) == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif get_configs('queues'): # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in get_configs('queues').items(): job_threads[queue_name] = [{ 'interval': int(sleep) } for sleep in sleeps.split(',')] if not queue_config.get(queue_name, []): queue_config[queue_name] = { 'name': queue_name, 'title': queue_name, 'widget': 5 } init_job_threads(job_threads) loggers = get_configs('log') initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()