def init_job_threads(config_dict): # 启动工作线程 set_job_threads(config_dict) # 保存信息 config = {} for queue_name, values in config_dict.items(): sleeps = [str(x['interval']) for x in values] config[queue_name] = ','.join(sleeps) CONFIG['queues'].update(config) # 将一些信息补全,让监视界面认为这个worker已经启动 alias = CONFIG['server']['alias'] # 初始化 dispatcher_config = ztq_core.get_dispatcher_config() if not dispatcher_config: dispatcher_config = {'queue_weight':{},'worker_weight':{}} # set dispatcher config worker_weight = dispatcher_config['worker_weight'] if not worker_weight.get(alias, 0): worker_weight[alias] = 5 ztq_core.set_dispatcher_config(dispatcher_config) # set worker config worker_config = ztq_core.get_worker_config() worker_config[alias] = config_dict
def run(self): self.init() # 监听指令 commands = ztq_core.get_command_queue(self.worker_name) while True: try: command = commands.pop() if command['command'] == 'report': worker_state = ztq_core.get_worker_state() worker_state[self.worker_name] = report(self.login_time) elif command['command'] == 'updatedriver': # TODO #async_drive_config() pass elif command['command'] == 'updateworker' and \ CONFIG['server'].get('active_config', 'false').lower() == 'true': queue = ztq_core.get_worker_config() set_job_threads(queue[self.worker_name]) elif command['command'] == 'kill': kill_transform(pid=command['pid'], timestamp=command['timestamp']) elif command['command'] == 'cancel': cancel_transform(pid=command['pid'], timestamp=command['timestamp']) except ztq_core.ConnectionError, e: logger.error('ERROR: redis command connection error: %s' % str(e)) time.sleep(3) except ztq_core.ResponseError, e: logger.error('ERROR: redis command response error: %s' % str(e)) time.sleep(3)
def run(self): self.init() # 监听指令 commands = ztq_core.get_command_queue(self.worker_name) while True: try: command = commands.pop() if command['command'] == 'report': worker_state = ztq_core.get_worker_state() worker_state[self.worker_name] = report(self.login_time) elif command['command'] == 'updatedriver': # TODO #async_drive_config() pass elif command['command'] == 'updateworker': queue = ztq_core.get_worker_config() set_job_threads(queue[self.worker_name]) elif command['command'] == 'kill': kill_transform(pid=command['pid'], timestamp=command['timestamp']) elif command['command'] == 'cancel': cancel_transform(pid=command['pid'], timestamp=command['timestamp']) except KeyboardInterrupt: import os # 实际上调用的是command_execute.clear_thread os.sys.exitfunc() os._exit(0)
def main(): """ 主函数 """ conf_file = '' # 用户指定一个配置文件 if len(sys.argv) > 1: conf_file = sys.argv[1] init_config(conf_file) server = get_configs('server') alias = safe_get_host('server', 'alias') active_config = server.get('active_config', 'false') # 动态注册task for module in server['modules'].split(): try: importlib.import_module(module) except ImportError: raise Exception('Not imported the %s module' % module) # 连结服务器 ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db'])) # 开启一个命令线程 command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif get_configs('queues'): # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} queue_config = ztq_core.get_queue_config() for queue_name, sleeps in get_configs('queues').items(): queue_config[queue_name] = {'title': queue_name} # for ztq_app job_threads[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} init_job_threads(job_threads) loggers = get_configs('log') initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()
def dispatch_single_queue(queue_name, from_right=True): """调整特定队列, 让worker从队列头或者从队列尾获取数据 """ worker_config = ztq_core.get_worker_config() for worker_name, queue_config in worker_config.items(): if queue_config.get(queue_name, None): queue_config[queue_name] = update_queue_config(queue_config[queue_name], from_right) worker_config[worker_name]= queue_config send_sync_command(worker_name)
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config["server"] # 动态注册task for module in server["modules"].split(): try: __import__(module) except ImportError: modules = module.split(".") __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server["host"] redis_port = int(server["port"]) redis_db = int(server["db"]) ztq_core.setup_redis("default", host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get("alias", "") if not alias: alias = get_ip() server["alias"] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write("Starting server in PID %s\n" % os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get("active_config", "false") if active_config.lower() == "true" and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif config["queues"]: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in config["queues"].items(): job_threads[queue_name] = [{"interval": int(sleep)} for sleep in sleeps.split(",")] if not queue_config.get(queue_name, []): queue_config[queue_name] = {"name": queue_name, "title": queue_name, "widget": 5} init_job_threads(job_threads) loggers = config["log"] initlog(loggers.get("key", "ztq_worker"), loggers.get("handler_file"), loggers.get("level", "ERROR")) # 不是以线程启动 command_thread.run()
def dispatch_single_queue(queue_name, from_right=True): """调整特定队列, 让worker从队列头或者从队列尾获取数据 """ worker_config = ztq_core.get_worker_config() for worker_name, queue_config in worker_config.items(): if queue_config.get(queue_name, None): queue_config[queue_name] = update_queue_config( queue_config[queue_name], from_right) worker_config[worker_name] = queue_config send_sync_command(worker_name)
def config_queue(request): """管理队列线程数量 传入参数:http://server/taskqueues/q01/config?action=queue_down """ queue_id = request.matchdict['id'] url_action = request.params.get('action','') # 对所有的worker的队列调整数量 for worker_name in ztq_core.get_worker_config(): utils.update_queue_threads(work_name, queue_id, action=url_action) return HTTPFound(location = '/taskqueues')
def init_job_threads(config_dict, force=True): # 如果首次注册,根据配置启动工作线程,否则根据之前的配置启动。 set_job_threads(config_dict) # 将一些信息补全,让监视界面认为这个worker已经启动 alias = CONFIG['server']['alias'] # set worker config worker_config = ztq_core.get_worker_config() if alias not in worker_config or force: worker_config[alias] = config_dict
def config_queue(request): """管理队列线程数量 传入参数:http://server/taskqueues/q01/config?action=queue_down """ queue_id = request.matchdict['id'] url_action = request.params.get('action', '') # 对所有的worker的队列调整数量 for worker_name in ztq_core.get_worker_config().keys(): utils.update_queue_threads(worker_name, queue_id, action=url_action) return HTTPFound(location='/taskqueues')
def get_taskqueues_list(): # 队列情况列表 dispatcher_config = ztq_core.get_dispatcher_config() queue_weight = dispatcher_config['queue_weight'] queues_list = ztq_core.get_queue_config() # 排序 sort_queue_name = {} for queue_name, queue_config in queues_list.items(): sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name)) for queue_name in sorted(sort_queue_name, key=lambda x: sort_queue_name[x], reverse=True): task_queue = {} task_queue['name'] = queue_name #task_queue['tags'] = queue_config.get('tags',()) queue = ztq_core.get_task_queue(queue_name) # 任务数/错误数 task_queue['length'] = len(queue) task_queue['error_length'] = sort_queue_name[queue_name] #任务首个时间 task_queue['error_end'] = task_queue['first'] = '' first_job = queue[0] first_job = ztq_core.get_task_hash(queue_name).get(first_job) if first_job: task_queue['first'] = datetime.datetime.fromtimestamp( first_job['runtime'].get('create', 0)) #错误最末一个的时间 error_first_job = ztq_core.get_error_queue(queue_name)[0] error_first_job = ztq_core.get_error_hash(queue_name).get( error_first_job) if error_first_job: task_queue['error_end'] = datetime.datetime.fromtimestamp( error_first_job['runtime'].get('create', 0)) task_queue['weight'] = queue_weight.get(queue_name, 0) # 获取worker工作线程配置 workers_config = ztq_core.get_worker_config() task_queue['from_right'] = True for worker_name, worker_config in workers_config.items(): task_queue['workers'] = [] for config in worker_config.get(queue_name, []): task_queue['workers'].append( [worker_name + ':', config['interval']]) if 'from_right' in config: task_queue['from_right'] = config['from_right'] task_queue['buffer_length'] = len( ztq_core.get_buffer_queue(queue_name)) yield task_queue
def get_taskqueues_list(): # 队列情况列表 dispatcher_config = ztq_core.get_dispatcher_config() queue_weight = dispatcher_config['queue_weight'] queues_list = ztq_core.get_queue_config() # 排序 sort_queue_name = {} for queue_name, queue_config in queues_list.items(): sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name)) for queue_name in sorted(sort_queue_name, key=lambda x: sort_queue_name[x], reverse=True): task_queue = {} task_queue['name'] = queue_name #task_queue['tags'] = queue_config.get('tags',()) queue = ztq_core.get_task_queue(queue_name) # 任务数/错误数 task_queue['length'] = len(queue) task_queue['error_length'] = sort_queue_name[queue_name] #任务首个时间 task_queue['error_end'] = task_queue['first'] = '' first_job = queue[0] first_job= ztq_core.get_task_hash(queue_name).get(first_job) if first_job: task_queue['first'] = datetime.datetime.fromtimestamp(first_job['runtime'].get('create', 0)) #错误最末一个的时间 error_first_job = ztq_core.get_error_queue(queue_name)[0] error_first_job = ztq_core.get_error_hash(queue_name).get(error_first_job) if error_first_job: task_queue['error_end'] = datetime.datetime.fromtimestamp(error_first_job['runtime'].get('create', 0)) task_queue['weight'] = queue_weight.get(queue_name, 0) # 获取worker工作线程配置 workers_config = ztq_core.get_worker_config() task_queue['from_right'] = True for worker_name,worker_config in workers_config.items(): task_queue['workers'] = [] for config in worker_config.get(queue_name,[]): task_queue['workers'].append([worker_name+':', config['interval']]) if 'from_right' in config: task_queue['from_right'] = config['from_right'] task_queue['buffer_length'] = len(ztq_core.get_buffer_queue(queue_name)) yield task_queue
def update_queue_threads(worker_name, queue_name, action): """调整特定队列线程数量,可以增加或者减少 """ worker_config = ztq_core.get_worker_config() queue_config = worker_config[worker_name] if queue_config.get(queue_name, None) is not None: _config = queue_config[queue_name] # 生成新的配置信息 if action == 'queue_down': _config.pop() elif action == 'queue_up' : _config.append({u'interval': 0}) queue_config[queue_name] = _config worker_config[worker_name]= queue_config send_sync_command(worker_name)
def update_queue_threads(worker_name, queue_name, action): """调整特定队列线程数量,可以增加或者减少 """ worker_config = ztq_core.get_worker_config() queue_config = worker_config[worker_name] if queue_config.get(queue_name, None) is not None: _config = queue_config[queue_name] # 生成新的配置信息 if action == 'queue_down': _config.pop() elif action == 'queue_up': _config.append({u'interval': 0}) queue_config[queue_name] = _config worker_config[worker_name] = queue_config send_sync_command(worker_name)
def gen_worker_config(worker_weight, queue_weight, worker_list): """生成新的转换器工作线程配置 输出参数:{'q01':[{ 'interval':5}], # 间隔时间 'q02':[{'interval':5},{'interval':3}], } """ #获取转换器的工作线程配置 worker_config = ztq_core.get_worker_config() for worker in worker_list: new_config = {} # 如果worker的权重为0则配置为空 if worker_weight.get(worker, '') == 0: worker_config[worker]= new_config send_sync_command(worker) continue #根据原子队列权重和转换器权重更新工作线程配置 for queue,queue_weight in queue_weight.iteritems(): new_config[queue] = [] #根据原子队列权重进程更新工作线程配置 if queue_weight < 1 : new_config[queue].append(gen_new_config(9)) elif queue_weight < 2 : new_config[queue].append(gen_new_config(7)) elif queue_weight < 3 : new_config[queue].append(gen_new_config(6)) elif queue_weight >= 3 and queue_weight < 5 : new_config[queue].append(gen_new_config(4)) elif queue_weight == 5: new_config[queue].append(gen_new_config(0)) elif queue_weight >= 5 and worker_weight[worker] <= 6: new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(0)) elif queue_weight >= 6 and worker_weight[worker] <= 7: new_config[queue].append(gen_new_config(1)) new_config[queue].append(gen_new_config(1)) new_config[queue].append(gen_new_config(1)) elif queue_weight >= 7 and worker_weight[worker] > 7: new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(1)) worker_config[worker]= new_config send_sync_command(worker)
def gen_worker_config(worker_weight, queue_weight, worker_list): """生成新的转换器工作线程配置 输出参数:{'q01':[{ 'interval':5}], # 间隔时间 'q02':[{'interval':5},{'interval':3}], } """ #获取转换器的工作线程配置 worker_config = ztq_core.get_worker_config() for worker in worker_list: new_config = {} # 如果worker的权重为0则配置为空 if worker_weight.get(worker, '') == 0: worker_config[worker] = new_config send_sync_command(worker) continue #根据原子队列权重和转换器权重更新工作线程配置 for queue, queue_weight in queue_weight.iteritems(): new_config[queue] = [] #根据原子队列权重进程更新工作线程配置 if queue_weight < 1: new_config[queue].append(gen_new_config(9)) elif queue_weight < 2: new_config[queue].append(gen_new_config(7)) elif queue_weight < 3: new_config[queue].append(gen_new_config(6)) elif queue_weight >= 3 and queue_weight < 5: new_config[queue].append(gen_new_config(4)) elif queue_weight == 5: new_config[queue].append(gen_new_config(0)) elif queue_weight >= 5 and worker_weight[worker] <= 6: new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(0)) elif queue_weight >= 6 and worker_weight[worker] <= 7: new_config[queue].append(gen_new_config(1)) new_config[queue].append(gen_new_config(1)) new_config[queue].append(gen_new_config(1)) elif queue_weight >= 7 and worker_weight[worker] > 7: new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(0)) new_config[queue].append(gen_new_config(1)) worker_config[worker] = new_config send_sync_command(worker)
def init_job_threads(config_dict): # 启动工作线程 set_job_threads(config_dict) # 保存信息 config = {} for queue_name, values in config_dict.items(): sleeps = [str(x['interval']) for x in values] config[queue_name] = ','.join(sleeps) set_config(config, 'queues') # 将一些信息补全,让监视界面认为这个worker已经启动 alias = safe_get_host('server', 'alias') # 初始化 dispatcher_config = ztq_core.get_dispatcher_config() if not dispatcher_config: dispatcher_config = {'queue_weight': {}, 'worker_weight': {}} # set dispatcher config worker_weight = dispatcher_config['worker_weight'] if not worker_weight.get(alias, 0): worker_weight[alias] = 5 ztq_core.set_dispatcher_config(dispatcher_config) # set worker config worker_config = ztq_core.get_worker_config() worker_config[alias] = config_dict
def main(config, thread=False): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n' % os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') # 计算那些是需要根据线上配置启动的队列 active_queue_config = {} if active_config.lower( ) == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 worker_config = ztq_core.get_worker_config() active_queue_config = worker_config.get(command_thread.worker_name, {}) # 根据本地配置,启动的队列 local_queue_config = {} if config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 for queue_name, sleeps in config['queues'].items(): # 线上配置稍后再设置 if queue_name in active_queue_config: continue local_queue_config[queue_name] = [{ 'interval': int(sleep) } for sleep in sleeps.split(',')] if not queue_config.get(queue_name, []): queue_config[queue_name] = { 'name': queue_name, 'title': queue_name, 'widget': 5 } # 合并线上和线下的配置 active_queue_config.update(local_queue_config) init_job_threads(active_queue_config) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 if thread: command_thread.setDaemon(True) command_thread.start() else: command_thread.run()
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') # 计算那些是需要根据线上配置启动的队列 active_queue_config = {} if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 worker_config = ztq_core.get_worker_config() active_queue_config = worker_config.get(command_thread.worker_name, {}) # 根据本地配置,启动的队列 local_queue_config = {} if config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 for queue_name, sleeps in config['queues'].items(): # 线上配置稍后再设置 if queue_name in active_queue_config: continue local_queue_config[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] if not queue_config.get(queue_name, []): queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} # 合并线上和线下的配置 active_queue_config.update(local_queue_config) init_job_threads(active_queue_config) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()
def main(): """ 主函数 """ conf_file = '' # 用户指定一个配置文件 if len(sys.argv) > 1: conf_file = sys.argv[1] init_config(conf_file) server = get_configs('server') alias = safe_get_host('server', 'alias') active_config = server.get('active_config', 'false') # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: raise Exception('Not imported the %s module' % module) # 连结服务器 ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db'])) # 开启一个命令线程 command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n' % os.getpid()) worker_state = ztq_core.get_worker_state() if active_config.lower( ) == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif get_configs('queues'): # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in get_configs('queues').items(): job_threads[queue_name] = [{ 'interval': int(sleep) } for sleep in sleeps.split(',')] if not queue_config.get(queue_name, []): queue_config[queue_name] = { 'name': queue_name, 'title': queue_name, 'widget': 5 } init_job_threads(job_threads) loggers = get_configs('log') initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] module_path = server['module_path'] sys.path.append(module_path) # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) # 是否启用 sentinel enable_sentinel = server['enable_sentinel'].lower() == 'true' # 对应的 sentinel service_name sentinel_name = server['sentinel_name'] if enable_sentinel: # 当启用 sentinel 时 # 配置的host, port, db等信息变为了sentinel的主机信息 ztq_core.setup_sentinel('default', [(redis_host, redis_port)], [sentinel_name], db = redis_db) else: ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in config['queues'].items(): job_threads[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] if not queue_config.get(queue_name, []): queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} init_job_threads(job_threads) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()