def get_sysinfo(): """ 取得系统资源使用情况""" import system_info from multiprocessing import cpu_count info = {} info['cpu'] = (cpu_count(),int(round(float(system_info.get_cpu_usage()))),system_info.get_cpu_style()) info['mem'] = system_info.get_mem_usage() info['ip'] = system_info.get_ip() return info
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config["server"] # 动态注册task for module in server["modules"].split(): try: __import__(module) except ImportError: modules = module.split(".") __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server["host"] redis_port = int(server["port"]) redis_db = int(server["db"]) ztq_core.setup_redis("default", host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get("alias", "") if not alias: alias = get_ip() server["alias"] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write("Starting server in PID %s\n" % os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get("active_config", "false") if active_config.lower() == "true" and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif config["queues"]: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in config["queues"].items(): job_threads[queue_name] = [{"interval": int(sleep)} for sleep in sleeps.split(",")] if not queue_config.get(queue_name, []): queue_config[queue_name] = {"name": queue_name, "title": queue_name, "widget": 5} init_job_threads(job_threads) loggers = config["log"] initlog(loggers.get("key", "ztq_worker"), loggers.get("handler_file"), loggers.get("level", "ERROR")) # 不是以线程启动 command_thread.run()
def safe_get_host(section, option): """ 得到worker的的host name 先找配置文件,是否配置了server节点下的alias 没有就返回worker机器的IP get_ip 方法已经缓存了结果 """ host = get_config(section, option) if not host: return get_ip() return host
# -*- encoding: utf-8 -*- import os from ConfigParser import ConfigParser from system_info import get_ip # 读取配置文件(app.ini),保存到CONFIG中,实际使用的都是CONFIG CONFIG = {'server':{'alias':get_ip()}, 'queues':{} } def read_config_file(location=None): """ 初始化配置管理 """ cfg = ConfigParser() if location: cfg.read(location) else: local_dir = os.path.dirname(os.path.realpath(__file__)) cfg.read( os.path.join(local_dir, 'config.cfg') ) global CONFIG for section in cfg.sections(): CONFIG[section] = {} for option in cfg.options(section): CONFIG[section][option] = cfg.get(section, option) return CONFIG def register_batch_queue(queue_name, batch_size, batch_func=None): """ 注册队列是批处理模式
def main(config, thread=False): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n' % os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') # 计算那些是需要根据线上配置启动的队列 active_queue_config = {} if active_config.lower( ) == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 worker_config = ztq_core.get_worker_config() active_queue_config = worker_config.get(command_thread.worker_name, {}) # 根据本地配置,启动的队列 local_queue_config = {} if config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 for queue_name, sleeps in config['queues'].items(): # 线上配置稍后再设置 if queue_name in active_queue_config: continue local_queue_config[queue_name] = [{ 'interval': int(sleep) } for sleep in sleeps.split(',')] if not queue_config.get(queue_name, []): queue_config[queue_name] = { 'name': queue_name, 'title': queue_name, 'widget': 5 } # 合并线上和线下的配置 active_queue_config.update(local_queue_config) init_job_threads(active_queue_config) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 if thread: command_thread.setDaemon(True) command_thread.start() else: command_thread.run()
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') # 计算那些是需要根据线上配置启动的队列 active_queue_config = {} if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 worker_config = ztq_core.get_worker_config() active_queue_config = worker_config.get(command_thread.worker_name, {}) # 根据本地配置,启动的队列 local_queue_config = {} if config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 for queue_name, sleeps in config['queues'].items(): # 线上配置稍后再设置 if queue_name in active_queue_config: continue local_queue_config[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] if not queue_config.get(queue_name, []): queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} # 合并线上和线下的配置 active_queue_config.update(local_queue_config) init_job_threads(active_queue_config) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()
from queue import Queue from system_info import get_ip from command_redis import CommandRedisThread from commandexecute import CommandThread from job_redis import JobRedisThread from jobexecute import JobThread from signal import signal, SIGINT, SIG_IGN,SIGTERM """初始化配置""" logging.config.fileConfig('./conf/logging.conf') logger = logging.getLogger('worker') config.read('./conf/worker.conf') #logger.info('test main logger') global_list.SERVER_NAME = 'wbsp' #服务名前缀 global_list.WORKNAME_NAME = '' #工人编号 global_list.LOCAL_IP = get_ip() #本机IP global_list.PROC_POOL = {} #线程列表 DOING_TASKES={} #正在进行的任务列表 global_list.G_EXIT = False #全局退出标记 SERVER = config.CONFIG['server'] #配置文件信息 #信号处理 def sig_exit(a,b): global_list.G_exit=True os._exit(0) signal(SIGINT,sig_exit) signal(SIGTERM,sig_exit) #注册redis消息中枢 setup_redis('default',SERVER['host'],int(SERVER['port']),SERVER['db']) #队列
# -*- encoding: utf-8 -*- import os from ConfigParser import RawConfigParser as ConfigParser from system_info import get_ip # 读取配置文件(app.ini),保存到CONFIG中,实际使用的都是CONFIG CONFIG = {'server': {'alias': get_ip()}, 'queues': {}} def read_config_file(location=None): """ 初始化配置管理 """ cfg = ConfigParser() if location: cfg.read(location) else: local_dir = os.path.dirname(os.path.realpath(__file__)) cfg.read(os.path.join(local_dir, 'config.cfg')) global CONFIG for section in cfg.sections(): CONFIG[section] = {} for option in cfg.options(section): CONFIG[section][option] = cfg.get(section, option) return CONFIG def register_batch_queue(queue_name, batch_size, batch_func=None): """ 注册队列是批处理模式
def main(config): """ 主函数 config: {'server': {host:, port:, db:} } """ server = config['server'] module_path = server['module_path'] sys.path.append(module_path) # 动态注册task for module in server['modules'].split(): try: __import__(module) except ImportError: modules = module.split('.') __import__(modules[0], globals(), locals(), modules[1]) # 连结服务器 redis_host = server['host'] redis_port = int(server['port']) redis_db = int(server['db']) # 是否启用 sentinel enable_sentinel = server['enable_sentinel'].lower() == 'true' # 对应的 sentinel service_name sentinel_name = server['sentinel_name'] if enable_sentinel: # 当启用 sentinel 时 # 配置的host, port, db等信息变为了sentinel的主机信息 ztq_core.setup_sentinel('default', [(redis_host, redis_port)], [sentinel_name], db = redis_db) else: ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db) # 开启一个命令线程 alias = server.get('alias', '') if not alias: alias = get_ip() server['alias'] = alias command_thread = CommandThread(worker_name=alias) sys.stdout.write('Starting server in PID %s\n'%os.getpid()) worker_state = ztq_core.get_worker_state() active_config = server.get('active_config', 'false') if active_config.lower() == 'true' and command_thread.worker_name in worker_state: # 如果服务器有这个机器的配置信息,需要自动启动工作线程 queue = ztq_core.get_worker_config() if command_thread.worker_name in queue: set_job_threads(queue[command_thread.worker_name]) elif config['queues']: # 把worker监视队列的情况上报到服务器 queue_config = ztq_core.get_queue_config() # 如果配置有queues,自动启动线程监视 job_threads = {} for queue_name, sleeps in config['queues'].items(): job_threads[queue_name] = [ {'interval': int(sleep)} for sleep in sleeps.split(',') ] if not queue_config.get(queue_name, []): queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5} init_job_threads(job_threads) loggers = config['log'] initlog( loggers.get('key', 'ztq_worker'), loggers.get('handler_file'), loggers.get('level', 'ERROR'), ) # 不是以线程启动 command_thread.run()