def get_sysinfo():
""" 取得系统资源使用情况"""
import system_info
from multiprocessing import cpu_count
info = {}
info['cpu'] = (cpu_count(),int(round(float(system_info.get_cpu_usage()))),system_info.get_cpu_style())
info['mem'] = system_info.get_mem_usage()
info['ip'] = system_info.get_ip()
return info
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config["server"]
# 动态注册task
for module in server["modules"].split():
try:
__import__(module)
except ImportError:
modules = module.split(".")
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server["host"]
redis_port = int(server["port"])
redis_db = int(server["db"])
ztq_core.setup_redis("default", host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get("alias", "")
if not alias:
alias = get_ip()
server["alias"] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write("Starting server in PID %s\n" % os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get("active_config", "false")
if active_config.lower() == "true" and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif config["queues"]:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
for queue_name, sleeps in config["queues"].items():
job_threads[queue_name] = [{"interval": int(sleep)} for sleep in sleeps.split(",")]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {"name": queue_name, "title": queue_name, "widget": 5}
init_job_threads(job_threads)
loggers = config["log"]
initlog(loggers.get("key", "ztq_worker"), loggers.get("handler_file"), loggers.get("level", "ERROR"))
# 不是以线程启动
command_thread.run()
def safe_get_host(section, option):
""" 得到worker的的host name
先找配置文件，是否配置了server节点下的alias
没有就返回worker机器的IP
get_ip 方法已经缓存了结果
"""
host = get_config(section, option)
if not host:
return get_ip()
return host
def safe_get_host(section, option):
""" 得到worker的的host name
先找配置文件，是否配置了server节点下的alias
没有就返回worker机器的IP
get_ip 方法已经缓存了结果
"""
host = get_config(section, option)
if not host:
return get_ip()
return host
# -*- encoding: utf-8 -*-
import os
from ConfigParser import ConfigParser
from system_info import get_ip
# 读取配置文件（app.ini），保存到CONFIG中，实际使用的都是CONFIG
CONFIG = {'server':{'alias':get_ip()},
'queues':{} }
def read_config_file(location=None):
""" 初始化配置管理
"""
cfg = ConfigParser()
if location:
cfg.read(location)
else:
local_dir = os.path.dirname(os.path.realpath(__file__))
cfg.read( os.path.join(local_dir, 'config.cfg') )
global CONFIG
for section in cfg.sections():
CONFIG[section] = {}
for option in cfg.options(section):
CONFIG[section][option] = cfg.get(section, option)
return CONFIG
def register_batch_queue(queue_name, batch_size, batch_func=None):
""" 注册队列是批处理模式
def main(config, thread=False):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
ztq_core.setup_redis('default',
host=redis_host,
port=redis_port,
db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n' % os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
# 计算那些是需要根据线上配置启动的队列
active_queue_config = {}
if active_config.lower(
) == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
worker_config = ztq_core.get_worker_config()
active_queue_config = worker_config.get(command_thread.worker_name, {})
# 根据本地配置，启动的队列
local_queue_config = {}
if config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
for queue_name, sleeps in config['queues'].items():
# 线上配置稍后再设置
if queue_name in active_queue_config: continue
local_queue_config[queue_name] = [{
'interval': int(sleep)
} for sleep in sleeps.split(',')]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {
'name': queue_name,
'title': queue_name,
'widget': 5
}
# 合并线上和线下的配置
active_queue_config.update(local_queue_config)
init_job_threads(active_queue_config)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
if thread:
command_thread.setDaemon(True)
command_thread.start()
else:
command_thread.run()
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n'%os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
# 计算那些是需要根据线上配置启动的队列
active_queue_config = {}
if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
worker_config = ztq_core.get_worker_config()
active_queue_config = worker_config.get(command_thread.worker_name, {})
# 根据本地配置，启动的队列
local_queue_config = {}
if config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
for queue_name, sleeps in config['queues'].items():
# 线上配置稍后再设置
if queue_name in active_queue_config: continue
local_queue_config[queue_name] = [
{'interval': int(sleep)} for sleep in sleeps.split(',')
]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}
# 合并线上和线下的配置
active_queue_config.update(local_queue_config)
init_job_threads(active_queue_config)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
from queue import Queue
from system_info import get_ip
from command_redis import CommandRedisThread
from commandexecute import CommandThread
from job_redis import JobRedisThread
from jobexecute import JobThread
from signal import signal, SIGINT, SIG_IGN,SIGTERM
"""初始化配置"""
logging.config.fileConfig('./conf/logging.conf')
logger = logging.getLogger('worker')
config.read('./conf/worker.conf')
#logger.info('test main logger')
global_list.SERVER_NAME = 'wbsp' #服务名前缀
global_list.WORKNAME_NAME = '' #工人编号
global_list.LOCAL_IP = get_ip() #本机IP
global_list.PROC_POOL = {} #线程列表
DOING_TASKES={} #正在进行的任务列表
global_list.G_EXIT = False #全局退出标记
SERVER = config.CONFIG['server'] #配置文件信息
#信号处理
def sig_exit(a,b):
global_list.G_exit=True
os._exit(0)
signal(SIGINT,sig_exit)
signal(SIGTERM,sig_exit)
#注册redis消息中枢
setup_redis('default',SERVER['host'],int(SERVER['port']),SERVER['db'])
#队列
# -*- encoding: utf-8 -*-
import os
from ConfigParser import RawConfigParser as ConfigParser
from system_info import get_ip
# 读取配置文件（app.ini），保存到CONFIG中，实际使用的都是CONFIG
CONFIG = {'server': {'alias': get_ip()}, 'queues': {}}
def read_config_file(location=None):
""" 初始化配置管理
"""
cfg = ConfigParser()
if location:
cfg.read(location)
else:
local_dir = os.path.dirname(os.path.realpath(__file__))
cfg.read(os.path.join(local_dir, 'config.cfg'))
global CONFIG
for section in cfg.sections():
CONFIG[section] = {}
for option in cfg.options(section):
CONFIG[section][option] = cfg.get(section, option)
return CONFIG
def register_batch_queue(queue_name, batch_size, batch_func=None):
""" 注册队列是批处理模式
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
module_path = server['module_path']
sys.path.append(module_path)
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
# 是否启用 sentinel
enable_sentinel = server['enable_sentinel'].lower() == 'true'
# 对应的 sentinel service_name
sentinel_name = server['sentinel_name']
if enable_sentinel:
# 当启用 sentinel 时
# 配置的host, port, db等信息变为了sentinel的主机信息
ztq_core.setup_sentinel('default', [(redis_host, redis_port)],
[sentinel_name], db = redis_db)
else:
ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n'%os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
for queue_name, sleeps in config['queues'].items():
job_threads[queue_name] = [
{'interval': int(sleep)} for sleep in sleeps.split(',')
]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}
init_job_threads(job_threads)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
