def init_job_threads(config_dict):
# 启动工作线程
set_job_threads(config_dict)
# 保存信息
config = {}
for queue_name, values in config_dict.items():
sleeps = [str(x['interval']) for x in values]
config[queue_name] = ','.join(sleeps)
CONFIG['queues'].update(config)
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = CONFIG['server']['alias']
# 初始化
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = {'queue_weight':{},'worker_weight':{}}
# set dispatcher config
worker_weight = dispatcher_config['worker_weight']
if not worker_weight.get(alias, 0):
worker_weight[alias] = 5
ztq_core.set_dispatcher_config(dispatcher_config)
# set worker config
worker_config = ztq_core.get_worker_config()
worker_config[alias] = config_dict
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker' and \
CONFIG['server'].get('active_config', 'false').lower() == 'true':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'], timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
except ztq_core.ConnectionError, e:
logger.error('ERROR: redis command connection error: %s' % str(e))
time.sleep(3)
except ztq_core.ResponseError, e:
logger.error('ERROR: redis command response error: %s' % str(e))
time.sleep(3)
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'], timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
except KeyboardInterrupt:
import os
# 实际上调用的是command_execute.clear_thread
os.sys.exitfunc()
os._exit(0)
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'],
timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'],
timestamp=command['timestamp'])
except KeyboardInterrupt:
import os
# 实际上调用的是command_execute.clear_thread
os.sys.exitfunc()
os._exit(0)
def main():
""" 主函数 """
conf_file = ''
# 用户指定一个配置文件
if len(sys.argv) > 1:
conf_file = sys.argv[1]
init_config(conf_file)
server = get_configs('server')
alias = safe_get_host('server', 'alias')
active_config = server.get('active_config', 'false')
# 动态注册task
for module in server['modules'].split():
try:
importlib.import_module(module)
except ImportError:
raise Exception('Not imported the %s module' % module)
# 连结服务器
ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db']))
# 开启一个命令线程
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n'%os.getpid())
worker_state = ztq_core.get_worker_state()
if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif get_configs('queues'):
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
queue_config = ztq_core.get_queue_config()
for queue_name, sleeps in get_configs('queues').items():
queue_config[queue_name] = {'title': queue_name} # for ztq_app
job_threads[queue_name] = [
{'interval': int(sleep)} for sleep in sleeps.split(',')
]
queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}
init_job_threads(job_threads)
loggers = get_configs('log')
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
def dispatch_single_queue(queue_name, from_right=True):
"""调整特定队列, 让worker从队列头或者从队列尾获取数据
"""
worker_config = ztq_core.get_worker_config()
for worker_name, queue_config in worker_config.items():
if queue_config.get(queue_name, None):
queue_config[queue_name] = update_queue_config(queue_config[queue_name], from_right)
worker_config[worker_name]= queue_config
send_sync_command(worker_name)
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config["server"]
# 动态注册task
for module in server["modules"].split():
try:
__import__(module)
except ImportError:
modules = module.split(".")
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server["host"]
redis_port = int(server["port"])
redis_db = int(server["db"])
ztq_core.setup_redis("default", host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get("alias", "")
if not alias:
alias = get_ip()
server["alias"] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write("Starting server in PID %s\n" % os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get("active_config", "false")
if active_config.lower() == "true" and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif config["queues"]:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
for queue_name, sleeps in config["queues"].items():
job_threads[queue_name] = [{"interval": int(sleep)} for sleep in sleeps.split(",")]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {"name": queue_name, "title": queue_name, "widget": 5}
init_job_threads(job_threads)
loggers = config["log"]
initlog(loggers.get("key", "ztq_worker"), loggers.get("handler_file"), loggers.get("level", "ERROR"))
# 不是以线程启动
command_thread.run()
def dispatch_single_queue(queue_name, from_right=True):
"""调整特定队列, 让worker从队列头或者从队列尾获取数据
"""
worker_config = ztq_core.get_worker_config()
for worker_name, queue_config in worker_config.items():
if queue_config.get(queue_name, None):
queue_config[queue_name] = update_queue_config(
queue_config[queue_name], from_right)
worker_config[worker_name] = queue_config
send_sync_command(worker_name)
def config_queue(request):
"""管理队列线程数量
传入参数:http://server/taskqueues/q01/config?action=queue_down
"""
queue_id = request.matchdict['id']
url_action = request.params.get('action','')
# 对所有的worker的队列调整数量
for worker_name in ztq_core.get_worker_config():
utils.update_queue_threads(work_name, queue_id, action=url_action)
return HTTPFound(location = '/taskqueues')
def init_job_threads(config_dict, force=True):
# 如果首次注册，根据配置启动工作线程，否则根据之前的配置启动。
set_job_threads(config_dict)
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = CONFIG['server']['alias']
# set worker config
worker_config = ztq_core.get_worker_config()
if alias not in worker_config or force:
worker_config[alias] = config_dict
def init_job_threads(config_dict, force=True):
# 如果首次注册，根据配置启动工作线程，否则根据之前的配置启动。
set_job_threads(config_dict)
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = CONFIG['server']['alias']
# set worker config
worker_config = ztq_core.get_worker_config()
if alias not in worker_config or force:
worker_config[alias] = config_dict
def config_queue(request):
"""管理队列线程数量
传入参数:http://server/taskqueues/q01/config?action=queue_down
"""
queue_id = request.matchdict['id']
url_action = request.params.get('action', '')
# 对所有的worker的队列调整数量
for worker_name in ztq_core.get_worker_config().keys():
utils.update_queue_threads(worker_name, queue_id, action=url_action)
return HTTPFound(location='/taskqueues')
def get_taskqueues_list():
# 队列情况列表
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
queues_list = ztq_core.get_queue_config()
# 排序
sort_queue_name = {}
for queue_name, queue_config in queues_list.items():
sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name))
for queue_name in sorted(sort_queue_name,
key=lambda x: sort_queue_name[x],
reverse=True):
task_queue = {}
task_queue['name'] = queue_name
#task_queue['tags'] = queue_config.get('tags',())
queue = ztq_core.get_task_queue(queue_name)
# 任务数/错误数
task_queue['length'] = len(queue)
task_queue['error_length'] = sort_queue_name[queue_name]
#任务首个时间
task_queue['error_end'] = task_queue['first'] = ''
first_job = queue[0]
first_job = ztq_core.get_task_hash(queue_name).get(first_job)
if first_job:
task_queue['first'] = datetime.datetime.fromtimestamp(
first_job['runtime'].get('create', 0))
#错误最末一个的时间
error_first_job = ztq_core.get_error_queue(queue_name)[0]
error_first_job = ztq_core.get_error_hash(queue_name).get(
error_first_job)
if error_first_job:
task_queue['error_end'] = datetime.datetime.fromtimestamp(
error_first_job['runtime'].get('create', 0))
task_queue['weight'] = queue_weight.get(queue_name, 0)
# 获取worker工作线程配置
workers_config = ztq_core.get_worker_config()
task_queue['from_right'] = True
for worker_name, worker_config in workers_config.items():
task_queue['workers'] = []
for config in worker_config.get(queue_name, []):
task_queue['workers'].append(
[worker_name + ':', config['interval']])
if 'from_right' in config:
task_queue['from_right'] = config['from_right']
task_queue['buffer_length'] = len(
ztq_core.get_buffer_queue(queue_name))
yield task_queue
def get_taskqueues_list():
# 队列情况列表
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
queues_list = ztq_core.get_queue_config()
# 排序
sort_queue_name = {}
for queue_name, queue_config in queues_list.items():
sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name))
for queue_name in sorted(sort_queue_name,
key=lambda x: sort_queue_name[x],
reverse=True):
task_queue = {}
task_queue['name'] = queue_name
#task_queue['tags'] = queue_config.get('tags',())
queue = ztq_core.get_task_queue(queue_name)
# 任务数/错误数
task_queue['length'] = len(queue)
task_queue['error_length'] = sort_queue_name[queue_name]
#任务首个时间
task_queue['error_end'] = task_queue['first'] = ''
first_job = queue[0]
first_job= ztq_core.get_task_hash(queue_name).get(first_job)
if first_job:
task_queue['first'] = datetime.datetime.fromtimestamp(first_job['runtime'].get('create', 0))
#错误最末一个的时间
error_first_job = ztq_core.get_error_queue(queue_name)[0]
error_first_job = ztq_core.get_error_hash(queue_name).get(error_first_job)
if error_first_job:
task_queue['error_end'] = datetime.datetime.fromtimestamp(error_first_job['runtime'].get('create', 0))
task_queue['weight'] = queue_weight.get(queue_name, 0)
# 获取worker工作线程配置
workers_config = ztq_core.get_worker_config()
task_queue['from_right'] = True
for worker_name,worker_config in workers_config.items():
task_queue['workers'] = []
for config in worker_config.get(queue_name,[]):
task_queue['workers'].append([worker_name+':', config['interval']])
if 'from_right' in config:
task_queue['from_right'] = config['from_right']
task_queue['buffer_length'] = len(ztq_core.get_buffer_queue(queue_name))
yield task_queue
def update_queue_threads(worker_name, queue_name, action):
"""调整特定队列线程数量，可以增加或者减少
"""
worker_config = ztq_core.get_worker_config()
queue_config = worker_config[worker_name]
if queue_config.get(queue_name, None) is not None:
_config = queue_config[queue_name]
# 生成新的配置信息
if action == 'queue_down':
_config.pop()
elif action == 'queue_up' :
_config.append({u'interval': 0})
queue_config[queue_name] = _config
worker_config[worker_name]= queue_config
send_sync_command(worker_name)
def update_queue_threads(worker_name, queue_name, action):
"""调整特定队列线程数量，可以增加或者减少
"""
worker_config = ztq_core.get_worker_config()
queue_config = worker_config[worker_name]
if queue_config.get(queue_name, None) is not None:
_config = queue_config[queue_name]
# 生成新的配置信息
if action == 'queue_down':
_config.pop()
elif action == 'queue_up':
_config.append({u'interval': 0})
queue_config[queue_name] = _config
worker_config[worker_name] = queue_config
send_sync_command(worker_name)
def gen_worker_config(worker_weight, queue_weight, worker_list):
"""生成新的转换器工作线程配置
输出参数:{'q01':[{ 'interval':5}], # 间隔时间
'q02':[{'interval':5},{'interval':3}],
}
"""
#获取转换器的工作线程配置
worker_config = ztq_core.get_worker_config()
for worker in worker_list:
new_config = {}
# 如果worker的权重为0则配置为空
if worker_weight.get(worker, '') == 0:
worker_config[worker]= new_config
send_sync_command(worker)
continue
#根据原子队列权重和转换器权重更新工作线程配置
for queue,queue_weight in queue_weight.iteritems():
new_config[queue] = []
#根据原子队列权重进程更新工作线程配置
if queue_weight < 1 :
new_config[queue].append(gen_new_config(9))
elif queue_weight < 2 :
new_config[queue].append(gen_new_config(7))
elif queue_weight < 3 :
new_config[queue].append(gen_new_config(6))
elif queue_weight >= 3 and queue_weight < 5 :
new_config[queue].append(gen_new_config(4))
elif queue_weight == 5:
new_config[queue].append(gen_new_config(0))
elif queue_weight >= 5 and worker_weight[worker] <= 6:
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(0))
elif queue_weight >= 6 and worker_weight[worker] <= 7:
new_config[queue].append(gen_new_config(1))
new_config[queue].append(gen_new_config(1))
new_config[queue].append(gen_new_config(1))
elif queue_weight >= 7 and worker_weight[worker] > 7:
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(1))
worker_config[worker]= new_config
send_sync_command(worker)
def gen_worker_config(worker_weight, queue_weight, worker_list):
"""生成新的转换器工作线程配置
输出参数:{'q01':[{ 'interval':5}], # 间隔时间
'q02':[{'interval':5},{'interval':3}],
}
"""
#获取转换器的工作线程配置
worker_config = ztq_core.get_worker_config()
for worker in worker_list:
new_config = {}
# 如果worker的权重为0则配置为空
if worker_weight.get(worker, '') == 0:
worker_config[worker] = new_config
send_sync_command(worker)
continue
#根据原子队列权重和转换器权重更新工作线程配置
for queue, queue_weight in queue_weight.iteritems():
new_config[queue] = []
#根据原子队列权重进程更新工作线程配置
if queue_weight < 1:
new_config[queue].append(gen_new_config(9))
elif queue_weight < 2:
new_config[queue].append(gen_new_config(7))
elif queue_weight < 3:
new_config[queue].append(gen_new_config(6))
elif queue_weight >= 3 and queue_weight < 5:
new_config[queue].append(gen_new_config(4))
elif queue_weight == 5:
new_config[queue].append(gen_new_config(0))
elif queue_weight >= 5 and worker_weight[worker] <= 6:
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(0))
elif queue_weight >= 6 and worker_weight[worker] <= 7:
new_config[queue].append(gen_new_config(1))
new_config[queue].append(gen_new_config(1))
new_config[queue].append(gen_new_config(1))
elif queue_weight >= 7 and worker_weight[worker] > 7:
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(0))
new_config[queue].append(gen_new_config(1))
worker_config[worker] = new_config
send_sync_command(worker)
def init_job_threads(config_dict):
# 启动工作线程
set_job_threads(config_dict)
# 保存信息
config = {}
for queue_name, values in config_dict.items():
sleeps = [str(x['interval']) for x in values]
config[queue_name] = ','.join(sleeps)
set_config(config, 'queues')
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = safe_get_host('server', 'alias')
# 初始化
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = {'queue_weight': {}, 'worker_weight': {}}
# set dispatcher config
worker_weight = dispatcher_config['worker_weight']
if not worker_weight.get(alias, 0):
worker_weight[alias] = 5
ztq_core.set_dispatcher_config(dispatcher_config)
# set worker config
worker_config = ztq_core.get_worker_config()
worker_config[alias] = config_dict
def main(config, thread=False):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
ztq_core.setup_redis('default',
host=redis_host,
port=redis_port,
db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n' % os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
# 计算那些是需要根据线上配置启动的队列
active_queue_config = {}
if active_config.lower(
) == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
worker_config = ztq_core.get_worker_config()
active_queue_config = worker_config.get(command_thread.worker_name, {})
# 根据本地配置，启动的队列
local_queue_config = {}
if config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
for queue_name, sleeps in config['queues'].items():
# 线上配置稍后再设置
if queue_name in active_queue_config: continue
local_queue_config[queue_name] = [{
'interval': int(sleep)
} for sleep in sleeps.split(',')]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {
'name': queue_name,
'title': queue_name,
'widget': 5
}
# 合并线上和线下的配置
active_queue_config.update(local_queue_config)
init_job_threads(active_queue_config)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
if thread:
command_thread.setDaemon(True)
command_thread.start()
else:
command_thread.run()
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n'%os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
# 计算那些是需要根据线上配置启动的队列
active_queue_config = {}
if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
worker_config = ztq_core.get_worker_config()
active_queue_config = worker_config.get(command_thread.worker_name, {})
# 根据本地配置，启动的队列
local_queue_config = {}
if config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
for queue_name, sleeps in config['queues'].items():
# 线上配置稍后再设置
if queue_name in active_queue_config: continue
local_queue_config[queue_name] = [
{'interval': int(sleep)} for sleep in sleeps.split(',')
]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}
# 合并线上和线下的配置
active_queue_config.update(local_queue_config)
init_job_threads(active_queue_config)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
def main():
""" 主函数 """
conf_file = ''
# 用户指定一个配置文件
if len(sys.argv) > 1:
conf_file = sys.argv[1]
init_config(conf_file)
server = get_configs('server')
alias = safe_get_host('server', 'alias')
active_config = server.get('active_config', 'false')
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
raise Exception('Not imported the %s module' % module)
# 连结服务器
ztq_core.setup_redis('default',
host=server['host'],
port=int(server['port']),
db=int(server['db']))
# 开启一个命令线程
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n' % os.getpid())
worker_state = ztq_core.get_worker_state()
if active_config.lower(
) == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif get_configs('queues'):
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
for queue_name, sleeps in get_configs('queues').items():
job_threads[queue_name] = [{
'interval': int(sleep)
} for sleep in sleeps.split(',')]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {
'name': queue_name,
'title': queue_name,
'widget': 5
}
init_job_threads(job_threads)
loggers = get_configs('log')
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
def main(config):
""" 主函数
config: {'server': {host:, port:, db:}
}
"""
server = config['server']
module_path = server['module_path']
sys.path.append(module_path)
# 动态注册task
for module in server['modules'].split():
try:
__import__(module)
except ImportError:
modules = module.split('.')
__import__(modules[0], globals(), locals(), modules[1])
# 连结服务器
redis_host = server['host']
redis_port = int(server['port'])
redis_db = int(server['db'])
# 是否启用 sentinel
enable_sentinel = server['enable_sentinel'].lower() == 'true'
# 对应的 sentinel service_name
sentinel_name = server['sentinel_name']
if enable_sentinel:
# 当启用 sentinel 时
# 配置的host, port, db等信息变为了sentinel的主机信息
ztq_core.setup_sentinel('default', [(redis_host, redis_port)],
[sentinel_name], db = redis_db)
else:
ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db)
# 开启一个命令线程
alias = server.get('alias', '')
if not alias:
alias = get_ip()
server['alias'] = alias
command_thread = CommandThread(worker_name=alias)
sys.stdout.write('Starting server in PID %s\n'%os.getpid())
worker_state = ztq_core.get_worker_state()
active_config = server.get('active_config', 'false')
if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
# 如果服务器有这个机器的配置信息，需要自动启动工作线程
queue = ztq_core.get_worker_config()
if command_thread.worker_name in queue:
set_job_threads(queue[command_thread.worker_name])
elif config['queues']:
# 把worker监视队列的情况上报到服务器
queue_config = ztq_core.get_queue_config()
# 如果配置有queues，自动启动线程监视
job_threads = {}
for queue_name, sleeps in config['queues'].items():
job_threads[queue_name] = [
{'interval': int(sleep)} for sleep in sleeps.split(',')
]
if not queue_config.get(queue_name, []):
queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}
init_job_threads(job_threads)
loggers = config['log']
initlog(
loggers.get('key', 'ztq_worker'),
loggers.get('handler_file'),
loggers.get('level', 'ERROR'),
)
# 不是以线程启动
command_thread.run()
