def config_queue(request):
"""管理队列权重,提升队列权重或降低队列权重
传入参数:http://server/taskqueues/q01/config?action=queue_down
"""
queue_id = request.matchdict['id']
url_action = request.params.get('action','')
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
# 根据操作类型进行权重调整,
if url_action == 'queue_down' :
queue_weight[queue_id] -= 1
# 队列权重最少为1
if queue_weight[queue_id] < 0: queue_weight[queue_id] = 0
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
elif url_action == 'queue_up' :
queue_weight[queue_id] += 1
if queue_weight[queue_id] > 10: queue_weight[queue_id] = 10
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
elif url_action == 'from_right' :
utils.dispatch_single_queue(queue_id, from_right=True)
elif url_action == 'from_left' :
utils.dispatch_single_queue(queue_id, from_right=False)
return HTTPFound(location = '/taskqueues')
def init_job_threads(config_dict):
# 启动工作线程
set_job_threads(config_dict)
# 保存信息
config = {}
for queue_name, values in config_dict.items():
sleeps = [str(x['interval']) for x in values]
config[queue_name] = ','.join(sleeps)
CONFIG['queues'].update(config)
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = CONFIG['server']['alias']
# 初始化
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = {'queue_weight':{},'worker_weight':{}}
# set dispatcher config
worker_weight = dispatcher_config['worker_weight']
if not worker_weight.get(alias, 0):
worker_weight[alias] = 5
ztq_core.set_dispatcher_config(dispatcher_config)
# set worker config
worker_config = ztq_core.get_worker_config()
worker_config[alias] = config_dict
def config_queue(request):
"""管理队列权重,提升队列权重或降低队列权重
传入参数:http://server/taskqueues/q01/config?action=queue_down
"""
queue_id = request.matchdict['id']
url_action = request.params.get('action', '')
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
# 根据操作类型进行权重调整,
if url_action == 'queue_down':
queue_weight[queue_id] -= 1
# 队列权重最少为1
if queue_weight[queue_id] < 0: queue_weight[queue_id] = 0
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
elif url_action == 'queue_up':
queue_weight[queue_id] += 1
if queue_weight[queue_id] > 10: queue_weight[queue_id] = 10
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
elif url_action == 'from_right':
utils.dispatch_single_queue(queue_id, from_right=True)
elif url_action == 'from_left':
utils.dispatch_single_queue(queue_id, from_right=False)
return HTTPFound(location='/taskqueues')
def main(global_config, redis_host='127.0.0.1', redis_port='6379', \
redis_db='0', frs_root='frs', init_dispatcher_config='true', \
frs_cache='frscache', addon_config=None, work_enable=True, **settings):
""" This function returns a Pyramid WSGI application.
"""
# 初始化Redis连接
ztq_core.setup_redis('default', redis_host, port=int(redis_port), db=int(redis_db),)
# 初始化权重数据数据,如果权重配置已经存在则pass
if init_dispatcher_config.lower() == 'true':
# init_dispatcher_config 是因为控制台可能没有运行服务， 这里去读取redis数据，会导致控制台起不来
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = weight = {'queue_weight':{},'worker_weight':{}}
ztq_core.set_dispatcher_config(weight)
queue_weight = dispatcher_config['queue_weight']
if not queue_weight:
queues_list = ztq_core.get_queue_config()
for queue_name, queue_config in queues_list.items():
queue_weight[queue_name] = queue_config.get('weight', 0)
ztq_core.set_dispatcher_config(dispatcher_config)
# # 开启后台服务
# 初始化fts_web配置
settings = dict(settings)
settings.setdefault('jinja2.directories', 'ztq_console:templates')
config = Configurator(settings=settings)
config.begin()
config.add_renderer('.html', pyramid_jinja2.renderer_factory)
config.add_static_view('static', 'ztq_console:static')
config.scan('ztq_console.views')
config.add_route('worker', '/worker/{id}',
view='ztq_console.views.config_worker')
config.add_route('end_thread', '/worker/{id}/{thread}/{pid}',
view='ztq_console.views.stop_working_job')
config.add_route('taskqueue', '/taskqueues/{id}')
config.add_route('taskqueues_config', '/taskqueues/{id}/config',
view='ztq_console.views.config_queue')
config.add_route('taskqueue_action', '/taskqueues_action/{id}')
config.add_route('errorqueues_job', '/errorqueues/{id}/job',
view='ztq_console.views.error_jobs_handler')
config.add_route('workerlog', '/workerlog/{page}')
config.add_route('syslog', '/syslog/{page}')
config.add_route('errorlog', '/errorlog/{page}')
config.add_route('errorqueue', '/errorqueue/{id}/{page}')
config.add_route('redo_all_error_for_queue', '/redo_all_error_for_queue/{id}')
config.add_route('del_all_error_for_queue', '/del_all_error_for_queue/{id}')
if addon_config is not None:
addon_config(config)
config.end()
return config.make_wsgi_app()
def config_worker(request):
"""对worker进行配置管理
"""
url_action = request.params.get('action', '')
dispatcher_config = ztq_core.get_dispatcher_config()
worker_weight = dispatcher_config['worker_weight']
# 获取用户请求操作
worker_id = request.matchdict['id']
# 根据操作类型进行权重调整,
if url_action == 'stop_worker':
#停止worker
worker_weight[worker_id] = 0
elif url_action == 'enable':
#启用worker
worker_weight[worker_id] = 5
elif url_action == 'worker_down':
#降低worker权重
worker_weight[worker_id] -= 1
if worker_weight[worker_id] < 1: worker_weight[worker_id] = 1
elif url_action == 'worker_up':
#提升worker权重
worker_weight[worker_id] += 1
if worker_weight[worker_id] > 10: worker_weight[worker_id] = 10
elif url_action == 'delete':
#删除还没启用的worker,删除操作不会导致调度配置更新
if worker_id in worker_weight: # 没有启用的情况
worker_weight.pop(worker_id)
workers_dict = ztq_core.get_worker_state()
del workers_dict[worker_id]
worker_job = ztq_core.get_job_state(worker_id)
for job_name, job_status in worker_job.items():
del worker_job[job_name]
ztq_core.set_dispatcher_config(dispatcher_config)
return HTTPFound(location='/workerstatus')
elif url_action == 'update':
# 发报告指令到各命令队列让worker报告自身状态
worker_list = ztq_core.get_all_worker()
for worker_name in worker_list:
if worker_name == worker_id:
utils.send_command(worker_name, 'report')
time.sleep(1)
return HTTPFound(location='/workerstatus')
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
return HTTPFound(location='/workerstatus')
def config_worker(request):
"""对worker进行配置管理
"""
url_action = request.params.get('action','')
dispatcher_config = ztq_core.get_dispatcher_config()
worker_weight = dispatcher_config['worker_weight']
# 获取用户请求操作
worker_id = request.matchdict['id']
# 根据操作类型进行权重调整,
if url_action == 'stop_worker':
#停止worker
worker_weight[worker_id] = 0
elif url_action == 'enable':
#启用worker
worker_weight[worker_id] = 5
elif url_action == 'worker_down' :
#降低worker权重
worker_weight[worker_id] -= 1
if worker_weight[worker_id] < 1: worker_weight[worker_id] = 1
elif url_action == 'worker_up' :
#提升worker权重
worker_weight[worker_id] += 1
if worker_weight[worker_id] >10: worker_weight[worker_id] = 10
elif url_action == 'delete':
#删除还没启用的worker,删除操作不会导致调度配置更新
if worker_id in worker_weight: # 没有启用的情况
worker_weight.pop(worker_id)
workers_dict = ztq_core.get_worker_state()
del workers_dict[worker_id]
worker_job = ztq_core.get_job_state(worker_id)
for job_name, job_status in worker_job.items():
del worker_job[job_name]
ztq_core.set_dispatcher_config(dispatcher_config)
return HTTPFound(location = '/workerstatus')
elif url_action == 'update':
# 发报告指令到各命令队列让worker报告自身状态
worker_list = ztq_core.get_all_worker()
for worker_name in worker_list:
if worker_name == worker_id:
utils.send_command(worker_name, 'report')
time.sleep(1)
return HTTPFound(location = '/workerstatus')
# 更新调度策略并进行调度
ztq_core.set_dispatcher_config(dispatcher_config)
utils.dispatch()
return HTTPFound(location = '/workerstatus')
def init_job_threads(config_dict):
# 启动工作线程
set_job_threads(config_dict)
# 保存信息
config = {}
for queue_name, values in config_dict.items():
sleeps = [str(x['interval']) for x in values]
config[queue_name] = ','.join(sleeps)
set_config(config, 'queues')
# 将一些信息补全，让监视界面认为这个worker已经启动
alias = safe_get_host('server', 'alias')
# 初始化
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = {'queue_weight': {}, 'worker_weight': {}}
# set dispatcher config
worker_weight = dispatcher_config['worker_weight']
if not worker_weight.get(alias, 0):
worker_weight[alias] = 5
ztq_core.set_dispatcher_config(dispatcher_config)
# set worker config
worker_config = ztq_core.get_worker_config()
worker_config[alias] = config_dict
def main(global_config, frs_root='frs', init_dispatcher_config='true', \
frs_cache='frscache', addon_config=None, work_enable=True, **settings):
""" This function returns a Pyramid WSGI application.
"""
# 是否启用sentinel
enable_sentinel = settings.get('enable_sentinel', 'false').lower() == 'true'
# 如果启用sentinel，则关于redis的host,port,db都为对sentinel的配置
if enable_sentinel:
# 主机列表
hosts = settings.get('sentinel_hosts', None)
assert(hosts)
# sentinel的所有services name
services = settings.get('sentinel_names', None)
assert(services)
# 使用的数据库
db = int(settings.get('sentinel_db', '0'))
assert(db >= 0)
services = services.split(',')
ztq_core.setup_sentinel('default',
map(lambda x: (x[0], int(x[1])), [host.split(':') for host in hosts.split(',')]),
services, db = db)
# 如果启用了sentinel
# servers 列表为所有的 services
MENU_CONFIG['servers'] = services
MENU_CONFIG['current_redis'] = services[0]
MENU_CONFIG['enable_sentinel'] = True
else:
# 初始化servers
# servers 格式
# name:host:port:db:title, ......
servers = settings.get('servers', None)
# servers 作为必须的配置项
# 取消原来的redis_host,redis_port,redis_db配置
assert(servers)
for server in servers.split(','):
texts = server.split(':')
# 单个server的配置项必须介于4-5之间
assert(len(texts) >= 4 and len(texts) <= 5)
# 添加到待管理的服务器列表中
MENU_CONFIG['servers'].append({
'name' : texts[0],
'host' : texts[1],
'port' : int(texts[2]),
'db' : int(texts[3]),
'title' : texts[4] if len(texts) == 5 else texts[0],
})
# 默认将列表中的第一个服务器作为默认服务器
current_redis = MENU_CONFIG['servers'][0]
MENU_CONFIG['current_redis'] = current_redis['name']
#
# 初始化Redis连接
ztq_core.setup_redis('default'
, current_redis['host']
, current_redis['port'], current_redis['db'])
MENU_CONFIG['enable_sentinel'] = False
# 初始化权重数据数据,如果权重配置已经存在则pass
if init_dispatcher_config.lower() == 'true':
# init_dispatcher_config 是因为控制台可能没有运行服务， 这里去读取redis数据，会导致控制台起不来
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = weight = {'queue_weight':{},'worker_weight':{}}
ztq_core.set_dispatcher_config(weight)
queue_weight = dispatcher_config['queue_weight']
if not queue_weight:
queues_list = ztq_core.get_queue_config()
for queue_name, queue_config in queues_list.items():
queue_weight[queue_name] = queue_config.get('weight', 0)
ztq_core.set_dispatcher_config(dispatcher_config)
# # 开启后台服务
# 初始化fts_web配置
authn_policy = AuthTktAuthenticationPolicy('sosecret', callback=groupfinder, hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
settings = dict(settings)
settings.setdefault('jinja2.directories', 'ztq_console:templates')
config = Configurator(settings=settings, root_factory='ztq_console.utils.models.RootFactory')
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
config.begin()
config.add_renderer('.html', pyramid_jinja2.renderer_factory)
config.add_static_view('static', 'ztq_console:static')
config.scan('ztq_console.views')
config.add_route('login', '/login')
config.add_route('logout', '/logout')
config.add_route('password', '/password' )
config.add_route('worker', '/worker/{id}',
view='ztq_console.views.config_worker')
config.add_route('end_thread', '/worker/{id}/{thread}/{pid}',
view='ztq_console.views.stop_working_job')
config.add_route('taskqueue', '/taskqueues/{id}')
config.add_route('taskqueues_config', '/taskqueues/{id}/config',
view='ztq_console.views.config_queue')
config.add_route('taskqueue_action', '/taskqueues_action/{id}')
config.add_route('errorqueues_job', '/errorqueues/{id}/job',
view='ztq_console.views.error_jobs_handler')
config.add_route('workerlog', '/workerlog/{page}')
config.add_route('syslog', '/syslog/{page}')
config.add_route('errorlog', '/errorlog/{page}')
config.add_route('errorqueue', '/errorqueue/{id}/{page}')
config.add_route('redo_all_error_for_queue', '/redo_all_error_for_queue/{id}')
config.add_route('del_all_error_for_queue', '/del_all_error_for_queue/{id}')
if addon_config is not None:
addon_config(config)
config.end()
return config.make_wsgi_app()
def main(global_config, redis_host='127.0.0.1', redis_port='6379', \
redis_db='0', frs_root='frs', init_dispatcher_config='true', \
frs_cache='frscache', addon_config=None, work_enable=True, **settings):
""" This function returns a Pyramid WSGI application.
"""
# 初始化Redis连接
ztq_core.setup_redis(
'default',
redis_host,
port=int(redis_port),
db=int(redis_db),
)
# 初始化权重数据数据,如果权重配置已经存在则pass
if init_dispatcher_config.lower() == 'true':
# init_dispatcher_config 是因为控制台可能没有运行服务， 这里去读取redis数据，会导致控制台起不来
dispatcher_config = ztq_core.get_dispatcher_config()
if not dispatcher_config:
dispatcher_config = weight = {
'queue_weight': {},
'worker_weight': {}
}
ztq_core.set_dispatcher_config(weight)
queue_weight = dispatcher_config['queue_weight']
if not queue_weight:
queues_list = ztq_core.get_queue_config()
for queue_name, queue_config in queues_list.items():
queue_weight[queue_name] = queue_config.get('weight', 0)
ztq_core.set_dispatcher_config(dispatcher_config)
# # 开启后台服务
# 初始化fts_web配置
settings = dict(settings)
settings.setdefault('jinja2.directories', 'ztq_console:templates')
config = Configurator(settings=settings)
config.begin()
config.add_renderer('.html', pyramid_jinja2.renderer_factory)
config.add_static_view('static', 'ztq_console:static')
config.scan('ztq_console.views')
config.add_route('worker',
'/worker/{id}',
view='ztq_console.views.config_worker')
config.add_route('end_thread',
'/worker/{id}/{thread}/{pid}',
view='ztq_console.views.stop_working_job')
config.add_route('taskqueue', '/taskqueues/{id}')
config.add_route('taskqueues_config',
'/taskqueues/{id}/config',
view='ztq_console.views.config_queue')
config.add_route('taskqueue_action', '/taskqueues_action/{id}')
config.add_route('errorqueues_job',
'/errorqueues/{id}/job',
view='ztq_console.views.error_jobs_handler')
config.add_route('workerlog', '/workerlog/{page}')
config.add_route('syslog', '/syslog/{page}')
config.add_route('errorlog', '/errorlog/{page}')
config.add_route('errorqueue', '/errorqueue/{id}/{page}')
config.add_route('redo_all_error_for_queue',
'/redo_all_error_for_queue/{id}')
config.add_route('del_all_error_for_queue',
'/del_all_error_for_queue/{id}')
if addon_config is not None:
addon_config(config)
config.end()
return config.make_wsgi_app()