예제 #1
0
파일: views.py 프로젝트: baojie/ztq
def config_queue(request):
    """管理队列权重,提升队列权重或降低队列权重
       传入参数:http://server/taskqueues/q01/config?action=queue_down
    """
    queue_id = request.matchdict['id']
    url_action = request.params.get('action','')
    dispatcher_config = ztq_core.get_dispatcher_config()
    queue_weight = dispatcher_config['queue_weight']

    # 根据操作类型进行权重调整,
    if url_action == 'queue_down' : 
        queue_weight[queue_id] -= 1
        # 队列权重最少为1
        if queue_weight[queue_id] < 0: queue_weight[queue_id] = 0
        # 更新调度策略并进行调度
        ztq_core.set_dispatcher_config(dispatcher_config)
        utils.dispatch()
    elif url_action == 'queue_up' : 
        queue_weight[queue_id] += 1
        if queue_weight[queue_id] > 10: queue_weight[queue_id] = 10
        # 更新调度策略并进行调度
        ztq_core.set_dispatcher_config(dispatcher_config)
        utils.dispatch()
    elif url_action == 'from_right' : 
        utils.dispatch_single_queue(queue_id, from_right=True)
    elif url_action == 'from_left' : 
        utils.dispatch_single_queue(queue_id, from_right=False)   
    return HTTPFound(location = '/taskqueues') 
예제 #2
0
def init_job_threads(config_dict):
    # 启动工作线程
    set_job_threads(config_dict)
    # 保存信息
    config = {}
    for queue_name, values in config_dict.items():
        sleeps = [str(x['interval']) for x in values]
        config[queue_name] = ','.join(sleeps)

    CONFIG['queues'].update(config)

    # 将一些信息补全,让监视界面认为这个worker已经启动
    alias = CONFIG['server']['alias']
    # 初始化
    dispatcher_config = ztq_core.get_dispatcher_config()
    if not dispatcher_config: 
        dispatcher_config = {'queue_weight':{},'worker_weight':{}}
    # set dispatcher config
    worker_weight = dispatcher_config['worker_weight']
    if not worker_weight.get(alias, 0):
        worker_weight[alias] = 5 
        ztq_core.set_dispatcher_config(dispatcher_config)
    # set worker config
    worker_config = ztq_core.get_worker_config()
    worker_config[alias] = config_dict 
예제 #3
0
파일: __init__.py 프로젝트: baojie/ztq
def main(global_config, redis_host='127.0.0.1', redis_port='6379', \
        redis_db='0', frs_root='frs', init_dispatcher_config='true', \
        frs_cache='frscache', addon_config=None, work_enable=True, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    # 初始化Redis连接
    ztq_core.setup_redis('default', redis_host, port=int(redis_port), db=int(redis_db),)

    # 初始化权重数据数据,如果权重配置已经存在则pass
    if init_dispatcher_config.lower() == 'true':
        # init_dispatcher_config 是因为控制台可能没有运行服务, 这里去读取redis数据,会导致控制台起不来
        dispatcher_config = ztq_core.get_dispatcher_config()
        if not dispatcher_config: 
            dispatcher_config = weight = {'queue_weight':{},'worker_weight':{}}
            ztq_core.set_dispatcher_config(weight)

        queue_weight = dispatcher_config['queue_weight']
        if not queue_weight:
            queues_list = ztq_core.get_queue_config()
            for queue_name, queue_config in queues_list.items():
                queue_weight[queue_name] = queue_config.get('weight', 0)
            ztq_core.set_dispatcher_config(dispatcher_config)

    # # 开启后台服务
    # 初始化fts_web配置
    settings = dict(settings)
    settings.setdefault('jinja2.directories', 'ztq_console:templates')
    config = Configurator(settings=settings)
    config.begin()
    config.add_renderer('.html', pyramid_jinja2.renderer_factory)
    config.add_static_view('static', 'ztq_console:static')
    config.scan('ztq_console.views')  
    config.add_route('worker', '/worker/{id}', 
                    view='ztq_console.views.config_worker')
    config.add_route('end_thread', '/worker/{id}/{thread}/{pid}', 
                    view='ztq_console.views.stop_working_job')                    
    config.add_route('taskqueue', '/taskqueues/{id}')
    config.add_route('taskqueues_config', '/taskqueues/{id}/config', 
                    view='ztq_console.views.config_queue')
    config.add_route('taskqueue_action', '/taskqueues_action/{id}')
    config.add_route('errorqueues_job', '/errorqueues/{id}/job',
                    view='ztq_console.views.error_jobs_handler')    
    config.add_route('workerlog', '/workerlog/{page}')
    config.add_route('syslog', '/syslog/{page}')
    config.add_route('errorlog', '/errorlog/{page}')
    config.add_route('errorqueue', '/errorqueue/{id}/{page}')
    config.add_route('redo_all_error_for_queue', '/redo_all_error_for_queue/{id}')
    config.add_route('del_all_error_for_queue', '/del_all_error_for_queue/{id}')
    if addon_config is not None:
        addon_config(config)
    config.end()

    return config.make_wsgi_app()
예제 #4
0
파일: dispatch.py 프로젝트: baojie/ztq
def dispatch():
    """根据调度参数进行转换器和队列的调度
    """
    #获取调度参数
    dispatcher_config = ztq_core.get_dispatcher_config()
    worker_weight = dispatcher_config['worker_weight']
    if not worker_weight: return
    queue_weight = dispatcher_config['queue_weight']
    worker_list = ztq_core.get_all_worker()
    
    #生成新的转换器工作线程配置
    if worker_list:
        gen_worker_config(worker_weight, queue_weight, worker_list) 
예제 #5
0
파일: views.py 프로젝트: baojie/ztq
def config_worker(request):
    """对worker进行配置管理
    """
    url_action = request.params.get('action','')

    dispatcher_config = ztq_core.get_dispatcher_config()
    worker_weight = dispatcher_config['worker_weight']
    # 获取用户请求操作
    worker_id = request.matchdict['id']
    # 根据操作类型进行权重调整,
    if url_action == 'stop_worker': 
        #停止worker
        worker_weight[worker_id] = 0
    elif url_action == 'enable': 
        #启用worker
        worker_weight[worker_id] = 5
    elif url_action == 'worker_down' : 
        #降低worker权重
        worker_weight[worker_id] -= 1
        if worker_weight[worker_id] < 1: worker_weight[worker_id] = 1
    elif url_action == 'worker_up' : 
        #提升worker权重
        worker_weight[worker_id] += 1
        if worker_weight[worker_id] >10: worker_weight[worker_id] = 10
    elif url_action == 'delete': 
        #删除还没启用的worker,删除操作不会导致调度配置更新
        if worker_id in worker_weight: # 没有启用的情况
            worker_weight.pop(worker_id)
        workers_dict = ztq_core.get_worker_state()
        del workers_dict[worker_id]
        worker_job = ztq_core.get_job_state(worker_id)
        for job_name, job_status in worker_job.items():
            del worker_job[job_name]
        ztq_core.set_dispatcher_config(dispatcher_config)
        return HTTPFound(location = '/workerstatus')
    elif url_action == 'update':
        # 发报告指令到各命令队列让worker报告自身状态
        worker_list = ztq_core.get_all_worker()
        for worker_name in worker_list:
            if worker_name == worker_id:
                utils.send_command(worker_name, 'report')
                time.sleep(1)
                return HTTPFound(location = '/workerstatus')
    # 更新调度策略并进行调度        
    ztq_core.set_dispatcher_config(dispatcher_config)
    utils.dispatch()
    return HTTPFound(location = '/workerstatus')
예제 #6
0
파일: get_fts_data.py 프로젝트: baojie/ztq
def get_taskqueues_list():
    # 队列情况列表
    dispatcher_config = ztq_core.get_dispatcher_config()
    queue_weight = dispatcher_config['queue_weight']
    queues_list = ztq_core.get_queue_config()

    # 排序
    sort_queue_name = {}
    for queue_name, queue_config in queues_list.items():
        sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name))
    
    for queue_name in sorted(sort_queue_name, 
                            key=lambda x: sort_queue_name[x], 
                            reverse=True):
        task_queue = {}
        task_queue['name'] = queue_name
        #task_queue['tags'] = queue_config.get('tags',())
        queue = ztq_core.get_task_queue(queue_name)
        # 任务数/错误数
        task_queue['length'] = len(queue)
        task_queue['error_length'] = sort_queue_name[queue_name]

        #任务首个时间
        task_queue['error_end'] = task_queue['first'] = ''
        first_job = queue[0]
        first_job= ztq_core.get_task_hash(queue_name).get(first_job)
        if first_job:
            task_queue['first'] = datetime.datetime.fromtimestamp(first_job['runtime'].get('create', 0))
        
        #错误最末一个的时间
        error_first_job = ztq_core.get_error_queue(queue_name)[0]
        error_first_job = ztq_core.get_error_hash(queue_name).get(error_first_job)
        if error_first_job:
            task_queue['error_end'] = datetime.datetime.fromtimestamp(error_first_job['runtime'].get('create', 0))

        task_queue['weight'] = queue_weight.get(queue_name, 0)
        # 获取worker工作线程配置
        workers_config = ztq_core.get_worker_config()
        task_queue['from_right'] = True
        for worker_name,worker_config in workers_config.items():
            task_queue['workers'] = []
            for config in worker_config.get(queue_name,[]):
                task_queue['workers'].append([worker_name+':', config['interval']])
                if 'from_right' in config:
                    task_queue['from_right'] = config['from_right']
        task_queue['buffer_length'] = len(ztq_core.get_buffer_queue(queue_name))
        yield task_queue
예제 #7
0
def get_worker_list():
    dispatcher_config = ztq_core.get_dispatcher_config()
    worker_weight = dispatcher_config['worker_weight']
    workers_dict = ztq_core.get_worker_state().items()
    for worker_name, worker_status in workers_dict:
        worker_status['_worker_name'] = worker_name
        worker_status['_started'] = \
            datetime.datetime.fromtimestamp(worker_status['started'])
        worker_status['_timestamp'] = \
            datetime.datetime.fromtimestamp(worker_status['timestamp'])
        worker_status['_worker_weight'] = worker_weight.get(worker_name, 0)
        # 检查worker是否在工作
        cmd_queue = ztq_core.get_command_queue(worker_name)
        # 如果指令队列不为空的话,意味着worker没工作,属于下线状态
        if cmd_queue: worker_status['_active'] = u'shutdown'
        elif worker_status['_worker_weight'] == 0:
            worker_status['_active'] = u'ldle'
        else:
            worker_status['_active'] = u'work'
        # 获取worker开了多少个线程
        worker_job = ztq_core.get_job_state(worker_name)
        worker_status['_threads'] = []
        for thread_name, thread_status in worker_job.items():
            thread_status['_detail'] = pprint.pformat(thread_status)
            thread_status['_name'] = thread_name
            thread_status['_comment'] = thread_status['kw'].get(
                'comment', thread_status['process'].get('comment', ''))
            thread_status['_pid'] = thread_status['process'].get('pid', -1)
            ident = unicode(thread_status['process'].get('ident', -1))
            if ident in worker_status['traceback']:
                thread_status['_thread_detail'] = pprint.pformat(
                    worker_status['traceback'][ident])
            # 任务进行了多少时间
            used_time = int(time.time()) - thread_status['process']['start']
            if used_time > 3600:
                used_time = u'%.2f小时' % (used_time / 3600.0)
            elif used_time > 60:
                used_time = u'%.2f分钟' % (used_time / 60.0)
            thread_status['_take_time'] = used_time

            worker_status['_threads'].append(thread_status)

        yield worker_status
예제 #8
0
파일: get_fts_data.py 프로젝트: baojie/ztq
def get_worker_list():
    dispatcher_config = ztq_core.get_dispatcher_config()
    worker_weight = dispatcher_config['worker_weight']
    workers_dict = ztq_core.get_worker_state().items()
    for worker_name, worker_status in workers_dict:
        worker_status['_worker_name'] = worker_name
        worker_status['_started'] = \
            datetime.datetime.fromtimestamp(worker_status['started'])
        worker_status['_timestamp'] = \
            datetime.datetime.fromtimestamp(worker_status['timestamp'])
        worker_status['_worker_weight'] = worker_weight.get(worker_name, 0)
        # 检查worker是否在工作
        cmd_queue = ztq_core.get_command_queue(worker_name)
        # 如果指令队列不为空的话,意味着worker没工作,属于下线状态
        if cmd_queue: worker_status['_active'] = u'shutdown'
        elif worker_status['_worker_weight'] == 0: 
            worker_status['_active'] = u'ldle'
        else: worker_status['_active'] = u'work'
        # 获取worker开了多少个线程
        worker_job = ztq_core.get_job_state(worker_name)
        worker_status['_threads'] = []
        for thread_name,thread_status in worker_job.items():
            thread_status['_detail'] = pprint.pformat(thread_status)
            thread_status['_name'] = thread_name
            thread_status['_comment'] = thread_status['kw'].get('comment',thread_status['process'].get('comment', ''))
            thread_status['_pid'] = thread_status['process'].get('pid', -1)
            ident = unicode(thread_status['process'].get('ident', -1))
            if ident in worker_status['traceback']:
                thread_status['_thread_detail'] = pprint.pformat(worker_status['traceback'][ident])
            # 任务进行了多少时间
            used_time = int(time.time())-thread_status['process']['start']
            if used_time > 3600:
                used_time = u'%.2f小时' % (used_time / 3600.0)
            elif used_time > 60:
                used_time = u'%.2f分钟' % (used_time / 60.0)
            thread_status['_take_time'] = used_time

            worker_status['_threads'].append(thread_status)

        yield worker_status
예제 #9
0
def init_job_threads(config_dict):
    # 启动工作线程
    set_job_threads(config_dict)
    # 保存信息
    config = {}
    for queue_name, values in config_dict.items():
        sleeps = [str(x['interval']) for x in values]
        config[queue_name] = ','.join(sleeps)
    set_config(config, 'queues')

    # 将一些信息补全,让监视界面认为这个worker已经启动
    alias = safe_get_host('server', 'alias')
    # 初始化
    dispatcher_config = ztq_core.get_dispatcher_config()
    if not dispatcher_config:
        dispatcher_config = {'queue_weight': {}, 'worker_weight': {}}
    # set dispatcher config
    worker_weight = dispatcher_config['worker_weight']
    if not worker_weight.get(alias, 0):
        worker_weight[alias] = 5
        ztq_core.set_dispatcher_config(dispatcher_config)
    # set worker config
    worker_config = ztq_core.get_worker_config()
    worker_config[alias] = config_dict
예제 #10
0
파일: __init__.py 프로젝트: uimeet/ztq
def main(global_config, frs_root='frs', init_dispatcher_config='true', \
        frs_cache='frscache', addon_config=None, work_enable=True, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    
    # 是否启用sentinel
    enable_sentinel = settings.get('enable_sentinel', 'false').lower() == 'true'
    # 如果启用sentinel,则关于redis的host,port,db都为对sentinel的配置
    if enable_sentinel:
        # 主机列表
        hosts = settings.get('sentinel_hosts', None)
        assert(hosts)
        # sentinel的所有services name
        services = settings.get('sentinel_names', None)
        assert(services)
        # 使用的数据库
        db = int(settings.get('sentinel_db', '0'))
        assert(db >= 0)

        services = services.split(',')
        ztq_core.setup_sentinel('default', 
                map(lambda x: (x[0], int(x[1])), [host.split(':') for host in hosts.split(',')]),
                services, db = db)
        # 如果启用了sentinel
        # servers 列表为所有的 services
        MENU_CONFIG['servers'] = services
        MENU_CONFIG['current_redis'] = services[0]
        MENU_CONFIG['enable_sentinel'] = True
    else:
        # 初始化servers
        # servers 格式
        # name:host:port:db:title, ......
        servers = settings.get('servers', None)
        # servers 作为必须的配置项
        # 取消原来的redis_host,redis_port,redis_db配置
        assert(servers)
        for server in servers.split(','):
            texts = server.split(':')
            # 单个server的配置项必须介于4-5之间
            assert(len(texts) >= 4 and len(texts) <= 5)
            # 添加到待管理的服务器列表中
            MENU_CONFIG['servers'].append({
                    'name'  : texts[0],
                    'host'  : texts[1],
                    'port'  : int(texts[2]),
                    'db'    : int(texts[3]),
                    'title' : texts[4] if len(texts) == 5 else texts[0],
                })
        # 默认将列表中的第一个服务器作为默认服务器
        current_redis = MENU_CONFIG['servers'][0]
        MENU_CONFIG['current_redis'] = current_redis['name']
        #
        # 初始化Redis连接
        ztq_core.setup_redis('default'
                , current_redis['host']
                , current_redis['port'], current_redis['db'])
        MENU_CONFIG['enable_sentinel'] = False

    # 初始化权重数据数据,如果权重配置已经存在则pass
    if init_dispatcher_config.lower() == 'true':
        # init_dispatcher_config 是因为控制台可能没有运行服务, 这里去读取redis数据,会导致控制台起不来
        dispatcher_config = ztq_core.get_dispatcher_config()
        if not dispatcher_config: 
            dispatcher_config = weight = {'queue_weight':{},'worker_weight':{}}
            ztq_core.set_dispatcher_config(weight)

        queue_weight = dispatcher_config['queue_weight']
        if not queue_weight:
            queues_list = ztq_core.get_queue_config()
            for queue_name, queue_config in queues_list.items():
                queue_weight[queue_name] = queue_config.get('weight', 0)
            ztq_core.set_dispatcher_config(dispatcher_config)

    # # 开启后台服务
    # 初始化fts_web配置
    authn_policy = AuthTktAuthenticationPolicy('sosecret', callback=groupfinder, hashalg='sha512')
    authz_policy = ACLAuthorizationPolicy()
    settings = dict(settings)
    settings.setdefault('jinja2.directories', 'ztq_console:templates')
    config = Configurator(settings=settings, root_factory='ztq_console.utils.models.RootFactory')
    config.set_authentication_policy(authn_policy)
    config.set_authorization_policy(authz_policy)
    config.begin()
    config.add_renderer('.html', pyramid_jinja2.renderer_factory)
    config.add_static_view('static', 'ztq_console:static')
    config.scan('ztq_console.views')  
    config.add_route('login', '/login')
    config.add_route('logout', '/logout')
    config.add_route('password', '/password' )
    config.add_route('worker', '/worker/{id}', 
                    view='ztq_console.views.config_worker')
    config.add_route('end_thread', '/worker/{id}/{thread}/{pid}', 
                    view='ztq_console.views.stop_working_job') 
    config.add_route('taskqueue', '/taskqueues/{id}')
    config.add_route('taskqueues_config', '/taskqueues/{id}/config', 
                    view='ztq_console.views.config_queue')
    config.add_route('taskqueue_action', '/taskqueues_action/{id}')
    config.add_route('errorqueues_job', '/errorqueues/{id}/job',
                    view='ztq_console.views.error_jobs_handler')    
    config.add_route('workerlog', '/workerlog/{page}')
    config.add_route('syslog', '/syslog/{page}')
    config.add_route('errorlog', '/errorlog/{page}')
    config.add_route('errorqueue', '/errorqueue/{id}/{page}')
    config.add_route('redo_all_error_for_queue', '/redo_all_error_for_queue/{id}')
    config.add_route('del_all_error_for_queue', '/del_all_error_for_queue/{id}')
    if addon_config is not None:
        addon_config(config)
    config.end()

    return config.make_wsgi_app()
예제 #11
0
def main(global_config, redis_host='127.0.0.1', redis_port='6379', \
        redis_db='0', frs_root='frs', init_dispatcher_config='true', \
        frs_cache='frscache', addon_config=None, work_enable=True, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    # 初始化Redis连接
    ztq_core.setup_redis(
        'default',
        redis_host,
        port=int(redis_port),
        db=int(redis_db),
    )

    # 初始化权重数据数据,如果权重配置已经存在则pass
    if init_dispatcher_config.lower() == 'true':
        # init_dispatcher_config 是因为控制台可能没有运行服务, 这里去读取redis数据,会导致控制台起不来
        dispatcher_config = ztq_core.get_dispatcher_config()
        if not dispatcher_config:
            dispatcher_config = weight = {
                'queue_weight': {},
                'worker_weight': {}
            }
            ztq_core.set_dispatcher_config(weight)

        queue_weight = dispatcher_config['queue_weight']
        if not queue_weight:
            queues_list = ztq_core.get_queue_config()
            for queue_name, queue_config in queues_list.items():
                queue_weight[queue_name] = queue_config.get('weight', 0)
            ztq_core.set_dispatcher_config(dispatcher_config)

    # # 开启后台服务
    # 初始化fts_web配置
    settings = dict(settings)
    settings.setdefault('jinja2.directories', 'ztq_console:templates')
    config = Configurator(settings=settings)
    config.begin()
    config.add_renderer('.html', pyramid_jinja2.renderer_factory)
    config.add_static_view('static', 'ztq_console:static')
    config.scan('ztq_console.views')
    config.add_route('worker',
                     '/worker/{id}',
                     view='ztq_console.views.config_worker')
    config.add_route('end_thread',
                     '/worker/{id}/{thread}/{pid}',
                     view='ztq_console.views.stop_working_job')
    config.add_route('taskqueue', '/taskqueues/{id}')
    config.add_route('taskqueues_config',
                     '/taskqueues/{id}/config',
                     view='ztq_console.views.config_queue')
    config.add_route('taskqueue_action', '/taskqueues_action/{id}')
    config.add_route('errorqueues_job',
                     '/errorqueues/{id}/job',
                     view='ztq_console.views.error_jobs_handler')
    config.add_route('workerlog', '/workerlog/{page}')
    config.add_route('syslog', '/syslog/{page}')
    config.add_route('errorlog', '/errorlog/{page}')
    config.add_route('errorqueue', '/errorqueue/{id}/{page}')
    config.add_route('redo_all_error_for_queue',
                     '/redo_all_error_for_queue/{id}')
    config.add_route('del_all_error_for_queue',
                     '/del_all_error_for_queue/{id}')
    if addon_config is not None:
        addon_config(config)
    config.end()

    return config.make_wsgi_app()