Пример #1
0
 def run(self):
     self.init()
     # 监听指令
     commands = ztq_core.get_command_queue(self.worker_name)
     while True:
         try:
             command = commands.pop()
             if command['command'] == 'report':
                 worker_state = ztq_core.get_worker_state()
                 worker_state[self.worker_name] = report(self.login_time)
             elif command['command'] == 'updatedriver':
                 # TODO
                 #async_drive_config()
                 pass
             elif command['command'] == 'updateworker':
                 queue = ztq_core.get_worker_config()
                 set_job_threads(queue[self.worker_name])
             elif command['command'] == 'kill':
                 kill_transform(pid=command['pid'], timestamp=command['timestamp'])
             elif command['command'] == 'cancel':
                 cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
         except KeyboardInterrupt:
             import os
             # 实际上调用的是command_execute.clear_thread
             os.sys.exitfunc()
             os._exit(0)
Пример #2
0
 def run(self):
     self.init()
     # 监听指令
     commands = ztq_core.get_command_queue(self.worker_name)
     while True:
         try:
             command = commands.pop()
             if command['command'] == 'report':
                 worker_state = ztq_core.get_worker_state()
                 worker_state[self.worker_name] = report(self.login_time)
             elif command['command'] == 'updatedriver':
                 # TODO
                 #async_drive_config()
                 pass
             elif command['command'] == 'updateworker' and \
                     CONFIG['server'].get('active_config', 'false').lower() == 'true':
                 queue = ztq_core.get_worker_config()
                 set_job_threads(queue[self.worker_name])
             elif command['command'] == 'kill':
                 kill_transform(pid=command['pid'], timestamp=command['timestamp'])
             elif command['command'] == 'cancel':
                 cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
         except ztq_core.ConnectionError, e:
             logger.error('ERROR: redis command connection error: %s' % str(e))
             time.sleep(3)
         except ztq_core.ResponseError, e:
             logger.error('ERROR: redis command response error: %s' % str(e))
             time.sleep(3)
Пример #3
0
 def run(self):
     self.init()
     # 监听指令
     commands = ztq_core.get_command_queue(self.worker_name)
     while True:
         try:
             command = commands.pop()
             if command['command'] == 'report':
                 worker_state = ztq_core.get_worker_state()
                 worker_state[self.worker_name] = report(self.login_time)
             elif command['command'] == 'updatedriver':
                 # TODO
                 #async_drive_config()
                 pass
             elif command['command'] == 'updateworker':
                 queue = ztq_core.get_worker_config()
                 set_job_threads(queue[self.worker_name])
             elif command['command'] == 'kill':
                 kill_transform(pid=command['pid'],
                                timestamp=command['timestamp'])
             elif command['command'] == 'cancel':
                 cancel_transform(pid=command['pid'],
                                  timestamp=command['timestamp'])
         except KeyboardInterrupt:
             import os
             # 实际上调用的是command_execute.clear_thread
             os.sys.exitfunc()
             os._exit(0)
Пример #4
0
def main():
    """ 主函数 """
    conf_file = ''
    # 用户指定一个配置文件
    if len(sys.argv) > 1:
        conf_file = sys.argv[1]

    init_config(conf_file)
    server = get_configs('server')

    alias = safe_get_host('server', 'alias')
    active_config = server.get('active_config', 'false')

    # 动态注册task
    for module in server['modules'].split():
        try:
            importlib.import_module(module)
        except ImportError:
            raise Exception('Not imported the %s module' % module)

    # 连结服务器
    ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db']))

    # 开启一个命令线程
    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write('Starting server in PID %s\n'%os.getpid())

    worker_state = ztq_core.get_worker_state()
    if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif get_configs('queues'):
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        queue_config = ztq_core.get_queue_config()
        for queue_name, sleeps in get_configs('queues').items():
            queue_config[queue_name] = {'title': queue_name} # for ztq_app
            job_threads[queue_name] = [
                    {'interval': int(sleep)} for sleep in sleeps.split(',')
                    ]
            queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}

        init_job_threads(job_threads)

    loggers = get_configs('log')
    initlog(
        loggers.get('key', 'ztq_worker'), 
        loggers.get('handler_file'), 
        loggers.get('level', 'ERROR'),
        )

    # 不是以线程启动
    command_thread.run()
Пример #5
0
def main(config):
    """ 主函数 

    config: {'server': {host:, port:, db:}
            }
    """

    server = config["server"]
    # 动态注册task
    for module in server["modules"].split():
        try:
            __import__(module)
        except ImportError:
            modules = module.split(".")
            __import__(modules[0], globals(), locals(), modules[1])

    # 连结服务器
    redis_host = server["host"]
    redis_port = int(server["port"])
    redis_db = int(server["db"])
    ztq_core.setup_redis("default", host=redis_host, port=redis_port, db=redis_db)

    # 开启一个命令线程
    alias = server.get("alias", "")
    if not alias:
        alias = get_ip()
        server["alias"] = alias

    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write("Starting server in PID %s\n" % os.getpid())

    worker_state = ztq_core.get_worker_state()
    active_config = server.get("active_config", "false")
    if active_config.lower() == "true" and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif config["queues"]:
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        for queue_name, sleeps in config["queues"].items():
            job_threads[queue_name] = [{"interval": int(sleep)} for sleep in sleeps.split(",")]
            if not queue_config.get(queue_name, []):
                queue_config[queue_name] = {"name": queue_name, "title": queue_name, "widget": 5}

        init_job_threads(job_threads)

    loggers = config["log"]
    initlog(loggers.get("key", "ztq_worker"), loggers.get("handler_file"), loggers.get("level", "ERROR"))

    # 不是以线程启动
    command_thread.run()
Пример #6
0
def main():
    """ 主函数 """
    conf_file = ''
    # 用户指定一个配置文件
    if len(sys.argv) > 1:
        conf_file = sys.argv[1]

    init_config(conf_file)
    server = get_configs('server')

    alias = safe_get_host('server', 'alias')
    active_config = server.get('active_config', 'false')

    # 动态注册task
    for module in server['modules'].split():
        try:
            __import__(module)
        except ImportError:
            raise Exception('Not imported the %s module' % module)

    # 连结服务器
    ztq_core.setup_redis('default',
                         host=server['host'],
                         port=int(server['port']),
                         db=int(server['db']))

    # 开启一个命令线程
    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write('Starting server in PID %s\n' % os.getpid())

    worker_state = ztq_core.get_worker_state()
    if active_config.lower(
    ) == 'true' and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif get_configs('queues'):
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        for queue_name, sleeps in get_configs('queues').items():
            job_threads[queue_name] = [{
                'interval': int(sleep)
            } for sleep in sleeps.split(',')]
            if not queue_config.get(queue_name, []):
                queue_config[queue_name] = {
                    'name': queue_name,
                    'title': queue_name,
                    'widget': 5
                }

        init_job_threads(job_threads)

    loggers = get_configs('log')
    initlog(
        loggers.get('key', 'ztq_worker'),
        loggers.get('handler_file'),
        loggers.get('level', 'ERROR'),
    )

    # 不是以线程启动
    command_thread.run()
Пример #7
0
Файл: main.py Проект: uimeet/ztq
def main(config):
    """ 主函数 

    config: {'server': {host:, port:, db:}
            }
    """

    server = config['server']
    module_path = server['module_path']
    sys.path.append(module_path)
    # 动态注册task
    for module in server['modules'].split():
        try:
            __import__(module)
        except ImportError:
            modules = module.split('.')
            __import__(modules[0], globals(), locals(), modules[1])

    # 连结服务器
    redis_host = server['host']
    redis_port = int(server['port'])
    redis_db = int(server['db'])
    # 是否启用 sentinel
    enable_sentinel = server['enable_sentinel'].lower() == 'true'
    # 对应的 sentinel service_name
    sentinel_name = server['sentinel_name']
    if enable_sentinel:
        # 当启用 sentinel 时
        # 配置的host, port, db等信息变为了sentinel的主机信息
        ztq_core.setup_sentinel('default', [(redis_host, redis_port)],
                [sentinel_name], db = redis_db)
    else:
        ztq_core.setup_redis('default', host=redis_host, port=redis_port, db=redis_db)

    # 开启一个命令线程
    alias = server.get('alias', '')
    if not alias:
        alias = get_ip()
        server['alias'] = alias

    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write('Starting server in PID %s\n'%os.getpid())

    worker_state = ztq_core.get_worker_state()
    active_config = server.get('active_config', 'false')
    if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif config['queues']:
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        for queue_name, sleeps in config['queues'].items():
            job_threads[queue_name] = [
                    {'interval': int(sleep)} for sleep in sleeps.split(',')
                    ]
            if not queue_config.get(queue_name, []):
                queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}

        init_job_threads(job_threads)

    loggers = config['log']
    initlog(
        loggers.get('key', 'ztq_worker'), 
        loggers.get('handler_file'), 
        loggers.get('level', 'ERROR'),
        )

    # 不是以线程启动
    command_thread.run()