Exemplo n.º 1
0
def main():
    """ 主函数 """
    conf_file = ''
    # 用户指定一个配置文件
    if len(sys.argv) > 1:
        conf_file = sys.argv[1]

    init_config(conf_file)
    server = get_configs('server')

    alias = safe_get_host('server', 'alias')
    active_config = server.get('active_config', 'false')

    # 动态注册task
    for module in server['modules'].split():
        try:
            importlib.import_module(module)
        except ImportError:
            raise Exception('Not imported the %s module' % module)

    # 连结服务器
    ztq_core.setup_redis('default', host=server['host'], port=int(server['port']), db=int(server['db']))

    # 开启一个命令线程
    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write('Starting server in PID %s\n'%os.getpid())

    worker_state = ztq_core.get_worker_state()
    if active_config.lower() == 'true' and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif get_configs('queues'):
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        queue_config = ztq_core.get_queue_config()
        for queue_name, sleeps in get_configs('queues').items():
            queue_config[queue_name] = {'title': queue_name} # for ztq_app
            job_threads[queue_name] = [
                    {'interval': int(sleep)} for sleep in sleeps.split(',')
                    ]
            queue_config[queue_name] = {'name':queue_name, 'title':queue_name, 'widget': 5}

        init_job_threads(job_threads)

    loggers = get_configs('log')
    initlog(
        loggers.get('key', 'ztq_worker'), 
        loggers.get('handler_file'), 
        loggers.get('level', 'ERROR'),
        )

    # 不是以线程启动
    command_thread.run()
Exemplo n.º 2
0
    def run(self):
        """ 阻塞方式找到任务,并自动调用"""
        # 如果上次有任务在运行还没结束,重新执行
        jobs = ztq_core.get_job_state(safe_get_host('server', 'alias'))
        if self.name in jobs:
            self.start_job(jobs[self.name])

        # 队列批处理模式
        # batch_size: 批处理的阀值,达到这个阀值,就执行一次batch_func
        # batch_func: 
        #    1, 执行一批batch_size 大小的任务后,后续自动执行这个方法方法
        #    2, 执行一批小于batch_size 大小的任务后,再得不到任务,后续自动执行这个方法
        batch_config = CONFIG.get("batch_queue", {}).get(self.queue_name, {})
        batch_size = batch_config.get('batch_size', None) or -1
        batch_func = batch_config.get('batch_func', None) or (lambda *args, **kw: -1)

        run_job_index = 0
        queue_tiemout = QUEUE_TIMEOUT
        # 循环执行任务
        while not self._stop:
            try:
                task = ztq_core.pop_task(
                        self.queue_name, 
                        timeout=queue_tiemout, 
                        from_right=self.from_right
                        )
            except ztq_core.ConnectionError:
                print 'ERROR: Not connected the server\n'
                task = None
                time.sleep(3)

            if task is None: 
                # 没有后续任务了。执行batch_func
                if run_job_index > 0:
                    run_job_index = 0
                    queue_tiemout = QUEUE_TIMEOUT
                    batch_func()
                continue

            self.start_job(task)

            if batch_size > 0: 
                if run_job_index >= batch_size - 1:
                    # 完成了一批任务。执行batch_func
                    run_job_index = 0
                    queue_tiemout = QUEUE_TIMEOUT
                    batch_func()
                else:
                    run_job_index += 1
                    queue_tiemout = -1

            if self.sleep_time:
                time.sleep(self.sleep_time)
Exemplo n.º 3
0
    def run(self):
        """ 阻塞方式找到任务,并自动调用"""
        # 如果上次有任务在运行还没结束,重新执行
        jobs = ztq_core.get_job_state(safe_get_host('server', 'alias'))
        if self.name in jobs:
            self.start_job(jobs[self.name])

        # 队列批处理模式
        # batch_size: 批处理的阀值,达到这个阀值,就执行一次batch_func
        # batch_func:
        #    1, 执行一批batch_size 大小的任务后,后续自动执行这个方法方法
        #    2, 执行一批小于batch_size 大小的任务后,再得不到任务,后续自动执行这个方法
        batch_config = CONFIG.get("batch_queue", {}).get(self.queue_name, {})
        batch_size = batch_config.get('batch_size', None) or -1
        batch_func = batch_config.get('batch_func',
                                      None) or (lambda *args, **kw: -1)

        run_job_index = 0
        queue_tiemout = QUEUE_TIMEOUT
        # 循环执行任务
        while not self._stop:
            try:
                task = ztq_core.pop_task(self.queue_name,
                                         timeout=queue_tiemout,
                                         from_right=self.from_right)
            except ztq_core.ConnectionError:
                print 'ERROR: Not connected the server\n'
                task = None
                time.sleep(3)

            if task is None:
                # 没有后续任务了。执行batch_func
                if run_job_index > 0:
                    run_job_index = 0
                    queue_tiemout = QUEUE_TIMEOUT
                    batch_func()
                continue

            self.start_job(task)

            if batch_size > 0:
                if run_job_index >= batch_size - 1:
                    # 完成了一批任务。执行batch_func
                    run_job_index = 0
                    queue_tiemout = QUEUE_TIMEOUT
                    batch_func()
                else:
                    run_job_index += 1
                    queue_tiemout = -1

            if self.sleep_time:
                time.sleep(self.sleep_time)
Exemplo n.º 4
0
    def start_job(self, task):
        self.start_job_time = int(time.time())
        task['runtime'].update({
            'worker': safe_get_host('server', 'alias'),
            'thread': self.getName(),
            'start': self.start_job_time,
        })
        # 记录当前在做什么
        task['process'] = {'ident': self.ident}
        thread_context.job = task
        try:
            self.run_task = ztq_core.task_registry[task['func']]
            # started report
            report_job(comment='start the job')

            try:
                self.run_task(*task['args'], **task['kw'])
            except TypeError:
                # keyword must string is a bug in python
                if sys.version[:5] < '2.6.5':
                    raise Exception, ("We not supported %s version of python,"
                                      "Please update it to 2.6.5 or later." %
                                      sys.version[:5])
                else:
                    raise

            if task.get('callback', None):
                callback_args = task.get('callback_args', ())
                callback_kw = task.get('callback_kw', {})
                ztq_core.push_task(task['callback'], *callback_args,
                                   **callback_kw)

        except Exception, e:
            reason = traceback.format_exception(*sys.exc_info())
            # 将错误信息记录到服务器
            return_code = str(e.args[0]) if len(e.args) > 1 else 300
            task['runtime']['return'] = return_code
            task['runtime']['reason'] = reason[-11:]
            task['runtime']['end'] = int(time.time())
            ztq_core.push_runtime_error(self.queue_name, task)
            # 错误回调
            if task.get('fcallback', None):
                callback_args = task.get('fcallback_args', ())
                callback_kw = task.get('fcallback_kw', {})
                callback_kw['return_code'] = return_code
                callback_kw['return_msg'] = reason[-1]
                ztq_core.push_task(task['fcallback'], *callback_args,
                                   **callback_kw)
            # 在终端打印错误信息
            #reason.insert(0, str(datetime.datetime.today()) + '\n')
            logger.error(''.join(reason))
Exemplo n.º 5
0
 def init(self):
     """ 开机初始化工作 """
     reboot = False
     worker_state = ztq_core.get_worker_state()
     if self.worker_name in worker_state:
         # 重启,读取服务器配置信息
         reboot = True
     # 记录系统日志
     system_log = ztq_core.get_system_log_queue()
     system_log.push(dict( host=safe_get_host('server', 'alias'),
                           alias=self.worker_name,
                           type=reboot and 'reboot' or 'power',
                           timestamp=self.login_time,))
     # 报告机器状态
     worker_state[self.worker_name] = report(self.login_time)
Exemplo n.º 6
0
    def start_job(self, task):
        self.start_job_time = int(time.time())
        task['runtime'].update({'worker': safe_get_host('server', 'alias'),
                                'thread': self.getName(),
                                'start': self.start_job_time, })
        # 记录当前在做什么
        task['process'] = {'ident':self.ident}
        thread_context.job = task
        try:
            self.run_task = ztq_core.task_registry[task['func']]
            # started report
            report_job(comment='start the job')

            try:
                self.run_task(*task['args'], **task['kw'])
            except TypeError:
                # keyword must string is a bug in python
                if sys.version[:5] < '2.6.5':
                    raise Exception,("We not supported %s version of python,"
                        "Please update it to 2.6.5 or later."%sys.version[:5])
                else:
                    raise

            if task.get('callback', None):
                callback_args = task.get('callback_args', ())
                callback_kw = task.get('callback_kw', {})
                ztq_core.push_task(task['callback'], *callback_args, **callback_kw)

        except Exception, e:
            reason = traceback.format_exception(*sys.exc_info())
            # 将错误信息记录到服务器
            return_code = str(e.args[0]) if len(e.args) > 1 else 300
            task['runtime']['return'] = return_code
            task['runtime']['reason'] = reason[-11:]
            task['runtime']['end'] = int( time.time() )
            ztq_core.push_runtime_error(self.queue_name, task)
            # 错误回调
            if task.get('fcallback', None):
                callback_args = task.get('fcallback_args', ())
                callback_kw = task.get('fcallback_kw', {})
                callback_kw['return_code'] = return_code
                callback_kw['return_msg'] = reason[-1]
                ztq_core.push_task(task['fcallback'], *callback_args, **callback_kw)
            # 在终端打印错误信息
            #reason.insert(0, str(datetime.datetime.today()) + '\n')
            logger.error(''.join(reason))
Exemplo n.º 7
0
 def init(self):
     """ 开机初始化工作 """
     reboot = False
     worker_state = ztq_core.get_worker_state()
     if self.worker_name in worker_state:
         # 重启,读取服务器配置信息
         reboot = True
     # 记录系统日志
     system_log = ztq_core.get_system_log_queue()
     system_log.push(
         dict(
             host=safe_get_host('server', 'alias'),
             alias=self.worker_name,
             type=reboot and 'reboot' or 'power',
             timestamp=self.login_time,
         ))
     # 报告机器状态
     worker_state[self.worker_name] = report(self.login_time)
Exemplo n.º 8
0
def report(start_time):
    """ 转换器向服务器报告状态 """
    cpu_style = get_cpu_style()
    cpu_percent = get_cpu_usage() 
    mem_percent, mem_total = get_mem_usage()
    ip = safe_get_host('server', 'alias')

    traceback_dict = {}
    for thread_id, frame in sys._current_frames().items():
        traceback_dict[thread_id] = traceback.format_stack(frame)

    # 向服务器报告
    return dict( ip=ip,
            cpu_style=cpu_style, 
            cpu_percent=cpu_percent, 
            mem_total=mem_total, 
            mem_percent=mem_percent, 
            started=start_time, 
            timestamp=int(time.time()),
            traceback=traceback_dict,
            )
Exemplo n.º 9
0
def report(start_time):
    """ 转换器向服务器报告状态 """
    cpu_style = get_cpu_style()
    cpu_percent = get_cpu_usage()
    mem_percent, mem_total = get_mem_usage()
    ip = safe_get_host('server', 'alias')

    traceback_dict = {}
    for thread_id, frame in sys._current_frames().items():
        traceback_dict[thread_id] = traceback.format_stack(frame)

    # 向服务器报告
    return dict(
        ip=ip,
        cpu_style=cpu_style,
        cpu_percent=cpu_percent,
        mem_total=mem_total,
        mem_percent=mem_percent,
        started=start_time,
        timestamp=int(time.time()),
        traceback=traceback_dict,
    )
Exemplo n.º 10
0
def init_job_threads(config_dict):
    # 启动工作线程
    set_job_threads(config_dict)
    # 保存信息
    config = {}
    for queue_name, values in config_dict.items():
        sleeps = [str(x['interval']) for x in values]
        config[queue_name] = ','.join(sleeps)
    set_config(config, 'queues')

    # 将一些信息补全,让监视界面认为这个worker已经启动
    alias = safe_get_host('server', 'alias')
    # 初始化
    dispatcher_config = ztq_core.get_dispatcher_config()
    if not dispatcher_config:
        dispatcher_config = {'queue_weight': {}, 'worker_weight': {}}
    # set dispatcher config
    worker_weight = dispatcher_config['worker_weight']
    if not worker_weight.get(alias, 0):
        worker_weight[alias] = 5
        ztq_core.set_dispatcher_config(dispatcher_config)
    # set worker config
    worker_config = ztq_core.get_worker_config()
    worker_config[alias] = config_dict
Exemplo n.º 11
0
def init_job_threads(config_dict):
    # 启动工作线程
    set_job_threads(config_dict)
    # 保存信息
    config = {}
    for queue_name, values in config_dict.items():
        sleeps = [str(x['interval']) for x in values]
        config[queue_name] = ','.join(sleeps)
    set_config(config, 'queues')

    # 将一些信息补全,让监视界面认为这个worker已经启动
    alias = safe_get_host('server', 'alias')
    # 初始化
    dispatcher_config = ztq_core.get_dispatcher_config()
    if not dispatcher_config: 
        dispatcher_config = {'queue_weight':{},'worker_weight':{}}
    # set dispatcher config
    worker_weight = dispatcher_config['worker_weight']
    if not worker_weight.get(alias, 0):
        worker_weight[alias] = 5 
        ztq_core.set_dispatcher_config(dispatcher_config)
    # set worker config
    worker_config = ztq_core.get_worker_config()
    worker_config[alias] = config_dict 
Exemplo n.º 12
0
 def __init__(self, worker_name=''):
     super(CommandThread, self).__init__()
     self.login_time = int(time.time())
     self.worker_name = worker_name or safe_get_host('server', 'alias')
Exemplo n.º 13
0
def main():
    """ 主函数 """
    conf_file = ''
    # 用户指定一个配置文件
    if len(sys.argv) > 1:
        conf_file = sys.argv[1]

    init_config(conf_file)
    server = get_configs('server')

    alias = safe_get_host('server', 'alias')
    active_config = server.get('active_config', 'false')

    # 动态注册task
    for module in server['modules'].split():
        try:
            __import__(module)
        except ImportError:
            raise Exception('Not imported the %s module' % module)

    # 连结服务器
    ztq_core.setup_redis('default',
                         host=server['host'],
                         port=int(server['port']),
                         db=int(server['db']))

    # 开启一个命令线程
    command_thread = CommandThread(worker_name=alias)

    sys.stdout.write('Starting server in PID %s\n' % os.getpid())

    worker_state = ztq_core.get_worker_state()
    if active_config.lower(
    ) == 'true' and command_thread.worker_name in worker_state:
        # 如果服务器有这个机器的配置信息,需要自动启动工作线程
        queue = ztq_core.get_worker_config()
        if command_thread.worker_name in queue:
            set_job_threads(queue[command_thread.worker_name])
    elif get_configs('queues'):
        # 把worker监视队列的情况上报到服务器
        queue_config = ztq_core.get_queue_config()
        # 如果配置有queues,自动启动线程监视
        job_threads = {}
        for queue_name, sleeps in get_configs('queues').items():
            job_threads[queue_name] = [{
                'interval': int(sleep)
            } for sleep in sleeps.split(',')]
            if not queue_config.get(queue_name, []):
                queue_config[queue_name] = {
                    'name': queue_name,
                    'title': queue_name,
                    'widget': 5
                }

        init_job_threads(job_threads)

    loggers = get_configs('log')
    initlog(
        loggers.get('key', 'ztq_worker'),
        loggers.get('handler_file'),
        loggers.get('level', 'ERROR'),
    )

    # 不是以线程启动
    command_thread.run()
Exemplo n.º 14
0
 def __init__(self, worker_name=''):
     super(CommandThread, self).__init__()
     self.login_time = int(time.time())
     self.worker_name = worker_name or safe_get_host('server', 'alias')