Beispiel #1
0
    def init_app(self):
        # init parent class, make oslo take effect
        super(LawrenceApplication, self).init_app()

        # init log
        logging.setup("lawrence instance.")
        LOG.info("server instance is runing.")

        # init redis cluster
        self.redis = RedisOperation()
        self.redis.connect()

        # init pool(max value is 50)
        self.pool = multiprocessing.Pool(4, init_signal_handler)

        # init socket long connect
        self.sock_obj = QSocketOperation()
        host_list = CONF.NODES.nodes
        proxy_port = CONF.PROXY.proxy_port
        timeout = 4
        self.sock_obj.connect(host_list, proxy_port, timeout)
Beispiel #2
0
class LawrenceApplication(QApplication):
    name = "lawrence"
    version = "v1.0"

    _last_update = 0

    def init_app(self):
        # init parent class, make oslo take effect
        super(LawrenceApplication, self).init_app()

        # init log
        logging.setup("lawrence instance.")
        LOG.info("server instance is runing.")

        # init redis cluster
        self.redis = RedisOperation()
        self.redis.connect()

        # init pool(max value is 50)
        self.pool = multiprocessing.Pool(4, init_signal_handler)

        # init socket long connect
        self.sock_obj = QSocketOperation()
        host_list = CONF.NODES.nodes
        proxy_port = CONF.PROXY.proxy_port
        timeout = 4
        self.sock_obj.connect(host_list, proxy_port, timeout)

    def run(self):
        while True:
            # heartbeat_check
            heartbeat = struct.pack("!II", HEARTBEAT, 0)
            send_status = self.sock_obj.sendmsg(heartbeat)
            if not send_status:
                LOG.warn("____proxy proxy master is badding, need alarm.")

            # request_task
            self._get_taskinfo()

            head_data = self.sock_obj.recvmsg(HEADER_BYTES)
            if head_data:
                head_msg = struct.unpack('!II', head_data)
                command_id = head_msg[0]
                if command_id == ACK_HEART:
                    pass
                elif command_id == ACK_NORMAL:
                    package_len = head_msg[1]
                    bodylen = package_len - HEADER_BYTES
                    body = self.sock_obj.recvmsg(bodylen)
                    LOG.info("receive body for task data is: %s" % body)
                    if not body:
                        time.sleep(2)
                        continue

                    # do task
                    task_list = json.loads(body)
                    self._dispatch_task(task_list)

            time.sleep(2)

    def _get_taskinfo(self):
        cur_time = time.time()
        timestamp = long(cur_time)
        timedelta = timestamp - self._last_update
        if timedelta < 50:
            return
        curSecond = long(time.strftime("%S", time.localtime(cur_time)))
        if curSecond < 30:
            return
        self._last_update = timestamp

        request_data = struct.pack("!II", NORMALMSG, 0)
        send_status = self.sock_obj.sendmsg(request_data)
        if not send_status:
            LOG.info("request task data is failed.........")

    def _dispatch_task(self, task_list):
        try:
            for item in task_list:
                group = item.get("group")
                is_exist = self.redis.exists(group)
                if is_exist:
                    self.redis.delete(group)
                    LOG.warn("find unfinished task, group is (%s)" % group)
                    continue
                self.redis.set_redis_cache(group, item)
                # Note(apply_async‘s args must be a single function.)
                self.pool.apply_async(collector, (item,))
            LOG.info("finish all tasks assigned.")
        except KeyboardInterrupt:
            self.pool.terminate()
            self.pool.close()