Exemplo n.º 1
0
class Core(object):

    def __init__(self, bot):
        self.bot = bot
        self.timeout = int(self.bot.config.get('timeout'))
        self.ping_queue = Queue(loop=bot.loop)

    def connection_made(self):
        self.bot.loop.call_later(self.timeout, self.check_ping)
        self.ping_queue.put_nowait(self.bot.loop.time())

    def check_ping(self):  # pragma: no cover
        # check if we received a ping
        # reconnect if queue is empty
        self.bot.log.debug(
            'Ping queue size: {}'.format(self.ping_queue.qsize()))
        if self.ping_queue.empty():
            self.bot.loop.call_soon(self.bot.protocol.transport.close)
        else:
            self.bot.loop.call_later(self.timeout, self.check_ping)
        while not self.ping_queue.empty():
            self.ping_queue.get_nowait()

    @event(rfc.PING)
    def pong(self, data):
        """PING reply"""
        self.ping_queue.put_nowait(self.bot.loop.time())
        self.bot.send('PONG ' + data)

    @event(rfc.NEW_NICK)
    def recompile(self, nick=None, new_nick=None, **kw):
        """recompile regexp on new nick"""
        if self.bot.nick == nick.nick:
            self.bot.config['nick'] = new_nick
            self.bot.recompile()

    @event(rfc.ERR_NICK)
    def badnick(self, me=None, nick=None, **kw):
        """Use alt nick on nick error"""
        if me == '*':
            self.bot.set_nick(self.bot.nick + '_')
        self.bot.log.debug('Trying to regain nickname in 30s...')
        self.bot.loop.call_later(30, self.bot.set_nick, self.bot.original_nick)

    @event(rfc.RPL_ENDOFMOTD)
    def autojoin(self, **kw):
        """autojoin at the end of MOTD"""
        self.bot.config['nick'] = kw['me']
        self.bot.recompile()
        channels = utils.as_list(self.bot.config.get('autojoins', []))
        for channel in channels:
            channel = utils.as_channel(channel)
            self.bot.log.info('Trying to join %s', channel)
            self.bot.join(channel)
Exemplo n.º 2
0
async def patched_alerta(service, stop_event, bot_alert_queue: Queue):
    alerta = AlertaRunner(msg_service=service,
                          stop_event=stop_event,
                          send_heartbeats=False)

    def _alert(item):
        bot_alert_queue.put_nowait(item)

    alerta.alert = _alert
    asyncio.create_task(alerta.start())
    await asyncio.sleep(.5)
    yield alerta
    alerta.stop_event.set()
    while not bot_alert_queue.empty():
        bot_alert_queue.get_nowait()
        bot_alert_queue.task_done()
Exemplo n.º 3
0
class Port(object):
    def __init__(self, tag="data", maxsize=1, name=None, loop=None):
        loop = loop if loop is not None else asyncio.get_event_loop()
        self.loop = loop
        self.name = name if name is not None else str(uuid1())
        self._queue = Queue(maxsize, loop=self.loop)
        self.default_value = None
        self.default_value_set = False
        self.connected = False
        self.belong_to_block = None
        self.data_tag = tag

    def set_default_value(self, value):
        if not isinstance(value, Payload):
            raise Exception("value should be Payload type")
        self.default_value = value
        self.default_value_set = True

    async def get(self):
        if self.default_value_set:
            if self._queue.empty():
                return self.default_value, self.default_value[self.data_tag]

        payload = await self._queue.get()
        return payload, payload[self.data_tag]

    def get_nowait(self):
        if self.default_value_set:
            if self._queue.empty():
                return self.default_value, self.default_value[self.data_tag]

        payload = self._queue.get_nowait()
        return payload, payload[self.data_tag]

    async def put(self, payload, item):
        if self.connected:
            payload[self.data_tag] = item
            await self._queue.put(payload)

    def put_nowait(self, payload, item):
        if self.connected:
            payload[self.data_tag] = item
            self._queue.put_nowait(payload)

    def empty(self):
        return self._queue.empty()

    def full(self):
        return self._queue.full()

    def set_buffer_size(self, maxsize):
        self._queue = Queue(maxsize, loop=self.loop)
Exemplo n.º 4
0
class FoundComponentIterator:
    def __init__(self, loop, components_file_request_list):
        self.loop = loop
        self.pending_tasks = components_file_request_list
        self._to_remove = None
        self.done = Queue(loop=self.loop)
        self.add_done_callback()

    def add_done_callback(self):
        for task in self.pending_tasks:
            task.add_done_callback(self.on_completion)

    def on_completion(self, task):
        self.pending_tasks.remove(task)
        self.done.put_nowait(task)

    async def cancel_pending_tasks(self):
        for task in self.pending_tasks:
            task.cancel()
        if len(self.pending_tasks):
            await asyncio.wait(self.pending_tasks)
        while not self.done.empty():
            task = self.done.get_nowait()
            try:
                task.result()
            except:
                pass

    def __aiter__(self):
        return self

    async def __anext__(self):
        try:
            while len(self.pending_tasks) > 0 or not self.done.empty():
                try:
                    future = await self.done.get()
                    component_key, fetched_files = await future
                    self._to_remove = future
                    if len(fetched_files) > 0:
                        return {'key': component_key, 'files': fetched_files}
                except (RejectRequest, StopRequest):
                    # Not fatal at all, just one of many
                    pass
        except OfflineHostException:
            await self.cancel_pending_tasks()
            raise
        raise StopAsyncIteration
Exemplo n.º 5
0
class IrcConnection(asyncio.Protocol):
    """asyncio protocol to handle an irc connection"""

    def connection_made(self, transport):
        self.transport = transport
        self.closed = False
        self.queue = Queue()

    def data_received(self, data):
        encoding = getattr(self, 'encoding', 'ascii')
        data = data.decode(encoding, 'ignore')
        if not self.queue.empty():
            data = self.queue.get_nowait() + data
        lines = data.split('\r\n')
        self.queue.put_nowait(lines.pop(-1))
        for line in lines:
            self.factory.dispatch(line)

    def write(self, data):
        if data is not None:
            if isinstance(data, text_type):
                data = data.encode(self.encoding)
            if not data.endswith(b'\r\n'):
                data = data + b'\r\n'
            self.transport.write(data)

    def connection_lost(self, exc):  # pragma: no cover
        self.factory.log.critical('connection lost (%s): %r',
                                  id(self.transport),
                                  exc)
        self.factory.notify('connection_lost')
        if not self.closed:
            self.close()
            # wait a few before reconnect
            self.factory.loop.call_later(
                2, self.factory.create_connection, self.__class__)

    def close(self):  # pragma: no cover
        if not self.closed:
            self.factory.log.critical('closing old transport (%r)',
                                      id(self.transport))
            try:
                self.transport.close()
            finally:
                self.closed = True
Exemplo n.º 6
0
class IrcConnection(asyncio.Protocol):
    """asyncio protocol to handle an irc connection"""
    def connection_made(self, transport):
        self.transport = transport
        self.closed = False
        self.queue = Queue()

    def data_received(self, data):
        encoding = getattr(self, 'encoding', 'ascii')
        data = data.decode(encoding, 'ignore')
        if not self.queue.empty():
            data = self.queue.get_nowait() + data
        lines = data.split('\r\n')
        self.queue.put_nowait(lines.pop(-1))
        for line in lines:
            self.factory.dispatch(line)

    def write(self, data):
        if data is not None:
            if isinstance(data, text_type):
                data = data.encode(self.encoding)
            if not data.endswith(b'\r\n'):
                data = data + b'\r\n'
            self.transport.write(data)

    def connection_lost(self, exc):  # pragma: no cover
        self.factory.log.critical('connection lost (%s): %r',
                                  id(self.transport), exc)
        self.factory.notify('connection_lost')
        if not self.closed:
            self.close()
            # wait a few before reconnect
            self.factory.loop.call_later(2, self.factory.create_connection,
                                         self.__class__)

    def close(self):  # pragma: no cover
        if not self.closed:
            self.factory.log.critical('closing old transport (%r)',
                                      id(self.transport))
            try:
                self.transport.close()
            finally:
                self.closed = True
class HandlerConnectionHandler:
    def __init__(self, host, port, _handler_id, _device_ids, log, app,
                 event) -> None:
        self._mqtt = None
        self._app = app
        self._log = log
        self.event = None
        self.command = None

        self._client_id = f'handler.{_handler_id}'
        self._host = host
        self._port = port
        self._device_ids = _device_ids
        self._handler_id = _handler_id
        self._event = event
        self._message_queue = Queue()

    def start(self):
        self._mqtt = ConnectionHandler(self._host, self._port, self._client_id,
                                       self._log)
        self._log.set_mqtt_client(self._mqtt)
        self._mqtt.init_mqtt_client_callbacks(self._on_connect,
                                              self._on_message,
                                              self._on_disconnect)

        self._mqtt.register_route(
            "Master", lambda topic, payload: self.dispatch_masterapp_message(
                topic, self._mqtt.decode_payload(payload)))
        self._mqtt.register_route(
            "Handler", lambda topic, payload: self.dispatch_masterapp_message(
                topic, self._mqtt.decode_payload(payload)))

        self._mqtt.set_last_will(
            self._generate_handler_status_topic(),
            self._mqtt.create_message(
                self._generate_status_message('crash', '')))
        self._mqtt.start_loop()

    async def stop(self) -> None:
        await self._mqtt.stop_loop()

    def subscribe(self, topic) -> None:
        self._mqtt.subscribe(topic)

    def publish(self, topic, payload, qos=0, retain=False) -> None:
        self._mqtt.publish(topic, json.dumps(payload), qos=qos, retain=retain)

    def publish_state(self, state, message) -> None:
        self._log.log_message(LogLevel.Info(), f'Handler state: {state}')
        self.publish(self._generate_handler_status_topic(),
                     self._generate_status_message(state, message),
                     qos=1,
                     retain=False)

    def _on_connect(self, client, userdata, flags, rc) -> None:
        self._app.startup_done('connection to broker is established')

        for device_id in self._device_ids:
            self.subscribe(self._generate_master_status_topic(device_id))
            self.subscribe(self._generate_command_topic(device_id))
            self.subscribe(self._generate_response_topic(device_id))

    def _on_message(self, client, userdata, message) -> None:
        self._mqtt.router.inject_message(message.topic, message.payload)

    def _on_disconnect(self, client, userdata, rc) -> None:
        self._log.debug("disconnected")

    def dispatch_masterapp_message(self, topic, message) -> None:
        master_id = topic.split('/')[1]
        if "status" in topic:
            master_state = message['state']
            self._app.on_master_state_changed(master_id, master_state)
            self._handle_master_state(message)
        elif "command" in topic or\
             "response" in topic:
            self._put_message(message)
        else:
            assert False

    def _handle_master_state(self, message):
        if message['state'] in INVISIBLE_TESTER_STATES:
            return

        if not self.command:
            return

        self._put_message(message)

    def _put_message(self, message):
        self._message_queue.put_nowait(message)
        self._event.set()

    def get_message(self):
        try:
            return self._message_queue.get_nowait()
        except QueueEmpty:
            return None

    def get_mqtt_client(self):
        return self._mqtt

    def send_command_message(self, message):
        for device_id in self._device_ids:
            self._mqtt.publish(self._generate_master_command_topic(device_id),
                               self._mqtt.create_message(message), False)

    def send_response_message(self, message):
        self._mqtt.publish(self._generate_handler_response_topic(),
                           self._mqtt.create_message(message), False)

    def handle_message(self, message):
        response = self._app.handle_message(message)
        if len(response):
            self._message_queue.put_nowait(response)
            self._event.set()
            return

        if message['type'] == 'temperature':
            self.send_response_message(message)
        else:
            self.send_command_message(message)
            self.command = message['type']

    @staticmethod
    def _generate_message(type, payload):
        return {'type': type, 'payload': payload}

    def _generate_status_message(self, state, message):
        payload = {'state': state, 'message': message}
        return self._generate_message('status', payload)

    @staticmethod
    def _generate_command_topic(device_id):
        return f"ate/{device_id}/Handler/command"

    @staticmethod
    def _generate_response_topic(device_id):
        return f"ate/{device_id}/Master/response"

    @staticmethod
    def _generate_log_message(log_message):
        return {"type": "log", "payload": log_message}

    def _handler_log_topic(self):
        return f'ate/{self._handler_id}/Handler/log/'

    def _generate_handler_status_topic(self):
        return f'ate/{self._handler_id}/Handler/status'

    def _generate_handler_response_topic(self):
        return f'ate/{self._handler_id}/Handler/response'

    @staticmethod
    def _generate_master_status_topic(device_id):
        return f'ate/{device_id}/Master/status'

    @staticmethod
    def _generate_master_command_topic(device_id):
        return f"ate/{device_id}/Master/command"
Exemplo n.º 8
0
class StreamConnection:
    def __init__(self, sr, sw, *, loop=None):
        if not loop:
            loop = asyncio.get_event_loop()
        self._loop = loop
        self._sr = sr
        self._sw = sw
        self._msgs = Queue(loop=loop)
        self._worker = loop.create_task(self._run())

    @asyncio.coroutine
    def _run(self):
        while self.alive():
            try:
                data = yield from self._sr.readline()
                if data and len(data):
                    self._msgs.put_nowait(self._convert(data))
            except asyncio.CancelledError:
                logger.debug("readline from stream reader was cancelled.")
            except ConnectionError:
                logger.debug("connection error")
                break
        logger.debug("connection closed")

    def _convert(self, data):
        return data.strip()

    @asyncio.coroutine
    def recv(self):
        try:
            return self._msgs.get_nowait()
        except QueueEmpty:
            pass

        # Wait for a message until the connection is closed
        next_message = self._loop.create_task(self._msgs.get())
        done, pending = yield from asyncio.wait(
                [next_message, self._worker],
                loop=self._loop, return_when=asyncio.FIRST_COMPLETED)
        if next_message in done:
            return next_message.result()
        else:
            next_message.cancel()

    def send(self, data):
        if not self.alive():
            raise ConnectionError("connection was closed.")
        try:
            data = data + b'\n'
            self._sw.write(data)
        except OSError:
            raise ConnectionError("can't send data.")
        except Exception:
            logger.debug("Q___Q")

    def alive(self):
        return not self._sr.at_eof()

    @asyncio.coroutine
    def drain():
        yield from self._sw.drain()
    
    @asyncio.coroutine
    def close(self):
        if self.alive():
            try:
                yield from self._sw.drain()
                self._sw.write_eof()
            except ConnectionError:
                pass
            else:
                self._sr.feed_eof()
                self._sw.close()
        self._worker.cancel()
Exemplo n.º 9
0
class OthelloPlayer:
    def __init__(self,
                 config: Config,
                 client,
                 mode="gui",
                 weight_table=0,
                 c=10,
                 mc=False):
        """
        :param config:
        :param agent.model.OthelloModel|None model:
        :param TreeNode mtcs_info:
        :parameter OthelloModelAPI api:
        """
        self.config = config
        self.client = client
        self.mode = mode
        self.play_config = self.config.play
        self.weight_table = weight_table
        self.c = c
        self.mc = mc

        # mc_tree
        self.num_tree, self.win_tree, self.policy_tree = createTrees()

        # expanded
        self.expanded = set()  #expanded存p(dict)的set形式
        self.now_expanding = set()

        # threads
        self.prediction_queue = Queue(
            self.play_config.prediction_queue_size)  #并行计算的信息队列queue大小
        self.sem = asyncio.Semaphore(
            self.play_config.parallel_search_num)  #限制并行搜索的线程数
        self.loop = asyncio.get_event_loop()

        # for gui
        if self.mode == 'gui':
            self.thinking_history = None  # for fun
            self.avalable = None
            self.allow_resign = False
        elif self.mode == 'self_play':
            self.moves = []
            self.allow_resign = True
        self.test_mode = False
        # params
        self.running_simulation_num = 0

        # solver
        self.solver = OthelloSolver()  #引入minmax树类

    def win_rate(self, node):
        return self.win_tree[node] / (self.num_tree[node] + 1e-5)

# think_and_play

    def think_and_play(self, own, enemy):
        """play tmd:方案:50步以前使用深度學習mctree,若tree到達50步深度后再用minmaxtree; 50步以後直接用minmaxtree
        若搜不到/超時再用之前構建的樹"""
        # renew env
        self.start_time = time.time()
        env = OthelloEnv().update(own, enemy, next_to_play=Stone.black)
        node = create_node(env)

        #五十步之后直接minmax树搜索,若搜索不到,再用深度學習
        if env.epoch >= self.play_config.use_solver_turn:
            logger.warning(f"Entering minmax_tree process")
            ret = self._solver(node)
            if ret:  # not save move as play data
                return ret
        else:  # 五十步之前直接用深度學習
            for t1 in range(self.play_config.thinking_loop
                            ):  # search moves for 3 times
                logger.warning(f"Entering {t1} thinking_loop")
                self._expand_tree(env, node)
                policy, action, value_diff = self._calc_policy_and_action(node)
                # if action 足够大 + n足够大 \ turn 很小
                if env.epoch <= self.play_config.start_rethinking_turn or \
                        (value_diff > -0.01 and self.num_tree[node][action] >= self.play_config.required_visit_to_decide_action):
                    break

            # record or return
            if self.mode == 'gui':
                self._update_thinking_history(own, enemy, action, policy)
                self._update_avalable(own, enemy, action, policy)
            elif self.mode == 'self_play':
                if self.allow_resign:  # resign win_rate 太小没有胜率。
                    if self.play_config.resign_threshold is not None and\
                        np.max(self.win_rate(node)-(self.num_tree[node]==0)*10) <= self.play_config.resign_threshold:
                        if env.epoch >= self.config.play.allowed_resign_turn:
                            return AcNQ(None, 0, 0)  # means resign
                        else:
                            logger.debug(
                                f"Want to resign but disallowed turn {env.epoch} < {self.config.play.allowed_resign_turn}"
                            )
                # save fuckers
                saved_policy = self.__calc_policy_by_prob(
                    node
                ) if self.config.play_data.save_policy_of_tau_1 else policy
                self.__save_data_to_moves(own, enemy, saved_policy)
            return AcNQ(action=action,
                        n=self.num_tree[node][action],
                        q=self.win_rate(node)[action])

    def _solver(self, node):
        # use solver to do minmax搜索
        action, point = self.solver.solve(node.black,
                                          node.white,
                                          Stone(node.next_to_play),
                                          exactly=True)
        if action is None:  #如果沒搜索到是不可以返回None的因爲要
            return None
        else:
            policy = np.zeros(64)
            policy[action] = 1
            update_num_tree_with_one_or_moresides(self.num_tree, node, action,
                                                  ["set"], [999])
            update_win_tree_with_one_or_moresides(self.win_tree, node, action,
                                                  ["set"],
                                                  [np.sign(point) * 999])
            update_policy_tree_with_one_or_moresides(self.policy_tree, node,
                                                     ["set"], [policy])
            self._update_thinking_history(node.black, node.white, action,
                                          policy)
        return AcNQ(action=action, n=999, q=np.sign(point))

    def _expand_tree(self, env, node):
        if env.epoch > 0:  # 对树进行拓展
            self._expand_tree_2(env.chessboard.black, env.chessboard.white)
        else:
            self._set_first_move(node)

    def _expand_tree_2(self, own, enemy):
        # params
        loop = self.loop
        self.running_simulation_num = 0
        # n simulation/move
        coroutine_list = []
        # 200 simulations
        for it in range(self.play_config.simulation_num_per_move):
            coroutine_list.append(self.__start_search_my_move(own, enemy))
        coroutine_list.append(self.__prediction_worker())
        loop.run_until_complete(asyncio.gather(*coroutine_list))

    async def __start_search_my_move(self, own, enemy):
        # set parmas
        self.running_simulation_num += 1
        # wait sems
        with await self.sem:  # 8綫程
            env = OthelloEnv().update(own, enemy, Stone.black)
            leaf_v = await self.___recursive_simulation(env, is_root_node=True)
            self.running_simulation_num -= 1
            return leaf_v

    async def ___recursive_simulation(self,
                                      env: OthelloEnv,
                                      is_root_node=False):
        "fertilize tree process"
        # get both keys
        node, another_side_node = create_both_nodes(env)
        if self.test_mode:
            if (node not in map.keys()):
                map[node] = env.epoch

        # return condition 1
        if env.done:
            if env.result == Result.black:
                return 1
            elif env.result == Result.white:
                return -1
            else:
                return 0

        # return condition 2 : get solver(大于50步,minmax)
        if env.epoch >= self.config.play.use_solver_turn_in_simulation:
            action, point = self.solver.solve(node.black,
                                              node.white,
                                              Stone(node.next_to_play),
                                              exactly=False)
            if action:
                point = point if env.next_to_play == Stone.black else -point
                leaf_v = np.sign(point)
                leaf_p = np.zeros(64)
                leaf_p[action] = 1
                # update tree
                update_num_tree_with_one_or_moresides(self.num_tree, node,
                                                      action, ["plus", "plus"],
                                                      [1, 1])  #走过的位置+1
                update_win_tree_with_one_or_moresides(
                    self.win_tree, node, action, ["plus", "minus"],
                    [leaf_v, leaf_v])  #走此步赢的次数+-1(win)
                update_policy_tree_with_one_or_moresides(
                    self.policy_tree, node, ["set", "set"],
                    [leaf_p, leaf_p])  #此节点应该走的位置(position)
                return np.sign(point)
            if time.time() - self.start_time >= 55:
                return 0
        #return condition 3 : expand tree(小於等於50步,用深度學習)
        while node in self.now_expanding:  # 兩個搜索綫程遇到同一個node,會有衝突的問題
            await asyncio.sleep(self.config.play.wait_for_expanding_sleep_sec)
        # is leaf
        if node not in self.expanded:  # reach leaf node
            leaf_v = await self.____expand_leaf_node(env)
            if env.next_to_play == Stone.black:
                return leaf_v  # Value for black
            else:
                return -leaf_v  # Value for white == -Value for black
        else:  # not leaf do
            virtual_loss_for_w = self.config.play.virtual_loss if env.next_to_play == Stone.black else -self.config.play.virtual_loss
            action_t = self.____decide_action(env, is_root_node)  #UCB公式
            update_num_tree_with_one_or_moresides(
                self.num_tree, node, action_t, ["plus"],
                [self.config.play.virtual_loss])
            update_win_tree_with_one_or_moresides(self.win_tree, node,
                                                  action_t, ["minus"],
                                                  [virtual_loss_for_w])
            env.do(action_t)
            leaf_v = await self.___recursive_simulation(env)  # next move
            # on returning search path
            update_num_tree_with_one_or_moresides(
                self.num_tree, node, action_t, ["plus", "plus"],
                [-self.config.play.virtual_loss + 1, 1])
            update_win_tree_with_one_or_moresides(
                self.win_tree, node, action_t, ["plus", "minus"],
                [virtual_loss_for_w + leaf_v, leaf_v])
            if self.test_mode:
                logger.warning(map[node], leaf_v)
        return leaf_v

    async def ____expand_leaf_node(self, env):
        "use to expand new leaf"
        node, another_side_node = create_both_nodes(env)
        self.now_expanding.add(node)

        # flip + rotate
        rotate_right_num, is_flip_vertical, black_ary, white_ary = flip_and_rotate_board_to_array(
            env.chessboard.black, env.chessboard.white)

        # predict
        state = [
            white_ary, black_ary
        ] if env.next_to_play == Stone.white else [black_ary, white_ary]
        future = await self.predict(np.array(state))  # type: Future
        await future
        leaf_p, leaf_v = future.result()

        # reverse rotate and flip about leaf_p
        leaf_p = flip_and_rotate_result(leaf_p, rotate_right_num,
                                        is_flip_vertical)
        if self.mc:
            black = env.chessboard.black
            leaf_v += np.sum(bit_to_array(black, 64) * self.weight_table)

        # update
        update_policy_tree_with_one_or_moresides(self.policy_tree, node,
                                                 ["set", "set"],
                                                 [leaf_p, leaf_p])
        self.expanded.add(node)
        self.now_expanding.remove(node)
        return leaf_v

    def ____decide_action(self, env, is_root_node):
        # find correct moves
        node = create_node(env)
        legal_moves = find_correct_moves(
            node.black, node.white
        ) if env.next_to_play == Stone.black else find_correct_moves(
            node.white, node.black)

        # vn = formula here
        vn = max(np.sqrt(np.sum(self.num_tree[node])),
                 1)  # SQRT of sum(N(s, b); for all b)

        # p = formula here  re-normalize in legal moves
        vp = self.policy_tree[node]
        vp = vp * bit_to_array(legal_moves, 64)
        temperature = 1
        if np.sum(vp) > 0:
            temperature = min(
                np.exp(1 -
                       np.power(env.epoch / self.config.play.policy_decay_turn,
                                self.config.play.policy_decay_power)), 1)
            vp = normalize(vp, temperature)
        # add noise 0.75*p + 0.25*noise
        if is_root_node and self.play_config.noise_eps > 0:  # Is it correct?? -> (1-e)p + e*Dir(alpha)
            noise = dirichlet_noise_of_mask(legal_moves,
                                            self.play_config.dirichlet_alpha)
            vp = (1 - self.play_config.noise_eps
                  ) * vp + self.play_config.noise_eps * noise

        # u_ = formula here
        vpn = vp * vn / (1 + self.num_tree[node])
        if env.next_to_play == Stone.black:
            vpn_with_weight = (self.win_rate(node) * self.c + vpn + 1000 +
                               self.weight_table) * bit_to_array(
                                   legal_moves, 64)
        else:
            vpn_with_weight = (-self.win_rate(node) * self.c + vpn + 1000 +
                               self.weight_table) * bit_to_array(
                                   legal_moves, 64)
        action_t = int(np.argmax(vpn_with_weight))
        return action_t

    async def __prediction_worker(self):
        " do prediction in this worker"
        margin = 10  # wait for at most 10 epochs x 0.0001
        while self.running_simulation_num > 0 or margin > 0:
            if self.prediction_queue.empty():
                if margin > 0:
                    margin -= 1
                await asyncio.sleep(
                    self.config.play.prediction_worker_sleep_sec)
                continue
            item_list = [
                self.prediction_queue.get_nowait()
                for _ in range(self.prediction_queue.qsize())
            ]  # type: list[QItem]
            data = np.array([x.state for x in item_list])
            policy_ary, value_ary = self.client.forward(
                data)  # shape=(N, 2, 8, 8)
            for p, v, item in zip(policy_ary, value_ary, item_list):
                item.future.set_result((p, v))

    def _set_first_move(self, node):
        # chose the random num_tree = [1] policy_tree = [每个可能的地方都是1/n]
        legal_array = bit_to_array(find_correct_moves(node.black, node.white),
                                   64)
        action = np.argmax(legal_array)
        update_num_tree_with_one_or_moresides(self.num_tree, node, action,
                                              ["set"], [1])
        update_win_tree_with_one_or_moresides(self.win_tree, node, action,
                                              ["set"], [0])
        update_policy_tree_with_one_or_moresides(
            self.policy_tree, node, ["set"],
            [legal_array / np.sum(legal_array)])

    def _calc_policy_and_action(self, node):
        policy = self._calc_policy(
            node.black, node.white
        )  # 先验 最大的n,  前四步p[n]=num_tree[key][n],后面p矩阵只有var_numb最大位置为1,其余为0.
        action = int(np.random.choice(range(64),
                                      p=policy))  #随机走一个点,p为随机取的各点概率(先验)
        action_by_value = int(
            np.argmax(self.win_rate(node) +
                      (self.num_tree[node] > 0) * 100))  #选走过的、q(胜率)最大的那个位置
        value_diff = self.win_rate(node)[action] - self.win_rate(node)[
            action_by_value]  #
        return policy, action, value_diff

    def _calc_policy(self, own, enemy):
        env = OthelloEnv().update(own, enemy, Stone.black)
        node = create_node(env)
        # if turn < 4
        if env.epoch < self.play_config.change_tau_turn:
            return self.__calc_policy_by_prob(node)  # p value
        else:
            return self.__calc_policy_by_max(node)

    def __calc_policy_by_prob(self, node):
        return self.num_tree[node] / np.sum(self.num_tree[node])  # tau = 1

    def __calc_policy_by_max(self, node):
        action = np.argmax(self.num_tree[node])  # tau = 0
        ret = np.zeros(64)  # one hot
        ret[action] = 1
        return ret

    def _update_thinking_history(self, black, white, action, policy):
        node = TreeNode(black, white, Stone.black.value)
        next_key = self.__get_next_key(black, white, action)
        self.thinking_history = \
            LastAcNQ(action, policy, list(self.win_rate(node)), list(self.num_tree[node]),
                        list(self.win_rate(next_key)), list(self.num_tree[next_key]))

    def _update_avalable(self, black, white, action, policy):
        node = TreeNode(black, white, Stone.black.value)
        next_key = self.__get_next_key(black, white, action)
        self.avalable = LastAva(
            find_correct_moves(node.black, node.white),
            find_correct_moves(next_key.white, next_key.black))

    def __get_next_key(self, own, enemy, action):
        env = OthelloEnv().update(own, enemy, Stone.black)
        env.do(action)
        return create_node(env)

    def __save_data_to_moves(self, own, enemy, policy):
        for flip in [False, True]:
            for rot_right in range(4):
                self.moves.append(
                    flip_and_rotate_right(flip, rot_right, own, enemy, policy))

    async def predict(self, x):
        future = self.loop.create_future()
        await self.prediction_queue.put(QItem(x, future))
        return future
Exemplo n.º 10
0
 async def consume(q: Queue):
     while not q.empty():
         json_file = q.get_nowait()
         await verify(config, json_file, mirror_base_path,
                      all_package_files, args, executor)
Exemplo n.º 11
0
class WebSocketCommonProtocol(asyncio.StreamReaderProtocol):
    """
    This class implements common parts of the WebSocket protocol.

    It assumes that the WebSocket connection is established. The handshake is
    managed in subclasses such as
    :class:`~websockets.server.WebSocketServerProtocol` and
    :class:`~websockets.client.WebSocketClientProtocol`.

    It runs a task that stores incoming data frames in a queue and deals with
    control frames automatically. It sends outgoing data frames and performs
    the closing handshake.

    The `host`, `port` and `secure` parameters are simply stored as attributes
    for handlers that need them.

    The `timeout` parameter defines the maximum wait time in seconds for
    completing the closing handshake and, only on the client side, for
    terminating the TCP connection. :meth:`close()` will complete in at most
    this time on the server side and twice this time on the client side.

    The `max_size` parameter enforces the maximum size for incoming messages
    in bytes. The default value is 1MB. ``None`` disables the limit. If a
    message larger than the maximum size is received, :meth:`recv()` will
    return ``None`` and the connection will be closed with status code 1009.

    Once the connection is closed, the status code is available in the
    :attr:`close_code` attribute and the reason in :attr:`close_reason`.
    """

    # There are only two differences between the client-side and the server-
    # side behavior: masking the payload and closing the underlying TCP
    # connection. This class implements the server-side behavior by default.
    # To get the client-side behavior, set is_client = True.

    is_client = False
    state = 'OPEN'

    def __init__(self, *,
                 host=None, port=None, secure=None, timeout=10, max_size=2 ** 20, loop=None):
        self.host = host
        self.port = port
        self.secure = secure

        self.timeout = timeout
        self.max_size = max_size

        super().__init__(asyncio.StreamReader(), self.client_connected, loop)

        self.close_code = None
        self.close_reason = ''

        # Futures tracking steps in the connection's lifecycle.
        self.opening_handshake = asyncio.Future()
        self.closing_handshake = asyncio.Future()
        self.connection_failed = asyncio.Future()
        self.connection_closed = asyncio.Future()

        # Queue of received messages.
        self.messages = Queue()

        # Mapping of ping IDs to waiters, in chronological order.
        self.pings = collections.OrderedDict()

        # Task managing the connection.
        self.worker = asyncio.async(self.run())

        # In a subclass implementing the opening handshake, the state will be
        # CONNECTING at this point.
        if self.state == 'OPEN':
            self.opening_handshake.set_result(True)

    # Public API

    @property
    def open(self):
        """
        This property is ``True`` when the connection is usable.

        It may be used to handle disconnections gracefully.
        """
        return self.state == 'OPEN'

    @asyncio.coroutine
    def close(self, code=1000, reason=''):
        """
        This coroutine performs the closing handshake.

        This is the expected way to terminate a connection on the server side.

        It waits for the other end to complete the handshake. It doesn't do
        anything once the connection is closed.

        It's usually safe to wrap this coroutine in `asyncio.async()` since
        errors during connection termination aren't particularly useful.

        The `code` must be an :class:`int` and the `reason` a :class:`str`.
        """
        if self.state == 'OPEN':
            # 7.1.2. Start the WebSocket Closing Handshake
            self.close_code, self.close_reason = code, reason
            yield from self.write_frame(OP_CLOSE, serialize_close(code, reason))
            # 7.1.3. The WebSocket Closing Handshake is Started
            self.state = 'CLOSING'

        # If the connection doesn't terminate within the timeout, break out of
        # the worker loop.
        try:
            yield from asyncio.wait_for(self.worker, timeout=self.timeout)
        except asyncio.TimeoutError:
            self.worker.cancel()

        # The worker should terminate quickly once it has been cancelled.
        yield from self.worker

    @asyncio.coroutine
    def recv(self):
        """
        This coroutine receives the next message.

        It returns a :class:`str` for a text frame and :class:`bytes` for a
        binary frame.

        When the end of the message stream is reached, or when a protocol
        error occurs, :meth:`recv` returns ``None``, indicating that the
        connection is closed.
        """
        # Return any available message
        try:
            return self.messages.get_nowait()
        except QueueEmpty:
            pass

        # Wait for a message until the connection is closed
        next_message = asyncio.async(self.messages.get())
        done, pending = yield from asyncio.wait(
                [next_message, self.worker],
                return_when=asyncio.FIRST_COMPLETED)
        if next_message in done:
            return next_message.result()
        else:
            next_message.cancel()

    @asyncio.coroutine
    def send(self, data):
        """
        This coroutine sends a message.

        It sends a :class:`str` as a text frame and :class:`bytes` as a binary
        frame.

        It raises a :exc:`TypeError` for other inputs and
        :exc:`InvalidState` once the connection is closed.
        """
        if isinstance(data, str):
            opcode = 1
            data = data.encode('utf-8')
        elif isinstance(data, bytes):
            opcode = 2
        else:
            raise TypeError("data must be bytes or str")
        yield from self.write_frame(opcode, data)

    @asyncio.coroutine
    def ping(self, data=None):
        """
        This coroutine sends a ping.

        It returns a Future which will be completed when the corresponding
        pong is received and which you may ignore if you don't want to wait.

        A ping may serve as a keepalive.
        """
        # Protect against duplicates if a payload is explicitly set.
        if data in self.pings:
            raise ValueError("Already waiting for a pong with the same data")
        # Generate a unique random payload otherwise.
        while data is None or data in self.pings:
            data = struct.pack('!I', random.getrandbits(32))

        self.pings[data] = asyncio.Future()
        yield from self.write_frame(OP_PING, data)
        return self.pings[data]

    @asyncio.coroutine
    def pong(self, data=b''):
        """
        This coroutine sends a pong.

        An unsolicited pong may serve as a unidirectional heartbeat.
        """
        yield from self.write_frame(OP_PONG, data)

    # Private methods - no guarantees.

    @asyncio.coroutine
    def run(self):
        # This coroutine guarantees that the connection is closed at exit.
        yield from self.opening_handshake
        while not self.closing_handshake.done():
            try:
                msg = yield from self.read_message()
                if msg is None:
                    break
                self.messages.put_nowait(msg)
            except asyncio.CancelledError:
                break
            except WebSocketProtocolError:
                yield from self.fail_connection(1002)
            except asyncio.IncompleteReadError:
                yield from self.fail_connection(1006)
            except UnicodeDecodeError:
                yield from self.fail_connection(1007)
            except PayloadTooBig:
                yield from self.fail_connection(1009)
            except Exception:
                yield from self.fail_connection(1011)
                raise
        yield from self.close_connection()

    @asyncio.coroutine
    def read_message(self):
        # Reassemble fragmented messages.
        frame = yield from self.read_data_frame(max_size=self.max_size)
        if frame is None:
            return
        if frame.opcode == OP_TEXT:
            text = True
        elif frame.opcode == OP_BINARY:
            text = False
        else:   # frame.opcode == OP_CONT
            raise WebSocketProtocolError("Unexpected opcode")

        # Shortcut for the common case - no fragmentation
        if frame.fin:
            return frame.data.decode('utf-8') if text else frame.data

        # 5.4. Fragmentation
        chunks = []
        max_size = self.max_size
        if text:
            decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
            if max_size is None:
                def append(frame):
                    nonlocal chunks
                    chunks.append(decoder.decode(frame.data, frame.fin))
            else:
                def append(frame):
                    nonlocal chunks, max_size
                    chunks.append(decoder.decode(frame.data, frame.fin))
                    max_size -= len(frame.data)
        else:
            if max_size is None:
                def append(frame):
                    nonlocal chunks
                    chunks.append(frame.data)
            else:
                def append(frame):
                    nonlocal chunks, max_size
                    chunks.append(frame.data)
                    max_size -= len(frame.data)
        append(frame)

        while not frame.fin:
            frame = yield from self.read_data_frame(max_size=max_size)
            if frame is None:
                raise WebSocketProtocolError("Incomplete fragmented message")
            if frame.opcode != OP_CONT:
                raise WebSocketProtocolError("Unexpected opcode")
            append(frame)

        return ('' if text else b'').join(chunks)

    @asyncio.coroutine
    def read_data_frame(self, max_size):
        # Deal with control frames automatically and return next data frame.
        # 6.2. Receiving Data
        while True:
            frame = yield from self.read_frame(max_size)
            # 5.5. Control Frames
            if frame.opcode == OP_CLOSE:
                self.close_code, self.close_reason = parse_close(frame.data)
                if self.state != 'CLOSING':
                    # 7.1.3. The WebSocket Closing Handshake is Started
                    self.state = 'CLOSING'
                    yield from self.write_frame(OP_CLOSE, frame.data, 'CLOSING')
                if not self.closing_handshake.done():
                    self.closing_handshake.set_result(True)
                return
            elif frame.opcode == OP_PING:
                # Answer pings.
                yield from self.pong(frame.data)
            elif frame.opcode == OP_PONG:
                # Do not acknowledge pings on unsolicited pongs.
                if frame.data in self.pings:
                    # Acknowledge all pings up to the one matching this pong.
                    ping_id = None
                    while ping_id != frame.data:
                        ping_id, waiter = self.pings.popitem(0)
                        if not waiter.cancelled():
                            waiter.set_result(None)
            # 5.6. Data Frames
            else:
                return frame

    @asyncio.coroutine
    def read_frame(self, max_size):
        is_masked = not self.is_client
        frame = yield from read_frame(self.reader.readexactly, is_masked, max_size=max_size)
        side = 'client' if self.is_client else 'server'
        logger.debug("%s << %s", side, frame)
        return frame

    @asyncio.coroutine
    def write_frame(self, opcode, data=b'', expected_state='OPEN'):
        # This may happen if a user attempts to write on a closed connection.
        if self.state != expected_state:
            raise InvalidState("Cannot write to a WebSocket "
                               "in the {} state".format(self.state))
        frame = Frame(True, opcode, data)
        side = 'client' if self.is_client else 'server'
        logger.debug("%s >> %s", side, frame)
        is_masked = self.is_client
        write_frame(frame, self.writer.write, is_masked)
        try:
            # Handle flow control automatically.
            yield from self.writer.drain()
        except ConnectionResetError:
            # Terminate the connection if the socket died,
            # unless it's already being closed.
            if expected_state != 'CLOSING':
                self.state = 'CLOSING'
                yield from self.fail_connection(1006)

    @asyncio.coroutine
    def close_connection(self):
        # 7.1.1. Close the WebSocket Connection
        if self.state == 'CLOSED':
            return

        # Defensive assertion for protocol compliance.
        if self.state != 'CLOSING':                         # pragma: no cover
            raise InvalidState("Cannot close a WebSocket connection "
                               "in the {} state".format(self.state))

        if self.is_client:
            try:
                yield from asyncio.wait_for(self.connection_closed,
                        timeout=self.timeout)
            except (asyncio.CancelledError, asyncio.TimeoutError):
                pass

            if self.state == 'CLOSED':
                return

        # Attempt to terminate the TCP connection properly.
        # If the socket is already closed, this will crash.
        try:
            if self.writer.can_write_eof():
                self.writer.write_eof()
        except Exception:
            pass

        self.writer.close()

        try:
            yield from asyncio.wait_for(self.connection_closed,
                    timeout=self.timeout)
        except (asyncio.CancelledError, asyncio.TimeoutError):
            pass

    @asyncio.coroutine
    def fail_connection(self, code=1011, reason=''):
        # Avoid calling fail_connection more than once to minimize
        # the consequences of race conditions between the two sides.
        if self.connection_failed.done():
            # Wait until the other coroutine calls connection_lost.
            yield from self.connection_closed
            return
        else:
            self.connection_failed.set_result(None)

        # Losing the connection usually results in a protocol error.
        # Preserve the original error code in this case.
        if self.close_code != 1006:
            self.close_code, self.close_reason = code, reason
        # 7.1.7. Fail the WebSocket Connection
        logger.info("Failing the WebSocket connection: %d %s", code, reason)
        if self.state == 'OPEN':
            yield from self.write_frame(OP_CLOSE, serialize_close(code, reason))
            self.state = 'CLOSING'
        if not self.closing_handshake.done():
            self.closing_handshake.set_result(False)
        yield from self.close_connection()

    # asyncio StreamReaderProtocol methods

    def client_connected(self, reader, writer):
        self.reader = reader
        self.writer = writer

    def connection_lost(self, exc):
        # 7.1.4. The WebSocket Connection is Closed
        self.state = 'CLOSED'
        if not self.connection_closed.done():
            self.connection_closed.set_result(None)
        if self.close_code is None:
            self.close_code = 1006
        super().connection_lost(exc)
Exemplo n.º 12
0
class DownloadingService(metaclass=Metaclass):
    def __init__(self, crawler_manager: CrawlerManager,
                 downloader_manager: DownloaderManager):
        self.crawler_manager = crawler_manager
        self.downloader_manager = downloader_manager

        self.running_crawler = {}

        self.start_time = int(time.time())

        self.schedule_queue = Queue()

        self.QUIT = False

    def get_running_messages(self):
        msgs = []
        for crawler_name, downloader in self.downloader_manager.downloaders.items(
        ):
            status = "Running" if downloader.is_running else "Not Running"
            msg = "Status: {}" \
                  " | Crawler Name: {}, " \
                  " | Concurrent Number: {}," \
                  " | Request Number: {}," \
                  " | Finished Number: {}," \
                  " | Failed Number: {}".\
                format(status, crawler_name, downloader.concurrent_number, downloader.pushed_request_count,
                       downloader.finished_request_count, downloader.failed_request_count)
            msgs.append(msg)

        msgs.append("运行时长:{}s".format(time.time() - self.start_time))
        return "\n".join(msgs)

    async def status_print_loop(self):
        while True:
            running_time = int(time.time()) - self.start_time
            if running_time > 0 and running_time % 60 == 0:
                log.info(self.get_running_messages())
            await asyncio.sleep(1)
            if self.QUIT is True:
                log.info(self.get_running_messages())
                break
        log.debug("Quit status print loop")

    async def append_new_crawling_mission(self, crawler_name, crawler_params):
        await self.schedule_queue.put((crawler_name, crawler_params))

    async def task_scheduling_loop(self):
        while True:
            try:
                crawler_name, crawler_params = self.schedule_queue.get_nowait()
            except QueueEmpty:
                await asyncio.sleep(1)
            else:
                crawler = self.crawler_manager.create_crawler(
                    crawler_name, crawler_params)
                downloader = self.downloader_manager.get_or_create_downloader(
                    crawler)
                await self.downloader_manager.schedule(crawler, downloader)
            finally:
                if self.QUIT and self.schedule_queue.qsize() == 0:
                    break
        log.debug("Quit task scheduling loop")

    async def start(self, forever=True):

        self.start_time = int(time.time())

        f1 = asyncio.ensure_future(self.task_scheduling_loop())
        f2 = asyncio.ensure_future(
            self.downloader_manager.reschedule_downloaders())
        f3 = asyncio.ensure_future(self.status_print_loop())

        if forever == False:

            def callback(_):
                log.debug("Downloading Mission Done!")
                self.QUIT = True

            f2.add_done_callback(callback)

        await asyncio.gather(f1, f2, f3)
Exemplo n.º 13
0
class WebSocketCommonProtocol(asyncio.StreamReaderProtocol):
    """
    This class implements common parts of the WebSocket protocol.

    It assumes that the WebSocket connection is established. The handshake is
    managed in subclasses such as
    :class:`~websockets.server.WebSocketServerProtocol` and
    :class:`~websockets.client.WebSocketClientProtocol`.

    It runs a task that stores incoming data frames in a queue and deals with
    control frames automatically. It sends outgoing data frames and performs
    the closing handshake.

    The `host`, `port` and `secure` parameters are simply stored as attributes
    for handlers that need them.

    The `timeout` parameter defines the maximum wait time in seconds for
    completing the closing handshake and, only on the client side, for
    terminating the TCP connection. :meth:`close()` will complete in at most
    this time on the server side and twice this time on the client side.

    Once the connection is closed, the status code is available in the
    :attr:`close_code` attribute and the reason in :attr:`close_reason`.
    """

    # There are only two differences between the client-side and the server-
    # side behavior: masking the payload and closing the underlying TCP
    # connection. This class implements the server-side behavior by default.
    # To get the client-side behavior, set is_client = True.

    is_client = False
    state = 'OPEN'

    def __init__(self,
                 *,
                 host=None,
                 port=None,
                 secure=None,
                 timeout=10,
                 loop=None):
        self.host = host
        self.port = port
        self.secure = secure

        self.timeout = timeout

        super().__init__(asyncio.StreamReader(), self.client_connected, loop)

        self.close_code = None
        self.close_reason = ''

        # Futures tracking steps in the connection's lifecycle.
        self.opening_handshake = asyncio.Future()
        self.closing_handshake = asyncio.Future()
        self.connection_closed = asyncio.Future()

        # Queue of received messages.
        self.messages = Queue()

        # Mapping of ping IDs to waiters, in chronological order.
        self.pings = collections.OrderedDict()

        # Task managing the connection.
        self.worker = asyncio. async (self.run())

        # In a subclass implementing the opening handshake, the state will be
        # CONNECTING at this point.
        if self.state == 'OPEN':
            self.opening_handshake.set_result(True)

    # Public API

    @property
    def open(self):
        """
        This property is ``True`` when the connection is usable.

        It may be used to handle disconnections gracefully.
        """
        return self.state == 'OPEN'

    @asyncio.coroutine
    def close(self, code=1000, reason=''):
        """
        This coroutine performs the closing handshake.

        This is the expected way to terminate a connection on the server side.

        It waits for the other end to complete the handshake. It doesn't do
        anything once the connection is closed.

        It's usually safe to wrap this coroutine in `asyncio.async()` since
        errors during connection termination aren't particularly useful.

        The `code` must be an :class:`int` and the `reason` a :class:`str`.
        """
        if self.state == 'OPEN':
            # 7.1.2. Start the WebSocket Closing Handshake
            self.close_code, self.close_reason = code, reason
            yield from self.write_frame(OP_CLOSE,
                                        serialize_close(code, reason))
            # 7.1.3. The WebSocket Closing Handshake is Started
            self.state = 'CLOSING'

        # If the connection doesn't terminate within the timeout, break out of
        # the worker loop.
        try:
            yield from asyncio.wait_for(self.worker, timeout=self.timeout)
        except asyncio.TimeoutError:
            self.worker.cancel()

        # The worker should terminate quickly once it has been cancelled.
        yield from self.worker

    @asyncio.coroutine
    def recv(self):
        """
        This coroutine receives the next message.

        It returns a :class:`str` for a text frame and :class:`bytes` for a
        binary frame.

        When the end of the message stream is reached, or when a protocol
        error occurs, :meth:`recv` returns ``None``, indicating that the
        connection is closed.
        """
        # Return any available message
        try:
            return self.messages.get_nowait()
        except QueueEmpty:
            pass

        # Wait for a message until the connection is closed
        next_message = asyncio.Task(self.messages.get())
        done, pending = yield from asyncio.wait(
            [next_message, self.worker], return_when=asyncio.FIRST_COMPLETED)
        if next_message in done:
            return next_message.result()

    @asyncio.coroutine
    def send(self, data):
        """
        This coroutine sends a message.

        It sends a :class:`str` as a text frame and :class:`bytes` as a binary
        frame.

        It raises a :exc:`TypeError` for other inputs and
        :exc:`InvalidState` once the connection is closed.
        """
        if isinstance(data, str):
            opcode = 1
            data = data.encode('utf-8')
        elif isinstance(data, bytes):
            opcode = 2
        else:
            raise TypeError("data must be bytes or str")
        yield from self.write_frame(opcode, data)

    @asyncio.coroutine
    def ping(self, data=None):
        """
        This coroutine sends a ping.

        It returns a Future which will be completed when the corresponding
        pong is received and which you may ignore if you don't want to wait.

        A ping may serve as a keepalive.
        """
        # Protect against duplicates if a payload is explicitly set.
        if data in self.pings:
            raise ValueError("Already waiting for a pong with the same data")
        # Generate a unique random payload otherwise.
        while data is None or data in self.pings:
            data = struct.pack('!I', random.getrandbits(32))

        self.pings[data] = asyncio.Future()
        yield from self.write_frame(OP_PING, data)
        return self.pings[data]

    @asyncio.coroutine
    def pong(self, data=b''):
        """
        This coroutine sends a pong.

        An unsolicited pong may serve as a unidirectional heartbeat.
        """
        yield from self.write_frame(OP_PONG, data)

    # Private methods - no guarantees.

    @asyncio.coroutine
    def run(self):
        # This coroutine guarantees that the connection is closed at exit.
        yield from self.opening_handshake
        while not self.closing_handshake.done():
            try:
                msg = yield from self.read_message()
                if msg is None:
                    break
                self.messages.put_nowait(msg)
            except asyncio.CancelledError:
                break
            except WebSocketProtocolError:
                yield from self.fail_connection(1002)
            except UnicodeDecodeError:
                yield from self.fail_connection(1007)
            except Exception:
                yield from self.fail_connection(1011)
                raise
        yield from self.close_connection()

    @asyncio.coroutine
    def read_message(self):
        # Reassemble fragmented messages.
        frame = yield from self.read_data_frame()
        if frame is None:
            return
        if frame.opcode == OP_TEXT:
            text = True
        elif frame.opcode == OP_BINARY:
            text = False
        else:  # frame.opcode == OP_CONT
            raise WebSocketProtocolError("Unexpected opcode")

        # Shortcut for the common case - no fragmentation
        if frame.fin:
            return frame.data.decode('utf-8') if text else frame.data

        # 5.4. Fragmentation
        chunks = []
        if text:
            decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
            append = lambda f: chunks.append(decoder.decode(f.data, f.fin))
        else:
            append = lambda f: chunks.append(f.data)
        append(frame)

        while not frame.fin:
            frame = yield from self.read_data_frame()
            if frame is None:
                raise WebSocketProtocolError("Incomplete fragmented message")
            if frame.opcode != OP_CONT:
                raise WebSocketProtocolError("Unexpected opcode")
            append(frame)

        return ('' if text else b'').join(chunks)

    @asyncio.coroutine
    def read_data_frame(self):
        # Deal with control frames automatically and return next data frame.
        # 6.2. Receiving Data
        while True:
            frame = yield from self.read_frame()
            # 5.5. Control Frames
            if frame.opcode == OP_CLOSE:
                self.close_code, self.close_reason = parse_close(frame.data)
                if self.state != 'CLOSING':
                    # 7.1.3. The WebSocket Closing Handshake is Started
                    self.state = 'CLOSING'
                    yield from self.write_frame(OP_CLOSE, frame.data,
                                                'CLOSING')
                self.closing_handshake.set_result(True)
                return
            elif frame.opcode == OP_PING:
                # Answer pings.
                yield from self.pong(frame.data)
            elif frame.opcode == OP_PONG:
                # Do not acknowledge pings on unsolicited pongs.
                if frame.data in self.pings:
                    # Acknowledge all pings up to the one matching this pong.
                    ping_id = None
                    while ping_id != frame.data:
                        ping_id, waiter = self.pings.popitem(0)
                        if not waiter.cancelled():
                            waiter.set_result(None)
            # 5.6. Data Frames
            else:
                return frame

    @asyncio.coroutine
    def read_frame(self):
        is_masked = not self.is_client
        frame = yield from read_frame(self.reader.readexactly, is_masked)
        side = 'client' if self.is_client else 'server'
        logger.debug("%s << %s", side, frame)
        return frame

    @asyncio.coroutine
    def write_frame(self, opcode, data=b'', expected_state='OPEN'):
        # This may happen if a user attempts to write on a closed connection.
        if self.state != expected_state:
            raise InvalidState("Cannot write to a WebSocket "
                               "in the {} state".format(self.state))
        frame = Frame(True, opcode, data)
        side = 'client' if self.is_client else 'server'
        logger.debug("%s >> %s", side, frame)
        is_masked = self.is_client
        write_frame(frame, self.writer.write, is_masked)
        # Handle flow control automatically.
        try:
            yield from self.writer.drain()
        except ConnectionResetError:
            pass

    @asyncio.coroutine
    def close_connection(self):
        # 7.1.1. Close the WebSocket Connection
        if self.state == 'CLOSED':
            return

        # Defensive assertion for protocol compliance.
        if self.state != 'CLOSING':  # pragma: no cover
            raise InvalidState("Cannot close a WebSocket connection "
                               "in the {} state".format(self.state))

        if self.is_client:
            try:
                yield from asyncio.wait_for(self.connection_closed,
                                            timeout=self.timeout)
            except (asyncio.CancelledError, asyncio.TimeoutError):
                pass

            if self.state == 'CLOSED':
                return

        if self.writer.can_write_eof():
            self.writer.write_eof()
        self.writer.close()

        try:
            yield from asyncio.wait_for(self.connection_closed,
                                        timeout=self.timeout)
        except (asyncio.CancelledError, asyncio.TimeoutError):
            pass

    @asyncio.coroutine
    def fail_connection(self, code=1011, reason=''):
        # Losing the connection usually results in a protocol error.
        # Preserve the original error code in this case.
        if self.close_code != 1006:
            self.close_code, self.close_reason = code, reason
        # 7.1.7. Fail the WebSocket Connection
        logger.info("Failing the WebSocket connection: %d %s", code, reason)
        if self.state == 'OPEN':
            yield from self.write_frame(OP_CLOSE,
                                        serialize_close(code, reason))
            self.state = 'CLOSING'
        if not self.closing_handshake.done():
            self.closing_handshake.set_result(False)
        yield from self.close_connection()

    # asyncio StreamReaderProtocol methods

    def client_connected(self, reader, writer):
        self.reader = reader
        self.writer = writer

    def connection_lost(self, exc):
        # 7.1.4. The WebSocket Connection is Closed
        self.state = 'CLOSED'
        if not self.connection_closed.done():
            self.connection_closed.set_result(None)
        if self.close_code is None:
            self.close_code = 1006
        super().connection_lost(exc)
Exemplo n.º 14
0
class Program(SoundCard):
    def __init__(self, id):
        super(Program, self).__init__()

        self.id = id
        self.other_program = None
        self.set_register('p', self.id)
        self.queue = Queue()
        self.values_sent = 0
        self.waiting = False

    async def send(self, reg):
        if reg.isdigit():
            val = int(reg)
        else:
            val = self.get_register(reg)

        self.other_program.queue.put_nowait(val)

        self.values_sent += 1

    async def receive(self, reg):
        self.waiting = True
        while True:
            try:
                val = self.queue.get_nowait()
                break
            except asyncio.QueueEmpty:
                if self.other_program.queue.empty() and self.other_program.waiting:
                    return
                await asyncio.sleep(1.0)
        self.waiting = False
        self.set_register(reg, val)

    def parse_instruction(self, instruction):

        func, params = super(Program, self).parse_instruction(instruction)

        if func == self.recover_last_sound:
            func = self.receive
        elif func == self.play_sound:
            func = self.send

        return func, params
    
    async def execute_program(self, program):
        if isinstance(program, str):
            program = program.replace('\r', '').split('\n')
        
        current_instruction = 0

        while current_instruction >= 0 and current_instruction < len(program):
            func, params = self.parse_instruction(program[current_instruction])

            if func == self.receive or func == self.send:
                res = await func(**params)
                if self.waiting:
                    break
            else:
                res = func(**params)

            if func == self.jump_instructions:
                current_instruction += res
            else:
                current_instruction += 1
Exemplo n.º 15
0
class Downloader(metaclass=Metaclass):
    '''下载器逻辑'''

    __polling_interval = 0.0001    # 轮询暂停间隔,防止CPU进行空循环

    def __init__(self, request_queue_max_size: int=1000, concurrent_number: int=1, delay: int=0):
        self.http_client = AsyncHTTPClient()

        self.request_queue = Queue(request_queue_max_size)  # 请求队列
        self.concurrent_number = concurrent_number  # 请求最大并发数
        self.delay = delay  # 每批请求之间的延时
        self.is_running = False  # 记录采集器的运行状态
        self.pushed_request_count = 0  # 记录采集器队列中已经添加的请求数量
        self.failed_request_count = 0  # 记录失败的请求数量
        self.finished_request_count = 0  # 记录总共完成的请求数量

    async def add_request(self, request: Request):
        '''
        添加请求
        :param request: 请求对象
        :return:
        '''
        await self.request_queue.put(request)
        self.pushed_request_count += 1

    async def fetch(self, request: Request):
        '''根据指定的请求对象发起HTTP请求'''
        log.debug("Send request {}".format(request))
        try:
            response = await self.http_client.fetch(request)
        except HTTPClientError as e:
            return Response(e.response, request)
        else:
            return Response(response, request)

    async def download(self, request: FileRequest):
        log.debug("Downloading '{}'".format(request.file_path))
        actual_length = 0
        file = open(request.file_path, "wb")
        def write_data(data):
            nonlocal actual_length
            actual_length += len(data)
            file.write(data)
        tornado_request = Request(
            url=request.url,
            callback=None,
            headers=request.headers,
            streaming_callback=write_data,
            request_timeout=60*10,
            connect_timeout=60*10
        )
        tornado_response = await self.http_client.fetch(tornado_request)
        file.close()

        if actual_length == int(tornado_response.headers["Content-Length"]):
            log.debug("Success download '{}'".format(request.file_path))
        else:
            raise Exception("Error download '{}': content-length error".format(request.file_path))
        return FileResponse(tornado_response, request, file_size=actual_length, file_path=request.file_path)

    async def handle_request(self, request):
        log.debug("Handle request {}".format(request))
        try:
            if isinstance(request, FileRequest):
                response = await self.download(request)
            else:
                response = await self.fetch(request)
            # 注意:callback方法必须是一个awaitable对象
            response_processed_result = request.callback(response)

            if isinstance(response_processed_result, typing.AsyncGenerator):
                async for result in response_processed_result:
                    if isinstance(result, Request):
                        await self.add_request(result)
            else:
                result = await response_processed_result
                if isinstance(result, Request):
                    await self.add_request(result)
        except Exception as e:
            log.exception(e)
            log.error("Catch exception: {} {}".format(e, request))
            self.failed_request_count += 1
        else:
            log.debug("Fetched response from {}".format(request))
            self.finished_request_count += 1

    async def run(self, downloader_name=None):
        downloader_name = downloader_name if downloader_name else ""
        log.debug("Run downloader {}".format(downloader_name))
        while self.is_running:
            try:
                request = self.request_queue.get_nowait()
            except QueueEmpty:
                await asyncio.sleep(self.__polling_interval)
            else:
                await self.handle_request(request)
                if self.pushed_request_count == self.finished_request_count + self.failed_request_count:
                    self.is_running = False
            await asyncio.sleep(self.delay)
        log.debug("Quit downloader {}".format(downloader_name))
Exemplo n.º 16
0
class GoogleReportState(iot_base.BaseIoT):
    """Report states to Google.

    Uses a queue to send messages.
    """
    def __init__(self, cloud: Cloud):
        """Initialize Google Report State."""
        super().__init__(cloud)
        self._connect_lock = asyncio.Lock()
        self._to_send = Queue(100)
        self._message_sender_task = None
        # Local code waiting for a response
        self._response_handler: Dict[str, asyncio.Future] = {}
        self.register_on_connect(self._async_on_connect)
        self.register_on_disconnect(self._async_on_disconnect)

        # Register start/stop
        cloud.register_on_stop(self.disconnect)

    @property
    def package_name(self) -> str:
        """Return the package name for logging."""
        return __name__

    @property
    def ws_server_url(self) -> str:
        """Server to connect to."""
        # https -> wss, http -> ws
        return f"ws{self.cloud.google_actions_report_state_url[4:]}/v1"

    async def async_send_message(self, msg):
        """Send a message."""
        msgid = uuid.uuid4().hex

        # Since connect is async, guard against send_message called twice in parallel.
        async with self._connect_lock:
            if self.state == iot_base.STATE_DISCONNECTED:
                self.cloud.run_task(self.connect())
                # Give connect time to start up and change state.
                await asyncio.sleep(0)

        if self._to_send.full():
            discard_msg = self._to_send.get_nowait()
            self._response_handler.pop(discard_msg["msgid"]).set_exception(
                ErrorResponse(ERR_DISCARD_CODE, ERR_DISCARD_MSG))

        fut = self._response_handler[msgid] = asyncio.Future()

        self._to_send.put_nowait({"msgid": msgid, "payload": msg})

        try:
            return await fut
        finally:
            self._response_handler.pop(msgid, None)

    def async_handle_message(self, msg):
        """Handle a message."""
        response_handler = self._response_handler.get(msg["msgid"])

        if response_handler is not None:
            if "error" in msg:
                response_handler.set_exception(
                    ErrorResponse(msg["error"], msg["message"]))
            else:
                response_handler.set_result(msg.get("payload"))
            return

        self._logger.warning("Got unhandled message: %s", msg)

    async def _async_on_connect(self):
        """On Connect handler."""
        self._message_sender_task = self.cloud.run_task(
            self._async_message_sender())

    async def _async_on_disconnect(self):
        """On disconnect handler."""
        self._message_sender_task.cancel()
        self._message_sender_task = None

    async def _async_message_sender(self):
        """Start sending messages."""
        self._logger.debug("Message sender task activated")
        try:
            while True:
                await self.async_send_json_message(await self._to_send.get())
        except asyncio.CancelledError:
            pass
        self._logger.debug("Message sender task shut down")
Exemplo n.º 17
0
class Connection:
    def __init__(self, cursor, conn: Database):
        self._cursor = cursor
        self._public_cursor = None
        self._connection = conn
        self._thread = ThreadPoolExecutor(max_workers=1)
        self._result_queue = Queue(maxsize=1)

    def _callback(self, result):
        res = result.result()
        self._result_queue.put_nowait(res)

    async def __get_result(self):
        while True:
            try:
                result = self._result_queue.get_nowait()
            except QueueEmpty:
                await asyncio.sleep(0.0001)
            except Exception as e:
                raise e
            else:
                return result

    def _execute(self, sql: str, parameters=None):
        if not parameters:
            parameters = []
        try:
            result = self._cursor.execute(sql, parameters)
            self._public_cursor = result
            return self
        except Exception as e:
            return e

    async def execute(self, sql, parameters=None):
        task = self._thread.submit(self._execute, sql, parameters=parameters)
        task.add_done_callback(self._callback)
        result = await self.__get_result()
        if isinstance(result, Exception):
            raise result
        return result

    def _execute_many(self, sql, parameters):
        try:
            result = self._cursor.executemany(sql, parameters)
            self._public_cursor = result
            return self
        except Exception as e:
            return e

    async def executemany(self, sql, parameters):
        task = self._thread.submit(self._execute_many,
                                   sql,
                                   parameters=parameters)
        task.add_done_callback(self._callback)
        result = await self.__get_result()
        if isinstance(result, Exception):
            raise result
        return result

    def _fetch_one(self):
        try:
            result = self._public_cursor.fetchone()
            return result
        except Exception as e:
            return e

    async def fetchone(self):
        if self._public_cursor is None:
            raise NoActiveCursor
        task = self._thread.submit(self._fetch_one)
        task.add_done_callback(self._callback)
        result = await self.__get_result()
        if isinstance(result, Exception):
            raise result
        return result

    def _fetch_all(self):
        try:
            result = self._public_cursor.fetchall()
            return result
        except Exception as e:
            return e

    async def fetchall(self):
        if self._public_cursor is None:
            raise NoActiveCursor
        task = self._thread.submit(self._fetch_all)
        task.add_done_callback(self._callback)
        result = await self.__get_result()
        if isinstance(result, Exception):
            raise result
        return result

    def _commit(self):
        try:
            self._cursor.commit()
            return None
        except Exception as e:
            return e

    async def commit(self):
        task = self._thread.submit(self._commit)
        task.add_done_callback(self._callback)
        result = await self.__get_result()
        if isinstance(result, Exception):
            raise result
        return result

    async def close(self):
        self._cursor.close()
        await self._connection.connection_semaphore.put({})
        return
Exemplo n.º 18
0
class Ant(abc.ABC):
    response_pipelines: typing.List[Pipeline] = []
    request_pipelines: typing.List[Pipeline] = []
    item_pipelines: typing.List[Pipeline] = []
    request_cls = Request
    response_cls = Response
    request_timeout = DEFAULT_TIMEOUT.total
    request_retries = 3
    request_retry_delay = 5
    request_proxies: typing.List[typing.Union[str, URL]] = []
    request_max_redirects = 10
    request_allow_redirects = True
    response_in_stream = False
    connection_limit = 100  # see "TCPConnector" in "aiohttp"
    connection_limit_per_host = 0
    concurrent_limit = 100

    def __init__(self,
                 loop: typing.Optional[asyncio.AbstractEventLoop] = None):
        self.loop = loop if loop is not None else asyncio.get_event_loop()
        self.logger = logging.getLogger(self.__class__.__name__)
        self.session: aiohttp.ClientSession = ClientSession(
            response_class=self.response_cls,
            connector=aiohttp.TCPConnector(
                limit=self.connection_limit,
                enable_cleanup_closed=True,
                limit_per_host=self.connection_limit_per_host))
        # coroutine`s concurrency support
        self._queue = Queue(loop=self.loop)
        self._done_queue = Queue(loop=self.loop)
        self._running_count = 0
        self._is_closed = False
        # report var
        self._reports: typing.DefaultDict[str, typing.List[
            int, int]] = defaultdict(lambda: [0, 0])
        self._drop_reports: typing.DefaultDict[str, typing.List[
            int, int]] = defaultdict(lambda: [0, 0])
        self._start_time = time.time()
        self._last_time = self._start_time
        self._report_slot = 60  # report once after one minute by default

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def is_running(self) -> bool:
        return self._running_count > 0

    async def request(
            self,
            url: typing.Union[str, URL],
            method: str = aiohttp.hdrs.METH_GET,
            params: typing.Optional[dict] = None,
            headers: typing.Optional[dict] = None,
            cookies: typing.Optional[dict] = None,
            data: typing.Optional[typing.Union[typing.AnyStr, typing.Dict,
                                               typing.IO]] = None,
            proxy: typing.Optional[typing.Union[str, URL]] = None,
            timeout: typing.Optional[typing.Union[int, float]] = None,
            retries: typing.Optional[int] = None,
            response_in_stream: typing.Optional[bool] = None) -> Response:
        if not isinstance(url, URL):
            url = URL(url)
        if proxy and not isinstance(proxy, URL):
            proxy = URL(proxy)
        elif proxy is None:
            proxy = self.get_proxy()
        if timeout is None:
            timeout = self.request_timeout
        if retries is None:
            retries = self.request_retries
        if response_in_stream is None:
            response_in_stream = self.response_in_stream

        req = self.request_cls(method,
                               url,
                               timeout=timeout,
                               params=params,
                               headers=headers,
                               cookies=cookies,
                               data=data,
                               proxy=proxy,
                               response_in_stream=response_in_stream)
        req = await self._handle_thing_with_pipelines(req,
                                                      self.request_pipelines)
        self.report(req)

        if retries > 0:
            res = await self.make_retry_decorator(
                retries, self.request_retry_delay)(self._request)(req)
        else:
            res = await self._request(req)

        res = await self._handle_thing_with_pipelines(res,
                                                      self.response_pipelines)
        self.report(res)
        return res

    async def collect(self, item: Item) -> None:
        self.logger.debug('Collect item: ' + str(item))
        await self._handle_thing_with_pipelines(item, self.item_pipelines)
        self.report(item)

    async def open(self) -> None:
        self.logger.info('Opening')
        for pipeline in itertools.chain(self.item_pipelines,
                                        self.response_pipelines,
                                        self.request_pipelines):
            obj = pipeline.on_spider_open()
            if asyncio.iscoroutine(obj):
                await obj

    async def close(self) -> None:
        await self.wait_scheduled_coroutines()

        for pipeline in itertools.chain(self.item_pipelines,
                                        self.response_pipelines,
                                        self.request_pipelines):
            obj = pipeline.on_spider_close()
            if asyncio.iscoroutine(obj):
                await obj

        await self.session.close()

        self._is_closed = True
        self.logger.info('Closed')

    @abc.abstractmethod
    async def run(self) -> None:
        """App custom entrance"""

    async def main(self) -> None:
        try:
            await self.open()
            await self.run()
        except Exception as e:
            self.logger.exception('Run ant with ' + e.__class__.__name__)
        try:
            await self.close()
        except Exception as e:
            self.logger.exception('Close ant with ' + e.__class__.__name__)
        # total report
        for name, counts in self._reports.items():
            self.logger.info('Get {:d} {:s} in total'.format(counts[1], name))
        for name, counts in self._drop_reports.items():
            self.logger.info('Drop {:d} {:s} in total'.format(counts[1], name))
        self.logger.info('Run {:s} in {:f} seconds'.format(
            self.__class__.__name__,
            time.time() - self._start_time))

    @staticmethod
    def make_retry_decorator(
            retries: int, delay: float
    ) -> typing.Callable[[typing.Callable], typing.Callable]:
        return retry(wait=wait_fixed(delay),
                     retry=(retry_if_result(lambda res: res.status >= 500)
                            | retry_if_exception_type(
                                exception_types=aiohttp.ClientError)),
                     stop=stop_after_attempt(retries + 1))

    def get_proxy(self) -> typing.Optional[URL]:
        """Chose a proxy, default by random"""
        try:
            return URL(random.choice(self.request_proxies))
        except IndexError:
            return None

    def schedule_coroutine(self, coroutine: typing.Coroutine) -> None:
        """Like "asyncio.ensure_future", it schedule coroutine in event loop
        and return immediately.

        Call "self.wait_scheduled_coroutines" make sure all coroutine has been
        done.
        """
        def _done_callback(f):
            self._running_count -= 1
            self._done_queue.put_nowait(f)
            try:
                if (self.concurrent_limit == -1
                        or self._running_count < self.concurrent_limit):
                    next_coroutine = self._queue.get_nowait()
                    self._running_count += 1
                    asyncio.ensure_future(
                        next_coroutine,
                        loop=self.loop).add_done_callback(_done_callback)
            except QueueEmpty:
                pass

        if self._is_closed:
            self.logger.warning('This pool has be closed!')
            return

        if (self.concurrent_limit == -1
                or self._running_count < self.concurrent_limit):
            self._running_count += 1
            asyncio.ensure_future(
                coroutine, loop=self.loop).add_done_callback(_done_callback)
        else:
            self._queue.put_nowait(coroutine)

    def schedule_coroutines(
            self, coroutines: typing.Iterable[typing.Coroutine]) -> None:
        """A short way to schedule many coroutines.
        """
        for coroutine in coroutines:
            self.schedule_coroutine(coroutine)

    async def wait_scheduled_coroutines(self):
        """Wait scheduled coroutines to be done, can be called many times.
        """
        while self._running_count > 0 or self._done_queue.qsize() > 0:
            await self._done_queue.get()

    def as_completed(
        self,
        coroutines: typing.Iterable[typing.Coroutine],
        limit: typing.Optional[int] = None
    ) -> typing.Generator[typing.Coroutine, None, None]:
        """Like "asyncio.as_completed",
        run and iter coroutines out of the pool.

        :param limit: set to "self.concurrent_limit" by default,
        this "limit" is not shared with pool`s limit
        """
        limit = self.concurrent_limit if limit is None else limit

        coroutines = iter(coroutines)
        queue = Queue(loop=self.loop)
        todo = []

        def _done_callback(f):
            queue.put_nowait(f)
            todo.remove(f)
            try:
                nf = asyncio.ensure_future(next(coroutines))
                nf.add_done_callback(_done_callback)
                todo.append(nf)
            except StopIteration:
                pass

        async def _wait_for_one():
            f = await queue.get()
            return f.result()

        if limit <= 0:
            fs = {
                asyncio.ensure_future(cor, loop=self.loop)
                for cor in coroutines
            }
        else:
            fs = {
                asyncio.ensure_future(cor, loop=self.loop)
                for cor in islice(coroutines, 0, limit)
            }
        for f in fs:
            f.add_done_callback(_done_callback)
            todo.append(f)

        while len(todo) > 0 or queue.qsize() > 0:
            yield _wait_for_one()

    async def as_completed_with_async(
        self,
        coroutines: typing.Iterable[typing.Coroutine],
        limit: typing.Optional[int] = None,
        raise_exception: bool = True,
    ) -> typing.AsyncGenerator[typing.Any, None]:
        """as_completed`s async version, can catch and log exception inside.
        """
        for coro in self.as_completed(coroutines, limit=limit):
            try:
                yield await coro
            except Exception as e:
                if raise_exception:
                    raise e
                else:
                    self.logger.exception('Get exception {:s} in '
                                          '"as_completed_with_async"'.format(
                                              str(e)))

    def report(self, thing: Things, dropped: bool = False) -> None:
        now_time = time.time()
        if now_time - self._last_time > self._report_slot:
            self._last_time = now_time
            for name, counts in self._reports.items():
                count = counts[1] - counts[0]
                counts[0] = counts[1]
                self.logger.info(
                    'Get {:d} {:s} in total with {:d}/{:d}s rate'.format(
                        counts[1], name, count, self._report_slot))
            for name, counts in self._drop_reports.items():
                count = counts[1] - counts[0]
                counts[0] = counts[1]
                self.logger.info(
                    'Drop {:d} {:s} in total with {:d}/{:d} rate'.format(
                        counts[1], name, count, self._report_slot))
        report_type = thing.__class__.__name__
        if dropped:
            reports = self._drop_reports
        else:
            reports = self._reports
        counts = reports[report_type]
        counts[1] += 1

    async def _handle_thing_with_pipelines(
            self, thing: Things, pipelines: typing.List[Pipeline]) -> Things:
        """Process thing one by one, break the process chain when get
        exception.
        """
        self.logger.debug('Process thing: ' + str(thing))
        raw_thing = thing
        for pipeline in pipelines:
            try:
                thing = pipeline.process(thing)
                if asyncio.iscoroutine(thing):
                    thing = await thing
            except Exception as e:
                if isinstance(e, ThingDropped):
                    self.report(raw_thing, dropped=True)
                raise e
        return thing

    async def _request(self, req: Request) -> Response:
        if req.proxy is not None:
            # proxy auth not work in one session with many requests,
            # add auth header to fix it
            if req.proxy.scheme == 'http' and req.proxy.user is not None:
                req.headers[aiohttp.hdrs.PROXY_AUTHORIZATION] = \
                    aiohttp.BasicAuth.from_url(req.proxy).encode()

        # cookies in headers, params in url
        req_kwargs = dict(headers=req.headers,
                          data=req.data,
                          timeout=req.timeout,
                          proxy=req.proxy,
                          max_redirects=self.request_max_redirects,
                          allow_redirects=self.request_allow_redirects)
        response = await self.session._request(req.method, req.url,
                                               **req_kwargs)

        if not req.response_in_stream:
            await response.read()
            response.close()
            await response.wait_for_close()
        return response
Exemplo n.º 19
0
async def radar(radar_Q: Queue = None):

    if radar_Q is None:
        print("Radar task is terminating. The radar_Q is None. Nothing to do")
        return

    try:
        q = Queue()

        while True:

            async with async_timeout.timeout(5) as tm:
                while True:
                    try:
                        incoming_action:INCOMING_ACTION_TYPE = await radar_Q.get()
                        q.put_nowait(incoming_action)
                        STATS['number_rx'] += 1
                        STATS['max_incoming_q_size'] = max(radar_Q.qsize(), STATS['max_incoming_q_size'])
                    except Exception as x:
                        break

            if not tm.expired:
                continue

            time_now = datetime.utcnow()

            while not q.empty():

                item = q.get_nowait()

                icao_rec = item.actionMsg

                # print(F"DB_WORKER_RXED: {action_types} {icao_rec}")

                # await db_process_sbs1_msg2(database, sbs1_msg)
                id = item.aircraftId
                last_seen = icao_rec['last_seen']
                data  = icao_rec['current']

                delta_sec = (time_now - last_seen).total_seconds()
                if delta_sec > 300:
                    print(F"@Radar: Got an old one: {id} {last_seen} {delta_sec}")
                    STATS['rx_expired'] += 1
                    remove_track(id)
                    del history[id]
                    continue

                history[id] = last_seen

                STATS['max_history_size'] = max(STATS['max_history_size'],len(history))

                if data['lon'] is not None and data['lat'] is not None:
                    update_track(id,data['lat'], data['lon'])
                    STATS['updates'] += 1

                update_table_store(id, last_seen, data)
                render_table_store()
            # endwhile

            # cleanup any lingering tracks:
            flush()

            refresh()


    except CancelledError:
            print("Cancellling radar task")

    except Exception as x:
        print(F"Exception {x}")