Example #1
0
def get_pages_with_weather_at_place(static_root: str, pages: deque) -> deque:
    """Function find all pages with link at a place or find all places in country and links for it."""

    links, another = [], []

    while pages:
        response = get(pages.popleft())
        soup = BeautifulSoup(response.text, 'lxml')
        cells = soup.find_all(class_="countryMap-cell")

        for cell in cells:
            for a in cell.find_all('a'):
                if a.text == '>>>':
                    pages.append(f"https://rp5.ru{a['href']}")
                elif a['href'].find('/Погода_в_') > -1:
                    links.append(f'https://rp5.ru{a["href"]}')
                elif a['href'].find('Погода_в_') > -1:
                    links.append(f'https://rp5.ru/{a["href"]}')
                else:
                    another.append(a["href"])

        for page in pages:
            if page in links:
                links.remove(page)
    return links, another
Example #2
0
    def _sample_next_cluster(self, history: deque) -> Tuple[deque, float]:
        """Sample next cluster and corresponding transition time.

        In the key resulting from the current history is not present in the
        list of available transition probabilities, the last cluster label
        in the given history is replaced with the label of the nearest cluster,
        and a new history is searched.

        :param history: current history/past/trajectory
        :type history: deque
        :return: the history with the new cluster appended and the transition time
        :rtype: Tuple[deque, float]
        """
        key = ",".join(map(str, list(history)))
        if not key in self._transition_prob:
            last_cluster = history.pop()
            history.append(self._find_closest_cluster(last_cluster))
            history = self._find_history(history)
            key = ",".join(map(str, list(history)))
        next_cluster = int(
            np.random.choice(self._transition_prob[key][:, 0],
                             p=self._transition_prob[key][:, 1]))
        key += ",{:d}".format(next_cluster)
        history.append(next_cluster)
        return history, self._transition_time[key]
Example #3
0
    def evaluate(
        self,
        obs_queue: deque,
        agent: Agent,
        num_episode: int = 3,
        render: bool = False,
    ) -> Tuple[float, List[GymImg], ]:
        """evaluate uses the given agent to run the game for a few episodes and
        returns the average reward and the captured frames."""
        self.__env = self.__env_eval
        ep_rewards = []
        frames = []
        for _ in range(self.get_eval_lives() * num_episode):
            observations, ep_reward, _frames = self.reset(render=render)
            for obs in observations:
                obs_queue.append(obs)
            if render:
                frames.extend(_frames)
            done = False

            while not done:
                state = self.make_state(obs_queue).to(self.__device).float()
                action = agent.run(state)
                obs, reward, done = self.step(action)

                ep_reward += reward
                obs_queue.append(obs)
                if render:
                    frames.append(self.get_frame())

            ep_rewards.append(ep_reward)

        self.__env = self.__env_train
        return np.sum(ep_rewards) / num_episode, frames
Example #4
0
def program(instructions: list, pid: int, out_queue: deque, in_queue: deque):
    registers = defaultdict(lambda: 0)
    registers["p"] = pid
    pointer = 0
    while 0 <= pointer < len(instructions):
        i, x, y, *_ = (instructions[pointer] + " !").split(" ")
        x_v = int(x) if is_digit(x) else registers[x]
        y = int(y) if is_digit(y) else registers[y]
        if i == "snd":
            out_queue.append(x_v)
            yield "send"
        elif i == "set":
            registers[x] = y
        elif i == "add":
            registers[x] += y
        elif i == "mul":
            registers[x] *= y
        elif i == "mod":
            registers[x] %= y
        elif i == "rcv":
            while len(in_queue) == 0:
                yield "wait"
            else:
                registers[x] = in_queue.popleft()
                yield "received"
        elif i == "jgz":
            if x_v > 0:
                pointer += (y - 1)

        pointer += 1
Example #5
0
 async def _part_uploader(
     self,
     upload_id: str,
     object_name: str,
     parts_queue: asyncio.Queue,
     results_queue: deque,
     part_upload_tries: int,
     **kwargs,
 ):
     backoff = asyncbackoff(
         None,
         None,
         max_tries=part_upload_tries,
         exceptions=(ClientError, ),
     )
     while True:
         msg = await parts_queue.get()
         if msg is DONE:
             break
         part_no, part_hash, part = msg
         etag = await backoff(self._put_part)(  # type: ignore
             upload_id=upload_id,
             object_name=object_name,
             part_no=part_no,
             data=part,
             content_sha256=part_hash,
             **kwargs,
         )
         log.debug(
             "Etag for part %d of %s is %s",
             part_no,
             upload_id,
             etag,
         )
         results_queue.append((part_no, etag))
Example #6
0
def add_rooms_to_explore(current_room: Room, to_explore: deque) -> deque:
    for direction in Direction:
        if current_room.door_is_open(direction):
            extended_path = current_room.code_with_current_path + direction.name[
                0]
            to_explore.append(Room(current_room.maze, extended_path))
    return to_explore
Example #7
0
def pop_three_addition_first(stack: deque):
    window = deque()
    remainder = deque()
    while len(stack) > 0:
        # fill window
        if len(stack) == 1:
            remainder.append(stack.pop())
            break
        for _ in range(3 - len(window)):
            window.append(stack.popleft())
        operand1, operator, operand2 = (window.popleft(), window.popleft(),
                                        window.popleft())
        if operator == '+':
            sum = int(operand1) + int(operand2)
            stack.appendleft(sum)
        else:
            # we have something like 5 * 3 here
            remainder.append(operand1)
            remainder.append(operator)
            window.append(operand2)
    for num in window:
        remainder.append(num)
    while len(remainder) > 1:
        pop_three(remainder)
    stack.append(remainder.pop())
Example #8
0
def bfs(initial: State):
    visited = {hash(initial)}
    state_queue = Queue([initial])

    # while thingies exist in the queue of states, keep going.
    while state_queue:
        curr_state = state_queue.popleft()
        # short-circuit the bfs is a solution is found.
        # returns the steps to reach the solution in a string.
        if curr_state.done:
            return curr_state.ops_string

        # tries to execute a Q operation
        if curr_state.operation_Q_allowed():
            next_state_q = curr_state.execute_operation_Q()
            q_hash = hash(next_state_q)
            if q_hash not in visited:
                visited.add(q_hash)
                state_queue.append(next_state_q)

        # tries to execute an S operation
        if curr_state.operation_S_allowed():
            next_state_s = curr_state.execute_operation_S()
            s_hash = hash(next_state_s)
            if s_hash not in visited:
                visited.add(s_hash)
                state_queue.append(next_state_s)

    raise Exception("BFS: No solution found.")
Example #9
0
    def evaluate(
        self,
        obs_queue: deque,
        agent: Agent,
        num_episode: int = 3,
        render: bool = False,
    ) -> Tuple[float, List[GymImg],
               ]:  # 使用给定的代理运行游戏几次(3个玩家5个回合),并返回平均奖励和捕获的帧(实际上不返回帧)。
        """evaluate uses the given agent to run the game for a few episodes and
        returns the average reward and the captured frames."""
        self.__env = self.__env_eval
        ep_rewards = []
        frames = []
        for _ in range(self.get_eval_lives() * num_episode):
            observations, ep_reward, _frames = self.reset(
                render=render)  # 初始化测试环境
            for obs in observations:
                obs_queue.append(obs)
            if render: frames.extend(_frames)
            done = False

            while not done:  # 开始测试
                state = self.make_state(obs_queue).to(self.__device).float()
                action = agent.run(state)  # 得到AI的下一个步骤
                obs, reward, done = self.step(action)  # 得到下一状态、收益、是否结束

                ep_reward += reward
                obs_queue.append(obs)
                if render: frames.append(self.get_frame())

            ep_rewards.append(ep_reward)  # 统计收益

        self.__env = self.__env_train
        return np.sum(ep_rewards) / num_episode, frames  # 返回平均收益
Example #10
0
    def __update_prob_recursive(self, start: GraphNode, probs: dict,
                                previous_words: deque):
        fixed_previous_words = previous_words
        for next_node in start.next:
            previous_words = fixed_previous_words.copy()
            pre_len = len(previous_words)

            next_node: GraphNode = next_node  # just for type declaration

            # prior probability
            prior_prob = probs[pre_len].probability(" ".join(previous_words))

            # union probability
            words = list(previous_words.copy())
            words.append(next_node.value)
            union_prob = probs[pre_len + 1].probability(" ".join(words))

            local_prob = union_prob / prior_prob
            accumulative_prob = local_prob * start.accumulative_prob

            if next_node.best_previous is None or accumulative_prob > next_node.accumulative_prob:
                next_node.accumulative_prob = accumulative_prob
                next_node.best_previous = start

            if start == self.end:
                start.next.append(self.end)
                self.end.previous.append(start)
                return

            # update previous words and continue recursion
            previous_words.append(next_node.value)
            if len(previous_words) >= self.ngram_size:
                previous_words.popleft()
            self.__update_prob_recursive(next_node, probs, previous_words)
    def __insertElement(self, queue: deque, node, lvl, side: SIDE, pinx):
        l = len(node.children)

        for inx in range(l):
            iinx = inx if side == SIDE.LEFT else l - inx - 1
            newNode = node.children[iinx]
            queue.append((newNode, lvl, inx, pinx))
Example #12
0
def checkBFSColourable(graph: Graph, queue: deque, array) -> bool:
    key = queue.popleft()
    if graph.matrix[key][key]:
        return False
    array[key][0] = True
    length = 0
    for i in range(graph.size):
        if graph.matrix[key][i]:
            if array[i][1] and array[key][2] == array[i][2]:
                return False
            if array[i][1]:
                continue
            queue.append(i)
            array[i][2] = array[key][2] + 1
            length += 1
            array[i][1] = True
    if length == 0:
        return True
    else:
        ret_val = True
        for _ in range(length):
            ret_val = ret_val and checkBFSColourable(graph, queue, array)
            if not ret_val:
                return False
        return ret_val
Example #13
0
        def bfs(q: deque, visit) -> list:
            nonlocal board
            minerals = []
            lowest_y = -1
            while q:
                y, x = q.popleft()
                lowest_y = max(lowest_y, y)
                minerals.append((y, x))

                for k in range(4):
                    ny, nx = y + dy[k], x + dx[k]

                    if not (0 <= ny < R) or not (0 <= nx < C):
                        continue

                    if board[ny][nx] == 'x' and not visit[ny][nx]:
                        visit[ny][nx] = True
                        q.append((ny, nx))

            # 가장 낮은 미네랄이 바닥에 닿아있으면 생략해도됨
            if lowest_y == R - 1:
                return []
            # 떠있으면 drop 대상이 됨
            else:
                return minerals
Example #14
0
	def bi_dry_helper(self, fringe: queue, visited: set, opp_visited: set, opp_prev: dict, prev: dict) -> list or None:


		c = fringe.popleft()

		# if the current front node that was taken off the fringe
		# has already been visited by the opposite path
		# then this crossed the a path the opp already explored
		if c in opp_visited or c in opp_prev:
			# opp bfs already crossed this path
			return c

		else:
			# mark curr node as visited
			visited.add(c)
			# if the current node has not been visited by the opp yet
			# then get all the valid adjacent nodes
			adjacent_nodes = self.find_valid_adjacent_nodes(c, visited, prev, fringe)
			# add them to the fringe
			# then associate them with f in the previous dict
			for n in adjacent_nodes:
				if n not in visited:
					fringe.append(n)
					prev[n] = c

		if self.single_mode and c == self.back:
			return c
		else:
			return None
def solution(a: deque):
    if len(a) == 1:
        return a[0]
    else:
        a.popleft()
        a.append(a.popleft())
        return solution(a)
Example #16
0
 def to_array(self):
     arr = Dq()
     cur = self.head
     while cur:
         arr.append(cur.val)
         cur = cur.next
     return arr
Example #17
0
def _traverse_order(G, r):
    in_degree = {v: G.in_degree[v] for v in G}
    should_visit = {v: in_degree[v] for v in G}
    pending = Que([r])
    pending_multi_in = Que()
    visit_order = list()
    while True:
        if pending:
            u = pending.popleft()
        elif pending_multi_in:
            u = pending_multi_in.popleft()
        else:
            break
        visit_order.append(u)
        for v in G.successors(u):
            should_visit[v] -= 1
            if should_visit[v] == 0:
                if in_degree[v] > 1:
                    pending_multi_in.append(v)
                else:
                    pending.append(v)
    if len(visit_order) != len(G):
        raise Exception(
            "Topological order is defined only for connected graphs")
    return visit_order
def search_solution(states: deque):
    current_state = states[-1]

    if current_state.is_final_state():
        print(" --- Solutions ---")
        visual_validation_bucket = [8, 0, 0]

        for state in states:
            action = state.action

            if action:
                get_water_quality_for_visual_validation(
                    visual_validation_bucket, action)
                print(
                    f"poll {action.amount_of_water}L of water from '{action.poll_from}L bucket' to '{action.add_to}L bucket' -> {visual_validation_bucket}"
                )
            else:
                print(
                    f"water start with 8L, 5L, 3L -> '{visual_validation_bucket}'"
                )

    # trying to use DFS to explore all solutions
    for next_move in explore_next_move(current_state):
        if not is_processed_state(states, next_move):
            states.append(next_move)
            search_solution(states)
            states.pop()
Example #19
0
 async def radio_receive_task(self, packet_deque: deque):
     print("started radio_receive_task")
     radio_buffer = bytes()
     while True:
         radio_buffer = await self.radio_receive()
         print("Received:", radio_buffer)
         packet_deque.append(radio_buffer)
Example #20
0
async def port_receive(recv_port: trio.SocketStream, packet_deque: deque):
    print("calling port_receive")
    async for data in recv_port:
        print("port_received", data)
        if data != b"":
            packet_deque.append(raw_to_base64(data))
            print("packet queue:", packet_deque)
        def is_same_tree(p: MaybeTreeNode, q: MaybeTreeNode) -> bool:
            """Tests `is_same_tree` iteratively, breadth-first."""

            queue = Deck([(p, q)])

            while queue:

                (p, q) = queue.popleft()

                # If both are empty trees, they are the same.
                if not p and not q:
                    continue

                # Now, if only one is empty, they are not the same.
                # If we got here, then we ruled out both being empty.
                if not p or not q:
                    return False

                # Now, both must be non-empty...

                # If their values are not equal, they are not the same. (Duh!)
                if p.val != q.val:
                    return False

                # Else, then append the left and right branches of `p` and `q` for testing.
                queue.append((p.left, q.left))
                queue.append((p.right, q.right))

            return True
Example #22
0
def get_new_states(acc_data: deque, gyro_data: deque, parameters: dict,
                   data: str, time: int, actions: dict) -> dict:
    """
    :param queue: queue with last ten measurements
    :param parameters: parameters of system state for current moment
    :param levels: constants to compare angular velocity and acceleration with
    :param data: new data from IMU
    :param time: current time counter
    :param actions:  current state of each action (swing, spin, clash, stab)
    :return: new actions state
    """

    data = data.split(';')
    accel = list(map(int, data[0].split()))
    acc_data.append(accel)
    gyro = list(map(int, data[1].split()))
    gyro_data.append(gyro)
    a_curr = sum([accel[i] * accel[i] for i in range(3)])
    w_curr = sum([gyro[i] * gyro[i] for i in range(3)])
    update_acc_data(parameters, actions, a_curr, time)
    update_gyro_data(parameters, actions, w_curr, time)
    actions['hit'] = check_hit_with_accelerometer_and_change(
        acc_data, time, parameters, actions['hit'])
    if not actions['swing']:
        actions['swing'] = check_swing(gyro_data, time, parameters)
    if not actions['stab']:
        actions['stab'] = check_stab(acc_data, gyro_data, time, parameters)
    parameters['w_prev'] = w_curr
    return actions
Example #23
0
    def _traversal(self, nodes: deque) -> List[List[int]]:
        if not nodes:
            return []

        res = []
        is_right_dir = True

        while nodes:
            init_len = len(nodes)
            new_level = []

            for _ in range(init_len):
                if is_right_dir:
                    node = nodes.popleft()
                    if node.left:
                        nodes.append(node.left)
                    if node.right:
                        nodes.append(node.right)
                else:
                    node = nodes.pop()
                    if node.right:
                        nodes.appendleft(node.right)
                    if node.left:
                        nodes.appendleft(node.left)
                new_level.append(node.val)

            res.append(new_level)
            is_right_dir = not is_right_dir

        return res
Example #24
0
def _read_data_thread(sensor: SDS011, q: deque, timeout: int):
    while True:
        meas = sensor.query()
        timestamp = int(time.time())
        q.append((meas, timestamp))

        time.sleep(timeout)
Example #25
0
def peek(stack: deque):
    try:
        top = stack.pop()
        stack.append(top)
    except IndexError:
        top = None
    return top
Example #26
0
    def get_random_solution(self, curr_time: float, individual: list, unvisited_nodes: list, visited_nodes: deque):
        if len(unvisited_nodes) == 0:
            first_node = self.get_node(individual, visited_nodes[0])
            last_node = self.get_node(individual, visited_nodes[-1])
            full_time = curr_time + last_node["dists"][first_node["index"]]
            cost = self.get_cost(individual, visited_nodes)
            return " ".join(str(visited_nodes[i]) for i in range(1, len(visited_nodes))), full_time, cost
        valid_nodes = self.possible_nodes(
            curr_time, individual, unvisited_nodes, visited_nodes[-1])
        while self.errors <= self.MAX_ERRORS:
            if len(valid_nodes) == 0 and len(unvisited_nodes) != 0:
                self.errors += 1
                return None

            elt = valid_nodes[randrange(len(valid_nodes))]
            last_node = self.get_node(individual, visited_nodes[-1])

            start, end = self.get_time_window(individual, elt)
            arrive_time = curr_time + last_node["dists"][elt]
            new_time = arrive_time if arrive_time > start else start

            unvisited_nodes.remove(elt)
            valid_nodes.remove(elt)
            visited_nodes.append(elt)

            result = self.get_random_solution(
                new_time, individual, unvisited_nodes, visited_nodes)
            if result == None:
                unvisited_nodes.append(visited_nodes.pop())
            else:
                return result
        return None
Example #27
0
def solution(q: deque, command: list):
    if command[0] == 'push':
        q.append(command[1])
    elif command[0] == 'pop':
        if q:
            print(q.popleft())
        else:
            print(-1)
    elif command[0] == 'size':
        print(len(q))
    elif command[0] == 'empty':
        if q:
            print(0)
        else:
            print(1)
    elif command[0] == 'front':
        if q:
            print(q[0])
        else:
            print(-1)
    elif command[0] == 'back':
        if q:
            print(q[-1])
        else:
            print(-1)
    def maxDepth__iterative__depth_first(self, root: TreeNode) -> int:
        """
        Solution to "maximum depth of binary tree" that...
        -   Uses iteration.
        -   Visits nodes in a depth-first order by using a stack.
        """

        from collections import deque as Deck

        #---------------------------------------

        max_depth = 0

        if not root:
            return max_depth

        stack = Deck([(root, 1)])

        while stack:

            (node, depth) = stack.pop()
            next_depth = depth + 1

            if node.left:
                stack.append((node.left, next_depth))

            if node.right:
                stack.append((node.right, next_depth))

            max_depth = max(max_depth, depth)

        return max_depth
Example #29
0
def try_to_queue_node(node: TreeNode, queue: collections.deque,
                      nodes_queued: int, max_nodes: int) -> bool:
    if max_nodes == -1 or nodes_queued < max_nodes:
        queue.append(node)
        return True
    else:
        return False
Example #30
0
class RK_hash_generator:
    """ Rabin & Karp fingerprint generator """
    def __init__(self, blockSize, hashRange):
        """ Initialise Rolling hash """
        if type(blockSize) is int:
            self.block_size = blockSize
        else:
            raise TypeError('Block Size should be an integer value')
        self.prev_hash = 0  # Previous used hash
        self.base = 10
        self.chars = None
        if type(hashRange) is int:
            self.hash_range = hashRange
        else:
            raise TypeError('Hash Range should be an integer value')

    def incremental(self, next_char):
        """ Calculates hash of byte sequence and stores it in the hash table. Use 'hash_block_with_history' method first to
        instantiate generator history before using this method. """
        try:
            previous_char = self.chars.popleft()
        except AttributeError:  # Self.chars not defined, no history defined
            raise RuntimeWarning(
                'No history defined, hash_block_with_history should be called first'
            )
        try:
            self.prev_hash = ((self.prev_hash - ord(previous_char) * pow(
                self.base, self.block_size - 1)) * self.base + ord(next_char))
        except TypeError:
            raise TypeError('Incremental buffer should be of size one')
        self.chars.append(next_char)
        return self.prev_hash % self.hash_range

    def hash_block(self, byte_sequence):
        """ Calculates hash of byte sequence """
        #if len(byte_sequence) != self.block_size:
        #    raise BufferError('Byte sequence is %s long instead of %s' % (len(byte_sequence), self.block_size))
        return self._hash_block_unconstrained(byte_sequence) % self.hash_range

    def hash_block_with_history(self, byte_sequence):
        """ Calculate hash of byte sequence """
        if len(
                byte_sequence
        ) != self.block_size:  # length of byte sequence must match specified block size
            raise BufferError('Byte sequence is %s long instead of %s' %
                              (len(byte_sequence), self.block_size))
        self.prev_hash = self._hash_block_unconstrained(byte_sequence)
        self.chars = Queue()  # Queue is used to store history
        for char in byte_sequence:
            self.chars.append(char)
        return self.prev_hash % self.hash_range

    def _hash_block_unconstrained(self, byte_sequence):
        """ Calculates hash of byte sequence without modulo"""
        h = 0  # initial hash value, starts at zero
        multiplier = self.block_size - 1
        for byte in byte_sequence:
            h += (ord(byte) * pow(self.base, multiplier))
            multiplier -= 1
        return h
Example #31
0
 def _expand_all(root):
     q = Queue()
     q.append((root, ''))
     paths = []
     while q:
         node, path = q.popleft()
         paths.append(path)
         for c in PathParser._search_graph.get(node, []):
             q.append((c, path + '/' + c))
     return paths
Example #32
0
class RK_hash_generator:
    """ Rabin & Karp fingerprint generator """
    def __init__(self, blockSize, hashRange):
        """ Initialise Rolling hash """
        if type(blockSize) is int:
            self.block_size = blockSize
        else:
            raise TypeError('Block Size should be an integer value')
        self.prev_hash = 0  # Previous used hash
        self.base = 10
        self.chars = None
        if type(hashRange) is int:
            self.hash_range = hashRange
        else:
            raise TypeError('Hash Range should be an integer value')

    def incremental(self, next_char):
        """ Calculates hash of byte sequence and stores it in the hash table. Use 'hash_block_with_history' method first to
        instantiate generator history before using this method. """
        try:
            previous_char = self.chars.popleft()
        except AttributeError:  # Self.chars not defined, no history defined
            raise RuntimeWarning('No history defined, hash_block_with_history should be called first')
        try:
            self.prev_hash = ((self.prev_hash - ord(previous_char) * pow(self.base, self.block_size - 1)) * self.base + ord(next_char))
        except TypeError:
            raise TypeError('Incremental buffer should be of size one')
        self.chars.append(next_char)
        return self.prev_hash % self.hash_range

    def hash_block(self, byte_sequence):
        """ Calculates hash of byte sequence """
        #if len(byte_sequence) != self.block_size:
        #    raise BufferError('Byte sequence is %s long instead of %s' % (len(byte_sequence), self.block_size))
        return self._hash_block_unconstrained(byte_sequence) % self.hash_range

    def hash_block_with_history(self, byte_sequence):
        """ Calculate hash of byte sequence """
        if len(byte_sequence) != self.block_size:  # length of byte sequence must match specified block size
            raise BufferError('Byte sequence is %s long instead of %s' % (len(byte_sequence), self.block_size))
        self.prev_hash = self._hash_block_unconstrained(byte_sequence)
        self.chars = Queue()  # Queue is used to store history
        for char in byte_sequence:
            self.chars.append(char)
        return self.prev_hash % self.hash_range

    def _hash_block_unconstrained(self, byte_sequence):
        """ Calculates hash of byte sequence without modulo"""
        h = 0      # initial hash value, starts at zero
        multiplier = self.block_size - 1
        for byte in byte_sequence:
            h += (ord(byte) * pow(self.base, multiplier))
            multiplier -= 1
        return h
Example #33
0
 def _expand_gap(root, end):
     q = Queue()
     q.append((root, ''))
     paths = []
     while q:
         node, path = q.popleft()
         if node == end:
             paths.append(path)
         else:
             for c in PathParser._search_graph.get(node, []):
                 q.append((c, path + '/' + c))
     return paths
Example #34
0
class spider:
    _plugins = []
    _baseUrl = ""
    _errors = {}

    def __init__(self, plugins, blacklist):
        self._plugins = plugins
        self._visited = set()
        self._queue = Queue()
        self._blacklist = set(blacklist)

    def spider(self, url):
        self._queue.append(url)
        self._baseUrl = url
        try:
            while 1:
                url = self._queue.pop()
                self._visit(url)
        except IndexError:
            pass

    def _visit(self, url):
        if url in self._visited:
            return
        print "visiting: " + url
        self._visited.add(url)
        br = mechanize.Browser()

        try:
            resp = br.open(url)
        except urllib2.HTTPError, e:
            self._errors[e.geturl()] = [e.getcode()]
            return

        if not br.viewing_html():
            return

        for plugin in self._plugins:
            plugin.parsePage(br)

        unique = set()

        for l in br.links():
            if l.absolute_url[0 : len(self._baseUrl)] == self._baseUrl and not l.absolute_url in self._blacklist:
                visitableUrl = l.absolute_url.split("#")[0]
                if not visitableUrl in unique and not visitableUrl in self._visited:
                    self._queue.append(visitableUrl)
                    unique.add(visitableUrl)
                    print "found: " + visitableUrl

        print "visited: " + url
def breadth_first_search(startnode, goalnode):
    queue = Queue()
    queue.append(startnode)

    nodesseen = set()
    nodesseen.add(startnode)

    while queue:
        node = queue.popleft()

        if node is goalnode:
            return True
        else:
            queue.extend(node for node in node.successors if node not in nodesseen)
            nodesseen.update(node.successors)

    return False
Example #36
0
def breadth_first_search(startnode, goalnode):
    """
    Input:
        startnode: A digraph node
        goalnode: A digraph node
    Output:
        Whether goalnode is reachable from startnode
    """
    queue = Queue()
    queue.append(startnode)
    nodesseen = set()
    nodesseen.add(startnode)
    while queue:
        node = queue.popleft()
        if node is goalnode:
            return True
        else:
            queue.extend(node for node in node.successors if node not in nodesseen)
            nodesseen.update(node.successors)
    return False
Example #37
0
def _load_queue(file_name: str, target: collections.deque):
    r = 0
    try:
        with gzip.open(file_name, 'rt', encoding='utf-8') as rep_file:
            for line in rep_file:
                j = json.loads(line)
                t = j.get('time')
                if t is None:
                    t = 0
                k = j.get('action')
                if k != 'game_end':
                    target.append((t, action.Action(line, 'instruction', None)))
                else:
                    r = t
                    target.append((t, action.Action(line, 'game_end', None)))
    except OSError:
        main.root_logger.error('Corrupted replay file: %s', file_name)
        target.append((0, action.Action('{"action":"game_end","ai_id":-2,"time":0}', 'game_end', None)))
    finally:
        return r
Example #38
0
def _append_bothsides(deq: collections.deque) -> Generator[None, Any, None]:
    """Alternately add to each side of a deque."""
    while True:
        deq.append((yield))
        deq.appendleft((yield))