def greedy_approx(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() path = [] '''Initialize Priority Queue which will help us find Farthest node after distance is calcualted from visited node''' for node in G.nodes(): pq.additem(node, float("-inf")) curr = pq.pop() vis.add(curr) path.append(curr) while len(pq) > 0: for s, nod, wt in G.edges(curr, data=True): '''Distance calculation''' if nod not in vis and -wt['weight'] > pq[nod]: pq.updateitem(nod, -wt['weight']) if len(pq) > 0: ''' Selection Step''' top = pq.top() vis.add(top) curr = pq.pop() ''' Insertion Step''' loc, cost = minCost(G, path, top) '''Insert into the location found by minCost()''' path.insert(loc, top) tot_weight += cost return path, tot_weight
def branchAndBound(G, cutoff, ftrace): queue = PQDict() bestSolution = INFINITY bestTour = [] totalNodes = len(G.nodes()) startNode = G.nodes()[0] nodeList = G.nodes() nodeList.remove(startNode) queue.additem(tuple([startNode]), lowerBound(G, [startNode])) start_time = time.time() while len(queue) != 0: coveredNodes, lowerBoundCurrent = queue.popitem() elapsed_time = time.time() - start_time if elapsed_time > cutoff: if bestSolution == INFINITY: return [], -1 return bestTour, bestSolution for neighbor in G.neighbors(coveredNodes[-1]): if not neighbor in coveredNodes: tempNodes = list(coveredNodes) tempNodes.append(neighbor) if len(tempNodes) == totalNodes: cost = findCost(G, tempNodes) + G.get_edge_data(neighbor, startNode)["weight"] if cost < bestSolution: bestSolution = cost bestTour = tempNodes ftrace.write("{0:.2f}".format(elapsed_time * 1.0) + "," + str(bestSolution) + "\n") else: tempLowerBound = lowerBound(G, tempNodes) if tempLowerBound < bestSolution: queue.additem(tuple(tempNodes), tempLowerBound) return bestTour, bestSolution
def AStar(problem): #Give path of data file path = problem.datapath #Data data = list() data = readFunc(path) #Initialization of a node. node = Node() node.state = problem.initState node.pathcost = 0 #Frontier is a priority queue dict of tuples:(node.pathcost, node), with key:node.state #Frontier is ordered by path cost. frontier = PQDict() frontier.additem(node.state, (node.pathcost,node)) #Explored begins as the empty set. explored = Set() while True: if len(frontier) == 0: return 'No route exists' node = frontier.popitem()[1][1] #Chooses the lowest-cost node in frontier if problem.goalState == node.state: return solution(node,problem) explored.add(node.state) possibleActions = findPossibleActions(node.state,data) for action in range(len(possibleActions)): child = childNode(problem,node,possibleActions[action],data) if (child.state not in explored) and (child.state not in frontier): frontier.additem(child.state, (child.pathcost, child)) elif (child.state in frontier) and (frontier[child.state][1].pathcost > child.pathcost): frontier[child.state] = (child.pathcost, child)
def dijkstra(g, s): vis = [False for i in range(len(g))] dist = [float('inf') for i in range(len(g))] prev = [None for i in range(len(g))] dist[s] = 0 ipq = PQDict() ipq.additem(s, 0) while len(ipq) != 0: index, minValue = ipq.popitem() vis[index] = True # mica optimizare, daca deja avem distanda mai buna pe alt drum fata de drumul direct dintre 2 noduri, pass if dist[index] < minValue: continue for edge in g[index]: # ia toate nodurile vecine nevizitate if vis[edge[0]]: continue # le calculaeza distanta newDist = dist[index] + edge[1] # si o modifica doar daca este mai mica decat cea care era deja if newDist < dist[edge[0]]: dist[edge[0]] = newDist # apoi modifica si in indexed priority queue, daca nu e, il adauga, daca e, ii updateaza valoarea if ipq.get(edge[0]) is None: ipq.additem(edge[0], newDist) else: ipq[edge[0]] = newDist # si seteaza si tatal de fiecare data pentru a ramane ultimul adica cel cu costul cel mai mic prev[edge[0]] = index return dist, prev
def greedy_approx(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() path = [] '''Initialize Priority Queue which will help us find Farthest node after distance is calcualted from visited node''' for node in G.nodes(): pq.additem(node, float("-inf")) curr = pq.pop() vis.add(curr) path.append(curr) while len(pq) > 0: for s,nod, wt in G.edges(curr, data=True): '''Distance calculation''' if nod not in vis and -wt['weight'] > pq[nod]: pq.updateitem(nod, -wt['weight']) if len(pq)>0: ''' Selection Step''' top = pq.top() vis.add(top) curr = pq.pop() ''' Insertion Step''' loc,cost = minCost(G,path,top) '''Insert into the location found by minCost()''' path.insert(loc, top) tot_weight += cost return path,tot_weight
def primMST(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() Gprime = nx.Graph() ''' Add all nodes to PQDict with infinite distance''' for node in G.nodes(): pq.additem(node, float("inf")) curr = pq.pop() #Select initial node vis.add(curr) while len(pq) > 0: for s, nod, wt in G.edges(curr, data=True): if nod not in vis and wt['weight'] < pq[nod]: pq.updateitem(nod, wt['weight']) if len(pq) > 0: top = pq.top() source, destination, dist = [ data for data in sorted(G.edges(top, data=True), key=lambda (source, target, data): data['weight']) if data[1] in vis ][0] Gprime.add_edge(source, destination, weight=dist['weight']) vis.add(top) tot_weight += pq[top] curr = pq.pop() return Gprime, tot_weight
def test_heapsort(self): # sequences of operations pq = PQDict() self.check_heap_invariant(pq) self.check_index(pq) items = generateData('int') # push in a sequence of items added_items = [] for dkey, pkey in items: pq.additem(dkey, pkey) self.check_heap_invariant(pq) self.check_index(pq) added_items.append( (dkey,pkey) ) # pop out all the items popped_items = [] while pq: dkey_pkey = pq.popitem() self.check_heap_invariant(pq) self.check_index(pq) popped_items.append(dkey_pkey) self.assertTrue(len(pq._heap)==0) self.check_index(pq)
def prim(G, start): """Function recives a graph and a starting node, and returns a MST""" stopN = G.number_of_nodes() - 1 current = start closedSet = set() pq = PQDict() mst = [] while len(mst) < stopN: # print " " # print "Current node :", current for node in G.neighbors(current): if node not in closedSet and current not in closedSet: # print " neigbors: ", node if (current, node) not in pq and (node, current) not in pq: w = G.edge[current][node]['weight'] pq.additem((current, node), w) closedSet.add(current) tup, wght = pq.popitem() while (tup[1] in closedSet): tup, wght = pq.popitem() mst.append(tup) current = tup[1] return mst
def primMST(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() Gprime = nx.Graph() ''' Add all nodes to PQDict with infinite distance''' for node in G.nodes(): pq.additem(node, float("inf")) curr = pq.pop() #Select initial node vis.add(curr) while len(pq) > 0: for s,nod, wt in G.edges(curr, data=True): if nod not in vis and wt['weight'] < pq[nod]: pq.updateitem(nod, wt['weight']) if len(pq)>0: top = pq.top() source,destination, dist = [data for data in sorted(G.edges(top, data=True), key=lambda (source,target,data): data['weight']) if data[1] in vis][0] Gprime.add_edge(source, destination, weight = dist['weight']) vis.add(top) tot_weight += pq[top] curr = pq.pop() return Gprime, tot_weight
def test_infpkey(self): dkeys = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] pkeys = [1, 2, 3, 4, 5, 6, 7] pq = PQDict(zip(dkeys, pkeys)) pq.additem('top', -float('inf')) pq.additem('bot', float('inf')) dkeys_sorted = [key for key in pq.iterkeys()] self.assertEqual(dkeys_sorted[0], 'top') self.assertEqual(dkeys_sorted[-1], 'bot')
def plan(self, start_state, goal_state): #PQ = pqdict() V = {} self.goal_state = goal_state h0 = self.heuristic(start_state, goal_state) n0 = node(start_state, None, None, 0, h0) key0 = np.around(n0.state, decimals=1).tostring() PQ = PQDict(key0=n0) i = 0 while PQ and i < 100000: current = PQ.popitem()[1] #print '\n' #print current.state[2] if (self.state_is_equal(current.path, goal_state)): path = self.reconstruct_path(current) return (path, current.f) V[np.around(current.state, decimals=1).tostring()] = copy.deepcopy(current) #get children children = self.getChildren( current) #do set parent, should return an array of nodes for child in children: i += 1 if i % 100 == 0: print 'A* iteration ' + str(i) child_key = np.around(child.state, decimals=1).tostring() if child_key in V: if child.f >= V[child_key].f: continue if (child_key in PQ): existing_child = PQ[child_key] if (child.g >= existing_child.g): continue else: PQ.updateitem(child_key, child) else: #print child.state, current.state #pdb.set_trace() #if(child.state[2] < 0): # pdb.set_trace() PQ.additem(child_key, child) print 'A* Failed' return (None, None)
def main(): # Main output dict output = {'results': []} # Read JSON from file with open('input.json', 'r') as f: inputjson = json.load(f) profiles = inputjson['profiles'] for profile in profiles: pq = PQDict() no_of_profiles = 0 matches = [] # The output for this profile in JSON format profile_output = {'profileId': profile['id'], 'matches': []} for other_profile in profiles: if other_profile['id'] == profile['id']: # dont calculate against our own profile continue # Calculate match percentage with OKCupid's formula match_score = math.sqrt( satisfaction(profile, other_profile) * satisfaction(other_profile, profile)) # Add the first ten matches to a min-heap (PQDict) if len(pq) < 11: pq.additem(other_profile['id'], match_score) else: if match_score > pq.popitem()[1]: pq.additem(other_profile['id'], match_score) else: continue for i in range(len(pq)): key, value = pq.popitem() temp = {'profileId': key, 'score': value} matches.append(temp) # Reverse the heap and store it in the output for that profile profile_output['matches'] = matches[::-1] output['results'].append(profile_output) # Write out the output JSON to file with open('output_optimized.json', 'w') as outf: json.dump(output, outf, indent=1) return 0
def main(): # Main output dict output = {'results': []} # Read JSON from file with open('input.json', 'r') as f: inputjson = json.load(f) profiles = inputjson['profiles'] for profile in profiles: pq = PQDict() no_of_profiles = 0 matches=[] # The output for this profile in JSON format profile_output = {'profileId': profile['id'], 'matches': []} for other_profile in profiles: if other_profile['id'] == profile['id']: # dont calculate against our own profile continue # Calculate match percentage with OKCupid's formula match_score = math.sqrt(satisfaction(profile, other_profile) * satisfaction(other_profile, profile)) # Add the first ten matches to a min-heap (PQDict) if len(pq) < 11: pq.additem(other_profile['id'],match_score) else: if match_score > pq.popitem()[1]: pq.additem(other_profile['id'],match_score) else: continue for i in range(len(pq)): key,value = pq.popitem() temp = {'profileId': key, 'score': value} matches.append(temp) # Reverse the heap and store it in the output for that profile profile_output['matches'] = matches[::-1] output['results'].append(profile_output) # Write out the output JSON to file with open('output_optimized.json', 'w') as outf: json.dump(output, outf, indent=1) return 0
def plan(self, start_state, goal_state): #PQ = pqdict() V = {} self.goal_state = goal_state h0 = self.heuristic(start_state, goal_state) n0 = node(start_state, None, None, 0, h0) key0 = np.around(n0.state, decimals = 1).tostring() PQ = PQDict(key0=n0) i = 0 while PQ and i < 100000: current = PQ.popitem()[1] #print '\n' #print current.state[2] if(self.state_is_equal(current.path, goal_state)): path = self.reconstruct_path(current) return (path, current.f) V[np.around(current.state, decimals = 1).tostring()] = copy.deepcopy(current) #get children children = self.getChildren(current)#do set parent, should return an array of nodes for child in children: i += 1 if i%100 == 0: print 'A* iteration '+str(i) child_key = np.around(child.state, decimals = 1).tostring() if child_key in V: if child.f >= V[child_key].f: continue if (child_key in PQ): existing_child = PQ[child_key] if(child.g >= existing_child.g): continue else: PQ.updateitem(child_key, child) else: #print child.state, current.state #pdb.set_trace() #if(child.state[2] < 0): # pdb.set_trace() PQ.additem(child_key,child) print 'A* Failed' return (None, None)
def prim(G: "networx Graph object", start: "1X2 Tuple(Node)") -> (list, list): """ Function recives a graph and a starting node, and returns the MST and the history of the algorithm """ MST_len = G.number_of_nodes() - 1 current = start visited = set() # Priority Queue #(keeps the item with the lowest value on the top - #in this case the weight of an edge) pq = PQDict() # Minimal spamming tree mst = [] steps = [] history = [] # While the MST has N - 1 edges (N is the total nodes) while len(mst) < MST_len: # Get the neighbors # history.append((current, G.neighbors(current))) for node in G.neighbors(current): if node not in visited and current not in visited: # Append the history steps.append((current, node)) if (current, node) not in pq and (node, current) not in pq: w = G.edge[current][node]['weight'] pq.additem((current, node), w) # We have visited the node visited.add(current) # Tup is the edge "(X, Y)", wght is the cost tup, wght = pq.popitem() # Get the lowest edge if we haven't visited the node while (tup[1] in visited): tup, wght = pq.popitem() # Append the edge to the minimum spanning tree mst.append(tup) history.append([tup, steps]) steps = list() # Update the current node to the one we visit based on the minimal weight current = tup[1] return mst, history
def branchAndBound(G, cutoff, ftrace): queue = PQDict() bestSolution = INFINITY bestTour = [] totalNodes = len(G.nodes()) startNode = G.nodes()[0] nodeList = G.nodes() nodeList.remove(startNode) queue.additem(tuple([startNode]), lowerBound(G, [startNode])) start_time = time.time() while (len(queue) != 0): # print count # count+=1 coveredNodes, lowerBoundCurrent = queue.popitem() # coveredNodes=list(coveredNodes) elapsed_time = time.time() - start_time if elapsed_time > cutoff: if bestSolution == INFINITY: return -1, [] return bestTour, bestSolution for neighbor in G.neighbors(coveredNodes[-1]): if not neighbor in coveredNodes: tempNodes = list(coveredNodes) tempNodes.append(neighbor) if (len(tempNodes) == totalNodes): cost = findCost(G, tempNodes) + G.get_edge_data( neighbor, startNode)['weight'] if (cost < bestSolution): bestSolution = cost bestTour = tempNodes ftrace.write("{0:.2f}".format(elapsed_time * 1.0) + ',' + str(bestSolution) + '\n') else: tempLowerBound = lowerBound(G, tempNodes) if tempLowerBound < bestSolution: queue.additem(tuple(tempNodes), tempLowerBound) # else: # print 'prune' return bestTour, bestSolution
def primWeight(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() for node in G.nodes(): pq.additem(node, float("inf")) curr = pq.pop() vis.add(curr) while len(pq) > 0: for s,nod, wt in G.edges(curr, data=True): if nod not in vis and wt['weight'] < pq[nod]: pq.updateitem(nod, wt['weight']) if len(pq)>0: top = pq.top() vis.add(top) tot_weight += pq[top] curr = pq.pop() return tot_weight
def primWeight(G): """ Return MST of the given undirected graph""" vis = set() tot_weight = 0 pq = PQDict() for node in G.nodes(): pq.additem(node, float("inf")) curr = pq.pop() vis.add(curr) while len(pq) > 0: for s, nod, wt in G.edges(curr, data=True): if nod not in vis and wt['weight'] < pq[nod]: pq.updateitem(nod, wt['weight']) if len(pq) > 0: top = pq.top() vis.add(top) tot_weight += pq[top] curr = pq.pop() return tot_weight
def executa_prim(G, primeiro_no): no_de_parada = G.number_of_nodes() - 1 atual = primeiro_no visitados = set() # Tira os repetidos e é uma Hash min_heap = PQDict() # É um min_heap mst = [] peso_total = 0 pesos = [] while len(mst) < no_de_parada: # print("Esse é o visitados", visitados) # print("Esse é o min_heap: ", min_heap) for noVizinho in G.neighbors(atual): # print("Esse é o no: ", noVizinho) # print("Esse é o atual: ", atual) if noVizinho not in visitados and atual not in visitados: if (atual, noVizinho) not in min_heap and (noVizinho, atual) not in min_heap: peso_aresta = G[atual][noVizinho]['weight'] min_heap.additem((atual, noVizinho), peso_aresta) visitados.add(atual) aresta, peso = min_heap.popitem() # Tira a raiz while (aresta[1] in visitados): # Aresta[1] é o vizinho aresta, peso = min_heap.popitem( ) # Vai tirando a raiz até encontrar um vizinho não visitado # print("Essa é a aresta: ", aresta) # print("Esse é o peso: ", peso) peso_total += peso pesos.append(peso) mst.append(aresta) atual = aresta[1] return mst, peso_total, pesos
def a_star(self, heuristic): node = self.tree.create_node(state=State(self.wrigglers), pathCost=0) node.heuristic = heuristic(node) frontier = PQDict() stateFrontier = {} explored = {} # Sacrifice memory to have a huge speed up being able to instantly check for state in frontier stateFrontier[str(node.state)] = node.heuristic frontier.additem(node._identifier, node.heuristic) while(True): if(len(frontier) == 0): return None nodeID = frontier.popitem()[0] node = self.tree.get_node(nodeID) nodeStateStr = str(node.state) del stateFrontier[nodeStateStr] if self.testGoal(node.state): return node explored[nodeStateStr] = -1 # we don't care what the hash matches actions = self.getActions(node.state) for action in actions: child = self.childNode(node, action) child.heuristic = heuristic(child) childStr = str(child.state) inExplored = False inFrontier = False if childStr in explored: inExplored = True bGreater = False if childStr in stateFrontier: if(stateFrontier[childStr] < child.heuristic + child.pathCost): bGreater = True inFrontier = True if(not inExplored and not inFrontier): stateFrontier[childStr] = child.heuristic frontier.additem(child._identifier, child.heuristic + child.pathCost) elif(bGreater): bHappened = False for key in frontier: if(str(self.tree.get_node(key).state) == childStr): bHappened = True frontier.pop(key) frontier.additem(child._identifier, child.heuristic + child.pathCost) break assert bHappened
node1 = edge[0] node2 = edge[1] cost = edge[2] if node1 == mst[0]: if costs[node2] > cost: costs[node2] = cost elif node2 == mst[0]: if costs[node1] > cost: costs[node1] = cost if test: print "initial costs is", costs # insert node to heap heap = PQDict() for node in range(2,n+1): heap.additem(node, costs[node]) if test: print "heap is", heap cost_sum = 0 while len(heap) != 0: next_node = heap.popitem(); # (node, cost) if test: print "next_node is %d" % next_node[0] cost_sum = cost_sum + next_node[1] if test: print "current cost_sum is %d" % cost_sum mst.append(next_node[0]) if test: print "current mst is", mst for edge in edges: node1 = edge[0]
def test_uncomparable(self): # non-comparable priority keys (Python 3 only) # Python 3 has stricter comparison than Python 2 pq = PQDict() pq.additem('a',[]) self.assertRaises(TypeError, pq.additem, 'b', 5)
def test_additem(self): pq = PQDict(self.items) pq.additem('new', 8.0) self.assertEqual(pq['new'], 8.0) self.assertRaises(KeyError, pq.additem, 'new', 1.5)
class Battle(object): """ Fight a battle between two teams of oeo """ _turn_stage_map = { 8: '02', 7: '03', 6: '04', 5: '05', 4: '06', 3: '07', 2: '08', 1: '09', 0: '10', -1: '11', -2: '12', -3: '13', -4: '14', -5: '15', -6: '16', -7: '17' } _turn_spi_map = { 0: '01', 1: '02', 2: '03', 3: '04', 4: '05', 5: '06', 6: '07', 7: '08', 8: '09', 9: '10', 10: '11', 11: '12' } def __init__(self, oeos, a_id, a, a_max_fielded, b_id, b, b_max_fielded): assert all(isinstance(oeo, Oeo) for oeo in oeos.values()), \ "oeos is not a dict of oeo_id:oeo" assert isinstance(a_id, str), "'a_id' is not a string" assert isinstance(a, set), "'a' is not a set of oeo_id" assert isinstance(a_max_fielded, int), "'a_max_fielded' is not an int" assert isinstance(b_id, str), "'b_id' is not a string" assert isinstance(b, set), "'b' is not a set of oeo_id" assert isinstance(b_max_fielded, int), "'b_max_fielded' is not an int" # Throw error if each oeo_id is not unique between teams if a & b: raise ValueError(f"{a & b} oeo_id(s) not unique between teams") self._oeo = oeos self._a_id = a_id self._a = a self._b_id = b_id self._b = b self._turn_number = 0 self._field = Field(a_id, a_max_fielded, b_id, b_max_fielded) self._pending_sim_events = PQDict() self._processed_sim_events = [] move_set = set() for o in itertools.chain(self._oeo.values()): for move in o.moves: move_set.add(move) moves = Move.load_moves(move_set) self._moves = moves # self._speed_priority_list = None self._setup_axel_events() @property def teams(self): return {self._a_id: self._a, self._b_id: self._b} @property def field(self): return self._field def _setup_axel_events(self): """ Initialise axel events """ # sim_output_message(msg) self.sim_output_message = Event() # event_choose_deployments(team_id, non_fielded_team, empty_positions) # non_fielded_team: oeo from the team who are not on the field # but are conscious # empty_positions: empty positions on the field into which an oeo # could be deployed self.event_choose_deployments = Event() # event_choose_actions(team_id, oeo_requiring_actions) # oeo_requiring_actions: oeo that are on the field and need to select # an action for the current turn self.event_choose_actions = Event() def run(self): """ Run the battle :return: id of the victor :rtype: str """ logger.info(f"{self._a_id} vs {self._b_id}...") ta = {oeo_id: self._oeo[oeo_id] for oeo_id in self._a} tb = {oeo_id: self._oeo[oeo_id] for oeo_id in self._b} logger.debug(f"{self._a_id}'s team:\n{ta}") logger.debug(f"{self._b_id}'s team:\n{tb}") # Add the BEGIN_TURN SimEvent for turn 1 self._pending_sim_events.additem(SimEvent(SimEventType.BeginTurn), 1) victor = None # While there are pending sim events, loop until we break when a # battle end condition is met while self._pending_sim_events: # Remove any oeo with 0 HP from the field self._remove_unconscious_oeo() # Check teams: if all oeo in the battle are unconscious, # then end battle as a draw # if all oeo in team A are unconscious, # then end battle as win for team B # if all oeo in team B are unconscious, # then end battle as win for team A if not any(oeo.conscious for oeo in self._oeo.values()): logger.info("All oeo on both sides of the battle are " "unconscious, the battle is a draw") victor = "DRAW" break elif not any(self._oeo[oeo_id].conscious for oeo_id in self._a): logger.info(f"{self._a_id}'s team are unconscious, " f"{self._b_id} wins the battle") victor = self._b_id break elif not any(self._oeo[oeo_id].conscious for oeo_id in self._b): logger.info(f"{self._b_id}'s team are unconscious, " f"{self._a_id} wins the battle") victor = self._a_id break # Let both sides choose oeo to deploy self._choose_deployments() logger.debug(f"{self._a_id}'s side: {self._field[self._a_id]}") logger.debug(f"{self._b_id}'s side: {self._field[self._b_id]}") # Check both sides of the field: # if team A side is empty, then end as win for team B # if team B side is empty, then end as win for team A if self._field[self._a_id].is_empty(): logger.info(f"{self._a_id} yields, " f"{self._b_id} wins the battle") victor = self._b_id break elif self._field[self._b_id].is_empty(): logger.info(f"{self._b_id} yields, " f"{self._a_id} wins the battle") victor = self._a_id break # Pop the next event to be processed, add it to the # processed events list, and process it event, event_priority = self._pending_sim_events.popitem() event_complete = 0 event_type = event.event_type if event_type is SimEventType.BeginTurn: event_complete = self._process_begin_turn() elif event_type is SimEventType.UseMove: event_complete = self._process_use_move(**event.data) elif event_type is SimEventType.UseItem: # event_complete = self._process_use_item pass elif event_type is SimEventType.Switch: # event_complete = self._process_switch_oeo pass elif event_type is SimEventType.Run: # event_complete = self._process_run pass else: raise ValueError(f"Invalid event_type: {event}") self._processed_sim_events.append( (event_priority, event, event_complete)) logger.debug(f"Pending events: {self._pending_sim_events}") logger.info(f"Processed events: {self._processed_sim_events}") return victor def _process_begin_turn(self): # Increment the turn number and add the BeginTurn SimEvent # for the next turn self._turn_number += 1 logger.debug(f"Processing BeginTurn({self._turn_number}) SimEvent") self._pending_sim_events.additem(SimEvent(SimEventType.BeginTurn), self._turn_number + 1) # TODO: Update status conditions - burn, poison, landing from flight, # then remove unconscious oeo from field # Choose the actions for the oeo on the field this turn, calculate the # order in which the actions should occur, and add them to the pending # sim events priority queue self._choose_actions() return 1 def _process_use_move(self, user_id, move_id, target_id): logger.debug("Processing UseMove SimEvent") user, move, target = self._oeo[user_id], self._moves[move_id], \ self._oeo[target_id] user_is_fielded = self._is_fielded(user_id) target_is_fielded = self._is_fielded(target_id) if user_is_fielded and target_is_fielded: logger.info(f"{user_id} attacks {target_id} using {move_id}") df_id = getattr(move, "df_id", "Standard") damage_function = get_damage_function(df_id) damage = damage_function(user, move, target) hp = target.current_hp target.current_hp -= damage logger.info(f"{target_id}'s HP = {hp}-{damage} " f"= {target.current_hp}") return 1 else: logger.debug(f"User on field = {user_is_fielded}, " f"Target on field = {target_is_fielded}") return -1 def _process_use_item(self, item, target): logger.debug("Processing UseItem SimEvent") return 0 def _process_switch(self, user, target): logger.debug("Processing Switch SimEvent") return 0 def _process_run(self, user, run_type): logger.debug("Processing Run SimEvent") return 0 def _calculate_event_priority(self, turn, priority, speed_priority): """ :param turn: the turn in which the event is to be actioned :param priority: the stage of the turn in which the event is \ to be actioned :param speed_priority: the speed_priority of the oeo undertaking \ the event :return: the priority of the event to be actioned """ return float(f"{turn}.{self._turn_stage_map[priority]}" f"{self._turn_spi_map[speed_priority]}") def _remove_unconscious_oeo(self): """ Withdraw unconscious oeo from the field """ for team_id in [self._a_id, self._b_id]: oeo_to_remove = [ oeo_id for oeo_id in self._field[team_id].fielded if self._oeo[oeo_id].conscious is False ] for oeo_id in oeo_to_remove: self._field.withdraw(team_id, oeo_id) def _is_fielded(self, oeo_id): """ :param oeo_id: :return: True if oeo_id is on the field else False """ if (oeo_id in self._field[self._a_id]) or \ (oeo_id in self._field[self._b_id]): return True else: return False def _choose_deployments(self): """ Choose and make deployments to the field """ for team_id in [self._a_id, self._b_id]: empty_positions = self._field[team_id].empty_positions logger.debug(f"Empty positions on {team_id}'s side: " f"{empty_positions}") if empty_positions: fielded = self._field[team_id].fielded benched = [ oeo_id for oeo_id in self.teams[team_id] if oeo_id not in fielded and self._oeo[oeo_id].conscious ] logger.debug(f"Benched on {team_id}'s side: {benched}") if benched: deployments = self._poll_deployments( team_id, benched, empty_positions) logger.debug(f"{team_id}'s oeo to deploy: {deployments}") for position, oeo_id in deployments.items(): self._field.deploy(team_id, oeo_id, position) def _poll_deployments(self, team_id, non_fielded_team, empty_positions): """ Polls for deployments for team_id :return: dict of position:oeo_id """ logger.debug(f"Polling for deployments from {team_id}") results = self.event_choose_deployments(team_id, list(non_fielded_team), empty_positions) flag, result, handler = results[0] if flag: for position, oeo_id in result.items(): # Ensure that each oeo is only deployed to one # field position at most if list(result.values()).count(oeo_id) > 1: raise Exception(f"{oeo_id} can not be deployed to more " "than one field position") # Ensure 0 >= position < len(self._field[team_id]) if position < 0 or position >= len(self._field[team_id]): raise Exception(f"Position {position} is out of bounds" f"(0-{len(self._field[team_id]) - 1})") # Ensure oeo_id is in self.team[team_id] if oeo_id not in self.teams[team_id]: raise Exception(f"{oeo_id} is not in {team_id}'s team") return result else: raise Exception("Exception in choose_deployments handler") \ from result def _choose_actions(self): """ Choose and schedule actions for oeo on the field """ # Create the speed_priority_list for this turn oeo_against_speed = [(oeo_id, oeo.speed) for oeo_id, oeo in self._oeo.items()] oeo_against_speed.sort(key=itemgetter(1), reverse=True) logger.debug(f"Speed Priority List: {oeo_against_speed}") speed_priority_list = [t[0] for t in oeo_against_speed] a_fielded = self._field[self._a_id].fielded b_fielded = self._field[self._b_id].fielded # Create action_map dictionary of oeo_id to action:None for # fielded oeo and update it from future_action dictionary action_map = { oeo_id: None for oeo_id in itertools.chain(a_fielded, b_fielded) } # todo: Update action_map from future_actions dictionary logger.debug(f"Initial action map for turn {self._turn_number}: " f"{action_map}") # Call event_choose_actions for each team for oeo that do not have # an action to perform (action is None) a_oeo_requiring_actions = [ oeo_id for oeo_id, action in action_map.items() if (oeo_id in a_fielded) and (action is None) ] a_oeo_requiring_actions.sort(key=lambda x: self._oeo[x].speed, reverse=True) b_oeo_requiring_actions = [ oeo_id for oeo_id, action in action_map.items() if (oeo_id in b_fielded) and (action is None) ] b_oeo_requiring_actions.sort(key=lambda x: self._oeo[x].speed, reverse=True) logger.debug(f"{self._a_id}'s oeo requiring actions: " f"{a_oeo_requiring_actions}") logger.debug(f"{self._b_id}'s oeo requiring actions: " f"{b_oeo_requiring_actions}") a_actions = self._poll_actions(self._a_id, a_oeo_requiring_actions) b_actions = self._poll_actions(self._b_id, b_oeo_requiring_actions) logger.info(f"{self._a_id}'s actions chosen: {a_actions}") logger.info(f"{self._b_id}'s actions chosen: {b_actions}") # For each {oeo_id: action} in dictionary returned by the event # handlers, add it to the action map if the oeo does not already have # an action for this turn in the action map for oeo_id, action in itertools.chain(a_actions.items(), b_actions.items()): if action_map[oeo_id] is not None: raise Exception(f"{oeo_id} already had an action " "for this turn") action_map[oeo_id] = action logger.debug(f"Final action map for turn {self._turn_number}: " f"{action_map}") # For each {oeo_id: action} in the action map add the SimEvent for # the action to the pending sim events queue for oeo_id, action in action_map.items(): if action: if action.event_type == SimEventType.UseMove: target = action.data["target_id"] move_id = action.data["move_id"] move_priority = self._moves[move_id].priority oeo_priority = speed_priority_list.index(oeo_id) s = SimEvent(SimEventType.UseMove, user_id=oeo_id, target_id=target, move_id=move_id) ep = self._calculate_event_priority( self._turn_number, move_priority, oeo_priority) self._pending_sim_events.additem(s, ep) if action.event_type == SimEventType.UseItem: pass def _poll_actions(self, team_id, oeo_requiring_actions): """ Polls for actions from team_id :return: dict of oeo_id:action """ logger.debug(f"Polling for actions from {team_id}") results = self.event_choose_actions(team_id, oeo_requiring_actions) flag, result, handler = results[0] if flag: for oeo_id, action in result.items(): # Ensure oeo is on the field if not self._is_fielded(oeo_id): raise Exception(f"{oeo_id} is not on the field") # Ensure oeo is in team_id if oeo_id not in self.teams[team_id]: raise Exception(f"{oeo_id} is not on {team_id}'s side") # Ensure action is SimEvent if not isinstance(action, SimEvent): raise Exception("Action is not a SimEvent") # Ensure action.event_type is UseMove, UseItem, Switch or Run if action.event_type not in [ SimEventType.UseMove, SimEventType.UseItem, SimEventType.Switch, SimEventType.Run ]: raise Exception("Action event type not UseMove, UseItem, " "Switch or Run") return result else: raise Exception("Exception in choose_actions handler") from result
class Crawler(): def __init__(self): self.query = input("Enter search query: ") self.webpages_limit = input( "Set total number of webpages to be crawled: ") self.limit = input( "Set limits on how many webpages be crawled from single site: ") self.priority_queue = PQDict().maxpq() self.queue = queue.Queue() self.downloader = Downloader() self.parser = Parser(self.query) self.calculator = Calculator(self.query) self.relevance = Relevance() self.webpages_crawled = 0 self.logger = logging.getLogger(__name__) self.visited_urls = set() self.sites_times = {} #fetch top 10 results from google search: def __fetch_google_results(self): service = build("customsearch", "v1", developerKey=API_KEY) res = service.cse().list(q=self.query, cx=SEARCH_ENGINE_ID).execute() return res #enqueue the 10 google search results def enqueue_seeds(self): res = self.__fetch_google_results() for item in res['items']: self.priority_queue.additem(item['link'], 10) self.queue.put(item['link']) self.logger.debug("Enqueued: " + item['link']) #check has this url been visited before #and has it reach the limit of each site #and Robot Exclusion Protocols def urlchecker(self, url): if url is None: return False normalized_url = urltools.normalize(url) robotparser = urllib.robotparser.RobotFileParser() try: url_comp = urlparse(normalized_url) base_url = url_comp.scheme + "://" + url_comp.netloc + "/" except: self.logger.error("Cannot parse: " + url) try: robotparser.set_url(base_url + "robots.txt") robotparser.read() if not robotparser.can_fetch("*", normalized_url): self.logger.error(url + " is excluded due to protocol") return False except: self.logger.error("Cannot determine robots exclusion protocol: " + url) if normalized_url in self.visited_urls: self.logger.debug(url + " Has been visited before! ") return False elif base_url in self.sites_times and self.sites_times[base_url] > int( self.limit): # self.logger.debug( url + " Times visiting this site have reach the limit ") return False elif 'cgi' in normalized_url: return False else: return True #the crawling process def crawl(self): try: harvest_rate_accum = 0 while self.webpages_crawled < int(self.webpages_limit): print(self.webpages_crawled) try: url = self.priority_queue.pop() except e: print("cannot pop") print(url) if self.urlchecker(url): try: content = self.downloader.download(url).decode('utf-8') if content is not None: self.webpages_crawled += 1 rel = self.relevance.relevance(content, self.query) harvest_rate_accum += rel self.crawled_log(" Harvest rate: " + str(harvest_rate_accum / self.webpages_crawled)) except: print("Failed in downloading") normalized_url = urltools.normalize(url) try: url_comp = urlparse(normalized_url) base_url = url_comp.scheme + "://" + url_comp.netloc + "/" except: self.logger.error("Cannot parse: " + url) if base_url in self.sites_times: self.sites_times[base_url] += 1 else: self.sites_times[base_url] = 1 self.visited_urls.add(normalized_url) if rel < 0.2: continue for link in self.parser.extract_all_links(content): full_link = self.parser.parse_links(url, link) if full_link is not None: link_promise = self.calculator.link_promise( full_link) + rel try: self.priority_queue.additem( full_link, link_promise) except: pass except KeyError: print("Queue is empty now") def bfs_crawl(self): try: harvest_rate_accum = 0 while self.webpages_crawled < int(self.webpages_limit): print(self.webpages_crawled) try: url = self.queue.get() except e: print("cannot pop") print(url) if self.urlchecker(url): try: content = self.downloader.download(url).decode('utf-8') if content is not None: self.webpages_crawled += 1 rel = self.relevance.relevance(content, self.query) harvest_rate_accum += rel self.crawled_log(" Harvest rate: " + str(harvest_rate_accum / self.webpages_crawled)) except: print("Failed in downloading") normalized_url = urltools.normalize(url) try: url_comp = urlparse(normalized_url) base_url = url_comp.scheme + "://" + url_comp.netloc + "/" except: self.logger.error("Cannot parse: " + url) self.visited_urls.add(normalized_url) for link in self.parser.extract_all_links(content): full_link = self.parser.parse_links(url, link) if full_link is not None: try: if base_url not in self.sites_times: self.sites_times[base_url] = 1 elif self.sites_times[base_url] < int( self.limit): self.sites_times[base_url] += 1 else: continue self.queue.put(full_link) except: pass except KeyError: print("Queue is empty now") def crawled_log(self, log): file = open('demo.log', 'a') file.write(log + '\n\n') file.close()