コード例 #1
0
ファイル: visualize.py プロジェクト: joehanchoi/CS292-Project
def graph(term):
	#make network graph with depth = 2 for term
	pages = Pages()
	id = pages.get_id(term)
	sim_pages = pages.query_page(id)

	#make networkx graph object
	G = nx.Graph()
	node_dict = {}
	title = pages.get_title(int(id))
	G.add_node(title)
	#original term is red
	node_dict[title] = ('red', 0.0)

	#add top 10 pages similar to term
	short_sim_pages = sorted(sim_pages, key=lambda x: x[1], reverse=True)[1:11]
	for page in short_sim_pages:
		page_name = pages.get_title(int(page[0]))
		G.add_node(page_name)
		#depth = 1 terms are blue
		node_dict[page_name] = ('blue', float(page[1]))
		G.add_edge(title,page_name)
		addl_nodes = add_x_nodes(pages, G, page[0], 10)
		#add next depth of pages
		for node in addl_nodes:
			if node not in node_dict:
				#depth = 2 terms are green
				node_dict[node] = ('green', float(page[1]))

	node_list = []
	node_color = []
	node_size = []
	#make node size proportional to similarity score of each node to original term
	for node in node_dict:
		node_list.append(node)
		node_color.append(node_dict[node][0])
		node_size.append(node_dict[node][1])
	node_size = [x*300/max(node_size) for x in node_size]
	node_size[node_size.index(0.0)]=300

	#draw and save network figure
	nx.draw_networkx(G=G,with_labels=True,nodelist=node_list,node_size=node_size,node_color=node_color,font_size=8)
	fig_name = "%s.png" %(term)
	plt.savefig(fig_name)
コード例 #2
0
def getDifferenceWikiPageRankLists(search_term, rank_list):
	pages = Pages()
	page_id = pages.get_id(search_term)
	wiki_page_links = set(pages.getPageLinksList(int(page_id)))
	rank_name_list = [pages.get_title(int(ranked_page[0])).lower() for ranked_page in rank_list]
	rank_set = set(rank_name_list)
	# get the links in both ranked list and wikipedea links
	print ("Intersection", rank_set.intersection(wiki_page_links))
	# get the links that exist only in ranked list
	print ("Difference", rank_set.difference(wiki_page_links))
	page_summary, page_content = pages.getPageSummaryContent(page_id)
	# get ranked pages that exist in summary and content of page
	links_in_summary_only = set()
	links_in_content_only = set()
	links_in_both = set()
	for rank_page in rank_name_list:
		in_summary = False
		in_content = False
		if re.search(r'\b%s\b' % rank_page, page_summary):
			in_summary = True
		if re.search(r'\b%s\b' % rank_page, page_content):
			in_content = True
		if in_summary and in_content:
			links_in_both.add(rank_page)
		elif in_summary:
			links_in_summary_only.add(rank_page)
		elif in_content:
			links_in_content_only.add(rank_page)
	# print out the summary for the comparison between the wikilinks, content and the ranked list
	print ("inks_in_both", links_in_both)
	print ("links_in_summary_only", links_in_summary_only)
	print ("links_in_content_only", links_in_content_only)
	PAGES_IN_WIKI_LINKS_AND_CONTENT = links_in_both.union(rank_set.intersection(wiki_page_links))
	print ("pages in both links and the content", PAGES_IN_WIKI_LINKS_AND_CONTENT)
	top_ranked_50 = set(rank_name_list[0:50])
	print ("print intersection with top ranks 50 and wiki", top_ranked_50.intersection(PAGES_IN_WIKI_LINKS_AND_CONTENT))
	print ("pages in top 50 and not in common set", top_ranked_50.difference(PAGES_IN_WIKI_LINKS_AND_CONTENT))
コード例 #3
0
            lambda url_urls_rank: computeContribsAssociation(url_urls_rank[1][0], url_urls_rank[1][1]))

        # Re-calculates page similarity ranks based on neighbor contributions.
        ranks = contribs.reduceByKey(add).mapValues(lambda rank: rank * 0.85 + 0.15/pages_num)
	# loop until it converge or it reaches the maximum number of iteration 
	if iteration > 0:
		print ("ieteration", iteration)
		converged_count = 0
		converge_norm = 0
		for (page_id, rank_value) in ranks.collect():
			if rank_value - prev_rank.lookup(page_id)[0] < 0.001:
				converged_count += 1
			converge_norm += (rank_value - prev_rank.lookup(page_id)[0])
		print ("converge.count()", converged_count)
		# If it converges, break
		if converge_norm <= 0.001:
			print ("Converged after ", iteration)
			break
	# store the current ranks to compare tem to the ranks of the next iterations		
	prev_rank = ranks
    # sort the pages based on the ranks similarity
    sorted_ranks = sorted(ranks.collect(), key = itemgetter(1), reverse=True)
    # Output the similarity ranks
    load_page = Pages()
    max_rank = sorted_ranks[0][1]
    for (link, rank) in sorted_ranks:
        print("%s,%s" % (load_page.get_title(int(link)), rank/max_rank))
    # get the difference between wikipedia links and similarty ranking list, and check if the ranked links exist in the content
    getDifferenceWikiPageRankLists(search_term, sorted_ranks)
    sc.stop()