def graph(term):
#make network graph with depth = 2 for term
pages = Pages()
id = pages.get_id(term)
sim_pages = pages.query_page(id)
#make networkx graph object
G = nx.Graph()
node_dict = {}
title = pages.get_title(int(id))
G.add_node(title)
#original term is red
node_dict[title] = ('red', 0.0)
#add top 10 pages similar to term
short_sim_pages = sorted(sim_pages, key=lambda x: x[1], reverse=True)[1:11]
for page in short_sim_pages:
page_name = pages.get_title(int(page[0]))
G.add_node(page_name)
#depth = 1 terms are blue
node_dict[page_name] = ('blue', float(page[1]))
G.add_edge(title,page_name)
addl_nodes = add_x_nodes(pages, G, page[0], 10)
#add next depth of pages
for node in addl_nodes:
if node not in node_dict:
#depth = 2 terms are green
node_dict[node] = ('green', float(page[1]))
node_list = []
node_color = []
node_size = []
#make node size proportional to similarity score of each node to original term
for node in node_dict:
node_list.append(node)
node_color.append(node_dict[node][0])
node_size.append(node_dict[node][1])
node_size = [x*300/max(node_size) for x in node_size]
node_size[node_size.index(0.0)]=300
#draw and save network figure
nx.draw_networkx(G=G,with_labels=True,nodelist=node_list,node_size=node_size,node_color=node_color,font_size=8)
fig_name = "%s.png" %(term)
plt.savefig(fig_name)
def getDifferenceWikiPageRankLists(search_term, rank_list):
pages = Pages()
page_id = pages.get_id(search_term)
wiki_page_links = set(pages.getPageLinksList(int(page_id)))
rank_name_list = [pages.get_title(int(ranked_page[0])).lower() for ranked_page in rank_list]
rank_set = set(rank_name_list)
# get the links in both ranked list and wikipedea links
print ("Intersection", rank_set.intersection(wiki_page_links))
# get the links that exist only in ranked list
print ("Difference", rank_set.difference(wiki_page_links))
page_summary, page_content = pages.getPageSummaryContent(page_id)
# get ranked pages that exist in summary and content of page
links_in_summary_only = set()
links_in_content_only = set()
links_in_both = set()
for rank_page in rank_name_list:
in_summary = False
in_content = False
if re.search(r'\b%s\b' % rank_page, page_summary):
in_summary = True
if re.search(r'\b%s\b' % rank_page, page_content):
in_content = True
if in_summary and in_content:
links_in_both.add(rank_page)
elif in_summary:
links_in_summary_only.add(rank_page)
elif in_content:
links_in_content_only.add(rank_page)
# print out the summary for the comparison between the wikilinks, content and the ranked list
print ("inks_in_both", links_in_both)
print ("links_in_summary_only", links_in_summary_only)
print ("links_in_content_only", links_in_content_only)
PAGES_IN_WIKI_LINKS_AND_CONTENT = links_in_both.union(rank_set.intersection(wiki_page_links))
print ("pages in both links and the content", PAGES_IN_WIKI_LINKS_AND_CONTENT)
top_ranked_50 = set(rank_name_list[0:50])
print ("print intersection with top ranks 50 and wiki", top_ranked_50.intersection(PAGES_IN_WIKI_LINKS_AND_CONTENT))
print ("pages in top 50 and not in common set", top_ranked_50.difference(PAGES_IN_WIKI_LINKS_AND_CONTENT))
lambda url_urls_rank: computeContribsAssociation(url_urls_rank[1][0], url_urls_rank[1][1]))
# Re-calculates page similarity ranks based on neighbor contributions.
ranks = contribs.reduceByKey(add).mapValues(lambda rank: rank * 0.85 + 0.15/pages_num)
# loop until it converge or it reaches the maximum number of iteration
if iteration > 0:
print ("ieteration", iteration)
converged_count = 0
converge_norm = 0
for (page_id, rank_value) in ranks.collect():
if rank_value - prev_rank.lookup(page_id)[0] < 0.001:
converged_count += 1
converge_norm += (rank_value - prev_rank.lookup(page_id)[0])
print ("converge.count()", converged_count)
# If it converges, break
if converge_norm <= 0.001:
print ("Converged after ", iteration)
break
# store the current ranks to compare tem to the ranks of the next iterations
prev_rank = ranks
# sort the pages based on the ranks similarity
sorted_ranks = sorted(ranks.collect(), key = itemgetter(1), reverse=True)
# Output the similarity ranks
load_page = Pages()
max_rank = sorted_ranks[0][1]
for (link, rank) in sorted_ranks:
print("%s,%s" % (load_page.get_title(int(link)), rank/max_rank))
# get the difference between wikipedia links and similarty ranking list, and check if the ranked links exist in the content
getDifferenceWikiPageRankLists(search_term, sorted_ranks)
sc.stop()
