def page(title=None,
pageid=None,
auto_suggest=True,
redirect=True,
preload=False):
'''
Get a WikipediaPage object for the page with title `title` or the pageid
`pageid` (mutually exclusive).
Keyword arguments:
* title - the title of the page to load
* pageid - the numeric pageid of the page to load
* auto_suggest - let Wikipedia find a valid page title for the query
* redirect - allow redirection without raising RedirectError
* preload - load content, summary, images, references, and links during initialization
'''
if title is not None:
if auto_suggest:
results, suggestion = search(title, results=1, suggestion=True)
try:
title = suggestion or results[0]
except IndexError:
# if there is no suggestion or search results, the page doesn't
# exist
raise PageError(title)
return WikipediaPage(title, redirect=redirect, preload=preload)
elif pageid is not None:
return WikipediaPage(pageid=pageid, preload=preload)
else:
raise ValueError("Either a title or a pageid must be specified")
def build_wiki_category_dataset():
readfile = codecs.open(
'/export/home/Dataset/wikipedia/parsed_output/tokenized_wiki/tokenized_wiki.txt',
'r', 'utf-8')
writefile = codecs.open(
'/export/home/Dataset/wikipedia/parsed_output/tokenized_wiki/tokenized_wiki2categories.txt',
'w', 'utf-8')
co = 0
for line in readfile:
try:
line_dic = json.loads(line)
except ValueError:
continue
try:
# title = line_dic.get('title')
title_id = line_dic.get('id')
article = WikipediaPage(pageid=title_id)
except AttributeError:
continue
type_list = article.categories
# print(type_list)
line_dic['categories'] = type_list
writefile.write(json.dumps(line_dic) + '\n')
co += 1
if co % 5 == 0:
print(co)
if co == 100000:
break
writefile.close()
readfile.close()
print('over')
def read_from_wiki(titles):
""" This function will except a list(list of Strings) of all the titles
and we will use these strings to get test from wikipedia."""
out_data_list = [] # List to append the dictionary elements(i.e. Required data with keys and values.) into one list.
for index, title in enumerate(titles):
out_data_dict = {'Title': title ,'Passage':'', "Question": [] ,"Error" : None } # Will store our processed text into dictionary. {key:'Passage', value:'Text'}
try:
get_wiki_data = WikipediaPage(title = title) # Get all the data from wikipedia.
except DisambiguationError:
# If there is any disambiguity in the Title name.
out_data_dict["Error"] = ("There is Disambigity in the title : " + title + ". Please provide more precise title.")
except PageError:
# If no page found with the given title.
out_data_dict["Error"] = ("Page id " + title + " does not match any pages. Try another id!")
if not out_data_dict["Error"]:
# If there is no error then store the passage.
content_only = get_wiki_data.content # Store main content into a variable.
processed_text = normalize_passage(content_only) # Process text using normalize_passge().
out_data_dict['Passage'] = processed_text # Store received text into dictionary.
out_data_list.append(out_data_dict) # Now append each dictionary into List.
return out_data_list
def main():
nlp = spacy.load("en_core_web_sm")
with open('test.txt', 'r') as file:
data = file.read().replace('\n','')
doc = nlp(data)
wiki = WikipediaPage("Indiana Jones and the Raiders of the Lost Ark")
map = dict()
for sent in doc.sents:
for token in sent:
if (token.is_alpha and not token.is_stop):
s = getName(token)
if (s = None):
s = token.lemma_
if (token.pos == PROPN):
flag = True
for name in names:
if (s in name):
flag = False
s = name
break
if (flag):
names.append(s)
if (s in map):
map[s] += 1
else:
map[s] = 1
print("---------------------------------------")
for k in map:
if(map[k] > 1):
print(k, map[k])
print("---------------------------------------")
print(names)
print("---------------------------------------")
for sent in doc.sents:
root = sent.root
line = ""
for token in root.lefts:
s = func(token)
print(s)
if(s != None and s in map and map[s] > 1):
line += " " + s
line += " " + root.lemma_
for token in root.rights:
s = token.lemma_
if (token.pos == PROPN):
s = str(token)
if (s in map and map[s] > 1):
line += " " + s
print(line)
def process_page(params):
id_link, link, depth = params
# Get the wiki object for the link
try:
wiki_obj = WikipediaPage(link)
links = wiki_obj.links
except (PageError, KeyError, DisambiguationError):
return 0, (id_link, None, None, link, None)
title = wiki_obj.title
summary = wiki_obj.summary[:300].replace("\"", "'")
return 1, (id_link, summary, depth + 1, title, links)
def get_articles(self, graph):
category = WikipediaPage(self.__primarylabel__ + ':' +
self.property_title)
for page in category.links:
article = Article(page, depth=self.property_depth + 1)
graph.push(article)
self.has_article.add(article)
graph.push(self)
def get_categories(self, graph):
category = WikipediaPage(self.__primarylabel__ + ':' +
self.property_title)
for cat in category.categories:
category = Category(cat, depth=self.property_depth + 1)
graph.push(category)
self.has_category.add(category)
graph.push(self)
def clever_life(response: wikipedia.WikipediaPage, quick_table=None):
res = {}
if quick_table is None:
quick_table = get_quick_table(response.html())
birthday = -1
death_day = -1
age = -1
alive = True
precision = True
birthday_words = {
"Дата рождения", "Рождение", "Родился", "Родилась", "Рождён"
}
death_words = {"Дата смерти", "Смерть", "Умер", "Умерла", "Убит"}
if "Дата рождения" in quick_table:
raw_birthday = find_4_digit_nums(quick_table["Дата рождения"])
if raw_birthday:
birthday = raw_birthday[0]
if raw_birthday and "Дата смерти" in quick_table:
death_day = find_4_digit_nums(quick_table["Дата смерти"])[0]
alive = False
age = death_day - birthday
elif raw_birthday:
alive = True
age = 2020 - birthday
if birthday == -1:
precision = False
summary = response.summary
bad_data = get_life_time(summary)
if "born" in bad_data:
birthday = bad_data["born"]
if "end" in bad_data:
death_day = bad_data["end"]
alive = False
else:
death_day = 2020
alive = True
else:
birthday = random.randint(1900, 2020)
death_day = random.randint(birthday, 2500)
alive = death_day > 2020
age = death_day - birthday
res = {
"alive": alive,
"birth_day": birthday,
"death_day": death_day,
"age": age,
"precision": precision
}
return res
def handle_noargs(self, **options):
DOCTORS_LIST = [
{'Number': 1, 'First': 'William', 'Second': 'Hartnell',
'Start_Date': '1963-11-23', 'End_Date': '1966-10-29', 'Slug': 'First_Doctor'},
{'Number': 2, 'First': 'Patrick', 'Second': 'Troughton',
'Start_Date': '1966-10-29', 'End_Date': '1969-07-21', 'Slug': 'Second_Doctor'},
{'Number': 3, 'First': 'Jon', 'Second': 'Pertwee',
'Start_Date': '1970-01-03', 'End_Date': '1974-07-08', 'Slug': 'Third_Doctor'},
{'Number': 4, 'First': 'Tom', 'Second': 'Baker',
'Start_Date': '1981-03-08', 'End_Date': '1981-03-21', 'Slug': 'Fourth_Doctor'},
{'Number': 5, 'First': 'Peter', 'Second': 'Davison',
'Start_Date': '1981-03-21', 'End_Date': '1984-03-16', 'Slug': 'Fifth_Doctor'},
{'Number': 6, 'First': 'Colin', 'Second': 'Baker',
'Start_Date': '1984-03-16', 'End_Date': '1986-12-06', 'Slug': 'Sixth_Doctor'},
{'Number': 7, 'First': 'Sylvester', 'Second': 'McCoy',
'Start_Date': '1987-09-07', 'End_Date': '1996-05-27', 'Slug': 'Seventh_Doctor'},
{'Number': 8, 'First': 'Paul', 'Second': 'McGann',
'Start_Date': '1996-05-27', 'End_Date': '1996-05-27', 'Slug': 'Eighth_Doctor'},
{'Number': 9, 'First': 'Christopher', 'Second': 'Eccleston',
'Start_Date': '2005-03-26', 'End_Date': '2005-07-18', 'Slug': 'Ninth_Doctor'},
{'Number': 10, 'First': 'David', 'Second': 'Tennant',
'Start_Date': '2005-07-18', 'End_Date': '2010-01-01', 'Slug': 'Tenth_Doctor'},
{'Number': 11, 'First': 'Matt', 'Second': 'Smith',
'Start_Date': '2010-01-01', 'End_Date': '2013-12-25', 'Slug': 'Eleventh_Doctor'},
{'Number': 12, 'First': 'Peter', 'Second': 'Capaldi',
'Start_Date': '2013-12-25', 'End_Date': datetime.datetime.now(), 'Slug': 'Twelfth_Doctor'},
]
for item in DOCTORS_LIST:
slug = item["Slug"]
page = WikipediaPage(slug)
doctor = Doctor.objects.create(
number=item["Number"],
first_name=item["First"],
second_name=item["Second"],
character_bio=page.section("Character biography"),
personality=page.section("Personality"),
appearance=page.section("Appearance"),
story_style=page.section("Story style"),
later_appearances=page.section("Later appearances"),
other_mentions=page.section("Other mentions"),
reception=page.section("Reception"),
short_desc="",
about=page.summary,
# doc_img=img,
start_date=item["Start_Date"],
end_date=item["End_Date"],
slug=slug)
doctor.save()
def get_categories(self):
article = WikipediaPage(self.title)
for category in article.categories:
title = prefix + category
category = WikiNode(title, self.graph, label="Category")
category.depth = self.depth + 1
self.graph.create(category)
self.graph.push(category)
self.has_category.add(category)
self.graph.push(self)
def get_plot(self):
"""gets the plot"""
# This is in case 'Plot' is not found
plot_names = ['Plot', 'Summary', 'Premise', 'Synopsis']
plot_section = None
self.full_plot_section = None
for i in plot_names:
try:
plot_section = WikipediaPage(self.title).section(i)
# saves this to use in get_keywords
self.full_plot_section = plot_section
# this is a brief SUMMARY, not the whole movie
plot_section = plot_section[:1000] + "..."
if plot_section is not None:
plot_section = plot_section.replace('\n',
' ') # remove newlines
return plot_section
except TypeError:
continue
if plot_section is None:
plot_section = 'none'
return plot_section
def process_co(message):
try:
co_msg = str(message.text)
set_lang("en")
lat, lon = WikipediaPage(co_msg).coordinates
bot.send_message(
chat_id=message.chat.id, text=str(round(lat, 5)) + ", " + str(round(lon, 5))
)
bot.send_location(
chat_id=message.chat.id,
latitude=lat,
longitude=lon,
reply_markup=main_keyboard(),
)
except Exception:
bot.send_message(
chat_id=message.chat.id,
text="Not a location.",
reply_markup=main_keyboard(),
)
def get_html_for_page(self, page: wikipedia.WikipediaPage):
"""Construct HTML for page
TODO: Fix HTML links so that images and external links work correctly.
For example, current wiki links are `href="/wiki/Mount_Lago"`, and that
should be changed to `href="wikipedia.org/wiki/Mount_Lago"`. Note that
it would be cool if you were able to check if the page pointed to is
also in the Wikipedia extract, because then you could link to it
offline!?!?
"""
if not isinstance(page, wikipedia.WikipediaPage):
raise TypeError('page must be of type wikipedia.WikipediaPage')
html = page.html()
# Download images
image_paths = self._download_images(page.images)
# Fix links
html = self._fix_links(html, image_paths)
return html
def get_rt_score(self):
"""gets Rotten Tomatoes score"""
full_page = WikipediaPage(self.title).content
sentences = nltk.sent_tokenize(full_page)
rt_score = [
x for x in sentences if "Rotten Tomatoes" in x and "%" in x
]
# for films without a Rotten Tomatoes score (ex. La Jetee)
if rt_score:
rt_score = rt_score[0]
else:
rt_score = 'none'
# To clean it up and remove everything before the last newline char:
rt_score = rt_score.strip() # remove trailing \n
nl_indx = 0 # counter for the last newline character
for i in range(len(rt_score)):
if rt_score[i] == '\n':
nl_indx = i
if nl_indx == 0:
nl_indx = -1 # exception
rt_score = rt_score[nl_indx + 1:]
return rt_score
def findSummaries(s):
global wikiFlag
global wikiChars
global nameSummaries
global soupChars
sum = ""
try:
if wikiFlag:
s1 = uniqueMatchAgainstLastName(s)
if s != s1:
nicknames[s] = s1
s = s1
wiki = WikipediaPage(s)
if "characters" in wiki.title:
# name brought up a list of characters wiki page
if not wikiFlag:
# this is the first time we've found this page
wikiFlag = True
wikiChars = wiki
soupChars = BeautifulSoup(wikiChars.html(),
'html.parser').find_all(
'span', class_='mw-headline')
catchUpOnSummaries()
summary = getSomeText(wiki.section(uniqueMatchAgainstLastName(s)))
if summary != None:
# the name is a section, so assign that summary
nameSummaries[s] = summary
return
s2 = uniqueMatchNameAgainstFirstName(s)
if s2 != s:
# the name (uniquely) matches to a first name, so grab that summary
summary = getSomeText(wiki.section(s2))
if summary != None:
nameSummaries[s] = summary
return
s3 = uniqueMatchNameAgainstLastName(s.split(' ')[-1])
if s3 != s:
nameSummaries[s] = getSomeText(wiki.section(s3))
else:
nameSummaries[s] = None
return
else:
nameSummaries[s] = getSomeText(wiki.summary)
findNicknames(s, wiki)
return
except wikipedia.exceptions.DisambiguationError as e:
if not wikiFlag:
namesToTry.append(s)
return
if verbose: print("-e1", s) #
# name matched by last name is causing wiki ambiguation
# try matching by first name
s2 = uniqueMatchAgainstFirstName(s)
if s2 != s:
if verbose: print("-e2", s2)
nicknames[s] = s2
findSummaries(s2)
return
# first name did not uniquely match, so try just s's last name
# and matching against another last name
s3 = uniqueMatchAgainstLastName(s.split(' ')[-1])
if s3 != s:
if verbose: print("-e3", s3) #
nicknames[s] = s3
findSummaries(s3)
return
# nothing is matching, so just take the first entry on wikipedia
if verbose: print("-ed", e.options[0]) #
nicknames[s] = e.options[0]
findSummaries(e.options[0])
return
except:
if not wikiFlag:
namesToTry.append(s)
return
if verbose: print("-m1", s) #
summary = getSomeText(wikiChars.section(s))
if summary != None:
nameSummaries[s] = summary
return
s2 = uniqueMatchAgainstFirstName(s)
if s2 != s:
nicknames[s] = s2
if verbose: print("-m2", s2) #
summary = getSomeText(wikiChars.section(s2))
if summary != None:
nameSummaries[s] = summary
return
s3 = uniqueMatchAgainstLastName(s.split(' ')[-1])
if s3 != s:
nicknames[s] = s3
if verbose: print("-m3", s3) #
summary = getSomeText(wikiChars.section(s3))
if summary != None:
nameSummaries[s] = summary
return
# Everything failed, description of this must be complex to find
if verbose: print("None", s)
nameSummaries[s] = None
import wikipedia
from wikipedia import WikipediaPage
wiki = WikipediaPage("Indiana Jones and the Raiders of the Lost Ark")
print("Ark" in wiki.links)
def send_summary(): # for Nick's microservice
"""exports the full wikipedia page"""
user_input = request.args.get("article")
full_page_2 = WikipediaPage(user_input).summary
return str(full_page_2)
("Applied Mathematics", "Statistics"),
("Applied Mathematics", "Game theory"),
("Applied Mathematics", "Information theory"),
("Applied Mathematics", "Computer science"),
("Applied Mathematics", "Theory of computation"),
("Applied Mathematics", "Control theory"), ("Others", "Order theory"),
("Others", "Graph theory")]
# Step 1. Get pages (and raw equations) from wikipedia
for pair in pages:
domain = pair[0]
method = pair[1]
if method not in results:
result = WikipediaPage(method)
# Show a visual check!
print("Matching %s to %s" % (method, result.title))
entry = {
'categories': result.categories,
'title': result.title,
'method': method,
'url': result.url,
'summary': result.summary,
'images': result.images
}
# We can use links to calculate relatedness
entry['links'] = get_attribute(result, 'links')
entry['references'] = get_attribute(result, 'references')