def main(self, question):
"""
Main method which instantiates the classes necessary for the operation
of the program, stores the collected data in parameters and returns a
formatted dictionary.
Args:
question(str): input user
Returns:
dict : Formatted response returned to AJAX
"""
initialize_parser = Parser()
self.parser = initialize_parser.clean(question)
initialize_google_client = GoogleClient(self.parser)
self.google_answer = initialize_google_client.get_location()
initialize_wiki_client = WikiClient()
self.wiki_answer = initialize_wiki_client.search_page(self.google_answer)
if self.wiki_answer['extract']:
self.wiki_answer['extract'] = layout(self.wiki_answer['extract'])
self.papy_answer = self._format_return_datas()
return self.papy_answer
def test_is_token_place_entity():
assert Parser.is_token_place_entity(
TestParser.MockToken(ent_type='LOC')) is True
assert Parser.is_token_place_entity(
TestParser.MockToken(ent_type='GPE')) is True
assert Parser.is_token_place_entity(
TestParser.MockToken(ent_type='DATE')) is False
def test_parser_can_retrieve_info_after_some_words(self):
""" Tests whether the parser is able to retrieve relevant data
(localisation) that follow a certain piece of words."""
parser = Parser(
"Bonjour Grandpy, peux-tu me dire l'adresse de la tour Eiffel?",
STOPWORDS, ACCENTS, QUESTIONS)
result = parser.get_end_of_question()
assert result == "la tour Eiffel"
def test_parser_can_return_clean_of_all_output(self):
""" Tests whether the parser is able to run all its methods in order
to get a clean localisation from a full sentence."""
parser = Parser(
"Bonjour Grandpy, peux-tu me dire l'adresse de la tour Eiffèl?",
STOPWORDS, ACCENTS, QUESTIONS)
result = parser.clean()
assert result == "tour eiffel"
def testParseArray():
parser = Parser()
el1 = RespBulkString(4, 'ping')
el2 = RespBulkString(6, 'foobar')
expected = [el1, el2]
expected = ['ping', 'foobar']
output = parser.decodeArray(b'*2\r\n$4\r\nping\r\n$6\r\nfoobar\r\n')
assert output == expected
def __init__(self,
result_save_directory,
save_to_json=False,
save_to_db=False,
*args,
**kwargs):
self.result_save_directory = result_save_directory
self.save_to_json = save_to_json
self.save_to_db = save_to_db
super().__init__(*args, **kwargs)
self.parser = Parser()
def __init__(self, model=None):
self.parser = Parser()
if model is None:
#TODO download if inexistent
self.model = KeyedVectors.load_word2vec_format(
'wiki.pt/wiki.pt.vec')
else:
self.model = model
self.status = "extracting words from website"
def test_parser():
my_parser = Parser("quelle est l'adresse de la poste la plus proche ?")
assert my_parser.parsed_string == "poste"
my_parser = Parser(
"salut grandpy comment ça va ? Tu sais ou est la tour eiffel par hasard ?"
)
assert my_parser.parsed_string == "tour eiffel"
my_parser = Parser("Hello quelle est l'adresse de openclassroom stp?")
assert my_parser.parsed_string == "openclassroom"
my_parser = Parser("")
assert my_parser.parsed_string == ""
my_parser = Parser(22)
assert my_parser.parsed_string == ""
def question():
"""
Get input, parse it, get position, get info, get story
and return latitude, longitude, adress and story in json
"""
question = request.form['question']
parser = Parser()
words = parser.clean_sentence(question)
latlon = GoogleMap().find_location(words)
adress = GoogleMap().find_adress(words)
location = GoogleMap().wiki_search(words)
infos = WikipediaInformation(location).summary_informations()
return jsonify(latlon=latlon, adress=adress, infos=infos)
def parse_question(whole_question: str) -> dict:
parsed_question = Parser(whole_question)
parsed_question.extract_interesting_tokens()
if len(parsed_question.interesting_tokens) == 0:
return None
return {
"formated_original_key_words":
" ".join(parsed_question.interesting_tokens),
"singled_key_words":
tuple((kw, Parser.word_reverse_case_first_char(kw))
for kw in parsed_question.interesting_tokens)
}
def test_if_communicant_can_get_gps_coordinates(self, monkeypatch):
""" Tests if we can retrieve the GPS coordinate out of the response."""
class MockGet():
"""This class is designed to mock a response object from the module
requests.
"""
def __init__(self, *args):
self.ok = True
self.response = {
"results" : [{
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
}
}
}],
"status" : "OK"
}
def json(self):
return self.response
parser = Parser("1600 Amphitheatre Parkway, Mountain View, CA",
STOPWORDS,
ACCENTS,
QUESTIONS
)
communicant = GAPICommunicant(parser)
monkeypatch.setattr(communicant,'response', MockGet())
result = communicant.get_data_from_geocoding()
assert result == {
"lat": 37.4224764,
"lng": -122.0842499
}
class TestParser:
PARSER = Parser()
def test_normalize(self):
self.PARSER.parsed_input = "Test and Retest! éùéî"
self.PARSER.normalize()
result = self.PARSER.parsed_input
assert result == "test and retest! euei"
def test_split_sentences(self):
self.PARSER.parsed_input = "First sentence. And a question? Wonderfull!"
self.PARSER.split_sentences()
result = self.PARSER.parsed_input
assert result == ["First sentence", "And a question", "Wonderfull"]
def test_find_question(self):
self.PARSER.parsed_input = [
"salut yoda, comment vas",
"tu",
"peux",
"tu me dire ou trouver la tour eiffel",
"passes une bonne journee",
]
self.PARSER.find_question()
result = self.PARSER.parsed_input
assert result == "tu me dire ou trouver la tour eiffel"
def test_clear_question(self):
self.PARSER.parsed_input = "tu me dire ou trouver la tour eiffel"
self.PARSER.clear_question()
result = self.PARSER.parsed_input
assert result == "trouver tour eiffel"
def result():
"""Method result called when user submit his input that help us
to retrieve data from JS to Python
result view recieves an HTTP request and sends an HTTP response"""
# object request allows me to access to data
user_input = request.get_data("user_input").decode() # decode unicode
# Get coordinates
parser = Parser(user_input)
geowrapper = GeoWrapper(parser.parsed_input)
# If one coord missing, stop here return error
if geowrapper.latitude is None or geowrapper.longitude is None:
return {"content": "Missing coordinates"}
# If coordinates, get wiki infos
wikiwrapper = Wikiwrapper()
wiki_details = wikiwrapper.get_wiki_info_by_long_lat(
geowrapper.longitude, geowrapper.latitude)
if wiki_details is None:
return {"content": "No wiki details found"}
else:
return {
"description": wiki_details["description"],
"url": wiki_details["url"],
"longitude": geowrapper.longitude,
"latitude": geowrapper.latitude,
}
def try_original_key_words(formated_original_key_words: str) -> dict:
"""
Requests API using original parsed key words.
To elevate results accuracy : if more relevant key words exists in map API response (see maps_position docstring)
they are firstly used for Wikipedia request. Else original parsed key words are tried for Wikipedia request.
"""
map_api_resp = maps_position(formated_original_key_words)
if map_api_resp["map_resp"]:
if map_api_resp["key_words"]:
wiki_api_resp = wiki_infos(
Parser.underscored_joined_words(map_api_resp["key_words"]))
if not wiki_api_resp["wiki_resp"]:
wiki_api_resp = wiki_infos(formated_original_key_words)
return {
**map_api_resp,
**wiki_api_resp # dicos concatenation
}
else:
wiki_api_resp = wiki_infos(formated_original_key_words)
return {
**map_api_resp,
**wiki_api_resp # dicos concatenation
}
else:
return map_api_resp
def send_answer():
user_text = request.form["userText"]#cherche dans le name de input html
parser = Parser(user_text)
result = Request(parser.cleaned)
answer = result.wiki_result
lat = result.lat
lng = result.lng
return jsonify({"answer": answer, "lat": lat, "lng": lng})
def api():
query = request.form["query"]
print(query)
message = Parser(query)
message.ponctuation()
message.list_it()
cleanMessage = message.delete_common_words()
print(cleanMessage)
rep = Geo()
coordonnees = rep.get_coordonnees(query)
if coordonnees is not None:
print(rep.get_address(coordonnees))
wiki = Wiki()
page_id = wiki.get_page_id(coordonnees[0], coordonnees[1])
summary = wiki.get_summary(page_id)
answer = botAnswer()
bot = answer.goodAnswer()
print(bot)
return jsonify({
"lat": coordonnees[0],
"lng": coordonnees[1],
"summary": summary,
"answer": bot,
})
else:
print("pas trouvé")
answer = botAnswer()
bot = answer.badAnswer()
print(bot)
return jsonify({
"answer": bot,
})
def test_exception_catch_for_an_unknown_place():
parser = Parser(
"Yop, donne moi les coordonnées du trou noir Sagittarius A*")
place = Location(parser)
with pytest_raises(UnknownPlaceError):
place.get_location()
def test_Location_return_correct_Location_for_several_places(
location, coordinates):
parser = Parser(location)
place = Location(parser)
place.get_location()
assert place.latitude, place.longitude == coordinates
class ParserTestCase(unittest.TestCase):
def setUp(self) -> None:
self.parser = Parser()
return super().setUp()
def tearDown(self) -> None:
self.parser = None
return super().tearDown()
def test_run(self) -> bool:
self.assertIsNone(self.parser.run("", ""))
def interpret(text):
lexer = Lexer(text)
current_token = lexer.get_next_token()
parser = Parser(lexer)
interpreter = Interpreter(parser)
executor = Executor(parser)
if EXECUTE:
return executor.execute()
else:
return interpreter.parse()
def parse_website(task_id):
task = Task.filter_by(id=task_id)[0]
task.update(1)
try:
Parser(task.web_url, task.depth, task.id).parse()
task.update(2)
dir = "{}/app/sites/{}".format(os.getcwd(), task_id)
zip_folder(dir)
except Exception as e:
print("Error while parsing site\n{}".format(e))
task.update(3)
def test_story_return_correct_extract_and_url_for_several_location(
input_user, extract_place, url_place):
parser = Parser(input_user)
location = Location(parser)
location.get_location()
story = Story(location)
story.about()
assert story.extract == extract_place
assert story.url == url_place
def main():
try:
config = cp.ConfigParser()
config.read('./app/conf/main.ini')
# database section
user = config.get('database', 'user')
password = config.get('database', 'password')
database = config.get('database', 'database')
host = config.get('database', 'host')
# other section
wall = config.get('other', 'wall')
update = config.getfloat('other', 'update')
except:
logger.error('Config file error')
sys.exit()
logger.info('Cakes thief: trying to connect to PostgreSQL database')
logger.info('User: %s' % user)
logger.info('Password: %s' % password)
logger.info('Database: %s' % database)
logger.info('Host: %s' % host)
logger.info('Wall: %s' % wall)
logger.info('Update in seconds: %s' % update)
db = Database(user=user,
password=password,
database=database,
host=host)
parser = Parser(wall, update, db)
parser.start()
parser.idle()
def app_grandpy(question):
"""
The main implementation of all the application logic
Create the parser for the questions asked analyzer
Using the API Google maps
Using the API Wikipedia
"""
parser = Parser(question)
question = parser.clean()
gmaps = GoogleMaps()
result_address = gmaps.search(question)
wikip = Wikipedia()
pages = wikip.search_for_pages(result_address["lat"],
result_address["lng"])
result_article, result_url = wikip.search_for_page_content(
pages[0]["pageid"]
)
return {
"parser": question,
"grandpy": random.choice(reponseListe),
"gmaps": result_address,
"wikip": result_article,
"wikip_url": result_url}
def process():
"""Process the user input"""
content = {}
if "message" in request.form and ":help" in request.form["message"]:
# Display instructions if the user type :help
content = BOT.instructions
elif "message" in request.form:
# Parse the user input
parser = Parser()
parser_response = parser.parse(request.form["message"])
if "parsed_input" in parser_response:
# Send the parsed input to the geo code api
geo_code = GeoCode()
geo_response = geo_code.api_request(
parser_response["parsed_input"])
if "place_name_fr" in geo_response:
content["map"] = geo_response
content["map"].update(BOT.found_place)
# Send the coordinates to wikipedia
wiki_search = WikiSearch()
wiki_response = wiki_search.geo_search_article(
content["map"]["latitude"], content["map"]["longitude"])
if "url" in wiki_response:
content["article"] = wiki_response
content["article"].update(BOT.found_article)
else:
content = BOT.not_found
else:
content = BOT.parse_error
return jsonify(content)
class ParserTest(unittest.TestCase):
def setUp(self):
self.p = Parser(None)
def test_links_compact(self):
data = [
(['http://ya.ru', 'https://ya.ru'], ['ya.ru'], ''),
(['http://www.ya.ru', 'https://ololo.ya.ru'], ['ya.ru'], ''),
(['http://www.ya.ru', 'https://ololo.ya.com'], ['ya.com', 'ya.ru'], ''),
(['http://www.ya.ru', 'https://ololo.ya.com'], ['ya.com'], 'ya.ru'),
]
for links, success, ignore_domain in data:
res = self.p._links_domain_filter(links, ignore_domain)
self.assertEqual(success, res)
class CSVParserTests(unittest.TestCase):
def setUp(self):
self.parser = Parser()
def test_find_operation_for_sqaure_field(self):
days = ['mon', 'tue', 'wed']
for day in days:
result = self.parser.find_operation(day)
self.assertEqual('square', result)
def test_find_operation_for_double_field(self):
days = ['thu', 'fri']
for day in days:
result = self.parser.find_operation(day)
self.assertEqual('double', result)
def test_find_operation_for_day_name_misspelled(self):
self.assertRaises(ValueError, self.parser.find_operation, 'abcd1234')
def test_parse_single_day_data(self):
result = self.parser.parse_single_day_data('fifth_desc', 'thu', '4')
expected_output = OrderedDict([('day', 'thu'),
('description', 'fifth_desc 8'),
('double', '8'), ('value', '4')])
self.assertEqual(expected_output, result)
def test_csv_reader_for_existing_file_name(self):
file_name = "files/1.csv"
self.assertIsInstance(self.parser.read_csv(file_name), object)
def test_csv_reader_for_non_existing_file_name(self):
file_name = "files/abc.csv"
self.assertRaises(IOError, self.parser.read_csv, '../files/abc.csv')
def test_parse_csv_for_multiple_days_with_processed_output(self):
test_file = open('test.csv', 'w')
test_file.write('description,tue-wed,thu-fri,mon,some_columnnth\n')
test_file.write('nth_desc,12,1,2,data\n')
test_file.close()
file_obj, reader = self.parser.read_csv('test.csv')
parsed_data = self.parser.parse_csv(reader)
result = self.parser.process_output(parsed_data)
expected_output = [OrderedDict([('day', 'mon'), ('description', 'nth_desc 4'), ('square', '4'), ('value', '2')]),\
OrderedDict([('day', 'tue'), ('description', 'nth_desc 144'), ('square', '144'), ('value', '12')]), \
OrderedDict([('day', 'wed'), ('description', 'nth_desc 144'), ('square', '144'), ('value', '12')]),\
OrderedDict([('day', 'thu'), ('description', 'nth_desc 2'), ('double', '2'), ('value', '1')]),\
OrderedDict([('day', 'fri'), ('description', 'nth_desc 2'), ('double', '2'), ('value', '1')])]
os.remove('test.csv')
self.assertEqual(result, json.dumps(expected_output))
class ParserTest(unittest.TestCase):
def setUp(self):
self.p = Parser(None)
def test_links_compact(self):
data = [
(['http://ya.ru', 'https://ya.ru'], ['ya.ru'], ''),
(['http://www.ya.ru', 'https://ololo.ya.ru'], ['ya.ru'], ''),
(['http://www.ya.ru', 'https://ololo.ya.com'], ['ya.com',
'ya.ru'], ''),
(['http://www.ya.ru',
'https://ololo.ya.com'], ['ya.com'], 'ya.ru'),
]
for links, success, ignore_domain in data:
res = self.p._links_domain_filter(links, ignore_domain)
self.assertEqual(success, res)
def test_requests_response_when_ok(self, monkeypatch):
""" Tests if we can get an answer in our API call"""
class MockGet():
"""This class is designed to mock a response object from the module
requests.
"""
def __init__(self, *args):
self.ok = True
parser = Parser("1600 Amphitheatre Parkway, Mountain View, CA",
STOPWORDS,
ACCENTS,
QUESTIONS
)
communicant = GAPICommunicant(parser)
# Mocking the API call
monkeypatch.setattr('requests.get', MockGet)
result = communicant.make_requests_to_geocoding_API()
assert result != None
class TestParser:
SENTENCE = Parser('this!, sentence; is: over? <<ponctuated>>')
def __init__(self, text):
self.text = text
self.ban = '!?#<>'
self.parse = self._parsing()
def test_ponctuation1(self):
assert ',' not in self.SENTENCE.ponctuation()
def test_ponctuation2(self):
assert '>' not in self.SENTENCE.ponctuation()
def test_list_it(self):
assert type(self.SENTENCE.list_it()) == list
def test_delete_common_words(self):
assert type(self.SENTENCE.delete_common_words()) == str
def _parsing(self):
return ''.join(' ' if c in self.ban else c for c in self.text)
def get_infos(sentence):
"""Parse an input sentence and try to get infos from it, coordinates,
extract from Wikipedia and url.
Args:
sentence (string): Sentence from user input
Returns:
tuple: Informations gathered from input parsed (coordinates, url to
wikipedia, random message from GranPy)
boolean: Does the parsing and finding informations occured normally
"""
parser = Parser(sentence)
location = Location(parser)
try:
location.get_location()
story = Story(location)
story.about()
except UnknownPlaceError:
# If Harold doesn't found requested location, returning random sentence
# of error
return sample(RANDOM_SENTENCE_UNKWON_PLACE, 1), False
return (
sample(RANDOM_SENTENCE_GRANPY, 1),
location.latitude,
location.longitude,
story.extract,
story.url,
), True
def setUp(self):
self.p = Parser(None)