def admin_dashboad(request):
context = {
"nb": {
"comment": CommentService().count(),
"website": WebSiteService().count(),
"user": UserService().count(),
"searcher": SearcherService().count(),
"expert": ExpertService().count(),
"hotel": HotelService().count(),
},
}
return render(request, "anaviadmin/dashboard.html", context)
def setUp(self):
# self.db_fd, flask.app.config['DATABASE'] = tempfile.mkstemp()
# self.app = flask.app.test_client()
# flask.init_db()
app = Flask(__name__)
env_config = DevelopmentConfig
app.config.from_object(env_config)
db = create_engine(
'postgresql://*****:*****@localhost/heartsync_data')
con = db.connect()
metadata = MetaData()
mail = Mail(app)
self.user_repo = UserRepo(con, metadata, db, app, mail)
self.user_service = UserService(self.user_repo)
self.file_repo = FileRepo(con, metadata, db, app)
self.file_service = FileService(self.file_repo)
self.scan_repo = ScanRepo(con, metadata, db, app)
self.scan_service = ScanService(self.scan_repo)
self.notes_repo = NoteRepo(con, metadata, db, app)
self.notes_service = NoteService(self.notes_repo)
self.comment_repo = CommentRepo(con, metadata, db, app)
self.comment_service = CommentService(self.comment_repo)
def parse(self):
self.add_website()
#driver=webdriver.Chrome()
driver = webdriver.Firefox()
driver.get(self.url)
sleep(20)
search_input = driver.find_element_by_id("qf-0q-destination")
search_input.send_keys(
"Cameroon") #maintenant que le cameroun a ete chosir
sleep(7)
random_element = driver.find_element_by_css_selector('h1')
if random_element:
random_element.click()
sleep(10)
#nav_list=driver.find_element_by_css_selector(".footer-nav-list")
#nav_list.click()
#envoyons des a present le boutton submit
form_search = driver.find_element_by_css_selector("form.cont-bd")
#submit_button =driver.find_elements_by_css_selector("button[type='submit']")[0]
clicked_per = False
i_click = 0
submit_button = form_search.find_element_by_css_selector(
"button[type='submit']")
submit_button.click()
sleep(20)
clicked_per = True
i_click = i_click + 1
#Faisons le scroll pour charger le plus de commentaire possible
i = 0
scroll_wait_time = 6
#last_height=driver.execute_script("return document.body.scrollHeight")
html = driver.find_element_by_tag_name("html")
while i < 20:
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
sleep(scroll_wait_time)
i = i + 1
#fin de l'envoies des scroll
#Maintenant nous devons avoir la liste des hotels qui ont ete charger
#list_hotel=driver.find_elements_by_css_selector("section.hotel-wrap")
list_hotel = driver.find_elements_by_css_selector("li.hotel")
final_hotel_list = [
] #Cette liste va contenir tous les objets hotels obteu grace au scroll
links_to_comment = [
] #dans cette liste on va mettre les diffrents liens qui mene aux commentaire
for hotel_item in list_hotel:
#ici nous devons d'abord definis un hotel, ensuite verifier s'il est deja dans la base de donnes
#Si il n'existe pas encore on l'ajoute sinon on recupere son identifiant
#Ensuite on definirt une liaisons entre le site web et l'hotel
#Si la liaisons n'existe pas on l'ajoute sinon on ne fait rien
nhotel = Hotel()
hotel_name = hotel_item.find_element_by_css_selector(
".hotel-wrap .description .p-name").text
hotel_web_id = hotel_item.get_attribute("data-hotel-id")
hotel_place = hotel_item.find_element_by_css_selector(
".hotel-wrap .description .contact .address").text
nhotel.name = hotel_name
nhotel.place = hotel_place
nhotel.state = True
hservice = HotelService()
response = {}
exist = hservice.if_exist_hotel(nhotel)
if (exist["status"] == False):
response = hservice.add_hotel(nhotel)
nhotel = response["hotel"]
else:
nhotel = exist["hotel"]
hotelwebsite = HotelWebSite()
hotelwebsite.hotel_id_in_website = hotel_web_id
hotelwebsite.hotel = nhotel
hotelwebsite.website = self.website
#definition du lien entre un hotel et un site web
hwservice = HotelWebSiteService()
hwservice.add(hotelwebsite)
#Maintenant que nous avons tout lancer, il est tant de recuperer les liens qui
#menent aux commentaire il se peut que cela genere des erreurs pour lien non existant
link = ""
try:
link = hotel_item.find_element_by_css_selector(
".reviews-box .guest-reviews-link").get_attribute("href")
except:
pass
links_to_comment.append(link)
final_hotel_list.append(nhotel)
#maintenant nous allons suivre les liens lies au commentaire
i = 0
for link in links_to_comment:
if (link != ""):
driver.get(link)
#all_reviews_btn=driver.find_element_by_css_selector(".see-all-reviews a")
#all_reviews_btn.click()
sleep(10)
comment_overlay = driver.find_element_by_css_selector(
".reviews-overlay")
#maintenant nous devons connaitre le nombre total de commentaire
total_comment = driver.find_element_by_css_selector(
".reviews-overlay .total-count-number").text
total_comment = int(total_comment)
loop_num = total_comment // 50
page_comment = 0
stop_condition = True
while stop_condition:
all_reviews_card = driver.find_elements_by_css_selector(
".brand-reviews-listing .review-card")
#Pour chaque carte on extrait le commentaire
commentservice = CommentService()
for review_card in all_reviews_card:
comment = Comment()
comment.text = review_card.find_element_by_css_selector(
"blockquote.description").text
comment.website = self.website
comment.hotel = nhotel
comment_date = review_card.find_element_by_css_selector(
".card-header .date").text
comment_date = comment_date.replace("Check-in ", "")
comment_date = dateparser.parse(comment_date)
comment.add_date = comment_date
comment_add = commentservice.add(comment)
#endfor
if loop_num == page_comment:
stop_condition = False
page_comment = page_comment + 1
#maintenant nous recherchons le boutton suivant
try:
link_to_next_comment_page = driver.find_element_by_css_selector(
".reviews-overlay .pagination-controls a.cta-next")
link_to_next_comment_page.click()
sleep(5)
except:
pass
i = i + 1
return driver
metadata = MetaData()
except:
logging.error("Failed to connect to database.")
mail = Mail(app)
user_repo = UserRepo(con, metadata, db, app, mail)
user_service = UserService(user_repo)
file_repo = FileRepo(con, metadata, db, app)
file_service = FileService(file_repo)
scan_repo = ScanRepo(con, metadata, db, app)
scan_service = ScanService(scan_repo)
notes_repo = NoteRepo(con, metadata, db, app)
notes_service = NoteService(notes_repo)
comment_repo = CommentRepo(con, metadata, db, app)
comment_service = CommentService(comment_repo)
@app.route('/api/auth/login', methods=['POST'])
def login():
user = json.loads(request.get_data())
if 'username' not in user.keys() or 'password' not in user.keys():
response = make_response('Invalid login parameters', 400)
else:
logging.info("Logging in for user " + user['username'])
token = user_service.login(user['username'], user['password'])
if token:
response = make_response(jsonify(accessToken=token), 200)
else:
response = make_response('Invalid login', 400)
def parse(self):
self.add_website()
driver = webdriver.Firefox()
driver.get(self.url)
sleep(25)
#search_input=driver.find_element_by_id("oA4zhb")
search_input = driver.find_elements_by_css_selector(
".whsOnd.zHQkBf")[1]
search_input.send_keys("Cameroon")
search_input.send_keys(Keys.ENTER)
sleep(10)
#on defnis les elements qui vont entrez dans les hotels que nous allons parcourir
links_hotel = []
names_hotel = []
#une liste de nom et une liste lien
total_hotel = driver.find_element_by_id("wKdiD").text
nb_iteration = 14
time_to_wait = 6
while nb_iteration > 0:
list_hotel = driver.find_elements_by_css_selector(
".uaTTDe.BcKagd.Ab08je.mlo5Vd")
for item_hotel in list_hotel:
link = ""
try:
link = item_hotel.find_element_by_css_selector(
".spNMC.lRagtb").get_attribute("href")
except:
pass
links_hotel.append(link)
try:
next_btn = driver.find_element_by_css_selector(
".U26fgb.O0WRkf.oG5Srb.C0oVfc.JDnCLc.yHhO4c.yNl8hd.zbLWdb"
)
next_btn.click()
sleep(time_to_wait)
except:
nb_iteration = 0
nb_iteration = nb_iteration - 1
#Maintenant nous allons suivre les liens que nous avons sauvegarder plutot
hservice = HotelService()
cservice = CommentService(
) #on definit celui avec qui on va ajouter les commentaire
print("Le nombre d'hotel est " + str(len(links_hotel)))
for link_item in links_hotel:
#print("Entrer dans for")
if (link_item != ""):
try:
#print("Weclome to my first hotel")
driver.get(link_item)
sleep(10)
nhotel = Hotel()
nhotel.name = driver.title
nhotel.place = "Cameroon"
hotel_add = hservice.add_hotel(nhotel)
nhotel = hotel_add["hotel"]
#maintenant nous devons associers a un siteweb
hwebsite = HotelWebSite()
hwservice = HotelWebSiteService()
hwebsite.hotel = nhotel
hwebsite.website = self.website
hwservice.add(hwebsite)
#maintenant que c'est ajouter nous devons recuperer les commentaires
#mais avant faisons d'abord un scrool infernal
scroll_number = 33
scroll_wait_time = 3
html = driver.find_element_by_tag_name("html")
while scroll_number > 0:
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
scroll_number = scroll_number - 1
sleep(scroll_wait_time)
#maintenant on va recuper
all_reviews_card = driver.find_elements_by_css_selector(
".Svr5cf.bKhjM")
for card_item in all_reviews_card:
comment = Comment()
comment.text = card_item.find_element_by_css_selector(
".kVathc .STQFb .K7oBsc div span").text
comment.hotel = nhotel
comment.website = self.website
date_comment = card_item.find_element_by_css_selector(
".iUtr1").text
date_comment = date_comment.split(" on ")[0]
try:
comment.add_date = dateparser.parse(date_comment)
cservice.add(comment)
except:
comment.add_date = datetime.datetime.now()
cservice.add(comment)
except:
print("error i don't like")
response = {
"links": links_hotel,
"names": names_hotel,
"website": self.website,
"driver": driver
}
return response
def parse_specific_links(self, links_hotel):
self.add_website()
hservice = HotelService()
cservice = CommentService(
) #on definit celui avec qui on va ajouter les commentaire
names_hotel = []
driver = webdriver.Firefox()
print("Le nombre d'hotel est " + str(len(links_hotel)))
for link_item in links_hotel:
#print("Entrer dans for")
if (link_item != ""):
try:
#print("Weclome to my first hotel")
driver.get(link_item)
sleep(10)
nhotel = Hotel()
nhotel.name = driver.title
nhotel.place = "Cameroon"
hotel_add = hservice.add_hotel(nhotel)
nhotel = hotel_add["hotel"]
#maintenant nous devons associers a un siteweb
hwebsite = HotelWebSite()
hwservice = HotelWebSiteService()
hwebsite.hotel = nhotel
hwebsite.website = self.website
hwservice.add(hwebsite)
#maintenant que c'est ajouter nous devons recuperer les commentaires
#mais avant faisons d'abord un scrool infernal
scroll_number = 33
scroll_wait_time = 3
html = driver.find_element_by_tag_name("html")
while scroll_number > 0:
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
scroll_number = scroll_number - 1
sleep(scroll_wait_time)
#maintenant on va recuper
all_reviews_card = driver.find_elements_by_css_selector(
".Svr5cf.bKhjM")
for card_item in all_reviews_card:
comment = Comment()
comment.text = card_item.find_element_by_css_selector(
".kVathc .STQFb .K7oBsc div span").text
comment.hotel = nhotel
comment.website = self.website
date_comment = card_item.find_element_by_css_selector(
".iUtr1").text
date_comment = date_comment.split(" on ")[0]
try:
comment.add_date = dateparser.parse(date_comment)
cservice.add(comment)
except:
comment.add_date = datetime.datetime.now()
cservice.add(comment)
except:
print("error i don't like")
response = {
"links": links_hotel,
"names": names_hotel,
"website": self.website,
"driver": driver
}
return response
class Test():
def setUp(self):
# self.db_fd, flask.app.config['DATABASE'] = tempfile.mkstemp()
# self.app = flask.app.test_client()
# flask.init_db()
app = Flask(__name__)
env_config = DevelopmentConfig
app.config.from_object(env_config)
db = create_engine(
'postgresql://*****:*****@localhost/heartsync_data')
con = db.connect()
metadata = MetaData()
mail = Mail(app)
self.user_repo = UserRepo(con, metadata, db, app, mail)
self.user_service = UserService(self.user_repo)
self.file_repo = FileRepo(con, metadata, db, app)
self.file_service = FileService(self.file_repo)
self.scan_repo = ScanRepo(con, metadata, db, app)
self.scan_service = ScanService(self.scan_repo)
self.notes_repo = NoteRepo(con, metadata, db, app)
self.notes_service = NoteService(self.notes_repo)
self.comment_repo = CommentRepo(con, metadata, db, app)
self.comment_service = CommentService(self.comment_repo)
def tearDown(self):
os.close(self.db_fd)
os.unlink(flask.app.config['DATABASE'])
def comment_repo_test(self):
scanid = 13
username = '******'
text = 'mytest'
date = '13/05/1999'
id = self.comment_repo.insert_comment(scanid, username, text, date)
assert (self.comment_repo.get_comment_by_id(id) is not None)
assert (self.comment_repo.get_comments_by_scan(scanid) != "")
self.comment_repo.delete_comment(id[0])
def comment_service_test(self):
scanid = 13
username = '******'
text = 'mytest'
date = '13/05/1999'
comment = self.comment_service.save_comment(scanid, username, text)
assert (comment['scanid'] == 13)
assert (comment['username'] == 'dariamarc')
assert (comment['text'] == 'mytest')
assert (self.comment_service.get_comments_by_scan(scanid) != "")
self.comment_repo.delete_comment(comment['id'])
def file_repo_test(self):
file_path = "test/path"
file_type = "obj"
file_size = 13
id = self.file_repo.insert_file(file_path, file_type, file_size)
assert (self.file_repo.get_file(id[0]) is not None)
self.file_repo.delete_file(id[0])
def file_service_test(self):
file_path = "../mytest/testfile/test.txt"
file_type = "obj"
file_size = 13
id = self.file_repo.insert_file(file_path, file_type, file_size)
assert (self.file_service.get_file(id[0]) is not None)
self.file_repo.delete_file(id[0])
def note_repo_test(self):
scanid = 13
text = 'mytest'
id = self.notes_repo.insert_note(scanid, text)
assert (self.notes_repo.get_note_by_id(id[0]) is not None)
assert (self.notes_repo.get_note_by_scan(scanid) is not None)
self.notes_repo.delete_note(id[0])
def note_service_test(self):
scanid = 13
text = 'mytest'
note = self.notes_service.save_note(scanid, text)
assert (self.notes_repo.get_note_by_id(note['id']) is not None)
assert (self.notes_service.get_note_by_scan(scanid) is not None)
self.notes_repo.delete_note(note['id'])
def scan_repo_test(self):
username = '******'
fileid = 13
name = 'testscan'
id = self.scan_repo.insert_scan(username, fileid, name)
assert (self.scan_repo.get_scan(id[0]) is not None)
assert (self.scan_repo.get_user_scans(username) != "")
self.scan_repo.delete_scan(id[0])
def scan_service_test(self):
username = '******'
fileid = 13
name = 'testscan'
scan = self.scan_service.save_scan(username, fileid, name)
assert (self.scan_service.get_scan_by_id(scan['id']) is not None)
assert (self.scan_service.get_user_scans(username) != "")
self.scan_repo.delete_scan(scan['id'])
def user_repo_test(self):
username = '******'
password = '******'
firstname = 'test'
lastname = 'test'
email = 'test'
self.user_repo.signup(username, password, email, firstname, lastname)
token = self.user_repo.login(username, password)
assert (token is not None)
self.user_repo.delete_user(username)
def user_service_test(self):
username = '******'
password = '******'
firstname = 'test'
lastname = 'test'
email = 'test'
self.user_repo.signup(username, password, email, firstname, lastname)
token = self.user_service.login(username, password)
assert (token is not None)
self.user_repo.delete_user(username)