def __get_data(self, ids_ready, reviews):
review_elements = self.driver.find_elements_by_class_name('reviewSelector')
for e in review_elements:
try:
_id = e.get_attribute('id')
if review_already_exists(self.db, _id):
break
if _id not in ids_ready:
set_locale(self.driver.current_url)
restaurant_id = get_id_by_url(self.driver.current_url)
date = e.find_element_by_class_name('ratingDate').get_attribute('title')
date = datetime.datetime.strptime(date,
I18N[get_language_by_url(self.driver.current_url)]['date_format'])
title = e.find_element_by_class_name('quote').find_element_by_tag_name(
'a').find_element_by_class_name(
'noQuotes').text
rating = RATING[
e.find_element_by_xpath(
'//*[@id="' + _id + '"]/div/div[2]/div/div[1]/div[1]/span[1]').get_attribute(
"class").split(' ')[1]]
try:
e.find_element_by_class_name('viaMobile')
via_mobile = True
except:
via_mobile = False
username = e.find_element_by_class_name('scrname').text
try:
more_button = e.find_element_by_class_name('taLnk')
if 'Expand' in more_button.get_attribute('onclick'):
more_button.click()
time.sleep(SECONDS_BETWEEN_REQUEST)
except:
pass
text = e.find_element_by_class_name('partial_entry').text.replace('\n', '')
language = detect(text)
ids_ready.append(_id)
reviews.append(
Review(_id, restaurant_id, date, title, username, text, rating, via_mobile, language))
self.__get_data(ids_ready, reviews)
break
except:
pass
return reviews
def fetch_reviews(self):
print('Fetching reviews...', flush=True)
self.lookup = {}
reviews = []
has_next = True
set_locale(self.url)
if not is_valid_url(self.url):
print('[ERROR] URL is not valid: ' + self.url, flush=True)
return None
self.driver.get(self.url)
try:
self.driver.find_element_by_id('taplc_location_review_filter_controls_0_filterLang_ALL').click()
except:
pass
while has_next:
time.sleep(SECONDS_BETWEEN_REQUEST + 0.5)
reviews_parsed = self.__parse_page()
if len(reviews_parsed) == 0:
break
reviews += reviews_parsed
print('Fetched reviews: ' + str(len(reviews)), flush=True)
try:
has_next = self.driver.execute_script(
'return !document.querySelector(".ui_pagination>.next").classList.contains("disabled")')
except:
has_next = False
if has_next:
self.driver.execute_script(
'document.querySelector(".ui_pagination>.next").click()')
return [r.__dict__ for r in reviews]
print "This script uses 'ElementTree' but it's not installed"
print "Look for 'python-elementtree' in your GNU/Linux distro and install it"
print "Windows users should loo at http://effbot.org/downloads/#elementtree"
sys.exit(1)
print __doc__
import sys, os, time
import shutil
# needed for certain SP modules we import
import SPLogging
SPLogging.set_level('error')
SPLogging.start()
import utils
utils.set_locale()
#---------------- Some functions used internally
def error(text):
print "=============== ERROR =========================================="
print " %s" % text
print " Quitting...\n"
print "================================================================"
sys.exit(1)
def check_for_q(ans):
if ans.lower() == 'q':
print "\nUser has hit 'q'"
print "Quitting...\n"
# This stuff must be called before importing worldMap,world and guiWorld
def SetLocale(lang=None):
global LocaleLang,LocaleMesg
# This also sets up the global rc dict in utils.py
rc_d = utils.parseRcFile()
try:
LocaleLang = rc_d['default']['lang']
except Exception,info:
module_logger.exception("Error reading rc dict, setting language to system")
module_logger.error("contents of rc_d: %s" % rc_d)
LocaleLang = 'system'
#This will set the locale used by gvr and returns the tuple (txt,loc)
# txt is a error message, if any else it will be an empty string.
# loc is a string with the locale country code set for gvr, this can be
# different from the systems locale.
LocaleMesg = utils.set_locale(LocaleLang)
try:
SetLocale()
except Exception,info:
module_logger.exception("Problems setting the locale.\n switching to English\nPlease inform the GvR developers about this.")
__builtin__.__dict__['_'] = lambda x:x
import Text
Text.set_summary(LocaleMesg[1])# needed to set summary file used to the current locale
Text.set_WBsummary(LocaleMesg[1])# idem for the worldbuilder
Text.set_Intro(LocaleMesg[1])# and the intro text
# when the frontend is not in sitepackages, as is the case for the org install
sys.path.append('gui-gtk')
def SetLocale(lang=None):
global LocaleLang, LocaleMesg
# This also sets up the global rc dict in utils.py
rc_d = utils.parseRcFile()
try:
LocaleLang = rc_d['default']['lang']
except Exception, info:
module_logger.exception(
"Error reading rc dict, setting language to system")
module_logger.error("contents of rc_d: %s" % rc_d)
LocaleLang = 'system'
#This will set the locale used by gvr and returns the tuple (txt,loc)
# txt is a error message, if any else it will be an empty string.
# loc is a string with the locale country code set for gvr, this can be
# different from the systems locale.
LocaleMesg = utils.set_locale(LocaleLang)
try:
SetLocale()
except Exception, info:
module_logger.exception(
"Problems setting the locale.\n switching to English\nPlease inform the GvR developers about this."
)
__builtin__.__dict__['_'] = lambda x: x
import Text
Text.set_summary(
LocaleMesg[1]) # needed to set summary file used to the current locale
Text.set_WBsummary(LocaleMesg[1]) # idem for the worldbuilder
Text.set_Intro(LocaleMesg[1]) # and the intro text
print "This script uses 'ElementTree' but it's not installed"
print "Look for 'python-elementtree' in your GNU/Linux distro and install it"
print "Windows users should loo at http://effbot.org/downloads/#elementtree"
sys.exit(1)
print __doc__
import sys,os,time
import shutil
# needed for certain SP modules we import
import SPLogging
SPLogging.set_level('error')
SPLogging.start()
import utils
utils.set_locale()
#---------------- Some functions used internally
def error(text):
print "=============== ERROR =========================================="
print " %s" % text
print " Quitting...\n"
print "================================================================"
sys.exit(1)
def check_for_q(ans):
if ans.lower() == 'q':
print "\nUser has hit 'q'"
print "Quitting...\n"
sys.exit(0)
if not os.path.exists(p):
if os.path.exists(CONTENTDBASE):
shutil.copy(CONTENTDBASE, p)
else:
from SPBasePaths import SHARELIBDATADIR
shutil.copy(os.path.join(SHARELIBDATADIR, CONTENTDBASE), p)
import pygame
## set a bigger buffer, seems that on win XP in conjuction with certain hardware
## the playback of sound is scrambled with the "normal" 1024 buffer.
### XXXX this still sucks, signed or unsigned that's the question :-(
pygame.mixer.pre_init(22050, -16, 2, 2048)
pygame.init()
# this will return the tuple (lang,rtl=bool)
LANG = utils.set_locale(lang=CMD_Options.lang)
if CMD_Options.checklog:
try:
import SPlogCheck
except (ImportError, utils.SPError):
sys.exit(1)
except utils.MyError:
sys.exit(1)
sys.exit(0)
if not utils._set_lock():
sys.exit(1)
import SPMainCore
def __init__(self, spgoodies, dbm):
self.logger = logging.getLogger("childsplay.SPDataManager.DataManager")
self.logger.debug("Starting")
self.SPG = spgoodies
self.cmd_options = self.SPG._cmd_options
self.current_user = self.cmd_options.user
self.current_user_id = None
self.COPxml = None# controlpanel stuff
atexit.register(self._cleanup)
self.content_engine, self.user_engine = dbm.get_engines()
self.metadata_contentdb, self.metadata_usersdb = dbm.get_metadatas()
self.all_orms = dbm.get_all_orms()
self.orms_content_db, self.orms_userdb = dbm.get_orms()
self.UserSession = sqlorm.sessionmaker(bind=self.user_engine)
self.ContentSession = sqlorm.sessionmaker(bind=self.content_engine)
# query which language we should use.
orm, session = self.get_orm('spconf', 'user')
row = session.query(orm).filter_by(activity_name = 'language_select')\
.filter_by(key = 'locale').first()
if not row:
language = self.cmd_options.lang
if not language:
language = self.cmd_options.default_language
row = orm(activity_name='language_select', key='locale', value=language, comment='locale used by the core')
session.add(row)
row = orm(activity_name='language_select', key='lang', value=language[:2], comment='language code used by the core')
session.add(row)
session.commit()
session.close()
language = set_locale(language)
elif not self.cmd_options.lang:
language = set_locale(row.value)
else:
language = self.cmd_options.lang
if not language:
language = self.cmd_options.default_language
language = set_locale(language)
self.language = language
self.SPG.localesetting = language
self._check_tables_uptodate()
# query to get all availabe cids, used to check served_content
orm, session = self.get_orm('game_available_content', 'content')
query = session.query(orm)
self.all_ids = [result.CID for result in query.all()]
session.close()
if self.cmd_options.no_login:
self.current_user = '******'
self._start_gdm_greeter()
elif self.cmd_options.user:
self.current_user = self.cmd_options.user
self._start_gdm_greeter()
elif self.SPG.get_theme() == 'braintrainer':
self.WeAreBTP = True
self._start_btp_screen()
else:
self.WeAreBTP = False
# we don't have a working login screen yet
self.current_user='******'
self._start_gdm_greeter()
# for childsplay we must make sure we have a content dbase in sqlite3 format present.
# if we want to use an sqlite content dbase we must make sure the default one is in the proper location.
p = os.path.join(HOMEDIR, CMD_Options.theme, CONTENTDBASE)
if not os.path.exists(p):
shutil.copy(CONTENTDBASE, p)
import pygame
## set a bigger buffer, seems that on win XP in conjuction with certain hardware
## the playback of sound is scrambled with the "normal" 1024 buffer.
### XXXX this still sucks, signed or unsigned that's the question :-(
pygame.mixer.pre_init(22050, -16, 2, 2048)
pygame.init()
# this will return the tuple (lang,rtl=bool)
LANG = utils.set_locale(lang=CMD_Options.lang)
if CMD_Options.checklog:
try:
import SPlogCheck
except (ImportError, utils.SPError):
sys.exit(1)
except utils.MyError:
sys.exit(1)
sys.exit(0)
if not utils._set_lock():
sys.exit(1)
import SPMainCore
def __init__(self, spgoodies, dbm):
self.logger = logging.getLogger("childsplay.SPDataManager.DataManager")
self.logger.debug("Starting")
self.SPG = spgoodies
self.cmd_options = self.SPG._cmd_options
self.current_user = self.cmd_options.user
self.current_user_id = None
self.COPxml = None # controlpanel stuff
atexit.register(self._cleanup)
self.content_engine, self.user_engine = dbm.get_engines()
self.metadata_contentdb, self.metadata_usersdb = dbm.get_metadatas()
self.all_orms = dbm.get_all_orms()
self.orms_content_db, self.orms_userdb = dbm.get_orms()
self.UserSession = sqlorm.sessionmaker(bind=self.user_engine)
self.ContentSession = sqlorm.sessionmaker(bind=self.content_engine)
# query which language we should use.
orm, session = self.get_orm('spconf', 'user')
row = session.query(orm).filter_by(activity_name = 'language_select')\
.filter_by(key = 'locale').first()
if not row:
language = self.cmd_options.lang
if not language:
language = self.cmd_options.default_language
row = orm(activity_name='language_select',
key='locale',
value=language,
comment='locale used by the core')
session.add(row)
row = orm(activity_name='language_select',
key='lang',
value=language[:2],
comment='language code used by the core')
session.add(row)
session.commit()
session.close()
language = set_locale(language)
elif not self.cmd_options.lang:
language = set_locale(row.value)
else:
language = self.cmd_options.lang
if not language:
language = self.cmd_options.default_language
language = set_locale(language)
self.language = language
self.SPG.localesetting = language
self._check_tables_uptodate()
# query to get all availabe cids, used to check served_content
orm, session = self.get_orm('game_available_content', 'content')
query = session.query(orm)
self.all_ids = [result.CID for result in query.all()]
session.close()
if self.cmd_options.no_login:
self.current_user = '******'
self._start_gdm_greeter()
elif self.cmd_options.user:
self.current_user = self.cmd_options.user
self._start_gdm_greeter()
elif self.SPG.get_theme() == 'braintrainer':
self.WeAreBTP = True
self._start_btp_screen()
else:
self.WeAreBTP = False
# we don't have a working login screen yet
self.current_user = '******'
self._start_gdm_greeter()