def get_standalone_info(game, name, document):
# middle left column
game_left_column = domparser.get_element(
document, 'div', class_='leftcol game_description_column')
purchase_block = domparser.get_element(game_left_column,
'div',
id='game_area_purchase')
game.store.category = get_game_category(purchase_block, document)
game.store.price = get_game_price(purchase_block, name)
game.store.os = get_game_os(game_left_column)
# top right header
glance_ctn_block = domparser.get_element(document,
'div',
class_='glance_ctn')
game.store.image = get_game_image(glance_ctn_block)
game.store.avg_review,\
game.store.cnt_review = get_game_review(glance_ctn_block, name)
game.store.release_date = get_game_release_date(glance_ctn_block, name)
game.store.tags = get_game_tags(glance_ctn_block)
if (game.store.category in [Category.Game, Category.Video]):
game.store.description = get_game_description(glance_ctn_block)
elif (game.store.category is Category.DLC):
game.store.description = get_dlc_description(document)
else:
game.store.description = ''
styledprint.print_info(
'The category {0} is not implemented yet!'.format(
game.store.category.name))
game.store.interface,\
game.store.audio,\
game.store.subtitles = get_game_languages(document)
def get_game_review(glance_ctn_block, name):
overall_block = domparser.get_element(glance_ctn_block,
'div',
class_='subtitle column all',
string='All Reviews:')
if (overall_block == None):
styledprint.print_info(
'Cannot find the review block for game: {0}'.format(name))
return '', '0'
user_reviews_block = domparser.get_parent(overall_block)
reviewCount = domparser.get_value(user_reviews_block,
'content',
'meta',
itemprop='reviewCount')
ratingValue = domparser.get_value(user_reviews_block,
'content',
'meta',
itemprop='ratingValue')
if (reviewCount):
return int(ratingValue), int(reviewCount)
return None, None
async def get_store_info(game, name, webSession):
page, description, url = await get_page(game.store.link, name, webSession)
if (not page):
game.store.description = description
return False
document, description = get_document(page, name)
if (document is None):
game.store.description = description
return False
game.store.link = re.sub(
r'(https://store.steampowered.com/[^/]*/[^/]*)/.*',
r'\1',
url,
flags=re.IGNORECASE)
if (('/sub/' in game.store.link) or ('/bundle/' in game.store.link)):
await get_collection_info(game, name, document, webSession)
elif ('/app/' in game.store.link):
get_standalone_info(game, name, document)
else:
styledprint.print_info('Unknown type of link {0}'.format(
game.store.link))
# middle right column
game.store.genres = get_game_genres(document)
game.store.details = get_game_details(document)
return True
def add_to_mapping(self, left, middle, right=None):
if (left not in self.mapping):
if (right):
self.mapping[left] = (middle.lower(), right)
else:
self.mapping[left] = (middle.lower(), )
else:
styledprint.print_info(
'Impossible to add mapping, {0} is already in the mapper'.
format(left))
async def refresh_applist(dryrun, skip, from_scratch=False, max_apps=None):
local_applist = await steam.get_applist_from_local()
if (skip):
return local_applist
async with web.Session(limit_per_host=20) as webSession:
foreign_applist = await steam.get_applist_from_server(
webSession, max_apps)
styledprint.print_info('Apps in server:', len(foreign_applist))
styledprint.print_info('Apps in cache at start:', len(local_applist))
calname = 'refresh_applist'
try:
tasks = []
for name in foreign_applist:
for app in foreign_applist[name]:
if ((not from_scratch) and (name in local_applist)
and (app in local_applist[name])):
continue
appid, typ = app
link = steam.get_store_link(appid, typ)
f = asyncio.ensure_future(
steam.get_page(link, name, webSession))
f.add_done_callback(
functools.partial(poolsubmit, calname,
get_newgame_info, name, appid, typ,
tasks))
tasks.append(f)
if (len(tasks)):
styledprint.print_info('async tasks:')
await asyncio.gather(progressbar.progress_bar(tasks))
except Exception as e:
styledprint.print_error(
'Error happened while running the async loop:', e)
styledprint.print_error(traceback.format_exc())
pass
fs = parallelism.wait_calname(calname)
for f in fs:
ext_applist = f.result()
merge_applists(local_applist, ext_applist)
styledprint.print_info(
'Apps in cache at end (duplicate names not in the count):',
len(local_applist))
if (not dryrun):
logging.debug('not dryrun so saving local_applist to disk')
await steam.save_applist_to_local(local_applist)
games = utils.DictCaseInsensitive()
for game in local_applist:
if (not game.lower().endswith('demo')):
games[game] = local_applist[game]
styledprint.print_info('Apps in cleaned cache:', len(games))
return games
def get_game_release_date(glance_ctn_block, name):
date = domparser.get_text(glance_ctn_block, 'div', class_='date')
if (date):
try:
return parser.parse(date.strip(), fuzzy_with_tokens=True)
except:
styledprint.print_info('date: {0} for game: {1} is no good'.format(
date, name))
pass
return None
def wait_calname(calname):
if (calname not in future):
return []
logging.debug('futures.wait() started at: ' + str(datetime.now().time()))
logging.debug('len(future[calname]: ' + str(len(future[calname])))
styledprint.print_info('pool tasks:')
for f in tqdm.tqdm(futures.as_completed(future[calname]),
total=len(future[calname])):
pass
logging.debug('futures.wait() done at: ' + str(datetime.now().time()))
fs = future[calname]
del future[calname]
return fs
def main():
CACHE_PATH = os.path.join('cache', 'games.p')
cache = Cache(CACHE_PATH)
cachedgames = cache.load_from_cache()
i = 0
for name in iter(cachedgames):
game = cachedgames[name]
if (not hasattr(game.store, 'interface')):
game.store.interface = list()
i = i + 1
if (not hasattr(game.store, 'audio')):
game.store.audio = list()
i = i + 1
if (not hasattr(game.store, 'subtitles')):
game.store.subtitles = list()
i = i + 1
styledprint.print_info('{0} games cleaned'.format(i))
cache.save_to_cache(cachedgames)
def get_appid_and_type_from_namematching(origname, name, games, appidstried,
matches):
while (True):
if (matches is None):
matches = get_clean_matches(name, games)
if (len(matches) == 0):
return None, None
matchedname = matches[0]
if (matchedname):
score = namematching.get_match_score(name, matchedname)
styledprint.print_info('Matched {0} with {1} at score {2}'.format(
origname, matchedname, score))
appid, typ = get_appid_and_type(matchedname, games, appidstried)
if (appid):
return appid, typ
else:
matches.pop(0)
else:
return None, None
def __init__(self, mappingfile):
if (os.path.exists(mappingfile)):
self.LINK = '<->'
self.mappingfile = mappingfile
self.mapping = utils.DictCaseInsensitive()
f = open(self.mappingfile, 'r', encoding='utf8')
for line in iter(f):
line = line.strip()
# remove the quotes needed to protect spaces
# at the end of names when steam messed up
line = line[1:-1]
if (self.LINK in line):
tup = line.split(self.LINK)
if (len(tup) <= 1):
styledprint.print_info(
'The line does not '
'contain enough members', line)
continue
key = tup[0]
value = tup[1].lower().strip('/')
if (key in self.mapping):
styledprint.print_info(
'The key {0} is already in the mapper!'.format(
key))
continue
if (len(tup) == 2):
self.mapping[key] = (value, )
elif (len(tup) == 3):
self.mapping[key] = (value, tup[2])
else:
styledprint.print_info(
'More members in the line than excepted!')
def get_game_price(purchase_block, name):
wrappers = domparser.get_elements(purchase_block,
'div',
class_='game_area_purchase_game_wrapper')
if (wrappers is not None):
names = list()
prices = list()
for wrapper in wrappers:
names.append(domparser.get_text(wrapper, 'h1')[4:].lower())
price = domparser.get_text(wrapper,
'div',
class_='game_purchase_price price')
if (not price):
price = domparser.get_text(wrapper,
'div',
class_='discount_final_price')
prices.append(price)
sortedprices = list()
if (len(names)):
matches = namematching.get_matches(name.lower(), names, len(names))
for match in matches:
index = names.index(match)
sortedprices.append(prices[index])
if (not len(sortedprices)):
sortedprices = prices
else:
sortedprices = domparser.get_texts(purchase_block,
'div',
class_='game_purchase_price price')
if (not sortedprices):
sortedprices = domparser.get_texts(purchase_block,
'div',
class_='discount_final_price')
for price in sortedprices:
if (price):
# we got the wrong div
if ('demo' in price.lower()):
continue
price = price.replace('$', '').replace('€', '').strip()
if (price.replace('.', '', 1).isdigit()):
return float(price)
elif (('free' in price.lower()) or ('play' in price.lower())):
return 0
else:
styledprint.print_info('Unexpected price: {0} for {1}'.format(
price, name))
return -1
play_game_span = domparser.get_element(purchase_block,
'span',
string='Play Game')
if (play_game_span is not None):
return 0
download_span = domparser.get_element(purchase_block,
'span',
string='Download')
if (download_span is not None):
return 0
styledprint.print_info('No price found for {0}'.format(name))
return -1
async def get_game_info(options, game, cachedgames, steamgames, winedb,
cleansteamgames, cleanwinedb, name, urlsmapping,
webSession):
try:
if ((not options.all) and (not game.hfr.is_available)):
# Ignoring not available games for now
# it may be better in the future to ignore them in output
# or allow the user to do so in the html page.
return
cleanname = None
#cache = webSession.cache
if (name in cachedgames):
# Whether the cache is ignored or not,
# if a game cached has a gift_date we keep it
if (cachedgames[name].hfr.gift_date):
game.hfr.gift_date = cachedgames[name].hfr.gift_date
if (not options.ignorecache):
game.store = cachedgames[name].store
game.wine = cachedgames[name].wine
# query the store if:
# - not cacheonly
# - not in cache
# - in cache and to be refreshed
if ((not options.cacheonly) and
((not game.store.link) or
((options.game) and (options.game.lower() in name.lower())))):
# keep the old information if there is no new one
if (game.store.link):
storeBU = copy.deepcopy(game.store)
worked = await steam.get_store_info(game, name, webSession)
if (worked):
styledprint.print_info(
'Info for game {0} was retrieved, {1}'.format(
name, str(datetime.now().time())))
else:
game.store = storeBU
else:
mapping = urlsmapping.get_mapping(name)
if (mapping == None):
appidstried = []
matches = None
while (True):
appid, typ = get_appid_and_type(
name, steamgames, appidstried)
if (appid):
styledprint.print_debug(
'The game {0} got its appid simply'.format(
name))
elif (options.fuzzymatching):
# it seems quicker to recompute it than use redis
#cleanname = await cache.function(namematching.nameclean, name)
cleanname = namematching.nameclean(name)
appid, typ = get_appid_and_type(
cleanname, cleansteamgames, appidstried)
if (appid):
styledprint.print_debug(
'The game {0} got its appid '
'by namecleaning'.format(name))
else:
appid, typ = get_appid_and_type_from_namematching(
name, cleanname, cleansteamgames,
appidstried, matches)
if ((appid in appidstried) or (not appid)):
game.store = StoreData()
game.store.description = 'The game was not found in the steam db'
styledprint.print_error('{0}: {1}'.format(
game.store.description, name))
cleanname = None
return
else:
appidstried.append(appid)
if (await steam.get_store_info_from_appid(
game, name, appid, typ, webSession)):
break
else:
url = mapping[0]
if (url == 'ignore'):
styledprint.print_debug(
'{0} cannot be found and is to be ignored'.format(
name))
return
styledprint.print_debug(
'URL mapping found for game {0}'.format(name))
await steam.get_store_info_from_url(
game, name, url, webSession)
# overwriting the steam provided category
if (len(mapping) == 2):
game.store.category = Category[mapping[1].upper()]
game.store.override = True
styledprint.print_info(
'Info for game {0} was retrieved, {1}'.format(
name, str(datetime.now().time())))
# TODO
# compute cleanname once
# cache heavy stuff
if (name in winedb):
game.wine = winedb[name]
elif (options.fuzzymatching):
if (cleanname is None):
# it seems quicker to recompute it than use redis
#cleanname = await cache.function(namematching.nameclean, name)
cleanname = namematching.nameclean(name)
if (cleanname in cleanwinedb):
game.wine = cleanwinedb[cleanname]
else:
cleanmatches = get_clean_matches(cleanname, cleanwinedb)
if (len(cleanmatches)):
game.wine = cleanwinedb[cleanmatches[0]]
except Exception as e:
styledprint.print_error('An exception was raised for', name)
raise e
def get_rows(games):
reviewMapping = dict()
reviewMapping[10] = 'Overwhelmingly Positive'
reviewMapping[9] = 'Very Positive'
reviewMapping[8] = 'Positive'
reviewMapping[7] = 'Mostly Positive'
reviewMapping[6] = 'Mixed'
reviewMapping[5] = 'Mostly Negative'
reviewMapping[4] = 'Negative'
reviewMapping[3] = 'Very Negative'
reviewMapping[2] = 'Overwhelmingly Negative'
justifyFormat = '<div class=\\"text\\">{0}</div>'
data = writeline('function getRows(){')
increase_indent_count()
data += writeline('rows=[];', True)
for gameName in sorted(games, key=str.lower):
game = games[gameName]
if (not game.hfr.is_available):
continue
data += writeline('var row = {')
increase_indent_count()
data += writeline('name: "{0}",'.format(gameName.replace('"', '\\"')))
#TODO make smallImage an input parameter?
smallImage = True
if (game.store.link):
if(game.store.image):
if (smallImage):
imageLink = re.sub(r'header\.jpg.*', 'capsule_sm_120.jpg', game.store.image)
else:
imageLink = game.store.image
data += writeline('nameFormat: "<a href=\\"{0}\\"><b>{1}</b><img src=\\"{2}\\" width=\\"100%\\"/></a>",'
.format(game.store.link, justifyFormat, imageLink))
else:
data += writeline('nameFormat: "<a href=\\"{0}\\"><b>{1}</b></a>",'
.format(game.store.link, justifyFormat))
else:
data += writeline('nameFormat: "<b>{0}</b>",'.format(justifyFormat))
if(game.store.description):
data += writeline('description: "{0}",'.format(
game.store.description.replace(os.linesep, '<br/>').replace('"', '\\"')))
if (game.store.category):
data += writeline('category: "{0}",'.format(game.store.category.name))
if (len(game.store.os) > 0):
data += writeline('os: "{0}",'.format(', '.join(game.store.os).replace('OS X', 'OS')))
if (game.store.price != None):
data += writeline('price: {0},'.format(str(game.store.price)))
if (game.store.price == 0):
data += writeline('priceFormat: "{0}",'
.format(justifyFormat.replace('{0}', 'Free')))
elif (game.store.price < 0):
data += writeline('priceFormat: "{0}",'
.format(justifyFormat.replace('{0}', 'N/A')))
else:
data += writeline('priceFormat: "{0}",'.
format(justifyFormat.replace('{0}', '${0}')))
if (len(game.store.genres) > 0):
data += writeline('genres: "{0}",'.format(', '.join(game.store.genres)))
if (game.store.release_date):
date, errors = game.store.release_date
# if errors has more than whitespace or empty strings
errors = tuple(filter(lambda x: x.strip(), errors))
if (len(errors)):
fmt = '%Y'
else:
fmt = '%Y-%m-%d'
data += writeline('date: "{0}",'.format(date.strftime(fmt).replace('-', '‑')))
if (game.store.avg_review) and (game.store.avg_review in reviewMapping):
avg_review_text = reviewMapping[game.store.avg_review]
MIN_POW = 6
avg_review_p = math.pow(10, MIN_POW) * game.store.avg_review
# if we have average negative reviews, we reduce the average review's power per review
if (game.store.avg_review < 6):
avg_review = str(avg_review_p - game.store.cnt_review)
# if we have average positive reviews, we increment the average review's power per review
else:
avg_review = str(avg_review_p + game.store.cnt_review)
if (game.store.avg_review < 10):
avg_review = '0{0}'.format(avg_review)
data += writeline('review: "{0} {1}",'.
format(avg_review, avg_review_text))
data += writeline('reviewFormat: "{0}",'.
format(justifyFormat.replace('{0}',
'{0} ({1})'.
format(avg_review_text,
game.store.cnt_review))))
else:
if (game.store.avg_review):
styledprint.print_info('The average review {0} for game {1} is not in the mapping!'.
format(game.store.avg_review, gameName))
if (game.hfr.requirements):
if (game.hfr.requirements.lower() == 'standard'):
stars = '*'
else:
stars = '**'
if (game.hfr.is_new):
requirements = '{0}: Nouveauté'.format(game.hfr.requirements)
else:
requirements = game.hfr.requirements
data += writeline('requirements: "{0}<sup><b>{1}</b></sup>",'.format(requirements, stars))
if (len(game.store.tags) > 0):
data += writeline('tags: "{0}",'.format(', '.join(game.store.tags)))
if (len(game.store.details) > 0):
data += writeline('details: "{0}",'.format(', '.join(game.store.details)))
if (game.hfr.gift_date):
data += writeline('giftdate: "{0}",'.format(game.hfr.gift_date.strftime("%Y-%m-%d")))
if (len(game.store.interface) > 0):
data += writeline('interface: "{0}",'.format(', '.join(game.store.interface)))
if (len(game.store.audio) > 0):
data += writeline('audio: "{0}",'.format(', '.join(game.store.audio)))
if (len(game.store.subtitles) > 0):
data += writeline('subtitles: "{0}",'.format(', '.join(game.store.subtitles)))
if (game.wine):
data += writeline('wine: "{0}",'.format(', '
.join([app.rating for app in game.wine])))
data += writeline('wineFormat: "{0}",'
.format(justifyFormat
.format(', '.join(
['<a href=\\"{0}\\"><b>{1}</b></a>'
.format(app.link, app.rating) for app in game.wine]))))
else:
data += writeline('wineFormat: "{0}",'
.format(justifyFormat
.format('<a href=\\"https://appdb.winehq.org/objectManager.php?sClass=application&sTitle=Browse+Applications&iappFamily-appNameOp=2&sappFamily-appNameData={0}\\"><i>?</i></a>'
.format(gameName.replace('"', '\\"')))))
decrease_indent_count()
data += writeline('};')
try:
if (game.store.override):
data += writeline('row[\'row-cls\'] = \'override\'')
except:
pass
data += writeline('rows.push(row);', True)
data += writeline('return rows;')
decrease_indent_count()
data += writeline('};')
return data