def get_games_info(options, games, steamgames, winedb):
styledprint.print_info_begin('Pulling Games Information')
CACHE_PATH = os.path.join('cache', 'games.p')
cache = Cache(CACHE_PATH)
cachedgames = cache.load_from_cache()
cleansteamgames = utils.DictCaseInsensitive()
cleanwinedb = utils.DictCaseInsensitive()
if (options.fuzzymatching):
parallelism.split_submit_job(steamgames, cleansteamgames, clean_names)
parallelism.split_submit_job(winedb, cleanwinedb, clean_names)
URLS_MAPPING = os.path.join('mappings', 'urlsmapping.txt')
urlsmapping = Mapper(URLS_MAPPING)
parallelism.split_submit_job(games, games, start_loop, options,
cachedgames, steamgames, winedb,
cleansteamgames, cleanwinedb, urlsmapping,
parallelism.get_number_of_cores())
if (not options.dryrun):
newcachedgames = cache.merge_old_new_cache(cachedgames, games)
cache.save_to_cache(newcachedgames)
urlsmapping.save_mapping()
styledprint.print_info_end('Pulling Games Information Done')
def __init__(self, mappingfile):
if (os.path.exists(mappingfile)):
self.LINK = '<->'
self.mappingfile = mappingfile
self.mapping = utils.DictCaseInsensitive()
f = open(self.mappingfile, 'r', encoding='utf8')
for line in iter(f):
line = line.strip()
# remove the quotes needed to protect spaces
# at the end of names when steam messed up
line = line[1:-1]
if (self.LINK in line):
tup = line.split(self.LINK)
if (len(tup) <= 1):
styledprint.print_info(
'The line does not '
'contain enough members', line)
continue
key = tup[0]
value = tup[1].lower().strip('/')
if (key in self.mapping):
styledprint.print_info(
'The key {0} is already in the mapper!'.format(
key))
continue
if (len(tup) == 2):
self.mapping[key] = (value, )
elif (len(tup) == 3):
self.mapping[key] = (value, tup[2])
else:
styledprint.print_info(
'More members in the line than excepted!')
async def parse_hfr():
URL = 'https://forum.hardware.fr/hfr/JeuxVideo/Achat-Ventes/gratuit-origin-download-sujet_171605_1.htm'
document = await get_document(URL)
games = utils.DictCaseInsensitive()
parse_hfr_std(games, document)
parse_hfr_donateur_and_premium(games, document)
return games
def load_from_cache(self):
db = utils.DictCaseInsensitive()
try:
if(os.path.exists(self.path)):
db = pickle.load(open(self.path, 'rb'))
except:
traceback.print_exc()
return db
async def refresh_applist(dryrun, skip, from_scratch=False, max_apps=None):
local_applist = await steam.get_applist_from_local()
if (skip):
return local_applist
async with web.Session(limit_per_host=20) as webSession:
foreign_applist = await steam.get_applist_from_server(
webSession, max_apps)
styledprint.print_info('Apps in server:', len(foreign_applist))
styledprint.print_info('Apps in cache at start:', len(local_applist))
calname = 'refresh_applist'
try:
tasks = []
for name in foreign_applist:
for app in foreign_applist[name]:
if ((not from_scratch) and (name in local_applist)
and (app in local_applist[name])):
continue
appid, typ = app
link = steam.get_store_link(appid, typ)
f = asyncio.ensure_future(
steam.get_page(link, name, webSession))
f.add_done_callback(
functools.partial(poolsubmit, calname,
get_newgame_info, name, appid, typ,
tasks))
tasks.append(f)
if (len(tasks)):
styledprint.print_info('async tasks:')
await asyncio.gather(progressbar.progress_bar(tasks))
except Exception as e:
styledprint.print_error(
'Error happened while running the async loop:', e)
styledprint.print_error(traceback.format_exc())
pass
fs = parallelism.wait_calname(calname)
for f in fs:
ext_applist = f.result()
merge_applists(local_applist, ext_applist)
styledprint.print_info(
'Apps in cache at end (duplicate names not in the count):',
len(local_applist))
if (not dryrun):
logging.debug('not dryrun so saving local_applist to disk')
await steam.save_applist_to_local(local_applist)
games = utils.DictCaseInsensitive()
for game in local_applist:
if (not game.lower().endswith('demo')):
games[game] = local_applist[game]
styledprint.print_info('Apps in cleaned cache:', len(games))
return games
def clean_names(names):
cleannames = utils.DictCaseInsensitive()
for name in names:
cleanname = namematching.nameclean(name)
if (cleanname in cleannames):
for t in names[name]:
if (t not in cleannames[cleanname]):
cleannames[cleanname].append(t)
else:
cleannames[cleanname] = names[name]
return cleannames
def split_submit_job(dict1, dict2, func, *args):
calname = utils.get_caller_name()
curCPU = 0
subDict = [utils.DictCaseInsensitive() for x in range(pool.ncpus)]
for key in dict1:
subDict[curCPU][key] = dict1[key]
curCPU += 1
if (curCPU == pool.ncpus):
curCPU = 0
future[calname] = []
for i in range(pool.ncpus):
submit_job_calname(calname, func, subDict[i], *args)
fs = wait_calname(calname)
for f in fs:
subDict = f.result()
for key in subDict:
dict2[key] = subDict[key]
async def get_ratings():
URL = 'https://appdb.winehq.org/objectManager.php?sClass=application&sTitle=Browse+Applications&iappVersion-ratingOp0=5&sOrderBy=appName&bAscending=true&iItemsPerPage=200&sappVersion-ratingData0='
ratings = utils.DictCaseInsensitive()
async with web.Session(limit_per_host=200) as webSession:
tasks = []
for e in Rating:
tasks.append(
asyncio.ensure_future(get_forOneRating(URL, e.name,
webSession)))
await asyncio.gather(progressbar.progress_bar(tasks))
for task in tasks:
apps = task.result()[0]
rating = task.result()[1]
for app in apps:
if (app[0] in ratings):
ratings[app[0]].append(WineApp(app[1], rating))
else:
ratings[app[0]] = [WineApp(app[1], rating)]
return ratings
async def parse_list(liste):
games = utils.DictCaseInsensitive()
async with aiofiles.open(liste, 'r') as f:
content = await f.read()
get_games(games, content.splitlines(), '')
return games
if __name__ == '__main__':
logging.basicConfig(
filename='mylog.log',
filemode='w',
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
#logging.getLogger('').addHandler(console)
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
#loop.set_debug(True)
options = get_parser().parse_args()
steamgames = utils.DictCaseInsensitive()
styledprint.set_verbosity(options.verbosity)
parallelism.create_pool(8, loop)
try:
tasks = [
asyncio.ensure_future(get_applist(options)),
asyncio.ensure_future(get_wine_ratings()),
asyncio.ensure_future(parse_list(options))
]
loop.run_until_complete(asyncio.gather(*tasks))
steamgames = tasks[0].result()
winedb = tasks[1].result()
games = tasks[2].result()