def get_games():
"""Actually retrieves the games and adds them to the datastore"""
stats = utils.retrieve_stats()
total_games = 0
games = Games()
all_games = games.get_all('us')
current_stack = []
for game in all_games:
if game.type == 'game':
name = game.name.encode('ascii', 'ignore')
store_url = create_store_url(game.appid)
search_name = utils.remove_specials(name.lower().replace(
'amp;', ''))
new_game = Games_DB(id=game.appid,
appid=int(game.appid),
game_name=name,
image_url=game.header_image,
store_url=store_url,
price=game.price,
search_name=search_name)
if int(game.appid) not in mp_games:
new_game.multiplayer_only = False
else:
new_game.multiplayer_only = True
current_stack.append(new_game)
if len(current_stack) > 250:
ndb.put_multi(current_stack)
current_stack = []
total_games += 1
stats.total_steam = total_games
stats.games_last_updated = datetime.now()
stats.put()
def get_games():
"""Actually retrieves the games and adds them to the datastore"""
stats = utils.retrieve_stats()
total_games = 0
games = Games()
all_games = games.get_all('us')
current_stack = []
for game in all_games:
if game.type == 'game':
name = game.name.encode('ascii', 'ignore')
store_url = create_store_url(game.appid)
search_name = utils.remove_specials(name.lower().replace('amp;', ''))
new_game = Games_DB(id=game.appid,appid=int(game.appid),game_name=name,image_url=game.header_image,
store_url=store_url,price=game.price,search_name=search_name)
if int(game.appid) not in mp_games:
new_game.multiplayer_only = False
else:
new_game.multiplayer_only = True
current_stack.append(new_game)
if len(current_stack) > 250:
ndb.put_multi(current_stack)
current_stack = []
total_games += 1
stats.total_steam = total_games
stats.games_last_updated = datetime.now()
stats.put()
def update_user(steam_id):
"""
Update users return codes:
2 - Full, sucessful update
5 - Private profile, user removed
6 - Update failed. Too soon since last update.
8 - Huh? That user doesn't exist.
"""
stats = utils.retrieve_stats()
user = SteamIds.get_by_id(steam_id)
if user:
if user.last_updated > datetime.now() - timedelta(minutes=1):
return user, 6
else:
info_to_update = SteamUsers(steam_id, api_key)
user, rc = _update_user(user, info_to_update, stats)
return user, rc
else:
return None, 8
def update_user(steam_id):
"""
Update users return codes:
2 - Full, sucessful update
5 - Private profile, user removed
6 - Update failed. Too soon since last update.
8 - Huh? That user doesn't exist.
"""
stats = utils.retrieve_stats()
user = SteamIds.get_by_id(steam_id)
if user:
if user.last_updated > datetime.now() - timedelta(minutes=1):
return user, 6
else:
info_to_update = SteamUsers(steam_id, api_key)
user, rc = _update_user(user, info_to_update, stats)
return user, rc
else:
return None, 8
def get_hltb():
stats = utils.retrieve_stats()
total_main_with_hours = 0
total_main = 0.0
total_completion_with_hours = 0
total_with_hours = 0
total_completion = 0.0
num_url = urllib2.urlopen("http://74.63.212.37/hltb/num.html").read()
file_range = range(1,int(num_url))
for i in file_range:
#for i in range(1,2): # Use for testing
curr_games = []
soup = BeautifulSoup(open_morpheus(i))
search_results = soup.find("div", {"class": "search_results"})
games = search_results.findAll("li", {"class": "backwhite radius shadow_box"})
for game in games:
title = game.find("a", {"class": "textwhite"})
try:
url = title['href']
except KeyError:
url = None
title = title.text
main = None
completion = None
combined = None
tidbits = game.findAll("div", {'class': tidbit_re})
if len(tidbits) > 1:
total_with_hours += 1
main_recorded = False
for i in range(len(tidbits)):
if tidbits[i].text == "Main Story":
main_recorded = True
main = tidbits[i+1].text
main = validate_hours(main)
if main is not None:
total_main += main
total_main_with_hours += 1
elif tidbits[i].text == "Completionist":
completion = tidbits[i+1].text
completion = validate_hours(completion)
if completion is not None:
total_completion += completion
total_completion_with_hours += 1
elif tidbits[i].text == "Combined":
combined = tidbits[i+1].text
combined = validate_hours(combined)
if main_recorded is False:
if combined is not None:
main = combined
this_game = {'title': title, 'url': url, 'main': main, 'completion': completion}
curr_games.append(this_game)
update_hltb(curr_games)
average_main = total_main / total_main_with_hours
average_completion = total_completion / total_completion_with_hours
stats.total_with_hours = total_with_hours
stats.average_main = average_main - 2
stats.average_completion = average_completion - 2
stats.hltb_last_updated = datetime.now()
stats.put()
return None, None
def get_user(steam_id, stats=None):
"""
Return codes for get_user():
1 - New user succesfully added
2 - User update succeeded
3 - New user was private, not added
4 - Current user, no need for update, sucesfully returned
5 - Update succeeded, but private profile
6 - Update failed - too soon since last update
7 - Bad Steam ID
"""
# If the user inputs a URL that doesn't have the 64 bit id, then we have to retrieve that
# with a call to Steam to see if it's valid. Since we don't always want to do that,
# we store the "orig_id" (the full url) in memcache, mapped to the actual id if needed.
# Cuts down on the amount of Steam API calls needed per user lookup.
if stats is None:
stats = utils.retrieve_stats()
orig_id = steam_id
id_retrieved = False
cached_id = memcache.get(orig_id)
if cached_id is not None:
id_retrieved = True
steam_id = cached_id
if id_retrieved is False:
steam_match = re.match(steam_re, steam_id)
if steam_match:
steam_id = steam_match.string
else:
if re.match(r'https?://steamcommunity.com/.*', steam_id):
try:
profile = urllib2.urlopen(steam_id)
except ValueError:
return None, 7
soup = BeautifulSoup(profile)
scripts = soup.findAll('script')
found = False
for script in scripts:
text = script.text.strip()
if text[:15] == 'g_rgProfileData':
json_info = json.loads(text[18:-1])
steam_id = json_info['steamid']
found = True
if found is False:
return None, 7
else:
try:
profile = urllib2.urlopen(
"http://steamcommunity.com/id/%s" % steam_id)
except ValueError:
return None, 7
soup = BeautifulSoup(profile)
scripts = soup.findAll('script')
found = False
for script in scripts:
text = script.text.strip()
if text[:15] == 'g_rgProfileData':
json_info = json.loads(text[18:-1])
steam_id = json_info['steamid']
found = True
if found is False:
return None, 7
memcache.add(orig_id, steam_id)
user = SteamIds.get_by_id(steam_id)
counters.pingpong_incr(queries_counter)
# User already exists, decide what to do
if user:
# If this is true, there have been updates to the db. Update the user, if possible.
if stats.games_last_updated > user.last_updated or stats.hltb_last_updated > user.last_updated:
info_to_update = SteamUsers(steam_id, api_key)
# User profile is invisible. Still update what we have on record, but warn the user
# to update w/public profile.
if info_to_update.visibility is False:
user, rc = _update_user(user, info_to_update, stats)
return user, rc
#User's profile was visible, fully sucessful update.
else:
user, rc = _update_user(user, info_to_update, stats)
return user, rc
# Current user, no need for update, just return for display.
else:
return user, 4
else:
user_info = SteamUsers(steam_id, api_key)
# This is not a Steam ID
if user_info.good_id is False:
return None, 7
# This is not a public profile. Can't view.
elif user_info.visibility is False:
return None, 3
# New user was succesfully added. FTW!
else:
user = add_user_to_ndb(user_info, stats)
#increment_steamids()
counters.pingpong_incr(steamids_counter)
return user, 1
def get_user(steam_id, stats=None):
"""
Return codes for get_user():
1 - New user succesfully added
2 - User update succeeded
3 - New user was private, not added
4 - Current user, no need for update, sucesfully returned
5 - Update succeeded, but private profile
6 - Update failed - too soon since last update
7 - Bad Steam ID
"""
# If the user inputs a URL that doesn't have the 64 bit id, then we have to retrieve that
# with a call to Steam to see if it's valid. Since we don't always want to do that,
# we store the "orig_id" (the full url) in memcache, mapped to the actual id if needed.
# Cuts down on the amount of Steam API calls needed per user lookup.
if stats is None:
stats = utils.retrieve_stats()
orig_id = steam_id
id_retrieved = False
cached_id = memcache.get(orig_id)
if cached_id is not None:
id_retrieved = True
steam_id = cached_id
if id_retrieved is False:
steam_match = re.match(steam_re, steam_id)
if steam_match:
steam_id = steam_match.string
else:
if re.match(r'https?://steamcommunity.com/.*', steam_id):
try:
profile = urllib2.urlopen(steam_id)
except ValueError:
return None, 7
soup = BeautifulSoup(profile)
scripts = soup.findAll('script')
found = False
for script in scripts:
text = script.text.strip()
if text[:15] == 'g_rgProfileData':
json_info = json.loads(text[18:-1])
steam_id = json_info['steamid']
found = True
if found is False:
return None, 7
else:
try:
profile = urllib2.urlopen("http://steamcommunity.com/id/%s" % steam_id)
except ValueError:
return None, 7
soup = BeautifulSoup(profile)
scripts = soup.findAll('script')
found = False
for script in scripts:
text = script.text.strip()
if text[:15] == 'g_rgProfileData':
json_info = json.loads(text[18:-1])
steam_id = json_info['steamid']
found = True
if found is False:
return None, 7
memcache.add(orig_id, steam_id)
user = SteamIds.get_by_id(steam_id)
counters.pingpong_incr(queries_counter)
# User already exists, decide what to do
if user:
# If this is true, there have been updates to the db. Update the user, if possible.
if stats.games_last_updated > user.last_updated or stats.hltb_last_updated > user.last_updated:
info_to_update = SteamUsers(steam_id, api_key)
# User profile is invisible. Still update what we have on record, but warn the user
# to update w/public profile.
if info_to_update.visibility is False:
user, rc = _update_user(user, info_to_update, stats)
return user, rc
#User's profile was visible, fully sucessful update.
else:
user, rc = _update_user(user, info_to_update, stats)
return user, rc
# Current user, no need for update, just return for display.
else:
return user, 4
else:
user_info = SteamUsers(steam_id, api_key)
# This is not a Steam ID
if user_info.good_id is False:
return None, 7
# This is not a public profile. Can't view.
elif user_info.visibility is False:
return None, 3
# New user was succesfully added. FTW!
else:
user = add_user_to_ndb(user_info, stats)
#increment_steamids()
counters.pingpong_incr(steamids_counter)
return user, 1
