def fix_incorrect_player_counts():
""" Finds and attempts to fix all replays where `Replay.human_players` does not match the quantity of `ReplayPlayer`
objects we have in the database.
Attempts to fix by deleting all ReplayPlayer objects and re-adding the replay to the job queue.
"""
_error = "PLAYER_COUNT_MISMATCH"
human_players_discrepancy = db.engine.execute(
text("""
SELECT
r.id,
r.human_players,
(
SELECT count(*) FROM {player_table} rp
WHERE rp.replay_id=r.id
and (rp.id is NULL or rp.account_id is not NULL) # Exclude bots from count (though there's the chance we have duplicate entries for bots? fack)
) as player_count,
(SELECT count(*) FROM {auto_fix_table} raf WHERE raf.replay_id=r.id) as fix_attempts
FROM {replay_table} r
WHERE
r.state != "GC_ERROR"
HAVING
r.human_players != player_count
and fix_attempts <= {max_fix_attempts}
""".format(
replay_table=Replay.__tablename__,
player_table=ReplayPlayer.__tablename__,
auto_fix_table=ReplayAutoFix.__tablename__,
max_fix_attempts=app.config['MAX_REPLAY_FIX_ATTEMPTS'])
)
)
for replay_id, human_count, player_count, auto_fix_attempts in human_players_discrepancy:
if not should_fix_be_attempted(replay_id, _error, {'human_count': human_count, 'player_count': player_count}):
continue
print("Replay {} has a human_count of {}, but we have {} player objects for this replay.".format(
replay_id,
human_count,
player_count
))
replay = Replay.query.filter(Replay.id == replay_id).one()
print("\tDeleting ReplayPlayer objects")
for player in replay.players:
db.session.delete(player)
db.session.commit()
print("\tRe-adding replay to GC queue")
Replay.add_gc_job(replay)
def fix_incorrect_player_counts():
""" Finds and attempts to fix all replays where `Replay.human_players` does not match the quantity of `ReplayPlayer`
objects we have in the database.
Attempts to fix by deleting all ReplayPlayer objects and re-adding the replay to the job queue.
"""
_error = "PLAYER_COUNT_MISMATCH"
human_players_discrepancy = db.engine.execute(
text("""
SELECT
r.id,
r.human_players,
(
SELECT count(*) FROM {player_table} rp
WHERE rp.replay_id=r.id
and (rp.id is NULL or rp.account_id is not NULL) # Exclude bots from count (though there's the chance we have duplicate entries for bots? fack)
) as player_count,
(SELECT count(*) FROM {auto_fix_table} raf WHERE raf.replay_id=r.id) as fix_attempts
FROM {replay_table} r
WHERE
r.state != "GC_ERROR"
HAVING
r.human_players != player_count
and fix_attempts <= {max_fix_attempts}
""".format(replay_table=Replay.__tablename__,
player_table=ReplayPlayer.__tablename__,
auto_fix_table=ReplayAutoFix.__tablename__,
max_fix_attempts=app.config['MAX_REPLAY_FIX_ATTEMPTS'])))
for replay_id, human_count, player_count, auto_fix_attempts in human_players_discrepancy:
if not should_fix_be_attempted(replay_id, _error, {
'human_count': human_count,
'player_count': player_count
}):
continue
print(
"Replay {} has a human_count of {}, but we have {} player objects for this replay."
.format(replay_id, human_count, player_count))
replay = Replay.query.filter(Replay.id == replay_id).one()
print("\tDeleting ReplayPlayer objects")
for player in replay.players:
db.session.delete(player)
db.session.commit()
print("\tRe-adding replay to GC queue")
Replay.add_gc_job(replay)
def archive_subscriber_matches():
subscriptions = Subscription.get_valid_subscriptions()
print "Found {} valid subscribers".format(len(subscriptions))
for subscription in subscriptions:
webapi_params = {
"account_id": subscription.user_id,
"date_min": None,
"matches_requested": 100 # 100 Max
}
latest_match = SubscriptionLastMatch.query.\
filter(SubscriptionLastMatch.user_id == subscription.user_id,
SubscriptionLastMatch.replay_found == True).\
order_by(SubscriptionLastMatch.created_at.desc()).\
first()
if latest_match:
webapi_params["date_min"] = latest_match.created_at_timestamp
else:
webapi_params["date_min"] = subscription.created_at_timestamp
matches = steam.api.interface("IDOTA2Match_570").GetMatchHistory(
**webapi_params).get("result")
# Log this match check, as well as whether or not we found a match.
last_match_log = SubscriptionLastMatch(subscription.user_id,
len(matches.get("matches")) > 0)
db.session.add(last_match_log)
db.session.commit()
print "Found {} matches for {}".format(len(matches.get("matches")),
subscription.user_id)
for match in matches.get("matches"):
replay_exists = Replay.query.filter(
Replay.id == match["match_id"]).count() > 0
if not replay_exists:
replay = Replay(match["match_id"])
db.session.add(replay)
db.session.commit()
Replay.add_gc_job(replay)
print "Added {} to database and job queue".format(
match["match_id"])
else:
print "Match {} already in database, skipping.".format(
match["match_id"])
def process_match_list(matches):
""" Iterates through a list ofmatches and checks whether we already have them in our database. If we do not then
this code will add the match to our database and create an associated GC job. """
if len(matches) > 0:
for match in matches:
replay_exists = Replay.query.filter(Replay.id == match["match_id"]).count() > 0
if not replay_exists:
replay = Replay(match["match_id"])
db.session.add(replay)
db.session.commit()
Replay.add_gc_job(replay)
print "Added {} to database and job queue".format(match["match_id"])
else:
print "Match {} already in database, skipping.".format(match["match_id"])
def archive_subscriber_matches():
subscriptions = Subscription.get_valid_subscriptions()
print "Found {} valid subscribers".format(len(subscriptions))
for subscription in subscriptions:
webapi_params = {
"account_id": subscription.user_id,
"date_min": None,
"matches_requested": 100 # 100 Max
}
latest_match = SubscriptionLastMatch.query.\
filter(SubscriptionLastMatch.user_id == subscription.user_id,
SubscriptionLastMatch.replay_found == True).\
order_by(SubscriptionLastMatch.created_at.desc()).\
first()
if latest_match:
webapi_params["date_min"] = latest_match.created_at_timestamp
else:
webapi_params["date_min"] = subscription.created_at_timestamp
matches = steam.api.interface("IDOTA2Match_570").GetMatchHistory(**webapi_params).get("result")
# Log this match check, as well as whether or not we found a match.
last_match_log = SubscriptionLastMatch(subscription.user_id, len(matches.get("matches")) > 0)
db.session.add(last_match_log)
db.session.commit()
print "Found {} matches for {}".format(len(matches.get("matches")), subscription.user_id)
for match in matches.get("matches"):
replay_exists = Replay.query.filter(Replay.id == match["match_id"]).count() > 0
if not replay_exists:
replay = Replay(match["match_id"])
db.session.add(replay)
db.session.commit()
Replay.add_gc_job(replay)
print "Added {} to database and job queue".format(match["match_id"])
else:
print "Match {} already in database, skipping.".format(match["match_id"])
def process_match_list(matches):
""" Iterates through a list ofmatches and checks whether we already have them in our database. If we do not then
this code will add the match to our database and create an associated GC job. """
if len(matches) > 0:
for match in matches:
replay_exists = Replay.query.filter(
Replay.id == match["match_id"]).count() > 0
if not replay_exists:
replay = Replay(match["match_id"])
db.session.add(replay)
db.session.commit()
Replay.add_gc_job(replay)
print "Added {} to database and job queue".format(
match["match_id"])
else:
print "Match {} already in database, skipping.".format(
match["match_id"])
#!/srv/www/dotabank.com/dotabank-web/bin/python
"""
Downoad and archive TI4 matches
"""
from app import steam, db # .info
from app.replays.models import Replay, ReplayPlayer
THE_INTERNATIONAL_4_ID = 600
matches = steam.api.interface("IDOTA2Match_570").GetMatchHistory(league_id=THE_INTERNATIONAL_4_ID).get("result")
if matches:
for match in matches.get("matches"):
replay_exists = Replay.query.filter(Replay.id == match["match_id"]).count() > 0
if not replay_exists:
replay = Replay(match["match_id"])
db.session.add(replay)
db.session.commit()
Replay.add_gc_job(replay)
print "Added {} to database and job queue".format(match["match_id"])
else:
print "Match {} already in database, skipping.".format(match["match_id"])
