def batch_query_match_history(session, pids):
pids_list = list(pids)
total = len(pids_list)
batch = 50
player_matches = {}
any_error = False
log("Begin querying %s players" % total)
for idx in range(0, total, batch):
next_idx = idx + batch
bound = next_idx if next_idx < total else total
sub_pids = pids_list[idx:bound]
log('Attempting to query players %s-%s of %s' % (idx, bound, total))
is_error, matches, player_matches_dict = _bulk_query_match_history(
session, sub_pids
)
any_error = any_error or is_error
save_match_list(matches)
player_matches.update(player_matches_dict)
set_player_updated_at(player_matches.keys())
with DATABASE.atomic():
cache = MatchCache()
for pid, matches in player_matches.items():
cache.process_matches(pid, matches)
cache.save()
return any_error
def bulk_insert(model, data_source):
total = len(data_source)
batch = 1000
log("Trying to insert %s rows" % total)
with DATABASE.atomic():
for idx in range(0, total, batch):
(model.insert_many(data_source[idx:idx + batch]).execute())
log("Inserted %s" % (idx + batch))
def bulk_insert(model, data_source):
total = len(data_source)
batch = 1000
log("Trying to insert %s rows" % total)
with DATABASE.atomic():
for idx in range(0, total, batch):
(
model
.insert_many(data_source[idx:idx + batch])
.execute()
)
log("Inserted %s" % (idx + batch))
def batch_query_match_history(session, pids):
pids_list = list(pids)
total = len(pids_list)
batch = 50
player_matches = {}
any_error = False
log("Begin querying %s players" % total)
for idx in range(0, total, batch):
next_idx = idx + batch
bound = next_idx if next_idx < total else total
sub_pids = pids_list[idx:bound]
log('Attempting to query players %s-%s of %s' % (idx, bound, total))
is_error, matches, player_matches_dict = _bulk_query_match_history(
session, sub_pids)
any_error = any_error or is_error
save_match_list(matches)
player_matches.update(player_matches_dict)
set_player_updated_at(player_matches.keys())
with DATABASE.atomic():
cache = MatchCache()
for pid, matches in player_matches.items():
cache.process_matches(pid, matches)
cache.save()
return any_error