def aggregate_batched_matches(batch, region, summoner_id): try: # init cass.get_realms(region=region).load() #old = time.time() * 1000 matchlist = [] for m_id in batch: match_id = int(m_id) matchlist.append(cass.get_match(id=match_id, region=region)) pool = Pool(len(matchlist)) pool.map(load_match, matchlist) pool.close() pool.join() #print("fetch:", time.time()*1000 - old) #old = time.time() * 1000 aggregate_user_matches(matchlist, summoner_id, region) #print("total:", time.time()*1000 - old) except Exception as e: if pool is not None: pool.close() log.warn("Failed to aggregate batched matches", e, stack_info=True) aggregate_batched_matches.retry(exc=e)
def aggregate_users(summoner_id, region, max_aggregations=-1): try: # init cass.get_realms(region=region).load() summoner_id = summoner_id max_aggregations = int(max_aggregations) profile = ProfileStats.objects.get(user_id=summoner_id, region=region) summoner = cass.get_summoner(id=summoner_id, region=region) index = 0 updated = False count = 0 while True: if updated or index >= max_aggregations and max_aggregations > 0 or count >= max_aggregations: break recent_matches = cass.get_match_history( summoner=summoner, begin_index=index, end_index=index + BATCH_SIZE, seasons=[cass.data.Season.from_id(13)]) batch = [] for match in recent_matches: if profile.last_match_updated == match.creation.timestamp: updated = True break if (match.region.value == region): batch.append(match.id) if len(batch) == Consts.AGGREGATION_BATCH_SIZE: aggregate_batched_matches.delay(batch, region, summoner_id) batch = [] count += 1 if count >= max_aggregations: break index += BATCH_SIZE if len(batch) > 0: aggregate_batched_matches.delay(batch, region, summoner_id) if len(recent_matches) == 0: break return f"Aggregated {summoner_id}, {region}, {count} matches" except Exception as e: log.warn("Failed to aggregate_user", e, stack_info=True) aggregate_users.retry(exc=e)
def aggregate_batched_matches(batch, region, summoner_id): # init cass.get_realms(region=region).load() matchlist = [] for m_id in batch: match_id = int(m_id) matchlist.append(cass.get_match(id=match_id, region=region)) #pool.map(load_match, matchlist) loop = asyncio.get_event_loop() tasks = [asyncio.ensure_future(fetch(m)) for m in matchlist] loop.run_until_complete(asyncio.wait(tasks))
def get_versions(): versions = cass.get_versions(region="NA") print(versions[0]) print(versions.region) versions = cass.get_versions(region="NA") print(versions[0]) realms = cass.get_realms(region="NA") print(realms.latest_versions)
def test_realms(): realms = cass.get_realms(region="NA") realms.latest_versions
def test_realms(): realms = cass.get_realms(region="NA") realms.latest_versions