def knife(uuid, token, verify, roles): # pylint: disable=R0914 """Pull all ESI data for a character_id. Args: uuid: string uuid token token: SSO access token verify: dictionary return from /verify/ roles: list of corporation roles """ character_id = verify["CharacterID"] LOG.info("knife run started for character: %s", character_id) scopes = verify["Scopes"] _, _, public = utils.request_or_wait( "{}/latest/characters/{}/".format(ESI, character_id) ) if isinstance(public, str): CACHE.delete("{}{}".format(Keys.processing.value, uuid)) utils.write_data(uuid, {"public info failure": public}) return headers = {"Authorization": "Bearer {}".format(token)} results = get_results(public, character_id, scopes, roles, headers) utils.write_data(uuid, results) CACHE.delete("{}{}".format(Keys.processing.value, uuid)) CACHE.cache.inc(Keys.alltime.value, 1) LOG.info("completed character: %r", character_id)
def main(): """Main worker entrypoint.""" LOG.info("knife worker online") # until we can resume jobs for state in (Keys.processing.value, Keys.pending.value): CACHE.delete_many(*utils.list_keys(state)) while True: prune = [] for glet in WORKERS: if glet.successful(): prune.append(glet) elif glet.dead: LOG.warning( "worker crashed: %s", "".join(format_exception(*glet.exc_info)).strip(), ) prune.append(glet) for glet in prune: WORKERS.remove(glet) process_new() gc.collect() gevent.sleep(10)
def process_new(): """Process all new tokens, verify or we're done early.""" for new_key in utils.list_keys(Keys.new.value): uuid = new_key.split(".")[-1] LOG.info("processing new uuid: %r", uuid) token = CACHE.get(new_key) CACHE.delete(new_key) if not token: LOG.warning("no token stored for uuid: %r", uuid) continue pending_key = "{}{}".format(Keys.pending.value, uuid) CACHE.set( pending_key, "1", timeout=70, ) headers = {"Authorization": "Bearer {}".format(token)} _, _, res = utils.request_or_wait( "{}/verify/".format(ESI), headers=headers, ) failed = False if isinstance(res, str) or "CharacterID" not in res: utils.write_data(uuid, {"auth failure": res}) failed = True else: _, _, roles = utils.request_or_wait( "{}/latest/characters/{}/roles/".format( ESI, res["CharacterID"], ), headers=headers, ) if isinstance(roles, str): utils.write_data(uuid, {"roles failure": roles}) failed = True CACHE.delete(pending_key) if not failed: CACHE.set( "{}{}".format(Keys.processing.value, uuid), res["CharacterID"], timeout=7200, ) WORKERS.append( gevent.spawn(knife, uuid, token, res, roles) )