def process_new():
"""Process all new tokens, verify or we're done early."""
for new_key in utils.list_keys(Keys.new.value):
uuid = new_key.split(".")[-1]
LOG.warning("processing new uuid: %r", uuid)
token = CACHE.get(new_key)
CACHE.delete(new_key)
if not token:
LOG.warning("no token stored for uuid: %r", uuid)
continue
pending_key = "{}{}".format(Keys.pending.value, uuid)
CACHE.set(
pending_key,
"1",
timeout=70,
)
headers = {"Authorization": "Bearer {}".format(token)}
_, res = utils.request_or_wait(
"{}/verify/".format(ESI),
headers=headers,
)
failed = False
if isinstance(res, str) or "CharacterID" not in res:
utils.write_data(uuid, {"auth failure": res})
failed = True
else:
_, roles = utils.request_or_wait(
"{}/latest/characters/{}/roles/".format(
ESI,
res["CharacterID"],
),
headers=headers,
)
if isinstance(roles, str):
utils.write_data(uuid, {"roles failure": roles})
failed = True
CACHE.delete(pending_key)
if not failed:
CACHE.set(
"{}{}".format(Keys.processing.value, uuid),
res["CharacterID"],
timeout=7200,
)
WORKERS.append(
gevent.spawn(knife, uuid, token, res, roles)
)
def knife(uuid, token, verify, roles): # pylint: disable=R0914
"""Pull all ESI data for a character_id.
Args:
uuid: string uuid token
token: SSO access token
verify: dictionary return from /verify/
roles: list of corporation roles
"""
character_id = verify["CharacterID"]
LOG.info("knife run started for character: %s", character_id)
scopes = verify["Scopes"]
_, _, public = utils.request_or_wait(
"{}/latest/characters/{}/".format(ESI, character_id)
)
if isinstance(public, str):
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
utils.write_data(uuid, {"public info failure": public})
return
headers = {"Authorization": "Bearer {}".format(token)}
results = get_results(public, character_id, scopes, roles, headers)
utils.write_data(uuid, results)
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
CACHE.cache.inc(Keys.alltime.value, 1)
LOG.info("completed character: %r", character_id)
def _get_names(ids):
"""Resolve ids to names."""
resolved = {}
failed = []
for i in range(0, len(ids), 1000):
batch = ids[i:i+1000]
_, _, res = utils.request_or_wait(
"{}/latest/universe/names/".format(ESI),
method="post",
json=batch,
)
if isinstance(res, list):
for _res in res:
resolved[_res["id"]] = _res["name"]
else:
failed.extend(batch)
while failed:
still_failed = []
random.shuffle(failed)
batch_size = max(min(int(len(failed) / 2), 500), 1)
for i in range(0, len(failed), batch_size):
batch = failed[i:i+batch_size]
_, _, res = utils.request_or_wait(
"{}/latest/universe/names/".format(ESI),
method="post",
json=batch,
)
if isinstance(res, list):
for _res in res:
resolved[_res["id"]] = _res["name"]
else:
still_failed.extend(batch)
failed = still_failed
if batch_size == 1 and still_failed:
LOG.warning("failed to resolve: %r", still_failed)
break
return resolved
def verify_token(headers):
"""Verify the access token.
Args:
headers: http request headers
Returns:
integer character ID, list of scopes
Raises:
SystemExit on error
"""
_, _, res = request_or_wait("{}/verify/".format(ESI), headers=headers)
if isinstance(res, dict) and "Scopes" in res and res["Scopes"] \
and "CharacterID" in res and res["CharacterID"]:
return res["CharacterID"], res["Scopes"]
raise SystemExit("Could not find scopes in access token")
def run(args):
"""Create a new knife file."""
token = get_access_token(args["--client-id"], args["--port"])
headers = {"Authorization": "Bearer {}".format(token)}
character_id, scopes = verify_token(headers)
_, _, public = request_or_wait(
"{}/latest/characters/{}/".format(ESI, character_id)
)
if isinstance(public, str):
raise SystemExit("Failed to look up public info for: {}".format(
character_id
))
roles = get_roles(headers, character_id)
results = get_results(public, character_id, scopes, roles, headers)
write_results(results, character_id)
def get_roles(headers, character_id):
"""Determine the character's roles.
Args:
headers: http request headers
character_id: integer character ID
Returns:
list of roles
Raises:
SystemExit on error
"""
_, _, res = request_or_wait(
"{}/latest/characters/{}/roles/".format(ESI, character_id),
headers=headers,
)
if isinstance(res, dict):
return res.get("roles", [])
raise SystemExit("Could not deterine character's corporation roles")
def knife(uuid, token, verify, roles): # pylint: disable=R0914
"""Pull all ESI data for a character_id.
Args:
uuid: string uuid token
token: SSO access token
verify: dictionary return from /verify/
roles: list of corporation roles
"""
character_id = verify["CharacterID"]
LOG.warning("knife run started for character: %s", character_id)
scopes = verify["Scopes"]
_, public = utils.request_or_wait(
"{}/latest/characters/{}/".format(ESI, character_id)
)
if isinstance(public, str):
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
utils.write_data(uuid, {"public info failure": public})
return
all_params = copy.deepcopy(ADDITIONAL_PARAMS)
known_params = {"character_id": character_id}
if public["corporation_id"] > 2000000:
known_params["corporation_id"] = public["corporation_id"]
else:
all_params.pop("corporation_id")
if "alliance_id" in public:
known_params["alliance_id"] = public["alliance_id"]
spec = utils.refresh_spec()
headers = {"Authorization": "Bearer {}".format(token)}
results = expand_params(
scopes,
roles,
spec,
known_params,
all_params,
headers,
)
urls = build_urls(scopes, roles, spec, known_params, all_params)
with ThreadPoolExecutor(max_workers=20) as pool:
futures = []
for url in urls:
futures.append(pool.submit(
utils.request_or_wait,
url,
headers=headers,
))
for future in as_completed(futures):
url, result = future.result()
results[url] = result
utils.write_data(uuid, results)
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
LOG.warning("completed character: %r", character_id)