def knife(uuid, token, verify, roles): # pylint: disable=R0914
"""Pull all ESI data for a character_id.
Args:
uuid: string uuid token
token: SSO access token
verify: dictionary return from /verify/
roles: list of corporation roles
"""
character_id = verify["CharacterID"]
LOG.info("knife run started for character: %s", character_id)
scopes = verify["Scopes"]
_, _, public = utils.request_or_wait(
"{}/latest/characters/{}/".format(ESI, character_id)
)
if isinstance(public, str):
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
utils.write_data(uuid, {"public info failure": public})
return
headers = {"Authorization": "Bearer {}".format(token)}
results = get_results(public, character_id, scopes, roles, headers)
utils.write_data(uuid, results)
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
CACHE.cache.inc(Keys.alltime.value, 1)
LOG.info("completed character: %r", character_id)
def main():
"""Main worker entrypoint."""
LOG.warning("worker online")
# until we can resume jobs
for state in (Keys.processing.value, Keys.pending.value):
CACHE.delete_many(*utils.list_keys(state))
while True:
prune = []
for glet in WORKERS:
if glet.successful():
prune.append(glet)
elif glet.dead:
LOG.warning(
"worker crashed: %s",
"".join(format_exception(*glet.exc_info)).strip(),
)
prune.append(glet)
for glet in prune:
WORKERS.remove(glet)
process_new()
gc.collect()
gevent.sleep(10)
def rate_limit():
"""Apply a rate limit."""
key = "".join((Keys.rate_limit.value, get_ip()))
reqs = CACHE.get(key) or 0
if reqs >= 20:
return True
CACHE.set(key, reqs + 1, timeout=60)
return False
def process_new():
"""Process all new tokens, verify or we're done early."""
for new_key in utils.list_keys(Keys.new.value):
uuid = new_key.split(".")[-1]
LOG.warning("processing new uuid: %r", uuid)
token = CACHE.get(new_key)
CACHE.delete(new_key)
if not token:
LOG.warning("no token stored for uuid: %r", uuid)
continue
pending_key = "{}{}".format(Keys.pending.value, uuid)
CACHE.set(
pending_key,
"1",
timeout=70,
)
headers = {"Authorization": "Bearer {}".format(token)}
_, res = utils.request_or_wait(
"{}/verify/".format(ESI),
headers=headers,
)
failed = False
if isinstance(res, str) or "CharacterID" not in res:
utils.write_data(uuid, {"auth failure": res})
failed = True
else:
_, roles = utils.request_or_wait(
"{}/latest/characters/{}/roles/".format(
ESI,
res["CharacterID"],
),
headers=headers,
)
if isinstance(roles, str):
utils.write_data(uuid, {"roles failure": roles})
failed = True
CACHE.delete(pending_key)
if not failed:
CACHE.set(
"{}{}".format(Keys.processing.value, uuid),
res["CharacterID"],
timeout=7200,
)
WORKERS.append(
gevent.spawn(knife, uuid, token, res, roles)
)
def character_knife():
"""Start a new knife run for a character."""
if "access_token" in request.args and "state" in request.args:
# verify token/start knife process for character
# do this all out of band, we might be error limited right now
if CACHE.get("authstate.{}".format(request.args["state"])):
CACHE.delete("authstate.{}".format(request.args["state"]))
token = uuid.uuid4()
CACHE.set("new.{}".format(token), request.args["access_token"])
return redirect("/view/{}/".format(token))
# start sso flow
state = uuid.uuid4()
CACHE.set("authstate.{}".format(state), "1", timeout=300)
return redirect(
("https://login.eveonline.com/oauth/authorize?response_type=token"
"&redirect_uri={callback}&client_id={client}"
"&scope={scopes}&state={state}").format(
callback=CALLBACK_URL,
client=CLIENT_ID,
scopes=SCOPES,
state=state,
))
def write_data(uuid, data):
"""Try to store the data, log errors."""
try:
CACHE.set(
"{}{}".format(Keys.complete.value, uuid),
codecs.decode(
base64.b64encode(
gzip.compress(codecs.encode(
ujson.dumps(data),
"utf-8",
))),
"utf-8",
),
timeout=EXPIRY,
)
except Exception as error:
LOG.warning("Failed to save data: %r", error)
def refresh_spec():
"""Refresh the ESI spec.
Returns:
dictionary: JSON loaded swagger spec
"""
try:
spec_details = CACHE.get(Keys.spec.value)
except redis.exceptions.ConnectionError:
spec_details = None
save_results = False
else:
save_results = True
if spec_details is None:
spec_details = {"timestamp": 0}
if time.time() - spec_details["timestamp"] > 300:
headers = {}
if spec_details.get("etag"):
headers["If-None-Match"] = spec_details["etag"]
_, _, res = request_or_wait(
"{}/latest/swagger.json".format(ESI),
_as_res=True,
headers=headers,
)
if isinstance(res, str):
LOG.warning("failed to refresh spec: %s", res)
return spec_details.get("spec", {})
spec_details["timestamp"] = time.time()
if res.status_code != 304:
spec_details["etag"] = res.headers.get("ETag")
spec_details["spec"] = JsonDeref().deref(res.json())
if save_results:
CACHE.set(Keys.spec.value, spec_details, timeout=3600)
return spec_details["spec"]
def metrics_index():
"""Display some metrics."""
return render_template(
"metrics.html",
new=len(utils.list_keys(Keys.new.value)),
pending=len(utils.list_keys(Keys.pending.value)),
processing=len(utils.list_keys(Keys.processing.value)),
completed=len(utils.list_keys(Keys.complete.value)),
alltime=CACHE.get(Keys.alltime.value) or 0,
worker=not APP.knife_worker.dead,
error_limited=APP.error_limited,
now=datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
)
def get_data(uuid):
"""Open and return the character's data."""
cache_key = "{}{}".format(Keys.complete.value, uuid)
try:
content = CACHE.get(cache_key)
except Exception as error:
LOG.warning("failed to get %s: %r", cache_key, error)
else:
if content is None:
return None
try:
return ujson.loads(gzip.decompress(base64.b64decode(content)))
except Exception as error:
LOG.warning("failed to decode %s: %r", content, error)
else:
CACHE.cache._client.expire( # pylint: disable=protected-access
cache_key,
EXPIRY,
)
return None
def knife(uuid, token, verify, roles): # pylint: disable=R0914
"""Pull all ESI data for a character_id.
Args:
uuid: string uuid token
token: SSO access token
verify: dictionary return from /verify/
roles: list of corporation roles
"""
character_id = verify["CharacterID"]
LOG.warning("knife run started for character: %s", character_id)
scopes = verify["Scopes"]
_, public = utils.request_or_wait(
"{}/latest/characters/{}/".format(ESI, character_id)
)
if isinstance(public, str):
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
utils.write_data(uuid, {"public info failure": public})
return
all_params = copy.deepcopy(ADDITIONAL_PARAMS)
known_params = {"character_id": character_id}
if public["corporation_id"] > 2000000:
known_params["corporation_id"] = public["corporation_id"]
else:
all_params.pop("corporation_id")
if "alliance_id" in public:
known_params["alliance_id"] = public["alliance_id"]
spec = utils.refresh_spec()
headers = {"Authorization": "Bearer {}".format(token)}
results = expand_params(
scopes,
roles,
spec,
known_params,
all_params,
headers,
)
urls = build_urls(scopes, roles, spec, known_params, all_params)
with ThreadPoolExecutor(max_workers=20) as pool:
futures = []
for url in urls:
futures.append(pool.submit(
utils.request_or_wait,
url,
headers=headers,
))
for future in as_completed(futures):
url, result = future.result()
results[url] = result
utils.write_data(uuid, results)
CACHE.delete("{}{}".format(Keys.processing.value, uuid))
LOG.warning("completed character: %r", character_id)