def resolve_extra_info_object(character, key, value):
if key == "structure_id" or key == "station_id":
Structure.verify_object_exists(value, character.pk)
elif key == "type_id":
ObjectType.verify_object_exists(value)
elif key == "character_id":
# fix inaccurate CCP data
# CCP uses character_id as a catch-all key for special extra_info types that aren't defined.
actual_type = CcpIdTypeResolver.get_id_type(value)
if actual_type == "character":
EVEPlayerCharacter.verify_object_exists(value)
else:
logger.error("getting incorrect character_id journal thing for type id %s actual_Type: %s" % (value, actual_type))
return None,None
elif key == "corporation_id":
EVEPlayerCorporation.verify_object_exists(value)
elif key == "alliance_id":
EVEPlayerAlliance.verify_object_exists(value)
elif key == "system_id":
System.verify_object_exists(value)
elif key == "market_transaction_id" or key == "industry_job_id" or key == "contract_id" or key == "planet_id" or key == "eve_system":
#idgaf
pass
else:
raise Exception("ccp returning some fancy new key object? %s %s" % (key,value) )
return (key,value)
def verify_generic_object_exists(object_type, ccp_object_id):
"""
Used by wallet/evemail endpoints since they have fields that are generic object ids
:param object_type:
:param object_pk:
:return:
"""
if object_type == "character":
EVEPlayerCharacter.verify_object_exists(ccp_object_id)
elif object_type == "corporation":
EVEPlayerCorporation.verify_object_exists(ccp_object_id)
elif object_type == "alliance":
EVEPlayerAlliance.verify_object_exists(ccp_object_id)
elif object_type == "system":
System.verify_object_exists(ccp_object_id)
elif object_type == "mailing_list":
CcpIdTypeResolver.add_type(ccp_object_id, "mailing_list")
pass
elif object_type == "eve_system":
return
elif object_type == "faction":
CcpIdTypeResolver.add_type(ccp_object_id, "faction")
else:
raise Exception("not implemented %s id %s" %
(object_type, ccp_object_id))
def get_character_location(char_id):
char = EVEPlayerCharacter.get_object(char_id)
if not char.has_esi_scope('esi-location.read_location.v1'):
return None
client = EsiClient(authenticating_character=char)
location, _ = client.get("/v1/characters/%s/location/" % char.pk)
system_id = location["solar_system_id"]
actual_type = CcpIdTypeResolver.get_id_type(system_id)
# fix for CCP-side bug
if actual_type != "system":
structure_id = system_id
structure = Structure.get_object(structure_id, char.pk)
system = structure.location.system if structure.location else None
else:
system = System.get_object(system_id)
# both structures and stations can be handled by structure.getobject
if "structure_id" in location:
structure = Structure.get_object(location["structure_id"], char.pk)
elif "station_id" in location:
structure = Structure.get_object(location["station_id"], char.pk)
else:
structure = None
return {"system": system, "structure": structure}
def get_character_transactions(character_ccp_id, oldest_entry=None, page=0):
character = EVEPlayerCharacter.get_object(character_ccp_id)
if not character.has_esi_scope('esi-wallet.read_character_wallet.v1'):
return None
client = EsiClient(authenticating_character=character)
if oldest_entry is None:
transaction_entries, _ = client.get(
"/v1/characters/%s/wallet/transactions/" % character_ccp_id)
else:
transaction_entries, _ = client.get(
"/v1/characters/%s/wallet/transactions/?from_id=%s" %
(character_ccp_id, oldest_entry))
oldest_transaction_entry = -1
transactions = []
for entry in transaction_entries:
# keep track of the oldest transaction entry we received for pagination
if entry[
"transaction_id"] < oldest_transaction_entry or oldest_transaction_entry == -1:
oldest_transaction_entry = entry["transaction_id"]
transactions.append(extract_transaction(character, entry))
# pagination logic
if oldest_transaction_entry != -1 and page < 5:
prev_transactions = get_character_transactions(
character_ccp_id=character_ccp_id,
oldest_entry=oldest_transaction_entry - 1,
page=page + 1)
else:
prev_transactions = []
return transactions + prev_transactions
def get_character_journal(character_ccp_id, page = 1, page_limit=5):
"""
:param self:
:param character_ccp_id:
:param oldest_entry:
:param page_limit:
:return:
"""
character = EVEPlayerCharacter.get_object(character_ccp_id)
if not character.has_esi_scope('esi-wallet.read_character_wallet.v1'):
return None
client = EsiClient(authenticating_character=character)
journal_entries, _ = client.get("/v4/characters/%s/wallet/journal/?page=%s" % (character_ccp_id,page))
formatted_entries = []
for entry in journal_entries:
e = verify_journal_entry(entry, character)
formatted_entries.append(e)
# pagination logic
if formatted_entries and page <= page_limit:
older_entries = get_character_journal(
character_ccp_id = character_ccp_id,
page = page + 1,
page_limit = page_limit
)
else:
older_entries = []
return journal_entries + older_entries
def _does_character_have_structure_access(character_id, structure_id):
has_access = True
# is this a station?
if structure_id >= 60000000 and structure_id <= 61000000:
return True
has_key = EsiKey.does_character_have_key(character_id)
if not has_key:
logger.info(
"Character {} no longer has access to structure {} due to a dead ESI key"
.format(character_id, structure_id))
has_access = False
else:
# check if we can query the structure using this character
char = EVEPlayerCharacter.get_object(character_id)
client = EsiClient(authenticating_character=char,
raise_application_errors=False,
log_application_errors=False)
res, err = client.get(
"/v2/universe/structures/{}/".format(structure_id))
if err == EsiError.EsiApplicationError:
logger.info(
"Character {} no longer has access to structure {} due to ESIApplicationError"
.format(character_id, structure_id))
has_access = False
if err == EsiError.InvalidRefreshToken:
logger.info(
"Character {} no longer has access to structure {} due to newly dead ESI key"
.format(character_id, structure_id))
has_access = False
return has_access
def structure_search(char_id, search_string):
char = EVEPlayerCharacter.get_object(char_id)
client = EsiClient(authenticating_character=char,
raise_application_errors=False)
logger.info("search string {}".format(search_string))
search_param = urllib.parse.urlencode({"search": search_string})
logger.info("search param {}".format(search_param))
res, err = client.get(
"/v3/characters/{}/search/?categories=structure,station&language=en-us&{}&strict=false"
.format(char_id, search_param))
if err == EsiError.EsiApplicationError:
return []
results = []
if "structure" in res:
if res["structure"]:
Structure.load_citadels_async(res["structure"], client)
results.extend(res["structure"])
if "station" in res:
if res["station"]:
stations_to_load = []
for station_id in res["station"]:
if not Structure.exists(station_id):
stations_to_load.append(station_id)
else:
results.append(station_id)
Structure.load_stations_async(stations_to_load, client)
results.extend(stations_to_load)
return results
def update_subscription_payments():
logger.info('LAUNCH_TASK update_subscription_payments')
with general_queue.lock_task('update_subscription_payments'):
char = EVEPlayerCharacter.get_object(settings.PAYMENT_CHARACTER_ID)
client = EsiClient(authenticating_character=char)
journal_entries = client.get_multiple_paginated(
"/v4/characters/{}/wallet/journal/".format(char.pk))
for entry in journal_entries:
if entry["ref_type"] == "player_donation" and entry.get(
"second_party_id") == char.pk:
if not SubscriptionPayment.exists(entry["id"]):
process_new_payment(
EVEPlayerCharacter.get_object(entry["first_party_id"]),
entry["amount"], entry["id"],
dateutil.parser.parse(entry["date"]),
entry.get("reason"))
logger.info('FINISH_TASK update_subscription_payments')
def create_user_account(char_name, char_id, owner_hash):
character = EVEPlayerCharacter.get_object(char_id)
username = format_character_name(char_name)
user = User.objects.create_user(username)
user.backend = 'django.contrib.auth.backends.ModelBackend'
user.save()
new_association = get_or_create_character_association(
char = character,
owner_hash = owner_hash,
user=user,
)
profile = user.profile
profile.primary_character = character
profile.save()
return user
def update_corporation_public_details(corporation_id, throttle_updates):
"""
Updates a corporations public details including CEO, member count, affiliation.
By passing throttle_updates=True, this task will only be fully executed once every hour.
If you pass throttle_updates=False, this task will fully complete all its esi calls no matter what.
:param corporation_id:
:param throttle_updates:
:return:
"""
if throttle_updates:
# do not add this key to EVEPlayerCorporation's post_save cache flush
c = cache.get(
"corporation_details_update_throttled_{}".format(corporation_id))
if c:
return
EVEPlayerCorporation_lazy = apps.get_model('eve_api',
'EVEPlayerCorporation')
corporation = EVEPlayerCorporation_lazy.get_object(corporation_id)
client = EsiClient()
corp_details, _ = client.get("/v4/corporations/{}/".format(corporation_id))
if corporation.alliance_id != corp_details.get("alliance_id"):
if corp_details.get("alliance_id"):
_ = EVEPlayerAlliance.get_object(corp_details["alliance_id"])
corporation.alliance_id = corp_details.get("alliance_id")
corporation.member_count = int(corp_details["member_count"])
corporation.tax_rate = float(corp_details["tax_rate"])
if not corporation.ceo_character or corporation.ceo_character.pk != corp_details[
"ceo_id"]:
corporation.ceo_character = EVEPlayerCharacter.get_object(
corp_details["ceo_id"])
corporation.save()
# we only set these once we're certain the update completed successfuly
cache.set("corporation_details_update_throttled_{}".format(corporation_id),
True,
timeout=3600)
_set_cached_corp_affiliation(corporation_id,
corp_details.get("alliance_id"))
def _get_key_for_structure(structure_id):
all_source_chars = TradingRoute.objects.filter(
source_character_has_access=True,
source_structure_id=structure_id).values_list('source_character',
flat=True)
all_dest_chars = TradingRoute.objects.filter(
destination_character_has_access=True,
destination_structure_id=structure_id).values_list(
'destination_character', flat=True)
all_structure_chars = list(
set(list(all_source_chars) + list(all_dest_chars)))
retry_count = -1
for char in all_structure_chars:
retry_count += 1
logger.info(
"The following character is yielded for structure {}, {}. Retry count: {}"
.format(structure_id, char, retry_count))
yield EVEPlayerCharacter.get_object(char)
def provision_esi_corporation(corp_id, force=False):
EVEPlayerCorporation_lazy = apps.get_model('eve_api',
'EVEPlayerCorporation')
corp, created = EVEPlayerCorporation_lazy.objects.get_or_create(pk=corp_id)
if not created and not force:
return corp
if force:
logger.warning("forcing reload of corporation {}".format(corp_id))
client = EsiClient()
corp_data, _ = client.get("/v4/corporations/{}/".format(corp_id))
corp.name = corp_data["name"]
corp.ticker = corp_data["ticker"]
#description = corp_data.get("description")
#corp.description = description.decode('ascii','ignore').encode("ascii") if description else None
#url = corp_data.get("url")
#corp.url = url.decode('ascii','ignore').encode("ascii") if url else None
corp.tax_rate = corp_data["tax_rate"]
corp.member_count = corp_data["member_count"]
# whoever made the corp model didnt use a bigint and i dont care enough to migrate it
corp.shares = 0
if "alliance_id" in corp_data:
corp.alliance = EVEPlayerAlliance.get_object(corp_data["alliance_id"])
corp.api_last_upated = timezone.now()
corp.save()
# Skip looking up the CEOs for NPC corps and ones with no CEO defined (dead corps)
# this MUST happen after the corp is initially saved.
if corp_id > 1000182 and int(corp_data['ceo_id']) > 1:
corp.ceo_character = EVEPlayerCharacter.get_object(corp_data["ceo_id"])
corp.save()
CcpIdTypeResolver.add_type(corp_id, "corporation")
return corp
def get_character_orders(character_ccp_id):
"""
:param character_ccp_id:
:return:
[
{
duration:int(total days order valid)
escrow: float(nullable)
is_buy_order: boolean (if CCP returns null for this, we assume sell order)
is_corporation: boolean
issued: datetime,
location: Structure
min_volume: int(nullable)
order_id: int
price: float
range: string options([ 1, 10, 2, 20, 3, 30, 4, 40, 5, region, solarsystem, station ])
type: object type
volume_remain: int
volume_total: int
region: Region
}
]
"""
character = EVEPlayerCharacter.get_object(character_ccp_id)
if not character.has_esi_scope('esi-markets.read_character_orders.v1'):
return None
client = EsiClient(authenticating_character=character)
order_entries, _ = client.get("/v2/characters/%s/orders/" %
character_ccp_id)
ret = []
for entry in order_entries:
ret.append(extract_order(character, entry))
return ret
def get_character_assets(character_ccp_id):
"""
:param character_ccp_id:
:return:
{
location_flag : varchar
location : ORM Location
is_in_container : bool
is_blueprint_copy: nullable bool
is_singleton: bool
item_id: bigint
location_type: varchar
quantity: bigint
type: ORM ObjectType
}
"""
character = EVEPlayerCharacter.get_object(character_ccp_id)
# just assume they have the f****n scope
#char_keys = EsiKey.objects.filter(
# Q(character_id = character_ccp_id) &
# Q(use_key=True)
#)
#for key in char_keys:
# if key.has_esi_scope('esi-assets.read_assets.v1'):
#if not character.has_esi_scope():
# return None
client = EsiClient(authenticating_character=character)
assets_pages = client.get_multiple_paginated(
"/v3/characters/{}/assets/".format(character_ccp_id))
assets = extract_assets(character, assets_pages)
return assets
def process_key_addition(request, sso_data, refresh_token):
char_id = int(sso_data["CharacterID"])
owner_hash = sso_data["CharacterOwnerHash"]
scope_list = sso_data.get('Scopes', None)
if not scope_list:
raise Exception("no scopes provided")
for scope in settings.ESI_SCOPES:
if scope not in scope_list:
raise Exception("not enough scopes provided")
# make sure we have this char
c = EVEPlayerCharacter.get_object(char_id)
# build/get association
association = get_or_create_character_association(
c,
owner_hash,
request.user
)
# create key
key = EsiKey.add_esi_key(c, refresh_token, request.user, owner_hash, scope_list)
return c.name
def update_player_assets(character_id):
logger.info("LAUNCH TASK update_player_assets {}".format(character_id))
with player_queue.lock_task(
'update-player-assets-{}'.format(character_id)):
char = EVEPlayerCharacter.get_object(character_id)
# double check to verify we actually need to scan this character right now
oldest_allowable_update = timezone.now() - player_assets_timedelta
if char.assets_last_updated and char.assets_last_updated > oldest_allowable_update:
logger.warning(
"{} was queued for assets update quickly over a given interval. killing followup task"
.format(character_id))
return
scan_log = PlayerAssetsScanLog.start_scan_log(char)
assets, containers = _get_player_assets_and_containers(char)
# extract hashes to figure out current configuration
current_assets_hash = AssetEntry.generate_hash(character_id)
new_assets_hash = AssetEntry.generate_has_from_list(assets)
force = False
if new_assets_hash != current_assets_hash or force:
logger.info("Assets hash changed for {}, updating assets".format(
character_id))
with transaction.atomic():
AssetEntry.objects.filter(character_id=character_id).delete()
AssetContainer.objects.filter(
character_id=character_id).delete()
container_objs = []
asset_objs = []
for c in containers:
container_objs.append(
AssetContainer(ccp_id=c["item_id"],
character=char,
structure=Structure.get_object(
c["location"].root_location_id,
char.pk),
name=c["name"]))
AssetContainer.objects.bulk_create(container_objs)
for a in assets:
asset_objs.append(
AssetEntry(
ccp_id=a["item_id"],
character=char,
structure=Structure.get_object(
a["location"].root_location_id, char.pk),
container_id=None
if not a["is_in_container"] else a["location"].pk,
quantity=a["quantity"],
object_type=a["type"]))
AssetEntry.objects.bulk_create(asset_objs)
char.assets_last_updated = timezone.now()
char.save()
scan_log.stop_scan_log()
logger.info("COMPLETE TASK update_player_assets {}".format(character_id))
return
def update_eve_character_esi(character_id, skip_token_calls=False):
"""
Updates public character data from the ESI API and returns the Character object when completed
CACHE - 3600 seconds
SCOPES - None
"""
logger.info("starting update for %s" % character_id)
character = EVEPlayerCharacter.get_object(character_id)
# todo: update corp history, attributes, etc as seen in import_eve_character_func
# update public data
client = EsiClient()
character_data, _ = client.get("/v4/characters/%s/" % character.pk)
corp_id = character_data['corporation_id']
corp = EVEPlayerCorporation.get_object(corp_id)
character.security_status = character_data.get("security_status")
character.corporation = corp
character.save()
if waffle.switch_is_active('esi-set-default-primary-character'):
set_default_primary_character(character)
# see if we can update private data
if character.esi_key and not skip_token_calls:
client = EsiClient(authenticating_character=character)
token, error = client.get_access_token()
if not error:
# nothing bad happened, lets update the character's esi_key_last_validated
character.esi_key_last_validated = timezone.now()
character.save()
# reset the ESI key's roles and repopulate them. Only use this if there's been problems with keys getting
# assigned incorrect permissions
if waffle.switch_is_active('esi-renew-key-roles-every-update'):
update_esi_key_roles(character)
# try to populate esi key's scopes
character._esi_roles = character.esi_scopes
character.save()
# skill/attrib/sp update
try:
update_character_esi_data(character)
except Exception as ex:
logger.exception("Error updating ESI character %s" % (ex))
pass
else:
if error is EsiError.InvalidRefreshToken:
logger.info(
"It looks like the user's esi token was rejected. updating services %s "
% character.owner.username)
# token removal is handled by EsiClient itself. all we need to do now is update user access
#update_user_access.delay(user=character.owner.id)
return None
elif error is EsiError.EsiNotResponding:
logger.info("ESI query seems to have timed out")
return None
elif error is EsiError.ConnectionError:
logger.info("ESI connection failed")
return None
elif error is EsiError.SsoError:
logger.info("SSO is s******g itself")
return None
else:
logger.warning(
"an unknown error was triggered when grabbing user access token err: %s char: %s"
% (error, character.pk))
return None
# update the character's corporation details
update_corporation_public_details(character.corporation.pk,
throttle_updates=True)
return character
def update_player_transactions(ccp_id):
logger.info("LAUNCH_TASK {} {}".format("update_player_transactions",
ccp_id))
with player_queue.lock_task(
'update-player-transactions-{}'.format(ccp_id)):
character = EVEPlayerCharacter.get_object(ccp_id)
# double check to verify we actually need to scan this character right now
oldest_allowable_update = timezone.now(
) - player_transactions_timedelta
if character.transactions_last_updated and character.transactions_last_updated > oldest_allowable_update:
logger.warning(
"{} was queued for transactions update quickly over a given interval. killing followup task"
.format(ccp_id))
return
scan_log = PlayerTransactionScanLog(character=character)
scan_log.save()
client = EsiClient(authenticating_character=character)
transactions = _get_transactions(client, character)
_verify_transaction_models_exist(transactions, character)
txn_models = []
logger.info("scanning {} transactions for {}".format(
len(transactions), ccp_id))
transaction_ids_being_added = []
for t in transactions:
# check if t exists
exists = PlayerTransaction.exists(t["transaction_id"])
if not exists and t[
"transaction_id"] not in transaction_ids_being_added:
transaction_ids_being_added.append(t["transaction_id"])
txn_models.append(
PlayerTransaction(
character=character,
ccp_id=t["transaction_id"],
client_id=t["client_id"],
timestamp=dateutil.parser.parse(t["date"]),
is_buy=t["is_buy"],
is_personal=t["is_personal"],
journal_ref_id=t["journal_ref_id"],
location_id=t["location_id"],
quantity=t["quantity"],
object_type_id=t["type_id"],
unit_price=t["unit_price"],
quantity_without_known_source=t["quantity"],
quantity_without_known_destination=t["quantity"]))
logger.info("creating {} transactions for {}".format(
len(txn_models), ccp_id))
PlayerTransaction.objects.bulk_create(txn_models)
trading_routes = TradingRoute.objects.filter(
Q(source_character=character) | Q(destination_character=character))
source_structures = trading_routes.filter(
source_character=character).values_list("source_structure_id",
flat=True)
destination_structures = trading_routes.filter(
destination_character=character).values_list(
"destination_structure_id", flat=True)
#structure_ids = list(set(source_structures).union(set(destination_structures)))
structure_ids = list(set(destination_structures))
# maybe log n this later? or just remove it and always trigger a scan
trigger_transaction_scan = True
if trigger_transaction_scan:
logger.info("trigger transaction scan for {}".format(character.pk))
transactions_needing_link = PlayerTransaction.objects.filter(
location_id__in=structure_ids,
quantity_without_known_source__gt=0,
is_buy=False,
character=character)
logger.info("checking {} transactions for linkages for {}".format(
len(transactions_needing_link), character.pk))
for t in transactions_needing_link:
t.link_transactions()
logger.info("Done checking for linkages")
character.transactions_last_updated = timezone.now()
character.save()
scan_log.scan_complete = timezone.now()
scan_log.save()
def import_eve_character_esi(refresh_token,
user,
logger=logging.getLogger(__name__)):
"""
Imports a EVE Character from the ESI API and returns the Character object when completed
"""
client_id = settings.EVEOAUTH['CONSUMER_KEY']
base_url = settings.EVEOAUTH['BASE_URL']
verify_url = base_url + "/verify/"
# Get character id/name
token = eve_oauth.get_access_token_from_refresh(refresh_token)
# check if something went wrong in token exchange
if token is None:
return False
esi_sso = OAuth2Session(client_id, token=token)
character_data = None
character_data = esi_sso.get(verify_url).json()
if 'error' in character_data.keys():
logger.error("ESI returned invalid data")
return False
# This happens when CCP SSO shits the f**k out. It happens more than you may think.
if character_data is None:
return False
character_id = character_data.get('CharacterID')
character_name = character_data.get('CharacterName')
owner_hash = character_data.get("CharacterOwnerHash")
if not character_id or not character_name or not owner_hash:
# more ccp oauth fuckups
return False
alliance_scopes = False
allied_scopes = False
scope_list = character_data.get('Scopes', None)
split_list = scope_list.split(' ')
if esi_scopes_valid(split_list, settings.ESI_SCOPES):
alliance_scopes = True
else:
if esi_scopes_valid(split_list, settings.ALLIED_ESI_SCOPES):
allied_scopes = True
if not alliance_scopes and not allied_scopes:
return None
# Get or build Character
character = EVEPlayerCharacter.get_object(character_id)
# force the character's corporation to update. this is meant to catch new corporations joining TEST
update_corporation_public_details.delay(character.corporation.pk,
throttle_updates=False)
# DO NOT ISSUE ANY ESI REQUESTS IN THE FOLLOWING BLOCK OF CODE. Any dumb network-related errors that terminate
# this code prematurely can result in a character with an ESI key that auth thinks is full-scoped, but is only
# allied scoped.
character.add_esi_key(refresh_token=refresh_token,
user_adding_key=user,
current_owner_hash=owner_hash,
scope_type=ESI_SCOPE_DEFAULT)
scopes = character.esi_scopes
scopes.reset()
if alliance_scopes:
character.authed_as_ally = False
else:
character.authed_as_ally = True
character.save()
scopes.update_notify_scopes(scope_list)
# end of critical block
# if this user does not have a primary character set, do it now
set_default_primary_character(character)
update_eve_character_esi.delay(character_id=character.pk)
return character.pk
