def import_item_type(type_id):
"""
Imports object type with the given type id. Does not return until after db transaction is committed.
Only updates if item is already imported.
:param self:
:param type_id:
:return:
"""
logger.info("import_item_type task start with typeid %s" % type_id)
client = EsiClient(raise_application_errors=False, log_application_errors=False)
item_data, _ = client.get("/v3/universe/types/%s/" % type_id)
if _ is not None:
logger.error("received error when querying universe/types for {} err: {}".format(type_id, _))
return
item, _ = ObjectType.objects.update_or_create(
ccp_id = type_id,
defaults={
"name" : item_data["name"],
"volume" : item_data.get("volume"),
"packaged_volume" : item_data.get("packaged_volume"),
"group" : TypeGroup.get_object(item_data.get("group_id")),
"icon_id": item_data.get("icon_id") if item_data.get("icon_id") else type_id,
"market_group": MarketGroup.get_object(item_data.get("market_group_id")),
"published": item_data["published"]
}
)
item.save()
return
def import_universe_systems():
client = EsiClient()
systems, _ = client.get("/v1/universe/systems/")
logger.info("dispatching system fetches")
system_data = client.get_multiple("/v4/universe/systems/{}/", systems)
system_objs = []
for r in system_data.values():
system_objs.append(
System(
ccp_id = r["system_id"],
constellation_id = r["constellation_id"],
name = r["name"],
security_status = r["security_status"]
)
)
System.objects.bulk_create(system_objs)
logger.info("systems created & committed")
# gen location data
location_objs = []
for system in System.objects.all():
location_objs.append(
Location(
ccp_id = system.pk,
system = system,
constellation = system.constellation,
region = system.constellation.region,
root_location_id = system.pk,
is_in_space = True
)
)
Location.objects.bulk_create(location_objs)
logger.info("system locations created & committed")
def get_character_journal(character_ccp_id, page = 1, page_limit=5):
"""
:param self:
:param character_ccp_id:
:param oldest_entry:
:param page_limit:
:return:
"""
character = EVEPlayerCharacter.get_object(character_ccp_id)
if not character.has_esi_scope('esi-wallet.read_character_wallet.v1'):
return None
client = EsiClient(authenticating_character=character)
journal_entries, _ = client.get("/v4/characters/%s/wallet/journal/?page=%s" % (character_ccp_id,page))
formatted_entries = []
for entry in journal_entries:
e = verify_journal_entry(entry, character)
formatted_entries.append(e)
# pagination logic
if formatted_entries and page <= page_limit:
older_entries = get_character_journal(
character_ccp_id = character_ccp_id,
page = page + 1,
page_limit = page_limit
)
else:
older_entries = []
return journal_entries + older_entries
def _does_character_have_structure_access(character_id, structure_id):
has_access = True
# is this a station?
if structure_id >= 60000000 and structure_id <= 61000000:
return True
has_key = EsiKey.does_character_have_key(character_id)
if not has_key:
logger.info(
"Character {} no longer has access to structure {} due to a dead ESI key"
.format(character_id, structure_id))
has_access = False
else:
# check if we can query the structure using this character
char = EVEPlayerCharacter.get_object(character_id)
client = EsiClient(authenticating_character=char,
raise_application_errors=False,
log_application_errors=False)
res, err = client.get(
"/v2/universe/structures/{}/".format(structure_id))
if err == EsiError.EsiApplicationError:
logger.info(
"Character {} no longer has access to structure {} due to ESIApplicationError"
.format(character_id, structure_id))
has_access = False
if err == EsiError.InvalidRefreshToken:
logger.info(
"Character {} no longer has access to structure {} due to newly dead ESI key"
.format(character_id, structure_id))
has_access = False
return has_access
def get_character_transactions(character_ccp_id, oldest_entry=None, page=0):
character = EVEPlayerCharacter.get_object(character_ccp_id)
if not character.has_esi_scope('esi-wallet.read_character_wallet.v1'):
return None
client = EsiClient(authenticating_character=character)
if oldest_entry is None:
transaction_entries, _ = client.get(
"/v1/characters/%s/wallet/transactions/" % character_ccp_id)
else:
transaction_entries, _ = client.get(
"/v1/characters/%s/wallet/transactions/?from_id=%s" %
(character_ccp_id, oldest_entry))
oldest_transaction_entry = -1
transactions = []
for entry in transaction_entries:
# keep track of the oldest transaction entry we received for pagination
if entry[
"transaction_id"] < oldest_transaction_entry or oldest_transaction_entry == -1:
oldest_transaction_entry = entry["transaction_id"]
transactions.append(extract_transaction(character, entry))
# pagination logic
if oldest_transaction_entry != -1 and page < 5:
prev_transactions = get_character_transactions(
character_ccp_id=character_ccp_id,
oldest_entry=oldest_transaction_entry - 1,
page=page + 1)
else:
prev_transactions = []
return transactions + prev_transactions
def update_character_affiliations(character_list, chunk_size=990):
if not character_list:
return
changed_corp_counter = 0
client = EsiClient()
# chunk into 990 sized lists
for post_body in [
character_list[x:x + chunk_size]
for x in xrange(0, len(character_list), chunk_size)
]:
affiliations, _ = client.post("/v1/characters/affiliation/",
post_body=post_body)
if not affiliations:
logger.exception("no affiliations returned from %s" % post_body)
return
for affiliation in affiliations:
char = EVEPlayerCharacter.objects.get(
pk=affiliation["character_id"])
if "corporation_id" in affiliation:
set_corporation_affiliation(affiliation["corporation_id"],
affiliation.get("alliance_id"))
if char.corporation.id != int(affiliation["corporation_id"]):
corp = EVEPlayerCorporation.get_object(
affiliation["corporation_id"])
changed_corp_counter += 1
char.corporation = corp
char.save()
logger.info("updated corporation of %s characters" % changed_corp_counter)
# now update the publicdata_last_updated in bulk
EVEPlayerCharacter.objects.filter(pk__in=character_list).update(
publicdata_last_updated=timezone.now())
def import_universe_constellation(constellation_id):
"""
Imports the constellation with the given constellation id along with the constellation's systems.
Attempts to import constellations systems even if constellation is already imported.
:param self:
:param constellation_id:
:return:
"""
client = EsiClient()
constellation_data, _ = client.get("/v1/universe/constellations/%s/" %
constellation_id)
if not Constellation.objects.filter(ccp_id=constellation_id).exists():
region = Region.get_object(constellation_data["region_id"])
constellation_obj = Constellation(ccp_id=constellation_id,
name=constellation_data["name"],
region=region)
constellation_obj.save()
raise Exception("redo this group code")
#group(
# import_universe_system.s(system)
# for system in constellation_data["systems"]
#)()
return
def structure_search(char_id, search_string):
char = EVEPlayerCharacter.get_object(char_id)
client = EsiClient(authenticating_character=char,
raise_application_errors=False)
logger.info("search string {}".format(search_string))
search_param = urllib.parse.urlencode({"search": search_string})
logger.info("search param {}".format(search_param))
res, err = client.get(
"/v3/characters/{}/search/?categories=structure,station&language=en-us&{}&strict=false"
.format(char_id, search_param))
if err == EsiError.EsiApplicationError:
return []
results = []
if "structure" in res:
if res["structure"]:
Structure.load_citadels_async(res["structure"], client)
results.extend(res["structure"])
if "station" in res:
if res["station"]:
stations_to_load = []
for station_id in res["station"]:
if not Structure.exists(station_id):
stations_to_load.append(station_id)
else:
results.append(station_id)
Structure.load_stations_async(stations_to_load, client)
results.extend(stations_to_load)
return results
def get_character_location(char_id):
char = EVEPlayerCharacter.get_object(char_id)
if not char.has_esi_scope('esi-location.read_location.v1'):
return None
client = EsiClient(authenticating_character=char)
location, _ = client.get("/v1/characters/%s/location/" % char.pk)
system_id = location["solar_system_id"]
actual_type = CcpIdTypeResolver.get_id_type(system_id)
# fix for CCP-side bug
if actual_type != "system":
structure_id = system_id
structure = Structure.get_object(structure_id, char.pk)
system = structure.location.system if structure.location else None
else:
system = System.get_object(system_id)
# both structures and stations can be handled by structure.getobject
if "structure_id" in location:
structure = Structure.get_object(location["structure_id"], char.pk)
elif "station_id" in location:
structure = Structure.get_object(location["station_id"], char.pk)
else:
structure = None
return {"system": system, "structure": structure}
def import_type_category(ccp_id):
client = EsiClient()
g, _ = client.get("/v1/universe/categories/%s/" % ccp_id)
category = TypeCategory(
ccp_id=ccp_id,
name=g["name"],
)
category.save()
def import_market_group(ccp_id):
client = EsiClient()
g, _ = client.get("/v1/markets/groups/%s/" % ccp_id)
group = MarketGroup(
ccp_id=ccp_id,
name=g["name"],
parent_id=g.get("parent_group_id"),
)
group.save()
def import_type_group(ccp_id):
client = EsiClient()
g,_ = client.get("/v1/universe/groups/%s/" % ccp_id)
group = TypeGroup(
ccp_id = ccp_id,
name = g["name"],
category_id = g["category_id"]
)
group.save()
def resolve_type(ccp_id):
client = EsiClient(log_application_errors=False,
raise_application_errors=False)
_, err = client.get("/v4/characters/%s/" % ccp_id)
if err is None:
return "character"
_, err = client.get("/v4/corporations/%s/" % ccp_id)
if err is None:
return "corporation"
_, err = client.get("/v3/alliances/%s/" % ccp_id)
if err is None:
return "alliance"
return None
def import_station(structure_id):
"""
Imports a station.
:param structure_id:
:return:
"""
client = EsiClient()
structure, _ = client.get("/v2/universe/stations/%s/" % structure_id)
if "error" in structure:
if structure["error"] == "Station not found":
return None
import_station_from_data(structure, structure_id)
def _get_station_orders(station_id):
client = EsiClient()
station = Structure.get_object(station_id, origin_character_id=None)
results = client.get_multiple_paginated("/v1/markets/" +
str(station.location.region.pk) +
"/orders/")
# filter out orders not in our target station
filtered_orders = []
for order in results:
if order["location_id"] == station_id:
filtered_orders.append(order)
return filtered_orders
def import_citadel(structure_id, character):
"""
Imports the citadel with the given structure_id, using esi token of the provided character.
If character doesn't have docking access to the citadel, structure is created with the name "Unknown Structure#<ccpid>"
Note that every Structure object also has a corresponding "Location". If a citadel has to be imported as an unknown
structure, the structure's Location will have jack shit information.
:param structure_id:
:param character:
:return:
"""
client = EsiClient(authenticating_character=character,
log_application_errors=False,
raise_application_errors=False)
structure, err = client.get("/v2/universe/structures/%s/" % structure_id)
import_citadel_from_data(structure, structure_id)
def get_character_balance(character_ccp_id):
"""
Creates a new snapshot of the given character's wallet balance.
:param self:
:param character_ccp_id:
:return:
"""
character = EVEPlayerCharacter.objects.get(pk=character_ccp_id)
if not character.has_esi_scope('esi-wallet.read_character_wallet.v1'):
return None
client = EsiClient(authenticating_character=character)
balance, _ = client.get("/v1/characters/%s/wallet/" % character_ccp_id)
return balance
def import_universe_constellations():
client = EsiClient()
constellations, _ = client.get("/v1/universe/constellations/")
logger.info("dispatching constellation fetches")
constellations_data = client.get_multiple(
"/v1/universe/constellations/{}/", constellations)
constellation_objs = []
for r in constellations_data.values():
constellation_objs.append(
Constellation(ccp_id=r["constellation_id"],
region_id=r["region_id"],
name=r["name"]))
Constellation.objects.bulk_create(constellation_objs)
logger.info("constellations created & committed")
def import_universe_regions():
client = EsiClient()
regions, _ = client.get("/v1/universe/regions/")
logger.info("dispatching region fetches")
regions_data = client.get_multiple("/v1/universe/regions/{}/", regions)
regions_objs = []
for r in regions_data.values():
regions_objs.append(
Region(
ccp_id = r["region_id"],
name = r["name"]
)
)
Region.objects.bulk_create(regions_objs)
logger.info("regions created & committed")
def import_all_type_categories():
client = EsiClient()
category_ids, _ = client.get("/v1/universe/categories/")
group_data = client.get_multiple("/v1/universe/categories/{}/",
category_ids)
logger.info("all category data downloaded")
category_objects = []
for data in group_data.values():
m = TypeCategory(ccp_id=data["category_id"], name=data["name"])
category_objects.append(m)
logger.info("category data objects created")
TypeCategory.objects.bulk_create(category_objects)
logger.info("category data objects committed")
def update_subscription_payments():
logger.info('LAUNCH_TASK update_subscription_payments')
with general_queue.lock_task('update_subscription_payments'):
char = EVEPlayerCharacter.get_object(settings.PAYMENT_CHARACTER_ID)
client = EsiClient(authenticating_character=char)
journal_entries = client.get_multiple_paginated(
"/v4/characters/{}/wallet/journal/".format(char.pk))
for entry in journal_entries:
if entry["ref_type"] == "player_donation" and entry.get(
"second_party_id") == char.pk:
if not SubscriptionPayment.exists(entry["id"]):
process_new_payment(
EVEPlayerCharacter.get_object(entry["first_party_id"]),
entry["amount"], entry["id"],
dateutil.parser.parse(entry["date"]),
entry.get("reason"))
logger.info('FINISH_TASK update_subscription_payments')
def import_all_market_groups():
client = EsiClient()
group_ids, _ = client.get("/v1/markets/groups/")
group_data = client.get_multiple("/v1/markets/groups/{}/", group_ids)
logger.info("all group data downloaded")
group_objects = []
for data in group_data.values():
# notice that this directly references category_id object, we expect categories to already be imported.
m = MarketGroup(ccp_id=data["market_group_id"],
parent_id=data.get("parent_group_id"),
name=data["name"])
group_objects.append(m)
logger.info("group data objects created")
MarketGroup.objects.bulk_create(group_objects)
logger.info("group data objects committed")
def update_corporation_public_details(corporation_id, throttle_updates):
"""
Updates a corporations public details including CEO, member count, affiliation.
By passing throttle_updates=True, this task will only be fully executed once every hour.
If you pass throttle_updates=False, this task will fully complete all its esi calls no matter what.
:param corporation_id:
:param throttle_updates:
:return:
"""
if throttle_updates:
# do not add this key to EVEPlayerCorporation's post_save cache flush
c = cache.get(
"corporation_details_update_throttled_{}".format(corporation_id))
if c:
return
EVEPlayerCorporation_lazy = apps.get_model('eve_api',
'EVEPlayerCorporation')
corporation = EVEPlayerCorporation_lazy.get_object(corporation_id)
client = EsiClient()
corp_details, _ = client.get("/v4/corporations/{}/".format(corporation_id))
if corporation.alliance_id != corp_details.get("alliance_id"):
if corp_details.get("alliance_id"):
_ = EVEPlayerAlliance.get_object(corp_details["alliance_id"])
corporation.alliance_id = corp_details.get("alliance_id")
corporation.member_count = int(corp_details["member_count"])
corporation.tax_rate = float(corp_details["tax_rate"])
if not corporation.ceo_character or corporation.ceo_character.pk != corp_details[
"ceo_id"]:
corporation.ceo_character = EVEPlayerCharacter.get_object(
corp_details["ceo_id"])
corporation.save()
# we only set these once we're certain the update completed successfuly
cache.set("corporation_details_update_throttled_{}".format(corporation_id),
True,
timeout=3600)
_set_cached_corp_affiliation(corporation_id,
corp_details.get("alliance_id"))
def provision_esi_character(char_id, force=False):
# Get or build Character
char, created = EVEPlayerCharacter.objects.get_or_create(id=char_id)
if not created and not force:
return char
esi_client = EsiClient()
character_data, _ = esi_client.get("/v4/characters/%s/" % char_id)
char.name = character_data["name"]
char.security_status = character_data.get("security_status")
char.corporation = EVEPlayerCorporation.get_object(
character_data["corporation_id"])
char.save()
CcpIdTypeResolver.add_type(char_id, "character")
return char
def import_universe_factions():
"""
Imports all EVE Factions
:return:
"""
client = EsiClient()
factions, _ = client.get("/v2/universe/factions/")
for faction in factions:
if Faction.objects.filter(ccp_id=faction["faction_id"]).exists():
continue
else:
f = Faction(
ccp_id=faction["faction_id"],
name=faction["name"],
is_unique=faction["is_unique"],
size_factor=float(faction["size_factor"]),
)
f.save()
def import_all_item_types():
"""
Imports all eve item types. This should only be called once when the database is initially set up.
:param self:
:return:
"""
client = EsiClient()
page_count = client.get_page_count("/v1/universe/types/")
logger.info("{} pages of items to download".format(page_count))
data = client.get_multiple("/v1/universe/types/?page={}", [p+1 for p in range(page_count)])
logger.info("all pages downloaded")
item_ids = []
for page_data in data.values():
item_ids.extend(page_data)
item_data = client.get_multiple("/v3/universe/types/{}/", item_ids)
logger.info("all item data downloaded")
item_objs = []
for item in item_data.values():
i = ObjectType.objects.update_or_create(
ccp_id = item["type_id"],
defaults= {
"name" :item["name"],
"volume" : item.get("volume"),
"packaged_volume" : item.get("packaged_volume"),
"group_id" : item["group_id"],
"icon_id" : item.get("icon_id") if item.get("icon_id") else item["type_id"],
"market_group" : MarketGroup.get_object(item.get("market_group_id")),
"published":item["published"]
}
)
#i.save()
item_objs.append(i)
logger.info("all item data has objects created")
ObjectType.objects.bulk_create(item_objs)
logger.info("all item data committed")
def import_universe_region(region_id):
"""
Imports the region with the given region_id, along with its constellations.
Always imports region's constellations, even if region already existed.
:param self:
:param region_id:
:return:
"""
client = EsiClient()
region_data, _ = client.get("/v1/universe/regions/%s/" % region_id)
if not Region.objects.filter(ccp_id=region_id).exists():
region_obj = Region(ccp_id=region_id, name=region_data["name"])
region_obj.save()
#group(
# import_universe_constellation.s(constellation)
#
#)()
return
def update_corp_member_info(corporation_id):
d_chars = EVEPlayerCharacter.objects.filter(corporation__id=corporation_id, esi_key_may_grant_services=True, roles__name="roleDirector", _esi_roles__esi_corporations_read_corporation_membership_v1=True).all()
if not d_chars:
logger.warning("no available character for membership checking %s" % corporation_id)
return
d_char = d_chars.all()[randint(0, d_chars.count() - 1)]
if not d_char.has_esi_scope('esi-corporations.read_corporation_membership.v1'):
logger.warning("%s [%s] does not have scopes to read membership" % (d_char, d_char.corporation.name))
return
if d_char:
client = EsiClient(authenticating_character=d_char)
res, _ = client.get("/v3/corporations/%s/members/" % d_char.corporation.pk)
if res:
corpmembers = EVEPlayerCharacter.objects.filter(pk__in=res).all()
currentmembers = [c.pk for c in corpmembers]
provision_esi_corporation(corp_id=d_char.corporation.pk, force=True)
missing = list(set(res) - set(currentmembers))
if missing:
for misc in missing:
provision_esi_character(misc)
update_character_affiliations(currentmembers)
def provision_esi_corporation(corp_id, force=False):
EVEPlayerCorporation_lazy = apps.get_model('eve_api',
'EVEPlayerCorporation')
corp, created = EVEPlayerCorporation_lazy.objects.get_or_create(pk=corp_id)
if not created and not force:
return corp
if force:
logger.warning("forcing reload of corporation {}".format(corp_id))
client = EsiClient()
corp_data, _ = client.get("/v4/corporations/{}/".format(corp_id))
corp.name = corp_data["name"]
corp.ticker = corp_data["ticker"]
#description = corp_data.get("description")
#corp.description = description.decode('ascii','ignore').encode("ascii") if description else None
#url = corp_data.get("url")
#corp.url = url.decode('ascii','ignore').encode("ascii") if url else None
corp.tax_rate = corp_data["tax_rate"]
corp.member_count = corp_data["member_count"]
# whoever made the corp model didnt use a bigint and i dont care enough to migrate it
corp.shares = 0
if "alliance_id" in corp_data:
corp.alliance = EVEPlayerAlliance.get_object(corp_data["alliance_id"])
corp.api_last_upated = timezone.now()
corp.save()
# Skip looking up the CEOs for NPC corps and ones with no CEO defined (dead corps)
# this MUST happen after the corp is initially saved.
if corp_id > 1000182 and int(corp_data['ceo_id']) > 1:
corp.ceo_character = EVEPlayerCharacter.get_object(corp_data["ceo_id"])
corp.save()
CcpIdTypeResolver.add_type(corp_id, "corporation")
return corp
def import_all_type_groups():
client = EsiClient()
# very lazy will regret
group_ids, _ = client.get("/v1/universe/groups/?page=1")
group_ids2, _ = client.get("/v1/universe/groups/?page=2")
group_ids.extend(group_ids2)
group_data = client.get_multiple("/v1/universe/groups/{}/", group_ids)
logger.info("all group data downloaded")
group_objects = []
for data in group_data.values():
m = TypeGroup(
ccp_id = data["group_id"],
name = data["name"],
category_id = data["category_id"]
)
group_objects.append(m)
logger.info("group data objects created")
TypeGroup.objects.bulk_create(group_objects)
logger.info("group data objects committed")