def import_universe_systems(): client = EsiClient() systems, _ = client.get("/v1/universe/systems/") logger.info("dispatching system fetches") system_data = client.get_multiple("/v4/universe/systems/{}/", systems) system_objs = [] for r in system_data.values(): system_objs.append( System( ccp_id = r["system_id"], constellation_id = r["constellation_id"], name = r["name"], security_status = r["security_status"] ) ) System.objects.bulk_create(system_objs) logger.info("systems created & committed") # gen location data location_objs = [] for system in System.objects.all(): location_objs.append( Location( ccp_id = system.pk, system = system, constellation = system.constellation, region = system.constellation.region, root_location_id = system.pk, is_in_space = True ) ) Location.objects.bulk_create(location_objs) logger.info("system locations created & committed")
def import_all_item_types(): """ Imports all eve item types. This should only be called once when the database is initially set up. :param self: :return: """ client = EsiClient() page_count = client.get_page_count("/v1/universe/types/") logger.info("{} pages of items to download".format(page_count)) data = client.get_multiple("/v1/universe/types/?page={}", [p+1 for p in range(page_count)]) logger.info("all pages downloaded") item_ids = [] for page_data in data.values(): item_ids.extend(page_data) item_data = client.get_multiple("/v3/universe/types/{}/", item_ids) logger.info("all item data downloaded") item_objs = [] for item in item_data.values(): i = ObjectType.objects.update_or_create( ccp_id = item["type_id"], defaults= { "name" :item["name"], "volume" : item.get("volume"), "packaged_volume" : item.get("packaged_volume"), "group_id" : item["group_id"], "icon_id" : item.get("icon_id") if item.get("icon_id") else item["type_id"], "market_group" : MarketGroup.get_object(item.get("market_group_id")), "published":item["published"] } ) #i.save() item_objs.append(i) logger.info("all item data has objects created") ObjectType.objects.bulk_create(item_objs) logger.info("all item data committed")
def import_universe_constellations(): client = EsiClient() constellations, _ = client.get("/v1/universe/constellations/") logger.info("dispatching constellation fetches") constellations_data = client.get_multiple( "/v1/universe/constellations/{}/", constellations) constellation_objs = [] for r in constellations_data.values(): constellation_objs.append( Constellation(ccp_id=r["constellation_id"], region_id=r["region_id"], name=r["name"])) Constellation.objects.bulk_create(constellation_objs) logger.info("constellations created & committed")
def import_universe_regions(): client = EsiClient() regions, _ = client.get("/v1/universe/regions/") logger.info("dispatching region fetches") regions_data = client.get_multiple("/v1/universe/regions/{}/", regions) regions_objs = [] for r in regions_data.values(): regions_objs.append( Region( ccp_id = r["region_id"], name = r["name"] ) ) Region.objects.bulk_create(regions_objs) logger.info("regions created & committed")
def import_all_type_categories(): client = EsiClient() category_ids, _ = client.get("/v1/universe/categories/") group_data = client.get_multiple("/v1/universe/categories/{}/", category_ids) logger.info("all category data downloaded") category_objects = [] for data in group_data.values(): m = TypeCategory(ccp_id=data["category_id"], name=data["name"]) category_objects.append(m) logger.info("category data objects created") TypeCategory.objects.bulk_create(category_objects) logger.info("category data objects committed")
def import_all_market_groups(): client = EsiClient() group_ids, _ = client.get("/v1/markets/groups/") group_data = client.get_multiple("/v1/markets/groups/{}/", group_ids) logger.info("all group data downloaded") group_objects = [] for data in group_data.values(): # notice that this directly references category_id object, we expect categories to already be imported. m = MarketGroup(ccp_id=data["market_group_id"], parent_id=data.get("parent_group_id"), name=data["name"]) group_objects.append(m) logger.info("group data objects created") MarketGroup.objects.bulk_create(group_objects) logger.info("group data objects committed")
def import_all_type_groups(): client = EsiClient() # very lazy will regret group_ids, _ = client.get("/v1/universe/groups/?page=1") group_ids2, _ = client.get("/v1/universe/groups/?page=2") group_ids.extend(group_ids2) group_data = client.get_multiple("/v1/universe/groups/{}/", group_ids) logger.info("all group data downloaded") group_objects = [] for data in group_data.values(): m = TypeGroup( ccp_id = data["group_id"], name = data["name"], category_id = data["category_id"] ) group_objects.append(m) logger.info("group data objects created") TypeGroup.objects.bulk_create(group_objects) logger.info("group data objects committed")
def update_region_market_history(region_id): logger.info("LAUNCH_TASK {} {}".format("update_region_market_history", region_id)) with history_queue.lock_task( 'update-region-market-history-{}'.format(region_id)): logger.info( "update_region_market_history {} LOCK ACQUIRED".format(region_id)) # drop dbconn here logger.info( "resetting django db connection for update_region_market_history {}" .format(region_id)) connection.connect() scan_log = MarketHistoryScanLog(region=Region.get_object(region_id)) scan_log.save() items = ObjectType.get_all_tradeable_items() client = EsiClient(raise_application_errors=False) esi_url = "/v1/markets/{}/history/".format(region_id) + "?type_id={}" item_histories = client.get_multiple(esi_url, items) # check for items removed from the game/trading to_delete = [] logger.info( "Done pulling item market history, pruning unpublished items") for object_id, h in item_histories.items(): if type(h) is dict: if "error" in h: if h["error"] == 'Type not found!': ObjectType.verify_object_exists(object_id, force=True) object = ObjectType.get_object(object_id) if not object.published or not object.market_group: to_delete.append(object_id) logger.info( "Setting object_id {} to unpublished".format( object_id)) else: msg = "receiving unrecognized application error from market query. object id {} error {}".format( object_id, h) logger.error(msg) raise Exception(msg) else: logger.warning( "dict-based item history detected {} {}".format( object_id, h)) for i in to_delete: del item_histories[i] logger.info( "Done pruneing unpublished items. Yanking existing markethistory entries out of the database" ) # grab existing entries existing_entries = _get_existing_region_entries(region_id, 30) logger.info("Done yanking old data. Building table of new data...") new_entries_to_commit = [] new_entry_count = 0 for object_id, history in item_histories.items(): existing = existing_entries[ object_id] if object_id in existing_entries else [] # only use the last 31 days of history data (ccp sorts for us) history = history[-31:] new_entries = update_market_history_for_item( region_id, object_id, history, existing) new_entries_to_commit.extend(new_entries) new_entry_count += len(new_entries) if new_entry_count > 10000: # early commit logger.info("performing early commit of market history data") MarketHistory.objects.bulk_create(new_entries_to_commit) new_entries_to_commit = [] new_entry_count = 0 if new_entry_count > 0: logger.info("performing final market history data commit") MarketHistory.objects.bulk_create(new_entries_to_commit) logger.info("done creating history items, purging cache") MarketPriceDAO.purge_region_dao_cache(region_id, items) scan_log.scan_complete = timezone.now() scan_log.save() logger.info("market history cache purged. all done")