def update_gw2db(settings): logger.info('Starting GW2DB import.') source_items = get_raw_data_from_settings(settings) now = datetime.datetime.now() # import the items logger.info('Import items...') i = 0 to_create = [] amount_per_block = 5000 grouped = itertools.groupby(enumerate(source_items), lambda (index, si): index / amount_per_block) for _, elements in grouped: session = DBSession() with transaction.manager: elements = list(elements) logger.info("Analyzing %d elements (%d%%)...", len(elements), elements[-1][0] * 100 / len(source_items)) min_data_id = elements[0][1]['DataID'] max_data_id = elements[-1][1]['DataID'] existing_items = dict(session.query(Item.data_id, Item).\ filter(Item.data_id >= min_data_id).\ filter(Item.data_id <= max_data_id)) for idx, source_item in elements: try: # update / create object item = _create_item(source_item, now, existing_items) session.add(item) except: logger.error('Error while importing element "%(Name)s" (id = %(ID)d)', source_item) raise logger.info('Elements analyzed. Committing...') logger.info('Committed.') logger.info('GW2DB import completed.')
def load_skill_point_recipes(): logger.info("Importing skill point recipes in database...") session = DBSession() with transaction.manager: for recipe in get_recipe_list(): # compute the object skill_point_recipe = recipe.get_skill_point_recipe(session) # look if it already exists try: session.query(SkillPointRecipe).filter( SkillPointRecipe.ingredient_1_item_id == skill_point_recipe.ingredient_1_item_id, SkillPointRecipe.ingredient_2_item_id == skill_point_recipe.ingredient_2_item_id, SkillPointRecipe.ingredient_3_item_id == skill_point_recipe.ingredient_3_item_id, SkillPointRecipe.ingredient_4_item_id == skill_point_recipe.ingredient_4_item_id, ).one() # it already exists: nothing more to do logger.debug("Recipe for %s already existing: skip.", recipe.item.name) continue except NoResultFound: # it didn't exist: save it in the session logger.info("Saving recipe for %s in the session.", recipe.item.name) session.add(skill_point_recipe) logger.debug("Committing...") logger.info("All known skill point recipes imported.")