def update_market_history():
session = DBSession()
with transaction.manager:
last_tick = session.query(func.max(MarketHistoryElement.ticks)).scalar()
current_date = datetime.datetime.now(tzlocal())
current_tick = int((current_date - MarketHistoryElement.START_ERA).total_seconds() / MarketHistoryElement.TICK_SECONDS_LENGTH)
assert last_tick <= current_tick
if last_tick == current_tick:
logger.debug("Skipping update to market history: tick %d already saved.", current_tick)
return
origin_select = session.\
query(Item.data_id,
literal_column(str(current_tick)),
Item.buy_count,
Item.buy_price,
Item.sell_count,
Item.sell_price).\
filter(Item.buy_count > 0, Item.sell_count > 0)
i = insert(MarketHistoryElement).from_select([
MarketHistoryElement.item_id,
MarketHistoryElement.ticks,
MarketHistoryElement.buy_count,
MarketHistoryElement.buy_price,
MarketHistoryElement.sell_count,
MarketHistoryElement.sell_price
], origin_select)
logger.debug("Executing market history insert...")
i.execute()
logger.debug("Saved market data for tick %d.", current_tick)
def update_gw2db(settings):
logger.info('Starting GW2DB import.')
source_items = get_raw_data_from_settings(settings)
now = datetime.datetime.now()
# import the items
logger.info('Import items...')
i = 0
to_create = []
amount_per_block = 5000
grouped = itertools.groupby(enumerate(source_items), lambda (index, si): index / amount_per_block)
for _, elements in grouped:
session = DBSession()
with transaction.manager:
elements = list(elements)
logger.info("Analyzing %d elements (%d%%)...", len(elements), elements[-1][0] * 100 / len(source_items))
min_data_id = elements[0][1]['DataID']
max_data_id = elements[-1][1]['DataID']
existing_items = dict(session.query(Item.data_id, Item).\
filter(Item.data_id >= min_data_id).\
filter(Item.data_id <= max_data_id))
for idx, source_item in elements:
try:
# update / create object
item = _create_item(source_item, now, existing_items)
session.add(item)
except:
logger.error('Error while importing element "%(Name)s" (id = %(ID)d)',
source_item)
raise
logger.info('Elements analyzed. Committing...')
logger.info('Committed.')
logger.info('GW2DB import completed.')
def load_skill_point_recipes():
logger.info("Importing skill point recipes in database...")
session = DBSession()
with transaction.manager:
for recipe in get_recipe_list():
# compute the object
skill_point_recipe = recipe.get_skill_point_recipe(session)
# look if it already exists
try:
session.query(SkillPointRecipe).filter(
SkillPointRecipe.ingredient_1_item_id == skill_point_recipe.ingredient_1_item_id,
SkillPointRecipe.ingredient_2_item_id == skill_point_recipe.ingredient_2_item_id,
SkillPointRecipe.ingredient_3_item_id == skill_point_recipe.ingredient_3_item_id,
SkillPointRecipe.ingredient_4_item_id == skill_point_recipe.ingredient_4_item_id,
).one()
# it already exists: nothing more to do
logger.debug("Recipe for %s already existing: skip.", recipe.item.name)
continue
except NoResultFound:
# it didn't exist: save it in the session
logger.info("Saving recipe for %s in the session.", recipe.item.name)
session.add(skill_point_recipe)
logger.debug("Committing...")
logger.info("All known skill point recipes imported.")
def update_vendor(settings):
logger.debug("Importing vendor data...")
gw2db_data = get_raw_data_from_settings(settings)
session = DBSession()
with transaction.manager:
# get out all existing VendorData
existing_data_map = dict(session.query(VendorData.item_id, VendorData))
already_added = set()
# update or add the karma/gold cost data
for entry in gw2db_data:
if len(entry['SoldBy']) > 0:
data_id = entry['DataID']
karma_cost = None
copper_cost = None
for sb in entry['SoldBy']:
if 'KarmaCost' in sb:
karma_cost = sb['KarmaCost']
if 'GoldCost' in sb:
copper_cost = sb['GoldCost']
if karma_cost is not None or copper_cost is not None:
vd = _get_entry(data_id, existing_data_map, session, already_added)
vd.karma_cost = karma_cost
vd.copper_cost = copper_cost
# update skill point costs
for name, cost in _name_to_sp_cost.items():
data_id = session.query(Item.data_id).filter_by(name=name).first()[0]
vd = _get_entry(data_id, existing_data_map, session, already_added)
vd.skill_point_cost = cost
# update special karma costs (bundles, sun beads)
for entry in _karma_materials_start_data:
data_id = session.query(Item.data_id).filter_by(name=name).first()[0]
vd = _get_entry(data_id, existing_data_map, session, already_added)
vd.karma_cost = entry['cost']
# delete unneeded elements
for data_id, el in existing_data_map.items():
if data_id not in already_added:
session.delete(el)
logger.debug("Importing vendor data done.")