def draw_profit_charts() -> None:
"""Create charts of alltime and individual item profits."""
profits = io.reader("reporting", "profits", "parquet")
alltime_profit = (
profits.reset_index().groupby("date")["total_profit"].sum().cumsum() /
10000)
tot = int(alltime_profit[-1])
daily = int(tot / alltime_profit.shape[0])
plt.figure()
alltime_profit.plot(
title=f"Total profit over all items ({tot} gold, {daily} per day)")
io.writer(plt, "plots", "_alltime_profit", "png")
plt.close()
user_items = io.reader("", "user_items", "json")
item_names = {
item_id: v.get("name_enus")
for item_id, v in user_items.items()
}
for item_id, _ in user_items.items():
item = item_names[item_id]
if item in profits.index:
plt.figure()
(profits.loc[item, "total_profit"].cumsum() /
10000).plot(title=f"Profit {item}")
io.writer(plt, "plots", f"{item}_profit", "png")
plt.close()
def produce_activity_tracking() -> None:
"""Produce chart of item prices, sold and bought for."""
bean_results = io.reader("cleaned", "bean_results", "parquet")
bean_results["date"] = bean_results["timestamp"].dt.date.astype(
"datetime64")
bean_sales = bean_results.groupby(["item", "date"])["buyout_per"].mean()
bean_sales.name = "sell_price"
bean_purchases = io.reader("cleaned", "bean_purchases", "parquet")
bean_purchases["date"] = bean_purchases["timestamp"].dt.date.astype(
"datetime64")
bean_buys = bean_purchases.groupby(["item", "date"])["buyout_per"].mean()
bean_buys.name = "buy_price"
bb_history = io.reader("cleaned", "bb_history", "parquet")
bb_history = bb_history[bb_history["date"] >= bean_results["date"].min()]
bb_history = bb_history.set_index(["item", "date"])
activity = bb_history.join(bean_buys).join(bean_sales)
cols = ["silveravg", "buy_price", "sell_price"]
user_items = io.reader("", "user_items", "json")
item_names = {
item_id: v.get("name_enus")
for item_id, v in user_items.items()
}
for item_id, _ in user_items.items():
item = item_names[item_id]
if item in activity.index:
plt.figure()
activity.loc[item][cols].plot(title=f"Historic activity {item}")
io.writer(plt, "plots", f"{item}_activity", "png")
plt.close()
def update_items() -> None:
"""Check current inventory for items not included in master table."""
driver = start_driver()
ark_inventory = io.reader("cleaned",
"ark_inventory",
"parquet",
self_schema=True)
user_items = io.reader(folder="", name="user_items", ftype="json")
items_character = _character_most_items(ark_inventory)
update_items = list(set(items_character) - set(user_items))
with tqdm(total=len(update_items), desc="Items for update") as pbar:
for item_id in update_items:
user_items[item_id] = _get_item_facts(driver, item_id)
user_items[item_id]["ahm"] = items_character[item_id]
user_items[item_id]["active"] = True
user_items[item_id]["ignore"] = False
user_items[item_id]["Sell"] = False
user_items[item_id]["Buy"] = False
user_items[item_id]["make_pass"] = True
pbar.update(1)
io.writer(user_items, folder="", name="user_items", ftype="json")
driver.close()
def clean_bb_data() -> None:
"""Parses all Booty Bay item json into tabular formats."""
"""Parses all Booty Bay item json into tabular formats."""
item_data = io.reader("raw", "bb_data", "json")
user_items = io.reader("", "user_items", "json")
bb_fortnight: List = []
bb_history: List = []
bb_alltime: List = []
for item_id, data in item_data.items():
item_name = user_items[item_id].get("name_enus")
bb_fortnight_data = pd.DataFrame(utils.get_bb_fields(data, "history"))
bb_fortnight_data["snapshot"] = pd.to_datetime(
bb_fortnight_data["snapshot"], unit="s")
bb_fortnight_data["item"] = item_name
bb_fortnight.append(bb_fortnight_data)
bb_history_data = pd.DataFrame(data["daily"])
bb_history_data["item"] = item_name
bb_history.append(bb_history_data)
bb_alltime_data = pd.DataFrame(utils.get_bb_fields(data, "monthly"))
bb_alltime_data["item"] = item_name
bb_alltime.append(bb_alltime_data)
bb_fortnight_df = pd.concat(bb_fortnight)
bb_fortnight_df["snapshot"] = pd.to_datetime(bb_fortnight_df["snapshot"])
bb_history_df = pd.concat(bb_history)
for col in bb_history_df.columns:
if col != "date" and col != "item":
bb_history_df[col] = bb_history_df[col].astype(int)
bb_history_df["date"] = pd.to_datetime(bb_history_df["date"])
bb_alltime_df = pd.concat(bb_alltime)
bb_alltime_df["date"] = pd.to_datetime(bb_alltime_df["date"])
io.writer(
bb_fortnight_df,
"cleaned",
"bb_fortnight",
"parquet",
self_schema=True,
)
io.writer(
bb_history_df,
"cleaned",
"bb_history",
"parquet",
self_schema=True,
)
io.writer(
bb_alltime_df,
"cleaned",
"bb_alltime",
"parquet",
self_schema=True,
)
def produce_item_reporting() -> None:
"""Collate item information and prepare feasibility chart."""
item_table = io.reader("intermediate", "item_table", "parquet")
buy_policy = io.reader("outputs", "buy_policy",
"parquet").set_index("item")
sell_policy = io.reader("outputs", "sell_policy",
"parquet").set_index("item")
make_policy = io.reader("outputs", "make_policy", "parquet")
item_info = (item_table.join(buy_policy[[
x for x in buy_policy if x not in item_table
]]).join(sell_policy[[
x for x in sell_policy if x not in item_table
]]).join(make_policy[[x for x in make_policy if x not in item_table]]))
item_info = item_info[sorted(item_info.columns)]
# item_reporting = {
# item: pd.DataFrame(item_info.loc[item]).to_html() for item in item_info.index
# }
io.writer(item_info, "reporting", "item_info", "parquet")
listing_profits = io.reader("reporting", "listing_profits", "parquet")
MAX_LISTINGS = cfg.analysis["MAX_LISTINGS_PROBABILITY"]
for item in listing_profits.columns:
plt.figure()
listing_profits[item].plot(title=f"List profit {item}")
pd.Series([sell_policy.loc[item, "profit_feasible"] * MAX_LISTINGS
]).plot()
io.writer(plt, "plots", f"{item}_feasible", "png")
plt.close()
def analyse_listings() -> None:
"""Convert live listings into single items."""
auc_listings = io.reader("cleaned", "auc_listings", "parquet")
user_items = io.reader("", "user_items", "json")
auc_listings = auc_listings[auc_listings["item_id"].isin(user_items)]
predicted_prices = io.reader("intermediate", "predicted_prices", "parquet")
ranges = pd.merge(
auc_listings,
predicted_prices,
how="left",
left_on="item",
right_index=True,
validate="m:1",
)
ranges["price_z"] = (ranges["price_per"] -
ranges["bbpred_price"]) / ranges["bbpred_std"]
cols = ["item", "price_per", "price_z"]
listing_each = utils.enumerate_quantities(ranges, cols=cols)
listing_each.columns = ["item", "list_price_per", "list_price_z"]
io.writer(listing_each, "intermediate", "listing_each", "parquet")
def predict_item_prices() -> None:
"""Analyse exponential average mean and std of items given 14 day, 2 hour history."""
bb_fortnight = io.reader("cleaned", "bb_fortnight", "parquet")
user_items = io.reader("", "user_items", "json")
predicted_prices = _predict_item_prices(bb_fortnight, user_items)
io.writer(predicted_prices, "intermediate", "predicted_prices", "parquet")
def create_item_inventory() -> None:
"""Convert Arkinventory tabular data into dataframe of counts for user items."""
ark_inventory = io.reader("cleaned", "ark_inventory", "parquet")
item_skeleton = io.reader("cleaned", "item_skeleton", "parquet")
user_ahm = item_skeleton.set_index("item_id")["user_ahm"]
ark_inventory["ahm"] = ark_inventory["item_id"].replace(user_ahm)
ark_inventory["role"] = (
ark_inventory["character"] == ark_inventory["ahm"]).replace({
True:
"ahm",
False:
"char"
})
item_inventory = ark_inventory
role_types = ["ahm", "char"]
assert item_inventory["role"].isin(role_types).all()
location_rename = {
"Inventory": "bag",
"Bank": "bank",
"Auctions": "auc",
"Mailbox": "mail",
}
item_inventory["loc_short"] = item_inventory["location"].replace(
location_rename)
item_inventory["inv"] = ("inv_" + item_inventory["role"] + "_" +
item_inventory["loc_short"])
item_inventory = item_inventory.groupby(["inv", "item"
]).sum()["count"].unstack().T
# Ensure 9x grid of columns
for role in role_types:
for loc in location_rename.values():
col = f"inv_{role}_{loc}"
if col not in item_inventory.columns:
item_inventory[col] = 0
item_inventory = item_inventory.fillna(0).astype(int)
# Analyse aggregate; ordering important here
item_inventory["inv_total_all"] = item_inventory.sum(axis=1)
cols = [x for x in item_inventory.columns if "ahm" in x]
item_inventory["inv_total_ahm"] = item_inventory[cols].sum(axis=1)
io.writer(item_inventory, "intermediate", "item_inventory", "parquet")
def _get_item_facts(driver: webdriver, item_id: int) -> Dict[str, Any]:
"""Given an item_id get info from BB and icon."""
# Get Booty Bay basic data
if Path(cfg.data_path, "item_info", f"{item_id}.json").exists():
result = io.reader("item_info", str(item_id), "json")
else:
result = get_bb_item_page(driver, item_id)
io.writer(result, folder="item_info", name=str(item_id), ftype="json")
if not result:
logger.debug(f"No item info for {item_id}")
# continue
data = utils.get_bb_fields(result, "stats")
history = utils.get_bb_fields(result, "history")
item_info = {k: v for k, v in data.items() if k in cfg.item_info_fields}
item_info["true_auctionable"] = (bool("vendor_price" not in item_info)
and bool(item_info["auctionable"])
and not bool(item_info["vendornpccount"])
and bool(item_info["price"])
and bool(history))
# Get icon
if not Path(cfg.data_path, "item_icons",
f"{item_info['icon']}.jpg").exists():
url = cfg.icons_path + item_info["icon"] + ".jpg"
r = requests.get(url)
io.writer(r.content, "item_icons", item_info["icon"], "jpg")
return item_info
def analyse_rolling_buyout() -> None:
"""Builds rolling average of user's auction purchases using beancounter data."""
bean_purchases = io.reader("cleaned", "bean_purchases", "parquet")
bean_buys = bean_purchases["item"].isin(utils.user_item_filter("Buy"))
bean_purchases = bean_purchases[bean_buys].sort_values(
["item", "timestamp"])
cols = ["item", "buyout_per"]
purchase_each = utils.enumerate_quantities(bean_purchases,
cols=cols,
qty_col="qty")
# Needed to ensure that groupby will work for a single item
purchase_each.loc[purchase_each.index.max() + 1] = ("dummy", 0)
SPAN = cfg.analysis["ROLLING_BUYOUT_SPAN"]
ewm = (purchase_each.groupby("item").apply(
lambda x: x["buyout_per"].ewm(span=SPAN).mean()).reset_index())
# Get the latest item value
latest_item = ewm.groupby("item")["level_1"].max()
bean_rolling_buyout = ewm.loc[latest_item].set_index("item")[[
"buyout_per"
]]
bean_rolling_buyout = bean_rolling_buyout.drop("dummy").astype(int)
bean_rolling_buyout.columns = ["bean_rolling_buyout"]
io.writer(bean_rolling_buyout, "intermediate", "bean_rolling_buyout",
"parquet")
def merge_item_table() -> None:
"""Combine item information into single master table."""
item_skeleton = io.reader("cleaned", "item_skeleton", "parquet")
mat_prices = io.reader("intermediate", "mat_prices", "parquet")
# item_facts = io.reader("cleaned", "item_facts", "parquet")
item_inventory = io.reader("intermediate", "item_inventory", "parquet")
predicted_prices = io.reader("intermediate", "predicted_prices", "parquet")
replenish = io.reader("intermediate", "replenish", "parquet")
item_table = (
item_skeleton.join(mat_prices).join(predicted_prices).join(
item_inventory)
# .join(item_facts)
.join(replenish)).fillna(0)
io.writer(item_table, "intermediate", "item_table", "parquet")
def have_in_bag() -> str:
"""Prints expected profits, make sure its in your bag."""
sell_policy = io.reader("outputs", "sell_policy", "parquet")
sell_policy = sell_policy.set_index("item")
sell_policy = sell_policy[
sell_policy["sell_estimated_profit"] > sell_policy["profit_feasible"]]
return sell_policy[["sell_estimated_profit"]].astype(int).to_html()
def profit_per_item() -> str:
"""Profits per item as HTML."""
profits = io.reader("reporting", "profits", "parquet")
item_profits = ((profits.groupby("item")[["total_profit"]].sum() /
10000).astype(int).sort_values(by="total_profit",
ascending=False))
return item_profits.to_html()
def analyse_buy_policy(MAX_BUY_STD: int = 2) -> None:
"""Create buy policy."""
logger.debug(f"max buy std {MAX_BUY_STD}")
item_table = io.reader("intermediate", "item_table", "parquet")
buy_policy = item_table[item_table["user_Buy"] == True]
subset_cols = [
"bbpred_price",
"bbpred_std",
"inv_total_all",
"replenish_qty",
"user_std_holding",
"replenish_z",
]
buy_policy = buy_policy[subset_cols]
listing_each = io.reader("intermediate", "listing_each", "parquet")
listing_each = listing_each.sort_values("list_price_per")
rank_list = listing_each.join(buy_policy, on="item").dropna()
rank_list["sell_rank"] = rank_list.groupby("item")["list_price_per"].rank(
method="max"
)
rank_list = rank_list.drop_duplicates()
rank_list["updated_rank"] = rank_list["replenish_qty"] - rank_list["sell_rank"]
rank_list["updated_replenish_z"] = (
rank_list["updated_rank"] / rank_list["user_std_holding"]
)
rank_list["updated_replenish_z"] = rank_list["updated_replenish_z"].clip(
upper=MAX_BUY_STD
)
rank_list = rank_list[rank_list["updated_replenish_z"] > rank_list["list_price_z"]]
io.writer(rank_list, "reporting", "buy_rank", "parquet")
buy_policy["buy_price_cap"] = rank_list.groupby("item")["list_price_per"].max()
buy_policy["buy_price_cap"] = buy_policy["buy_price_cap"].fillna(1).astype(int)
buy_policy.index.name = "item"
buy_policy = buy_policy.reset_index()
io.writer(buy_policy, "outputs", "buy_policy", "parquet")
def get_beancounter_data() -> None:
"""Reads WoW Addon Beancounter lua and saves to local json."""
"""Reads Ark Inventory json and parses into tabular format."""
beancounter_data: dict = {}
for account_name in cfg.wow.get("accounts", {}):
path = utils.make_lua_path(account_name, "BeanCounter")
bean = io.reader(name=path, ftype="lua")
beancounter_data = utils.source_merge(beancounter_data, bean).copy()
io.writer(beancounter_data, "raw", "beancounter_data", "json")
def clean_auctioneer_data() -> None:
"""Cleans Auctioneer json data into tablular format."""
aucscan_data = io.reader("raw", "aucscan_data", "json")
auc_listings_raw = pd.DataFrame(aucscan_data)
auc_listings = _process_auctioneer_data(auc_listings_raw)
# Saves latest scan to intermediate (immediate)
io.writer(auc_listings, "cleaned", "auc_listings", "parquet")
def grand_total() -> Dict[str, int]:
"""Returns total inventory and money value."""
ark_monies = io.reader("cleaned", "ark_monies", "parquet")
inventory_valuation = io.reader("reporting", "inventory_valuation",
"parquet")
inventory_value_total = ((inventory_valuation["inv_total_all"].sum() /
10000).round(0).astype(int))
money_total = (ark_monies["monies"].sum() / 10000).round(0).astype(int)
grand_total = inventory_value_total + money_total
total_holdings = {
"Inventory": inventory_value_total,
"Money": money_total,
"Grand Total": grand_total,
}
return total_holdings
def write_buy_policy() -> None:
"""Writes the buy policy to all accounts."""
buy_policy = io.reader("outputs", "buy_policy", "parquet")
cols = ["item", "buy_price_cap"]
new_snatch = encode_buy_campaign(buy_policy[cols])
# Read client lua, replace with
for account in cfg.wow.get("accounts", []):
path = utils.make_lua_path(account_name=account, datasource="Auc-Advanced")
data = io.reader(name=path, ftype="lua")
snatch = data["AucAdvancedData"]["UtilSearchUiData"]["Current"]
snatch["snatch.itemsList"] = {}
snatch = snatch["snatch.itemsList"]
data["AucAdvancedData"]["UtilSearchUiData"]["Current"][
"snatch.itemsList"
] = new_snatch
io.writer(data, name=path, ftype="lua")
def inventory_valuation() -> str:
"""Profits per item as HTML."""
inventory_valuation = io.reader("reporting", "inventory_valuation",
"parquet")
inventory_valuation = ((inventory_valuation["inv_total_all"] /
10000).round(0).astype(int).sort_values(
ascending=False))
inventory_valuation.name = "inventory_valuation"
inventory_valuation = pd.DataFrame(inventory_valuation)
return inventory_valuation.to_html()
def calculate_inventory_valuation() -> None:
"""Get total inventory value based on current market price."""
item_inventory = io.reader("intermediate", "item_inventory", "parquet")
predicted_prices = io.reader("intermediate", "predicted_prices", "parquet")
user_items = io.reader("", "user_items", "json")
item_ids = {
v.get("name_enus"): item_id
for item_id, v in user_items.items()
}
item_trade = item_inventory.loc[item_inventory.index.isin(item_ids)]
bbpred_price = predicted_prices["bbpred_price"]
bbpred_price.name = "item"
inventory_valuation = item_trade.multiply(bbpred_price, axis=0)
inventory_valuation = inventory_valuation.fillna(0).astype(int)
io.writer(inventory_valuation, "reporting", "inventory_valuation",
"parquet")
def write_make_policy() -> None:
"""Writes the make policy to all accounts."""
make_policy = io.reader("outputs", "make_policy", "parquet")
new_craft_queue, item_groups = encode_make_policy(make_policy)
ahm = utils.get_ahm()
path = utils.make_lua_path(
account_name=ahm["account"], datasource="TradeSkillMaster"
)
content = io.reader(name=path, ftype="lua", custom="rb")
craft_mark = (
f'f@Alliance - {cfg.wow["booty_server"]["server_name"]}@internalData@crafts'
)
start, end = utils.find_tsm_marker(content, f'["{craft_mark}"]'.encode("ascii"))
crafting_dict = lua.decode("{" + content[start:end].decode("ascii") + "}")
for _, item_data in crafting_dict[craft_mark].items():
item_name = item_data.get("name", "_no_name")
queued = new_craft_queue.get(item_name, 0)
if "queued" in item_data:
item_data["queued"] = queued
new_craft = utils.dict_to_lua(crafting_dict).encode("ascii")
new_craft = new_craft.replace(
f"\n{craft_mark}".encode("ascii"), f'\n["{craft_mark}"]'.encode("ascii"),
)
content = content[:start] + new_craft + content[end:]
# Update item groups
groups_mark = '["p@Default@userData@items"]'
item_text = f"{groups_mark} = " + "{"
for item_code, group in item_groups.items():
item_text += f'["i:{item_code}"] = "{group}", '
item_text += "}"
start, end = utils.find_tsm_marker(content, groups_mark.encode("ascii"))
content = content[:start] + item_text.encode("ascii") + content[end:]
io.writer(content, name=path, ftype="lua", custom="wb")
def write_sell_policy() -> None:
"""Writes the sell policy to accounts."""
sell_policy = io.reader("outputs", "sell_policy", "parquet")
cols = [
"item",
"sell_buyout",
"sell_bid",
"sell_count",
"sell_stack",
"sell_duration",
]
new_appraiser = encode_sell_campaign(sell_policy[cols])
# Read client lua, replace with
for account in cfg.wow.get("accounts", []):
path = utils.make_lua_path(account_name=account, datasource="Auc-Advanced")
data = io.reader(name=path, ftype="lua")
data["AucAdvancedConfig"]["profile.Default"]["util"][
"appraiser"
] = new_appraiser
io.writer(data, name=path, ftype="lua")
def analyse_material_cost() -> None:
"""Analyse cost of materials for items, using purchase history or BB predicted price."""
bean_rolling_buyout = io.reader("intermediate", "bean_rolling_buyout",
"parquet")
item_prices = io.reader("intermediate", "predicted_prices", "parquet")
mat_prices = item_prices.join(bean_rolling_buyout)
user_items = io.reader("", "user_items", "json")
auctionable_items = {
item_id: v
for item_id, v in user_items.items() if v["true_auctionable"]
}
r = cfg.analysis["BB_MAT_PRICE_RATIO"]
# Material costs are taken as a ratio of booty bay prices, and (recent) actual buyouts
mat_prices["material_buyout_cost"] = (
mat_prices["bean_rolling_buyout"].fillna(mat_prices["bbpred_price"]) *
(1 - r) + (mat_prices["bbpred_price"] * r)).astype(int)
mat_prices["material_make_cost"] = 0
# Determine raw material cost for manufactured items
for _, item_details in auctionable_items.items():
item_name = item_details.get("name_enus")
material_cost = 0
user_made_from = item_details.get("made_from", {})
if user_made_from:
for ingredient, count in user_made_from.items():
material_cost += (
mat_prices.loc[ingredient, "material_buyout_cost"] * count)
else:
material_cost = mat_prices.loc[item_name, "material_buyout_cost"]
mat_prices.loc[item_name, "material_make_cost"] = int(material_cost)
mat_prices = mat_prices[["material_buyout_cost", "material_make_cost"]]
io.writer(mat_prices, "intermediate", "mat_prices", "parquet")
def analyse_replenishment() -> None:
"""Determine the demand for item replenishment."""
item_skeleton = io.reader("cleaned", "item_skeleton", "parquet")
item_inventory = io.reader("intermediate", "item_inventory", "parquet")
replenish = item_skeleton.join(item_inventory).fillna(0)
user_items = io.reader("", "user_items", "json")
item_ids = {
v.get("name_enus"): item_id
for item_id, v in user_items.items()
}
replenish["replenish_qty"] = (replenish["user_mean_holding"] -
replenish["inv_total_all"])
# Update replenish list with user_made_from
for item, row in replenish.iterrows():
item_id = item_ids[item]
if row["replenish_qty"] > 0:
for ingredient, count in user_items[item_id].get("made_from",
{}).items():
replenish.loc[ingredient,
"replenish_qty"] += (count *
row["replenish_qty"])
else: # pragma: no cover
pass
replenish["replenish_z"] = (replenish["replenish_qty"] /
replenish["user_std_holding"])
replenish["replenish_z"] = (replenish["replenish_z"].replace([inf, -inf],
0).fillna(0))
replenish = replenish[["replenish_qty", "replenish_z"]]
io.writer(replenish, "intermediate", "replenish", "parquet")
def produce_listing_items() -> None:
"""Generte the item listing on current AH."""
listing_each = io.reader("intermediate", "listing_each", "parquet")
item_info = io.reader("reporting", "item_info", "parquet")
user_items = io.reader("", "user_items", "json")
item_names = {
item_id: v.get("name_enus")
for item_id, v in user_items.items()
}
for item_id, _ in user_items.items():
item = item_names[item_id]
plt.figure()
list_item = listing_each[(listing_each["item"] == item)
& (listing_each["list_price_z"] < 10)]
list_item = list_item["list_price_per"].sort_values().reset_index(
drop=True)
list_item.plot(title=f"Current AH listings {item}")
pd.Series([item_info.loc[item, "material_make_cost"]] *
list_item.shape[0]).plot()
io.writer(plt, "plots", f"{item}_listing", "png")
plt.close()
def get_auctioneer_data() -> None:
"""Reads WoW Addon Auctioneer lua and parses text file into json."""
ahm = utils.get_ahm()
path = utils.make_lua_path(ahm["account"], "Auc-ScanData")
ropes = io.reader(name=path, ftype="lua", custom="Auc-ScanData")
listings = []
for rope in ropes:
if len(rope) < 10:
continue
listings_part = rope.split("},{")
listings_part[0] = listings_part[0].split("{{")[1]
listings_part[-1] = listings_part[-1].split("},}")[0]
listings.extend(listings_part)
aucscan_data = [x.split("|")[-1].split(",") for x in listings]
io.writer(aucscan_data, "raw", "aucscan_data", "json")
def get_bb_data() -> None:
"""Reads Booty Bay web API data using selenium and blizzard login."""
driver = start_driver()
# Get item_ids for user specified items of interest
user_items = io.reader("", "user_items", "json")
auctionable_items = [
item_id for item_id, v in user_items.items() if v["true_auctionable"]
]
# Get bb data from API
bb_data: Dict[str, Dict[Any, Any]] = defaultdict(dict)
with tqdm(total=len(auctionable_items), desc="Booty Items") as pbar:
for item_id in auctionable_items:
bb_data[item_id] = get_bb_item_page(driver, item_id)
pbar.update(1)
driver.close()
io.writer(bb_data, "raw", "bb_data", "json")
def get_arkinventory_data() -> None:
"""Reads WoW Addon Ark Inventory lua data and saves local copy as json."""
acc_inv: dict = {}
for account_name in cfg.wow.get("accounts", {}):
path = utils.make_lua_path(account_name, "ArkInventory")
data = io.reader(name=path, ftype="lua")
player_data = data["ARKINVDB"]["global"]["player"]["data"]
# Ensure character data does belong to account
character_match = []
for server, characters in cfg.wow["accounts"][account_name][
"servers"].items():
for character in characters["characters"]:
character_match.append(f"{character} - {server}")
for character in player_data.keys():
if character in character_match:
acc_inv[character] = player_data[character]
io.writer(acc_inv, "raw", "arkinventory_data", "json")
def encode_make_policy(
make_policy: pd.DataFrame,
) -> Tuple[Dict[str, int], Dict[int, str]]:
"""Encodes make campaign dataframe into dictionary."""
make_policy = io.reader("outputs", "make_policy", "parquet")
new_craft_queue = make_policy[
(make_policy["user_make_pass"] == 0) & (make_policy["make_actual"] > 0)
]["make_actual"].to_dict()
# Ordering important here for overwrites
make_policy["group"] = "Other"
make_policy.loc[make_policy[make_policy["user_Sell"] == 1].index, "group"] = "Sell"
make_policy.loc[
make_policy[make_policy["make_mat_flag"] == 1].index, "group"
] = "Materials"
make_policy = make_policy[make_policy["item_id"] > 0]
item_groups = make_policy.set_index(["item_id"])["group"].to_dict()
return new_craft_queue, item_groups
def clean_beancounter_data() -> None:
"""Reads Beancounter json and parses into tabular format."""
data = io.reader("raw", "beancounter_data", "json")
item_ids = cfg.get_item_ids_fixed()
# Parses all listings into flat python list
parsed = []
for server, server_data in data["BeanCounterDB"].items():
for character, auction_data in server_data.items():
for auction_type, item_listings in auction_data.items():
for item_id, listings in item_listings.items():
for _, listing in listings.items():
for auction in listing:
parsed.append([auction_type] + [int(item_id)] +
[server] + [item_ids[int(item_id)]] +
[character] + auction.split(";"))
# Setup as pandas dataframe, remove irrelevant columns
df = pd.DataFrame(parsed)
bean_purchases = _clean_beancounter_purchases(df)
io.writer(bean_purchases, "cleaned", "bean_purchases", "parquet")
failed = _clean_beancounter_failed(df)
success = _clean_beancounter_success(df)
bean_results = success.append(failed)
bean_results["success"] = bean_results["auction_type"].replace({
"completedAuctions":
1,
"failedAuctions":
0
})
io.writer(
bean_results,
"cleaned",
"bean_results",
"parquet",
self_schema=True,
)
