def update_items() -> None: """Check current inventory for items not included in master table.""" driver = start_driver() ark_inventory = io.reader("cleaned", "ark_inventory", "parquet", self_schema=True) user_items = io.reader(folder="", name="user_items", ftype="json") items_character = _character_most_items(ark_inventory) update_items = list(set(items_character) - set(user_items)) with tqdm(total=len(update_items), desc="Items for update") as pbar: for item_id in update_items: user_items[item_id] = _get_item_facts(driver, item_id) user_items[item_id]["ahm"] = items_character[item_id] user_items[item_id]["active"] = True user_items[item_id]["ignore"] = False user_items[item_id]["Sell"] = False user_items[item_id]["Buy"] = False user_items[item_id]["make_pass"] = True pbar.update(1) io.writer(user_items, folder="", name="user_items", ftype="json") driver.close()
def analyse_rolling_buyout() -> None: """Builds rolling average of user's auction purchases using beancounter data.""" bean_purchases = io.reader("cleaned", "bean_purchases", "parquet") bean_buys = bean_purchases["item"].isin(utils.user_item_filter("Buy")) bean_purchases = bean_purchases[bean_buys].sort_values( ["item", "timestamp"]) cols = ["item", "buyout_per"] purchase_each = utils.enumerate_quantities(bean_purchases, cols=cols, qty_col="qty") # Needed to ensure that groupby will work for a single item purchase_each.loc[purchase_each.index.max() + 1] = ("dummy", 0) SPAN = cfg.analysis["ROLLING_BUYOUT_SPAN"] ewm = (purchase_each.groupby("item").apply( lambda x: x["buyout_per"].ewm(span=SPAN).mean()).reset_index()) # Get the latest item value latest_item = ewm.groupby("item")["level_1"].max() bean_rolling_buyout = ewm.loc[latest_item].set_index("item")[[ "buyout_per" ]] bean_rolling_buyout = bean_rolling_buyout.drop("dummy").astype(int) bean_rolling_buyout.columns = ["bean_rolling_buyout"] io.writer(bean_rolling_buyout, "intermediate", "bean_rolling_buyout", "parquet")
def _get_item_facts(driver: webdriver, item_id: int) -> Dict[str, Any]: """Given an item_id get info from BB and icon.""" # Get Booty Bay basic data if Path(cfg.data_path, "item_info", f"{item_id}.json").exists(): result = io.reader("item_info", str(item_id), "json") else: result = get_bb_item_page(driver, item_id) io.writer(result, folder="item_info", name=str(item_id), ftype="json") if not result: logger.debug(f"No item info for {item_id}") # continue data = utils.get_bb_fields(result, "stats") history = utils.get_bb_fields(result, "history") item_info = {k: v for k, v in data.items() if k in cfg.item_info_fields} item_info["true_auctionable"] = (bool("vendor_price" not in item_info) and bool(item_info["auctionable"]) and not bool(item_info["vendornpccount"]) and bool(item_info["price"]) and bool(history)) # Get icon if not Path(cfg.data_path, "item_icons", f"{item_info['icon']}.jpg").exists(): url = cfg.icons_path + item_info["icon"] + ".jpg" r = requests.get(url) io.writer(r.content, "item_icons", item_info["icon"], "jpg") return item_info
def produce_activity_tracking() -> None: """Produce chart of item prices, sold and bought for.""" bean_results = io.reader("cleaned", "bean_results", "parquet") bean_results["date"] = bean_results["timestamp"].dt.date.astype( "datetime64") bean_sales = bean_results.groupby(["item", "date"])["buyout_per"].mean() bean_sales.name = "sell_price" bean_purchases = io.reader("cleaned", "bean_purchases", "parquet") bean_purchases["date"] = bean_purchases["timestamp"].dt.date.astype( "datetime64") bean_buys = bean_purchases.groupby(["item", "date"])["buyout_per"].mean() bean_buys.name = "buy_price" bb_history = io.reader("cleaned", "bb_history", "parquet") bb_history = bb_history[bb_history["date"] >= bean_results["date"].min()] bb_history = bb_history.set_index(["item", "date"]) activity = bb_history.join(bean_buys).join(bean_sales) cols = ["silveravg", "buy_price", "sell_price"] user_items = io.reader("", "user_items", "json") item_names = { item_id: v.get("name_enus") for item_id, v in user_items.items() } for item_id, _ in user_items.items(): item = item_names[item_id] if item in activity.index: plt.figure() activity.loc[item][cols].plot(title=f"Historic activity {item}") io.writer(plt, "plots", f"{item}_activity", "png") plt.close()
def produce_item_reporting() -> None: """Collate item information and prepare feasibility chart.""" item_table = io.reader("intermediate", "item_table", "parquet") buy_policy = io.reader("outputs", "buy_policy", "parquet").set_index("item") sell_policy = io.reader("outputs", "sell_policy", "parquet").set_index("item") make_policy = io.reader("outputs", "make_policy", "parquet") item_info = (item_table.join(buy_policy[[ x for x in buy_policy if x not in item_table ]]).join(sell_policy[[ x for x in sell_policy if x not in item_table ]]).join(make_policy[[x for x in make_policy if x not in item_table]])) item_info = item_info[sorted(item_info.columns)] # item_reporting = { # item: pd.DataFrame(item_info.loc[item]).to_html() for item in item_info.index # } io.writer(item_info, "reporting", "item_info", "parquet") listing_profits = io.reader("reporting", "listing_profits", "parquet") MAX_LISTINGS = cfg.analysis["MAX_LISTINGS_PROBABILITY"] for item in listing_profits.columns: plt.figure() listing_profits[item].plot(title=f"List profit {item}") pd.Series([sell_policy.loc[item, "profit_feasible"] * MAX_LISTINGS ]).plot() io.writer(plt, "plots", f"{item}_feasible", "png") plt.close()
def draw_profit_charts() -> None: """Create charts of alltime and individual item profits.""" profits = io.reader("reporting", "profits", "parquet") alltime_profit = ( profits.reset_index().groupby("date")["total_profit"].sum().cumsum() / 10000) tot = int(alltime_profit[-1]) daily = int(tot / alltime_profit.shape[0]) plt.figure() alltime_profit.plot( title=f"Total profit over all items ({tot} gold, {daily} per day)") io.writer(plt, "plots", "_alltime_profit", "png") plt.close() user_items = io.reader("", "user_items", "json") item_names = { item_id: v.get("name_enus") for item_id, v in user_items.items() } for item_id, _ in user_items.items(): item = item_names[item_id] if item in profits.index: plt.figure() (profits.loc[item, "total_profit"].cumsum() / 10000).plot(title=f"Profit {item}") io.writer(plt, "plots", f"{item}_profit", "png") plt.close()
def analyse_listings() -> None: """Convert live listings into single items.""" auc_listings = io.reader("cleaned", "auc_listings", "parquet") user_items = io.reader("", "user_items", "json") auc_listings = auc_listings[auc_listings["item_id"].isin(user_items)] predicted_prices = io.reader("intermediate", "predicted_prices", "parquet") ranges = pd.merge( auc_listings, predicted_prices, how="left", left_on="item", right_index=True, validate="m:1", ) ranges["price_z"] = (ranges["price_per"] - ranges["bbpred_price"]) / ranges["bbpred_std"] cols = ["item", "price_per", "price_z"] listing_each = utils.enumerate_quantities(ranges, cols=cols) listing_each.columns = ["item", "list_price_per", "list_price_z"] io.writer(listing_each, "intermediate", "listing_each", "parquet")
def predict_item_prices() -> None: """Analyse exponential average mean and std of items given 14 day, 2 hour history.""" bb_fortnight = io.reader("cleaned", "bb_fortnight", "parquet") user_items = io.reader("", "user_items", "json") predicted_prices = _predict_item_prices(bb_fortnight, user_items) io.writer(predicted_prices, "intermediate", "predicted_prices", "parquet")
def clean_auctioneer_data() -> None: """Cleans Auctioneer json data into tablular format.""" aucscan_data = io.reader("raw", "aucscan_data", "json") auc_listings_raw = pd.DataFrame(aucscan_data) auc_listings = _process_auctioneer_data(auc_listings_raw) # Saves latest scan to intermediate (immediate) io.writer(auc_listings, "cleaned", "auc_listings", "parquet")
def get_beancounter_data() -> None: """Reads WoW Addon Beancounter lua and saves to local json.""" """Reads Ark Inventory json and parses into tabular format.""" beancounter_data: dict = {} for account_name in cfg.wow.get("accounts", {}): path = utils.make_lua_path(account_name, "BeanCounter") bean = io.reader(name=path, ftype="lua") beancounter_data = utils.source_merge(beancounter_data, bean).copy() io.writer(beancounter_data, "raw", "beancounter_data", "json")
def create_item_inventory() -> None: """Convert Arkinventory tabular data into dataframe of counts for user items.""" ark_inventory = io.reader("cleaned", "ark_inventory", "parquet") item_skeleton = io.reader("cleaned", "item_skeleton", "parquet") user_ahm = item_skeleton.set_index("item_id")["user_ahm"] ark_inventory["ahm"] = ark_inventory["item_id"].replace(user_ahm) ark_inventory["role"] = ( ark_inventory["character"] == ark_inventory["ahm"]).replace({ True: "ahm", False: "char" }) item_inventory = ark_inventory role_types = ["ahm", "char"] assert item_inventory["role"].isin(role_types).all() location_rename = { "Inventory": "bag", "Bank": "bank", "Auctions": "auc", "Mailbox": "mail", } item_inventory["loc_short"] = item_inventory["location"].replace( location_rename) item_inventory["inv"] = ("inv_" + item_inventory["role"] + "_" + item_inventory["loc_short"]) item_inventory = item_inventory.groupby(["inv", "item" ]).sum()["count"].unstack().T # Ensure 9x grid of columns for role in role_types: for loc in location_rename.values(): col = f"inv_{role}_{loc}" if col not in item_inventory.columns: item_inventory[col] = 0 item_inventory = item_inventory.fillna(0).astype(int) # Analyse aggregate; ordering important here item_inventory["inv_total_all"] = item_inventory.sum(axis=1) cols = [x for x in item_inventory.columns if "ahm" in x] item_inventory["inv_total_ahm"] = item_inventory[cols].sum(axis=1) io.writer(item_inventory, "intermediate", "item_inventory", "parquet")
def merge_item_table() -> None: """Combine item information into single master table.""" item_skeleton = io.reader("cleaned", "item_skeleton", "parquet") mat_prices = io.reader("intermediate", "mat_prices", "parquet") # item_facts = io.reader("cleaned", "item_facts", "parquet") item_inventory = io.reader("intermediate", "item_inventory", "parquet") predicted_prices = io.reader("intermediate", "predicted_prices", "parquet") replenish = io.reader("intermediate", "replenish", "parquet") item_table = ( item_skeleton.join(mat_prices).join(predicted_prices).join( item_inventory) # .join(item_facts) .join(replenish)).fillna(0) io.writer(item_table, "intermediate", "item_table", "parquet")
def get_auctioneer_data() -> None: """Reads WoW Addon Auctioneer lua and parses text file into json.""" ahm = utils.get_ahm() path = utils.make_lua_path(ahm["account"], "Auc-ScanData") ropes = io.reader(name=path, ftype="lua", custom="Auc-ScanData") listings = [] for rope in ropes: if len(rope) < 10: continue listings_part = rope.split("},{") listings_part[0] = listings_part[0].split("{{")[1] listings_part[-1] = listings_part[-1].split("},}")[0] listings.extend(listings_part) aucscan_data = [x.split("|")[-1].split(",") for x in listings] io.writer(aucscan_data, "raw", "aucscan_data", "json")
def analyse_buy_policy(MAX_BUY_STD: int = 2) -> None: """Create buy policy.""" logger.debug(f"max buy std {MAX_BUY_STD}") item_table = io.reader("intermediate", "item_table", "parquet") buy_policy = item_table[item_table["user_Buy"] == True] subset_cols = [ "bbpred_price", "bbpred_std", "inv_total_all", "replenish_qty", "user_std_holding", "replenish_z", ] buy_policy = buy_policy[subset_cols] listing_each = io.reader("intermediate", "listing_each", "parquet") listing_each = listing_each.sort_values("list_price_per") rank_list = listing_each.join(buy_policy, on="item").dropna() rank_list["sell_rank"] = rank_list.groupby("item")["list_price_per"].rank( method="max" ) rank_list = rank_list.drop_duplicates() rank_list["updated_rank"] = rank_list["replenish_qty"] - rank_list["sell_rank"] rank_list["updated_replenish_z"] = ( rank_list["updated_rank"] / rank_list["user_std_holding"] ) rank_list["updated_replenish_z"] = rank_list["updated_replenish_z"].clip( upper=MAX_BUY_STD ) rank_list = rank_list[rank_list["updated_replenish_z"] > rank_list["list_price_z"]] io.writer(rank_list, "reporting", "buy_rank", "parquet") buy_policy["buy_price_cap"] = rank_list.groupby("item")["list_price_per"].max() buy_policy["buy_price_cap"] = buy_policy["buy_price_cap"].fillna(1).astype(int) buy_policy.index.name = "item" buy_policy = buy_policy.reset_index() io.writer(buy_policy, "outputs", "buy_policy", "parquet")
def write_buy_policy() -> None: """Writes the buy policy to all accounts.""" buy_policy = io.reader("outputs", "buy_policy", "parquet") cols = ["item", "buy_price_cap"] new_snatch = encode_buy_campaign(buy_policy[cols]) # Read client lua, replace with for account in cfg.wow.get("accounts", []): path = utils.make_lua_path(account_name=account, datasource="Auc-Advanced") data = io.reader(name=path, ftype="lua") snatch = data["AucAdvancedData"]["UtilSearchUiData"]["Current"] snatch["snatch.itemsList"] = {} snatch = snatch["snatch.itemsList"] data["AucAdvancedData"]["UtilSearchUiData"]["Current"][ "snatch.itemsList" ] = new_snatch io.writer(data, name=path, ftype="lua")
def get_bb_data() -> None: """Reads Booty Bay web API data using selenium and blizzard login.""" driver = start_driver() # Get item_ids for user specified items of interest user_items = io.reader("", "user_items", "json") auctionable_items = [ item_id for item_id, v in user_items.items() if v["true_auctionable"] ] # Get bb data from API bb_data: Dict[str, Dict[Any, Any]] = defaultdict(dict) with tqdm(total=len(auctionable_items), desc="Booty Items") as pbar: for item_id in auctionable_items: bb_data[item_id] = get_bb_item_page(driver, item_id) pbar.update(1) driver.close() io.writer(bb_data, "raw", "bb_data", "json")
def get_arkinventory_data() -> None: """Reads WoW Addon Ark Inventory lua data and saves local copy as json.""" acc_inv: dict = {} for account_name in cfg.wow.get("accounts", {}): path = utils.make_lua_path(account_name, "ArkInventory") data = io.reader(name=path, ftype="lua") player_data = data["ARKINVDB"]["global"]["player"]["data"] # Ensure character data does belong to account character_match = [] for server, characters in cfg.wow["accounts"][account_name][ "servers"].items(): for character in characters["characters"]: character_match.append(f"{character} - {server}") for character in player_data.keys(): if character in character_match: acc_inv[character] = player_data[character] io.writer(acc_inv, "raw", "arkinventory_data", "json")
def calculate_inventory_valuation() -> None: """Get total inventory value based on current market price.""" item_inventory = io.reader("intermediate", "item_inventory", "parquet") predicted_prices = io.reader("intermediate", "predicted_prices", "parquet") user_items = io.reader("", "user_items", "json") item_ids = { v.get("name_enus"): item_id for item_id, v in user_items.items() } item_trade = item_inventory.loc[item_inventory.index.isin(item_ids)] bbpred_price = predicted_prices["bbpred_price"] bbpred_price.name = "item" inventory_valuation = item_trade.multiply(bbpred_price, axis=0) inventory_valuation = inventory_valuation.fillna(0).astype(int) io.writer(inventory_valuation, "reporting", "inventory_valuation", "parquet")
def clean_beancounter_data() -> None: """Reads Beancounter json and parses into tabular format.""" data = io.reader("raw", "beancounter_data", "json") item_ids = cfg.get_item_ids_fixed() # Parses all listings into flat python list parsed = [] for server, server_data in data["BeanCounterDB"].items(): for character, auction_data in server_data.items(): for auction_type, item_listings in auction_data.items(): for item_id, listings in item_listings.items(): for _, listing in listings.items(): for auction in listing: parsed.append([auction_type] + [int(item_id)] + [server] + [item_ids[int(item_id)]] + [character] + auction.split(";")) # Setup as pandas dataframe, remove irrelevant columns df = pd.DataFrame(parsed) bean_purchases = _clean_beancounter_purchases(df) io.writer(bean_purchases, "cleaned", "bean_purchases", "parquet") failed = _clean_beancounter_failed(df) success = _clean_beancounter_success(df) bean_results = success.append(failed) bean_results["success"] = bean_results["auction_type"].replace({ "completedAuctions": 1, "failedAuctions": 0 }) io.writer( bean_results, "cleaned", "bean_results", "parquet", self_schema=True, )
def clean_bb_data() -> None: """Parses all Booty Bay item json into tabular formats.""" """Parses all Booty Bay item json into tabular formats.""" item_data = io.reader("raw", "bb_data", "json") user_items = io.reader("", "user_items", "json") bb_fortnight: List = [] bb_history: List = [] bb_alltime: List = [] for item_id, data in item_data.items(): item_name = user_items[item_id].get("name_enus") bb_fortnight_data = pd.DataFrame(utils.get_bb_fields(data, "history")) bb_fortnight_data["snapshot"] = pd.to_datetime( bb_fortnight_data["snapshot"], unit="s") bb_fortnight_data["item"] = item_name bb_fortnight.append(bb_fortnight_data) bb_history_data = pd.DataFrame(data["daily"]) bb_history_data["item"] = item_name bb_history.append(bb_history_data) bb_alltime_data = pd.DataFrame(utils.get_bb_fields(data, "monthly")) bb_alltime_data["item"] = item_name bb_alltime.append(bb_alltime_data) bb_fortnight_df = pd.concat(bb_fortnight) bb_fortnight_df["snapshot"] = pd.to_datetime(bb_fortnight_df["snapshot"]) bb_history_df = pd.concat(bb_history) for col in bb_history_df.columns: if col != "date" and col != "item": bb_history_df[col] = bb_history_df[col].astype(int) bb_history_df["date"] = pd.to_datetime(bb_history_df["date"]) bb_alltime_df = pd.concat(bb_alltime) bb_alltime_df["date"] = pd.to_datetime(bb_alltime_df["date"]) io.writer( bb_fortnight_df, "cleaned", "bb_fortnight", "parquet", self_schema=True, ) io.writer( bb_history_df, "cleaned", "bb_history", "parquet", self_schema=True, ) io.writer( bb_alltime_df, "cleaned", "bb_alltime", "parquet", self_schema=True, )
def write_make_policy() -> None: """Writes the make policy to all accounts.""" make_policy = io.reader("outputs", "make_policy", "parquet") new_craft_queue, item_groups = encode_make_policy(make_policy) ahm = utils.get_ahm() path = utils.make_lua_path( account_name=ahm["account"], datasource="TradeSkillMaster" ) content = io.reader(name=path, ftype="lua", custom="rb") craft_mark = ( f'f@Alliance - {cfg.wow["booty_server"]["server_name"]}@internalData@crafts' ) start, end = utils.find_tsm_marker(content, f'["{craft_mark}"]'.encode("ascii")) crafting_dict = lua.decode("{" + content[start:end].decode("ascii") + "}") for _, item_data in crafting_dict[craft_mark].items(): item_name = item_data.get("name", "_no_name") queued = new_craft_queue.get(item_name, 0) if "queued" in item_data: item_data["queued"] = queued new_craft = utils.dict_to_lua(crafting_dict).encode("ascii") new_craft = new_craft.replace( f"\n{craft_mark}".encode("ascii"), f'\n["{craft_mark}"]'.encode("ascii"), ) content = content[:start] + new_craft + content[end:] # Update item groups groups_mark = '["p@Default@userData@items"]' item_text = f"{groups_mark} = " + "{" for item_code, group in item_groups.items(): item_text += f'["i:{item_code}"] = "{group}", ' item_text += "}" start, end = utils.find_tsm_marker(content, groups_mark.encode("ascii")) content = content[:start] + item_text.encode("ascii") + content[end:] io.writer(content, name=path, ftype="lua", custom="wb")
def write_sell_policy() -> None: """Writes the sell policy to accounts.""" sell_policy = io.reader("outputs", "sell_policy", "parquet") cols = [ "item", "sell_buyout", "sell_bid", "sell_count", "sell_stack", "sell_duration", ] new_appraiser = encode_sell_campaign(sell_policy[cols]) # Read client lua, replace with for account in cfg.wow.get("accounts", []): path = utils.make_lua_path(account_name=account, datasource="Auc-Advanced") data = io.reader(name=path, ftype="lua") data["AucAdvancedConfig"]["profile.Default"]["util"][ "appraiser" ] = new_appraiser io.writer(data, name=path, ftype="lua")
def analyse_material_cost() -> None: """Analyse cost of materials for items, using purchase history or BB predicted price.""" bean_rolling_buyout = io.reader("intermediate", "bean_rolling_buyout", "parquet") item_prices = io.reader("intermediate", "predicted_prices", "parquet") mat_prices = item_prices.join(bean_rolling_buyout) user_items = io.reader("", "user_items", "json") auctionable_items = { item_id: v for item_id, v in user_items.items() if v["true_auctionable"] } r = cfg.analysis["BB_MAT_PRICE_RATIO"] # Material costs are taken as a ratio of booty bay prices, and (recent) actual buyouts mat_prices["material_buyout_cost"] = ( mat_prices["bean_rolling_buyout"].fillna(mat_prices["bbpred_price"]) * (1 - r) + (mat_prices["bbpred_price"] * r)).astype(int) mat_prices["material_make_cost"] = 0 # Determine raw material cost for manufactured items for _, item_details in auctionable_items.items(): item_name = item_details.get("name_enus") material_cost = 0 user_made_from = item_details.get("made_from", {}) if user_made_from: for ingredient, count in user_made_from.items(): material_cost += ( mat_prices.loc[ingredient, "material_buyout_cost"] * count) else: material_cost = mat_prices.loc[item_name, "material_buyout_cost"] mat_prices.loc[item_name, "material_make_cost"] = int(material_cost) mat_prices = mat_prices[["material_buyout_cost", "material_make_cost"]] io.writer(mat_prices, "intermediate", "mat_prices", "parquet")
def analyse_replenishment() -> None: """Determine the demand for item replenishment.""" item_skeleton = io.reader("cleaned", "item_skeleton", "parquet") item_inventory = io.reader("intermediate", "item_inventory", "parquet") replenish = item_skeleton.join(item_inventory).fillna(0) user_items = io.reader("", "user_items", "json") item_ids = { v.get("name_enus"): item_id for item_id, v in user_items.items() } replenish["replenish_qty"] = (replenish["user_mean_holding"] - replenish["inv_total_all"]) # Update replenish list with user_made_from for item, row in replenish.iterrows(): item_id = item_ids[item] if row["replenish_qty"] > 0: for ingredient, count in user_items[item_id].get("made_from", {}).items(): replenish.loc[ingredient, "replenish_qty"] += (count * row["replenish_qty"]) else: # pragma: no cover pass replenish["replenish_z"] = (replenish["replenish_qty"] / replenish["user_std_holding"]) replenish["replenish_z"] = (replenish["replenish_z"].replace([inf, -inf], 0).fillna(0)) replenish = replenish[["replenish_qty", "replenish_z"]] io.writer(replenish, "intermediate", "replenish", "parquet")
def produce_listing_items() -> None: """Generte the item listing on current AH.""" listing_each = io.reader("intermediate", "listing_each", "parquet") item_info = io.reader("reporting", "item_info", "parquet") user_items = io.reader("", "user_items", "json") item_names = { item_id: v.get("name_enus") for item_id, v in user_items.items() } for item_id, _ in user_items.items(): item = item_names[item_id] plt.figure() list_item = listing_each[(listing_each["item"] == item) & (listing_each["list_price_z"] < 10)] list_item = list_item["list_price_per"].sort_values().reset_index( drop=True) list_item.plot(title=f"Current AH listings {item}") pd.Series([item_info.loc[item, "material_make_cost"]] * list_item.shape[0]).plot() io.writer(plt, "plots", f"{item}_listing", "png") plt.close()
def analyse_make_policy() -> None: """Prints what potions to make.""" item_table = io.reader("intermediate", "item_table", "parquet") item_table.index.name = "item" cols = [ "item_id", "user_Make", "user_Sell", "user_make_pass", "user_mean_holding", "inv_total_all", "inv_ahm_bag", "inv_ahm_bank", ] make_policy = item_table[cols] make_policy["make_ideal"] = ( make_policy["user_mean_holding"] - make_policy["inv_total_all"] ) make_policy["make_counter"] = make_policy["make_ideal"].apply(lambda x: max(x, 0)) make_policy["make_mat_available"] = ( make_policy["inv_ahm_bag"] + make_policy["inv_ahm_bank"] ) make_policy["make_actual"] = 0 make_policy["make_mat_flag"] = 0 user_items = io.reader("", "user_items", "json") item_ids = {v.get("name_enus"): item_id for item_id, v in user_items.items()} # Iterates through the table one at a time, to ensure fair distribution of mat usage # Tests if reached counter and is made from stuff # Checks the material count can go down first before decrementing # If after each check, append to list to see for any changes on any pass through change = [True] while any(change): change = [] for item, row in make_policy.iterrows(): item_id = item_ids[item] made_from = user_items[item_id].get("made_from", {}) under_counter = row["make_actual"] < row["make_counter"] user_make_pass = row["user_make_pass"] if made_from and under_counter and not (user_make_pass): item_increment = True for material, qty in made_from.items(): if "Vial" not in material: item_increment = ( make_policy.loc[material, "make_mat_available"] >= qty ) & item_increment if item_increment: for material, qty in user_items[item].get("made_from", {}).items(): make_policy.loc[material, "make_mat_available"] -= qty make_policy.loc[material, "make_mat_flag"] = 1 make_policy.loc[item, "make_actual"] += 1 change.append(item_increment) io.writer(make_policy, "outputs", "make_policy", "parquet")
def analyse_sell_policy( stack: int = 1, max_sell: int = 10, duration: str = "m", MAX_STD: int = 5, MIN_PROFIT: int = 300, MIN_PROFIT_PCT: float = 0.015, ) -> None: """Creates sell policy based on information.""" item_table = io.reader("intermediate", "item_table", "parquet") listing_each = io.reader("intermediate", "listing_each", "parquet") item_volume_change_probability = io.reader( "intermediate", "item_volume_change_probability", "parquet" ) cols = [ "item_deposit", "material_make_cost", "bbpred_std", "bbpred_price", "user_max_sell", "inv_ahm_bag", "replenish_qty", "replenish_z", ] sell_items = item_table[item_table["user_Sell"] == True][cols] sell_items["item_deposit"] = sell_items["item_deposit"] * ( utils.duration_str_to_mins(duration) / (60 * 24) ) sell_items["sell_exp_decay"] = 2 - sell_items["replenish_z"].apply( lambda x: norm.cdf(x) ) listing_each = listing_each[listing_each["list_price_z"] < MAX_STD] listing_each = listing_each.sort_values(["item", "list_price_per"]) listing_each["sell_rank"] = ( listing_each.groupby("item")["list_price_z"].rank(method="first").astype(int) - 1 ) listing_each = pd.merge( item_volume_change_probability, listing_each, how="left", on=["item", "sell_rank"], ) listing_each = listing_each.set_index(["item"]) listing_each["list_price_z"] = listing_each["list_price_z"].fillna(MAX_STD) gouge_price = sell_items["bbpred_price"] + (sell_items["bbpred_std"] * MAX_STD) listing_each["list_price_per"] = ( listing_each["list_price_per"].fillna(gouge_price).astype(int) ) listing_each = listing_each.reset_index().sort_values(["item", "sell_rank"]) listing_profits = pd.merge( listing_each, sell_items, how="left", left_on="item", right_index=True ) listing_profits["sell_buyout"] = listing_profits["list_price_per"] - 9 listing_profits["sell_estimated_profit"] = ( (listing_profits["sell_buyout"] * 0.95 - listing_profits["material_make_cost"]) * (listing_profits["sell_probability"] ** listing_profits["sell_exp_decay"]) ) - (listing_profits["item_deposit"] * (1 - listing_profits["sell_probability"])) best_profits_ind = listing_profits.groupby("item")["sell_estimated_profit"].idxmax() sell_policy = listing_profits.loc[best_profits_ind] sell_policy["profit_min"] = MIN_PROFIT sell_policy["profit_pct"] = MIN_PROFIT_PCT * sell_policy["bbpred_price"] sell_policy["profit_feasible"] = sell_policy[["profit_min", "profit_pct"]].max( axis=1 ) # sell_policy["profit_infeasible"] = ( # sell_policy["profit_feasible"] > sell_policy["sell_estimated_profit"] # ) # Shows the amount required to be profitable sell_policy["sell_bid"] = ( sell_policy["sell_buyout"] - sell_policy["sell_estimated_profit"] + sell_policy["profit_feasible"] ) low_bid_ind = sell_policy[ sell_policy["sell_bid"] < sell_policy["sell_buyout"] ].index sell_policy.loc[low_bid_ind, "sell_bid"] = sell_policy.loc[ low_bid_ind, "sell_buyout" ] sell_policy["sell_duration"] = utils.duration_str_to_mins(duration) sell_policy = sell_policy.sort_values("sell_estimated_profit", ascending=False) sell_policy["sell_stack"] = stack sell_policy["user_max_sell"] = sell_policy["user_max_sell"].replace(0, max_sell) sell_policy["sell_count"] = sell_policy[["inv_ahm_bag", "user_max_sell"]].min( axis=1 ) sell_policy["sell_count"] = ( sell_policy["sell_count"] / sell_policy["sell_stack"] ).astype(int) # TODO Here is likely where we can make it respect min_holding sell_policy["sell_min"] = sell_policy[["user_max_sell", "inv_ahm_bag"]].min(axis=1) adjust_stack = sell_policy[ sell_policy["sell_min"] < sell_policy["sell_stack"] ].index sell_policy.loc[adjust_stack, "sell_stack"] = 1 sell_policy.loc[adjust_stack, "sell_count"] = sell_policy.loc[ adjust_stack, "sell_min" ] io.writer(sell_policy, "outputs", "sell_policy", "parquet") listing_profits = listing_profits.set_index(["sell_rank", "item"])[ "sell_estimated_profit" ].unstack() io.writer(listing_profits, "reporting", "listing_profits", "parquet")
def report_profits() -> None: """Compare purchases and sales to expected value to derive profit from action.""" bean_results = io.reader("cleaned", "bean_results", "parquet") bean_purchases = io.reader("cleaned", "bean_purchases", "parquet") bb_history = io.reader("cleaned", "bb_history", "parquet") user_items = io.reader("", "user_items", "json") item_names = { item_id: v.get("name_enus") for item_id, v in user_items.items() } item_ids = { v.get("name_enus"): item_id for item_id, v in user_items.items() } bean_results["date"] = bean_results["timestamp"].dt.date.astype( "datetime64") bean_results["profit"] = bean_results["received"].fillna( -bean_results["item_deposit"]) bean_results["qty_change"] = bean_results.apply( lambda x: -x["qty"] if x["auction_type"] == "completedAuctions" else 0, axis=1) bean_purchases["date"] = bean_purchases["timestamp"].dt.date.astype( "datetime64") bean_purchases["qty_change"] = bean_purchases["qty"] bean_purchases["profit"] = -bean_purchases["buyout"] purchase_change = bean_purchases.groupby(["item", "date" ])[["qty_change", "profit"]].sum() purchase_change.columns = ["purchase_qty_change", "purchase_profit"] result_change = bean_results.groupby(["auction_type", "item", "date"])[["qty_change", "profit"]].sum() completed_change = result_change.loc["completedAuctions"] completed_change.columns = ["completed_qty_change", "completed_profit"] failed_change = result_change.loc["failedAuctions"] failed_change.columns = ["failed_qty_change", "failed_profit"] bb_history = bb_history[bb_history["date"] >= bean_results["date"].min()] bb_history = bb_history.set_index(["item", "date"]) profits = (bb_history.join(purchase_change).join(completed_change).join( failed_change).fillna(0).astype(int)) profits["total_action"] = profits[[x for x in profits if "_profit" in x]].sum(axis=1) profits["total_qty"] = profits[[x for x in profits if "_qty_change" in x]].sum(axis=1) # vector style material cost calculation material_update = [] for item_id, item_details in user_items.items(): item_name = item_names[item_id] if item_name in profits.index: material_cost = pd.Series(0, index=profits.loc[item_name].index, name="silveravg") user_made_from = item_details.get("made_from", {}) if user_made_from: for ingredient, count in user_made_from.items(): if ingredient in profits.index: material_cost += profits.loc[ingredient, "silveravg"] * count else: material_cost += ( user_items[item_ids[ingredient]]["vendor_price"] * count) else: material_cost = profits.loc[item_name, "silveravg"] material_cost = material_cost.reset_index() material_cost["item"] = item_name material_update.append(material_cost) material_updates = pd.concat(material_update) profits = profits.join(material_updates.set_index(["item", "date"])["silveravg"], rsuffix="_cost") profits[ "total_materials"] = -profits["silveravg_cost"] * profits["total_qty"] profits[ "total_profit"] = profits["total_action"] - profits["total_materials"] io.writer(profits, "reporting", "profits", "parquet")
def clean_item_skeleton() -> None: """Creates basic dataframe from user items information.""" user_items = io.reader("", "user_items", "json") item_facts = pd.DataFrame(user_items).T item_facts.index.name = "item_id" # Add made_from as a json string on item_id item_facts["made_from"] = False for item_id, facts in user_items.items(): item_facts.loc[item_id, "made_from"] = bool(facts.get("made_from", False)) item_facts = item_facts.reset_index() item_facts = item_facts.rename(columns={"name_enus": "item"}) item_facts = item_facts.set_index("item") # # Rename fields and set index user_columns = [ "ahm", "active", "ignore", "Sell", "Buy", "made_from", "max_holding", "max_sell", "mean_holding", "min_holding", "std_holding", "vendor_price", "make_pass", ] item_facts = item_facts.rename( columns={k: f"user_{k}" for k in user_columns}) item_fact_columns = [ "icon", "stacksize", "selltovendor", "auctionable", "price", "vendornpccount", "true_auctionable", ] item_facts = item_facts.rename( columns={k: f"item_{k}" for k in item_fact_columns}) # Ensure user columns exist user_items_ensure_columns = [ "user_min_holding", "user_max_holding", "user_max_sell", "user_Buy", "user_Sell", "user_Make", "user_made_from", "user_make_pass", "user_vendor_price", ] for col in user_items_ensure_columns: if col not in item_facts: item_facts[col] = nan # # Additional standardization and cleaning item_facts["item_deposit"] = (item_facts["item_selltovendor"] / 20 * 12).astype(int) int_cols = [ "user_min_holding", "user_max_holding", "user_vendor_price", "item_id" ] item_facts[int_cols] = item_facts[int_cols].fillna(0).astype(int) item_facts["user_std_holding"] = ( item_facts["user_max_holding"] - item_facts["user_min_holding"]) / cfg.analysis["USER_STD_SPREAD"] item_facts["user_mean_holding"] = (item_facts[[ "user_min_holding", "user_max_holding" ]].mean(axis=1).astype(int)) item_facts["user_Make"] = item_facts["user_made_from"] & ( item_facts["user_make_pass"] == False) item_facts = item_facts.drop("user_made_from", axis=1) bool_cols = ["user_Buy", "user_Sell", "user_Make", "user_make_pass"] item_facts[bool_cols] = item_facts[bool_cols].fillna(False).astype(int) io.writer(item_facts, "cleaned", "item_skeleton", "parquet")
def clean_arkinventory_data(run_dt: dt) -> None: """Reads Ark Inventory json and parses into tabular format.""" inventory_data = io.reader("raw", "arkinventory_data", "json") raw_data: list = [] monies: Dict[str, int] = {} for character, character_data in inventory_data.items(): character_name = character.split(" ")[0] character_money = int(character_data.get("info").get("money", 0)) monies[character] = character_money # Get Bank, Inventory, Character, Mailbox etc location_slots = character_data.get("location", []) for lkey in location_slots: items: Dict[str, int] = defaultdict(int) if str(lkey) not in cfg.location_info: # pragma: no cover continue else: loc_name = cfg.location_info[str(lkey)] location_slot = location_slots[lkey] if location_slot: bag_slots = location_slot["bag"] # Get the items from each of the bags, add to master list for bag in bag_slots: for item in bag.get("slot", []): # Must have item details, a count and must not be a soulbound item if item.get("h") and item.get( "count") and item.get("sb") != 3: item_name: str = item.get("h").split("[")[1].split( "]")[0] item_id: str = item.get("h").split( "tem:")[1].split(":")[0] items[f"{item_id}_{item_name}"] += item.get( "count") for item_details, item_count in items.items(): item_id, item_name = item_details.split("_", 1) raw_data.append( (character_name, loc_name, item_id, item_name, item_count)) # Convert information to dataframe cols = ["character", "location", "item_id", "item", "count"] ark_inventory = pd.DataFrame(raw_data) ark_inventory.columns = cols ark_inventory["item_id"] = ark_inventory["item_id"].astype(int) ark_inventory["timestamp"] = run_dt io.writer( ark_inventory, "cleaned", "ark_inventory", "parquet", self_schema=True, ) ark_monies = pd.Series(monies) ark_monies.name = "monies" ark_monies.index.name = "character" ark_monies = pd.DataFrame(ark_monies) ark_monies["timestamp"] = run_dt io.writer( ark_monies, "cleaned", "ark_monies", "parquet", self_schema=True, )