def generate_data(): """ Generate data for each shoppers for each week for each product combination in preparation for cross-validation :return: cv_dict Collected dictionary of X_train, y_train, X_test, y_test for given week folds """ from tqdm import tqdm from dataloader import Dataloader from dataloader import create_combined_dict import os n_shoppers = 10_000 weeks = [86, 87, 88, 89] shopper_list = list(range(2000)) shopper_chunks = [ shopper_list[i:i + 100] for i in range(0, len(shopper_list), 100) ] cv_dict = { 'X_train': list(), 'y_train': list(), 'X_test': list(), 'y_test': list() } path = os.getcwd() + "data/" data = Dataloader(path=path) data.create_category_table(n_shoppers=n_shoppers) for i, week in enumerate(weeks): print(f"week: {week}") X_train_list = list() y_train_list = list() X_test_list = list() y_test_list = list() for idx, shopper in enumerate(tqdm(shopper_chunks)): # print(f"shopper_chunk index: {idx}") # train-test-split data.train_test_split(week, shopper) # data add categories data.add_categories() # create features data.create_feature_dict() # combine everything if i == 0: X_train, y_train, X_test, y_test = data.make_featured_data() X_train_list.append(X_train) y_train_list.append(y_train) del X_train, y_train else: _, _, X_test, y_test = data.make_featured_data() X_test_list.append(X_test) y_test_list.append(y_test) del X_test, y_test cv_dict = create_combined_dict(X_train_list, y_train_list, X_test_list, y_test_list, cv_dict) return cv_dict
def generate_coupons(): """ Setting up dynamic tables for predictions in week 90 """ path = os.getcwd() + "/../data/" # create dataloader object which loads data which creates baskets and coupons data = Dataloader(path) # Create Categories for products data.create_category_table(10000) shopper_list = list(range(2000)) shopper_chunks = [ shopper_list[i:i + 100] for i in range(0, len(shopper_list), 100) ] X_train_list = list() y_train_list = list() # generates training date over weeks 0,...89 for idx, shopper in enumerate(shopper_chunks): print(f"shopper_chunk index: {idx}") X_train, y_train = data.baskets_train_prediction(shopper) X_train_list.append(X_train) y_train_list.append(y_train) del X_train, y_train X_train_df = pd.concat(X_train_list, ignore_index=True) X_train_df['shopper'] = X_train_df['shopper'].astype('category') X_train_df['product'] = X_train_df['product'].astype('category') y_train_df = pd.concat(y_train_list, ignore_index=True) y_train_df = y_train_df.astype('category') # training data with tuned hyperparameters with open(os.getcwd() + "/pickle/best_param.pickle", 'rb') as f: best_params = pickle.load(f) lgb_clf = LGBMClassifier(**best_params) lgb_clf.fit(X_train.drop('week'), y_train) # create table for prediction pred_table = pd.DataFrame({ 'week': np.full(250, 90), 'shopper': np.full(250, shopper), 'product': list(range(250)) }) n_coupons = 5 shoppers = list(range(2000)) discounts = [0.15, 0.2, 0.25, 0.3] # Randomly initialize (product, discount) prod_discount_dict = { 0: (0, 0.15), 1: (1, 0.2), 2: (2, 0.2), 3: (3, 0.25), 4: (4, 0.3) } # save all coupons for each shopper shopper_coupon_dict = {shopper: {} for shopper in shoppers} expected_revenue = 0 for shopper in shoppers: prod_discount_dict = { 0: (0, 0.15), 1: (1, 0.2), 2: (2, 0.2), 3: (3, 0.25), 4: (4, 0.3) } for coupon in range(n_coupons): for product in list(range(250)): for discount in discounts: prod_discount_dict_temp = prod_discount_dict.copy() prod_discount_dict_temp[coupon] = (product, discount) X_predict = create_prediction_tables( shopper, prod_discount_dict_temp) prob = lgb_clf.predict_proba(X_predict)[:, 1] expected_revenue_temp = np.matmul(X_predict['price'], prob) if expected_revenue_temp > expected_revenue: expected_revenue = expected_revenue_temp prod_discount_dict = prod_discount_dict_temp shopper_coupon_dict[shopper] = prod_discount_dict return shopper_coupon_dict