def target():
ID = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "id.pkl")
sales = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "sales.pkl")
df = pd.DataFrame({"id": ID, "sales": sales})
df = reduce_mem_usage(df)
df["shift_28_roll_365"] = df.groupby(
["id"])["sales"].transform(lambda x: x.shift(28).rolling(365).mean())
df["sales_residual_diff_28_roll_365"] = df["sales"] - df[
"shift_28_roll_365"]
df["shift_28_roll_365"].to_pickle(jsn['FEATURE_TARGET_DIR'] +
"shift_28_roll_365.pkl")
df["sales_residual_diff_28_roll_365"].to_pickle(
jsn['FEATURE_TARGET_DIR'] + "sales_residual_diff_28_roll_365.pkl")
del df, ID, sales
gc.collect()
def make_lag_df():
'''
Create data frames for Lag features
Create 28 dataframes from day 1 to day 28
'''
################################# Make DataFrame
#################################################################################
for SHIFT_DAY in range(1, 29):
if (SHIFT_DAY >= 1) & (SHIFT_DAY <= 7):
LAG_DAY = 7
elif (SHIFT_DAY >= 8) & (SHIFT_DAY <= 14):
LAG_DAY = 14
elif (SHIFT_DAY >= 15) & (SHIFT_DAY <= 21):
LAG_DAY = 21
elif (SHIFT_DAY >= 22) & (SHIFT_DAY <= 28):
LAG_DAY = 28
##Basic
#folder select
folder1 = jsn["FEATURE_BASIC_DIR"]
folder2 = jsn["FEATURE_ENCODING_DIR"]
folder4 = jsn["FEATURE_LAG_DIR"]
df = pd.DataFrame([])
######file select
print(f"make Lag-dataframe for day{SHIFT_DAY}")
if SHIFT_DAY in [7, 14, 21, 28]:
flist4 = []
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [2, 3, 4, 8, 12]:
for func in [
"mean",
]:
flist4.append(
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [
4,
8,
]:
for func in ["max", "min"]:
flist4.append(
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [
SHIFT_DAY,
]:
for multi in [2, 3, 5]:
for roll in [3, 6, 10]:
for func in [
"mean",
]:
flist4.append(
f"multi_{multi}_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [7, 14, 30, 60]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_mean.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_std.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_median.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_max.pkl"
)
for shift in [LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_min.pkl"
)
s = SHIFT_DAY
for shift in [
s,
s + 1,
s + 2,
s + 3,
s + 4,
s + 5,
s + 6,
s + 7,
s + 8,
s + 9,
s + 10,
s + 11,
s + 12,
s + 13,
]:
flist4.append(
f"sales_residual_diff_28_roll_365_shift_{shift}.pkl")
else:
flist4 = []
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [2, 3, 4, 8, 12]:
for func in [
"mean",
]:
flist4.append(
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [
4,
8,
]:
for func in ["max", "min"]:
flist4.append(
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [
SHIFT_DAY,
]:
for multi in [2, 3, 5]:
for roll in [3, 6, 10]:
for func in [
"mean",
]:
flist4.append(
f"multi_{multi}_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_{func}.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [7, 14, 30, 60]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_mean.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_std.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_median.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_max.pkl"
)
for shift in [SHIFT_DAY, LAG_DAY, LAG_DAY + 7]:
for roll in [7, 30]:
flist4.append(
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift}_roll_{roll}_min.pkl"
)
s = SHIFT_DAY
for shift in [
s,
s + 1,
s + 2,
s + 3,
s + 4,
s + 5,
s + 6,
s + 7,
s + 8,
s + 9,
s + 10,
s + 11,
s + 12,
s + 13,
]:
flist4.append(
f"sales_residual_diff_28_roll_365_shift_{shift}.pkl")
folders = [
# folder1,
# folder2,
# folder3,
folder4,
]
flists = [
# flist1,
# flist2,
# flist3,
flist4,
]
#make DATAFRAME
for folder, flist in zip(folders, flists):
for filename in flist:
row = pd.read_pickle(folder + filename)
filename = filename[:-4]
df[filename] = row
print("{}".format(folder))
# sales = pd.read_pickle("features/Target/sales_residual_diff_28_roll_365.pkl")
# df["sales"] = sales
df = reduce_mem_usage(df)
# df.to_pickle(f"dataframe/data_base_{term}_df.pkl")
df.to_pickle(jsn["DATAFRAME_DIR"] + f"data_day{SHIFT_DAY}_df.pkl")
print(f"Lag_DataFrame_day{SHIFT_DAY}")
def make_base_df():
'''
Create a basic data frame for training
'''
################################# Make DataFrame
#################################################################################
for term in ["private", "public", "validation", "semival"]:
print(f"make Basic-dataframe for {term}")
##Basic
#folder select
folder1 = jsn["FEATURE_BASIC_DIR"]
folder2 = jsn["FEATURE_ENCODING_DIR"]
folder4 = jsn["FEATURE_LAG_DIR"]
df = pd.DataFrame([])
######file select
## BASIC
flist1 = [
'month.pkl',
'day.pkl',
'price_momentum_y_item_store.pkl',
# 'w_serial.pkl',
'item_id.pkl',
# 'd_serial.pkl',
# 'snap_TX.pkl',
# '.DS_Store',
'last_sales.pkl',
# 'snap_CA.pkl',
'is_weekend.pkl',
# 'sales.pkl',
'sell_start_log.pkl',
'sell_price_minority12.pkl',
'week_of_year.pkl',
'price_momentum_m_item_state.pkl',
'Mothers_day.pkl',
'year.pkl',
'IndependenceDay.pkl',
'olympic_president_elec_year.pkl',
'event_name_1.pkl',
'dept_id.pkl',
'event_name_2.pkl',
'price_unique_item_state.pkl',
'date.pkl',
'id.pkl',
'moon.pkl',
'sell_price.pkl',
'national_holiday.pkl',
'state_id.pkl',
# 'event_type_1.pkl',
'price_momentum_y_item_state.pkl',
'event_type_statecat_labelenc.pkl',
'event_type_2.pkl',
'store_id.pkl',
'snap_total.pkl',
'cat_id.pkl',
# 'snap_WI.pkl',
'price_unique_item_store.pkl',
'sell_start.pkl',
# 'weekday.pkl',
'week_of_month.pkl',
'Easter.pkl',
'wday.pkl',
'price_momentum_m_item_store.pkl',
'OrthodoxEaster.pkl',
'id_serial.pkl',
'Ramadan_Starts.pkl',
'NBA_finals.pkl',
# 'wm_yr_wk.pkl',
# 'd.pkl',
]
# Encoding
flist2 = []
for level in [f"LEVEL{i}" for i in range(2, 13)]:
flist2.append(
f"{term}_sales_residual_diff_28_roll_365_enc_{level}_mean.pkl")
flist2.append(
f"{term}_sales_residual_diff_28_roll_365_enc_{level}_std.pkl")
for diff in [28]:
for level in [f"LEVEL{i}" for i in range(2, 13)]:
for wd in ["week", "day"]:
for func in ["mean", "std"]:
flist2.append(
f"{term}_sales_residual_diff_{diff}_roll_365_enc_{wd}_{level}_{func}.pkl"
)
folders = [
folder1,
folder2,
# folder3,
# folder4,
]
flists = [
flist1,
flist2,
# flist3,
# flist4,
]
#make DATAFRAME
for folder, flist in zip(folders, flists):
for filename in flist:
row = pd.read_pickle(folder + filename)
filename = filename[:-4]
df[filename] = row
print("{}".format(folder))
sales = pd.read_pickle(jsn["FEATURE_TARGET_DIR"] +
"sales_residual_diff_28_roll_365.pkl")
df["sales"] = sales
df = reduce_mem_usage(df)
order = [
'id_serial',
'id',
'is_weekend',
'sell_start',
'date',
'snap_total',
'day',
'sell_price',
'event_name_1',
'week_of_month',
'wday',
'week_of_year',
'sell_start_log',
'Mothers_day',
'national_holiday',
'NBA_finals',
'sell_price_minority12',
'year',
'month',
'olympic_president_elec_year',
'OrthodoxEaster',
'store_id',
'moon',
'cat_id',
'Ramadan_Starts',
'IndependenceDay',
'last_sales',
'event_name_2',
'event_type_2',
'event_type_statecat_labelenc',
'Easter',
'item_id',
'dept_id',
'state_id',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL2_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL2_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL3_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL3_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL4_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL4_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL5_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL5_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL6_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL6_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL7_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL7_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL8_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL8_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL9_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL9_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL10_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL10_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL11_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL11_std',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL12_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_LEVEL12_std',
'price_unique_item_state',
'price_momentum_m_item_state',
'price_momentum_y_item_state',
'price_unique_item_store',
'price_momentum_m_item_store',
'price_momentum_y_item_store',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL2_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL2_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL2_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL2_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL3_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL3_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL3_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL3_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL4_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL4_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL4_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL4_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL5_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL5_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL5_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL5_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL6_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL6_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL6_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL6_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL7_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL7_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL7_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL7_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL8_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL8_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL8_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL8_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL9_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL9_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL9_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL9_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL10_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL10_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL10_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL10_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL11_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL11_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL11_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL11_std',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL12_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_week_LEVEL12_std',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL12_mean',
f'{term}_sales_residual_diff_28_roll_365_enc_day_LEVEL12_std',
'sales',
]
df = df[order]
df.to_pickle(jsn["DATAFRAME_DIR"] + f"data_base_{term}_df.pkl")
print(f"Base_DataFrame_{term}")
def lag():
###### LAG Rolling
#############################
ID = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "id.pkl")
d_serial = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "d_serial.pkl")
sales = pd.read_pickle(jsn['FEATURE_TARGET_DIR'] +
"sales_residual_diff_28_roll_365.pkl")
df = pd.DataFrame({"id": ID, "d_serial": d_serial, "sales": sales})
df = reduce_mem_usage(df)
#### shift 1~ 56
for shift_day in range(1, 57):
df.groupby(
["id"])["sales"].transform(lambda x: x.shift(shift_day)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"sales_residual_diff_28_roll_365_shift_{shift_day}.pkl")
#### rolling
####################
for i in [1, 2, 3, 5, 7]:
df[f"multi_{i}"] = df["d_serial"].transform(lambda x: x % i)
for shift_day in range(1, 36):
target = f"sales_residual_diff_28_roll_365_shift_{shift_day}"
df[target] = pd.read_pickle(
jsn["FEATURE_LAG_DIR"] +
f"sales_residual_diff_28_roll_365_shift_{shift_day}.pkl")
#### multi 2, 3, 5 rolling
for roll_wind in [3, 6, 10]:
df.groupby([
'id', 'multi_2'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_2_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
df.groupby([
'id', 'multi_3'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_3_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
df.groupby([
'id', 'multi_5'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_5_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
#### multi 7 rolling
for roll_wind in [2, 3, 4, 8, 12]:
if roll_wind in [4, 8]:
df.groupby([
'id', 'multi_7'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
df.groupby([
'id', 'multi_7'
])[target].transform(lambda x: x.rolling(roll_wind).max(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_max.pkl"
)
df.groupby([
'id', 'multi_7'
])[target].transform(lambda x: x.rolling(roll_wind).min(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_min.pkl"
)
else:
df.groupby([
'id', 'multi_7'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_7_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
#### multi 1 rolling
for roll_wind in [7, 14, 30, 60]:
if roll_wind in [7, 30]:
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).std(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_std.pkl"
)
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).max(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_max.pkl"
)
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).min(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_min.pkl"
)
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).median(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_median.pkl"
)
else:
df.groupby([
'id', 'multi_1'
])[target].transform(lambda x: x.rolling(roll_wind).mean(
)).to_pickle(
jsn["FEATURE_LAG_DIR"] +
f"multi_1_sales_residual_diff_28_roll_365_shift_{shift_day}_roll_{roll_wind}_mean.pkl"
)
df = df.drop(target, axis=1)
def encoding():
ID = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "id.pkl")
item_id = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "item_id.pkl")
store_id = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "store_id.pkl")
dept_id = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "dept_id.pkl")
state_id = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "state_id.pkl")
cat_id = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "cat_id.pkl")
d_serial = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "d_serial.pkl")
sell_price = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "sell_price.pkl")
wday = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "wday.pkl")
day = pd.read_pickle(jsn['FEATURE_BASIC_DIR'] + "day.pkl")
sales = pd.read_pickle(jsn['FEATURE_TARGET_DIR'] +
"sales_residual_diff_28_roll_365.pkl")
df = pd.DataFrame({
"id": ID,
"item_id": item_id,
"store_id": store_id,
"dept_id": dept_id,
"state_id": state_id,
"cat_id": cat_id,
"d_serial": d_serial,
"sell_price": sell_price,
"wday": wday,
"day": day,
"sales": sales
})
df = reduce_mem_usage(df)
df["flag"] = 0
df.loc[df["sell_price"] >= 0, "flag"] = 1
df.loc[df["flag"] == 0, "sales"] = np.nan
pred_terms = ["private", "public", "validation", "semival"]
for i, term in enumerate(pred_terms):
df.loc[df["d_serial"] >= 1942 - i * 28, "sales"] = np.nan
LEVEL = {
"LEVEL2": ["state_id"],
"LEVEL3": ["store_id"],
"LEVEL4": ["cat_id"],
"LEVEL5": ["dept_id"],
"LEVEL6": ["state_id", "cat_id"],
"LEVEL7": ["state_id", "dept_id"],
"LEVEL8": ["store_id", "cat_id"],
"LEVEL9": ["store_id", "dept_id"],
"LEVEL10": ["item_id"],
"LEVEL11": ["state_id", "item_id"],
"LEVEL12": ["store_id", "item_id"]
}
for key, value in LEVEL.items():
df.groupby(value + ["wday"])["sales"].transform(np.mean).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_week_{key}_mean.pkl"
)
df.groupby(value + ["wday"])["sales"].transform(np.std).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_week_{key}_std.pkl"
)
df.groupby(value + ["day"])["sales"].transform(np.mean).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_day_{key}_mean.pkl"
)
df.groupby(value + ["day"])["sales"].transform(np.std).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_day_{key}_std.pkl"
)
df.groupby(value)["sales"].transform(np.mean).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_{key}_mean.pkl")
df.groupby(value)["sales"].transform(np.std).to_pickle(
jsn["FEATURE_ENCODING_DIR"] +
f"{term}_sales_residual_diff_28_roll_365_enc_{key}_std.pkl")
del df, ID, item_id, store_id, dept_id, state_id, cat_id, d_serial, sell_price, wday, day, sales
gc.collect()
def basic():
train_val_df = pd.read_csv(jsn["TRAIN_DATA_PATH"])
sell_price_df = pd.read_csv(jsn["SELL_PRICES_PATH"])
calendar_df = pd.read_csv(jsn["CALENDAR_PATH"])
### Basic Category ID
day_list = []
for i in range(1942, 1970):
day_list.append("d_{}".format(i))
for day in day_list:
train_val_df[day] = np.nan
train_val_df["id_serial"] = list(range(30490))
melt_sales = pd.melt(train_val_df,
id_vars=[
'id_serial', 'id', 'item_id', 'dept_id', 'cat_id',
'store_id', 'state_id'
],
var_name="d",
value_name="sales")
####### Calendar feature
#event2
calendar_df.loc[calendar_df["event_name_2"] == "Cinco De Mayo",
"event_name_2"] = 0
calendar_df.loc[calendar_df["event_name_2"] == "Easter",
"event_name_2"] = 1
calendar_df.loc[calendar_df["event_name_2"] == "Father's day",
"event_name_2"] = 2
calendar_df.loc[calendar_df["event_name_2"] == "OrthodoxEaster",
"event_name_2"] = 3
calendar_df.loc[calendar_df["event_type_2"] == "Cultural",
"event_type_2"] = 0
calendar_df.loc[calendar_df["event_type_2"] == "Religious",
"event_type_2"] = 1
calendar_df["event_name_2"] = calendar_df["event_name_2"].astype(
np.float16)
calendar_df["event_type_2"] = calendar_df["event_type_2"].astype(
np.float16)
#weekend
calendar_df["is_weekend"] = 0
calendar_df.loc[calendar_df["weekday"] == "Saturday", "is_weekend"] = 1
calendar_df.loc[calendar_df["weekday"] == "Sunday", "is_weekend"] = 1
# d_serial
calendar_df["d_serial"] = calendar_df["d"].apply(lambda x: int(x[2:]))
# w_serial
calendar_df["w_serial"] = 0
cnt = 1
for i in range(calendar_df.shape[0]):
if i % 7 == 0:
cnt += 1
calendar_df.loc[i, "w_serial"] = cnt
# date
calendar_df['date'] = pd.to_datetime(calendar_df['date'],
format='%Y-%m-%d')
# day
calendar_df["day"] = calendar_df["date"].apply(lambda x: x.day)
calendar_df["month"] = calendar_df["date"].apply(lambda x: x.month)
calendar_df["year"] = calendar_df["date"].apply(lambda x: x.year)
# Mooon
dec = decimal.Decimal
def get_moon_phase(d): # 0=new, 4=full; 4 days/phase
diff = d - datetime(2001, 1, 1)
days = dec(diff.days) + (dec(diff.seconds) / dec(86400))
lunations = dec("0.20439731") + (days * dec("0.03386319269"))
phase_index = math.floor((lunations % dec(1) * dec(8)) + dec('0.5'))
return int(phase_index) % 8
calendar_df['moon'] = calendar_df.date.apply(get_moon_phase)
# week of month, year
calendar_df[
"week_of_month"] = calendar_df["w_serial"] - calendar_df.groupby(
["year", "month"])["w_serial"].transform("min") + 1
calendar_df[
"week_of_year"] = calendar_df["w_serial"] - calendar_df.groupby(
["year"])["w_serial"].transform("min") + 1
# olympic_president_elec_year
calendar_df["olympic_president_elec_year"] = 0
calendar_df.loc[calendar_df["year"] == 2012,
"olympic_president_elec_year"] = 1
calendar_df.loc[calendar_df["year"] == 2016,
"olympic_president_elec_year"] = 1
# NBA_Finals
calendar_df["NBA_finals"] = 0
for day in [
"d_123", "d_124", "d_125", "d_126", "d_127", "d_128", "d_129",
"d_130", "d_131", "d_132", "d_133", "d_134", "d_135", "d_501",
"d_502", "d_503", "d_504", "d_505", "d_506", "d_507", "d_508",
"d_509", "d_510", "d_860", "d_861", "d_862", "d_863", "d_864",
"d_865", "d_866", "d_867", "d_868", "d_869", "d_870", "d_871",
"d_872", "d_873", "d_874", "d_1224", "d_1225", "d_1226", "d_1227",
"d_1228", "d_1229", "d_1230", "d_1231", "d_1232", "d_1233",
"d_1234", "d_1588", "d_1589", "d_1590", "d_1591", "d_1592",
"d_1593", "d_1594", "d_1595", "d_1596", "d_1597", "d_1598",
"d_1599", "d_1600", "d_1952", "d_1953", "d_1954", "d_1955",
"d_1956", "d_1957", "d_1958", "d_1959", "d_1960", "d_1961",
"d_1962", "d_1963", "d_1964", "d_1965", "d_1966", "d_1967",
"d_1968", "d_1969"
]:
calendar_df.loc[calendar_df["d"] == day, "NBA_finals"] = 1
# Ramadan start
day_list = []
for i in range(30):
day_list.append("d_{}".format(i + 185))
day_list.append("d_{}".format(i + 539))
day_list.append("d_{}".format(i + 893))
day_list.append("d_{}".format(i + 1248))
day_list.append("d_{}".format(i + 1602))
for i in range(13):
day_list.append("d_{}".format(i + 1957))
day_list
calendar_df["Ramadan_Starts"] = 0
for day in day_list:
calendar_df.loc[calendar_df["d"] == day, "Ramadan_Starts"] = 1
# Mothers day
day_list_1 = ["d_100", "d_471", "d_835", "d_1199", "d_1563", "d_1927"]
day_list_2 = ["d_99", "d_470", "d_834", "d_1198", "d_1562", "d_1926"]
calendar_df["Mothers_day"] = 0
for day in day_list_1:
calendar_df.loc[calendar_df["d"] == day, "Mothers_day"] = 1
for day in day_list_2:
calendar_df.loc[calendar_df["d"] == day, "Mothers_day"] = 2
# OrthodoxEaster
day_list_1 = ["d_86", "d_443", "d_828", "d_1178", "d_1535", "d_1920"]
day_list_2 = ["d_85", "d_442", "d_827", "d_1177", "d_1534", "d_1919"]
calendar_df["OrthodoxEaster"] = 0
for day in day_list_1:
calendar_df.loc[calendar_df["d"] == day, "OrthodoxEaster"] = 1
for day in day_list_2:
calendar_df.loc[calendar_df["d"] == day, "OrthodoxEaster"] = 2
# Easter
day_list_1 = ["d_86", "d_436", "d_793", "d_1178", "d_1528", "d_1885"]
day_list_2 = ["d_85", "d_435", "d_792", "d_1177", "d_1527", "d_1884"]
calendar_df["Easter"] = 0
for day in day_list_1:
calendar_df.loc[calendar_df["d"] == day, "Easter"] = 1
for day in day_list_2:
calendar_df.loc[calendar_df["d"] == day, "Easter"] = 2
# IndependenceDay
day_list_1 = ["d_151", "d_517", "d_882", "d_1247", "d_1612"]
day_list_2 = ["d_152", "d_518", "d_883", "d_1248", "d_1613"]
day_list_3 = ["d_153", "d_519", "d_884", "d_1249", "d_1614"]
day_list_4 = ["d_154", "d_520", "d_885", "d_1250", "d_1615"]
day_list_5 = ["d_155", "d_521", "d_886", "d_1251", "d_1616"]
day_list_6 = ["d_156", "d_522", "d_887", "d_1252", "d_1617"]
day_list_7 = ["d_157", "d_523", "d_888", "d_1253", "d_1618"]
day_list_8 = ["d_158", "d_524", "d_889", "d_1254", "d_1619"]
day_list_9 = ["d_159", "d_525", "d_890", "d_1255", "d_1620"]
day_list_10 = ["d_160", "d_526", "d_891", "d_1256", "d_1621"]
calendar_df["IndependenceDay"] = 0
for day in day_list_1:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 1
for day in day_list_2:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 2
for day in day_list_3:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 3
for day in day_list_4:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 4
for day in day_list_5:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 5
for day in day_list_6:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 6
for day in day_list_7:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 7
for day in day_list_8:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 8
for day in day_list_9:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 9
for day in day_list_10:
calendar_df.loc[calendar_df["d"] == day, "IndependenceDay"] = 10
###### price feature
##############################
sell_price_df["state_id"] = sell_price_df["store_id"].apply(
lambda x: x[:2])
sell_price_df["dept_id"] = sell_price_df["item_id"].apply(lambda x: x[:-4])
sell_price_df["cat_id"] = sell_price_df["dept_id"].apply(lambda x: x[:-2])
sell_price_df["price_unique_item_state"] = sell_price_df.groupby(
['state_id', 'item_id'])['sell_price'].transform('nunique')
sell_price_df["price_unique_item_store"] = sell_price_df.groupby(
['store_id', 'item_id'])['sell_price'].transform('nunique')
calendar_prices = calendar_df[['wm_yr_wk', 'month', 'year']]
calendar_prices = calendar_prices.drop_duplicates(subset=['wm_yr_wk'])
sell_price_df = sell_price_df.merge(
calendar_prices[['wm_yr_wk', 'month', 'year']],
on=['wm_yr_wk'],
how='left')
sell_price_df['price_momentum_m_item_state'] = sell_price_df[
'sell_price'] / sell_price_df.groupby(
['state_id', 'item_id', 'month'])['sell_price'].transform('mean')
sell_price_df['price_momentum_y_item_state'] = sell_price_df[
'sell_price'] / sell_price_df.groupby(
['state_id', 'item_id', 'year'])['sell_price'].transform('mean')
sell_price_df['price_momentum_m_item_store'] = sell_price_df[
'sell_price'] / sell_price_df.groupby(
['store_id', 'item_id', 'month'])['sell_price'].transform('mean')
sell_price_df['price_momentum_y_item_store'] = sell_price_df[
'sell_price'] / sell_price_df.groupby(
['store_id', 'item_id', 'year'])['sell_price'].transform('mean')
# sell_start
sell_price_df["one"] = 1
sell_price_df["sell_start"] = sell_price_df.groupby(
["store_id", "item_id"])["one"].transform(lambda x: x.cumsum())
sell_price_df["sell_start_log"] = sell_price_df.groupby(
["store_id", "item_id"])["sell_start"].transform(lambda x: np.log(x))
sell_price_df = sell_price_df.drop(
["month", "year", "one", "state_id", "dept_id", "cat_id"], axis=1)
####### merge
melt_sales = melt_sales.merge(calendar_df, on="d", how="left")
melt_sales = pd.merge(melt_sales,
sell_price_df,
on=["store_id", "item_id", "wm_yr_wk"],
how='left')
####### save by column
melt_sales = reduce_mem_usage(melt_sales)
for col in melt_sales.columns:
melt_sales[col].to_pickle(jsn['FEATURE_BASIC_DIR'] + f"{col}.pkl")
del melt_sales, sell_price_df, calendar_df, train_val_df
gc.collect()
def total_weight():
### make WEIGHT_SCALED_42840, WEIGHT_FORMAT, WEIGHT_SCALED_30490
train_val_df = pd.read_csv(jsn["TRAIN_DATA_PATH"])
sell_price_df = pd.read_csv(jsn["SELL_PRICES_PATH"])
calendar_df = pd.read_csv(jsn["CALENDAR_PATH"])
day_list = []
for i in range(1942, 1970):
day_list.append("d_{}".format(i))
for day in day_list:
train_val_df[day] = np.nan
melt_sales = pd.melt(
train_val_df,
id_vars=['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name="day",
value_name="sales")
calendar_df['date'] = pd.to_datetime(calendar_df['date'],
format='%Y-%m-%d')
melt_sales = melt_sales.merge(calendar_df, left_on="day", right_on="d")
melt_sales = pd.merge(melt_sales,
sell_price_df,
on=["store_id", "item_id", "wm_yr_wk"],
how='left')
melt_sales = melt_sales[["id", "date", "sales", "sell_price"]]
melt_sales = reduce_mem_usage(melt_sales)
############## Creat mat
# train_val_df = train_val_df[train_val_df["distribution"]=="poisson"]
NUM_ITEMS = train_val_df.shape[0]
product = train_val_df[[
'id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'
]].drop_duplicates()
# Creating weight mats
weight_mat = np.c_[
np.ones([NUM_ITEMS, 1]).astype(np.int8), # level 1
pd.get_dummies(product.state_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.store_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.cat_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.dept_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.cat_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.dept_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.store_id.astype(str) +
product.cat_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.store_id.astype(str) +
product.dept_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.item_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.item_id.astype(str),
drop_first=False).astype('int8').values,
np.identity(NUM_ITEMS).astype(np.int8) #item :level 12
].T
np.save(jsn["WEIGHTS_DIR"] + "weight_mat_total.npy", weight_mat)
weight_mat_csr = csr_matrix(weight_mat)
# del train_val_df, sell_price_df, calendar_df, weight_mat
# gc.collect()
############## Loss function weights are calculated and stored.
weight1, weight2 = weight_calc(melt_sales, product, weight_mat_csr)
# del df, train_val_df; gc.collect()
# del melt_sales; gc.collect()
############## SAVE WEIGHT
np.save(jsn["WEIGHTS_DIR"] + "weight1_total.npy", weight1)
np.save(jsn["WEIGHTS_DIR"] + "weight2_total.npy", weight2)
# train_val_df = pd.read_csv('/Users/hiroaki_ikeshita/myfolder/diveintocode-ml/Kaggle/Walmart/m5-forecasting-accuracy/sales_train_evaluation.csv')
# product = train_val_df[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
df = pd.DataFrame(np.ones([30490, 1]).astype(np.int8),
index=product.index,
columns=["total"])
df = pd.concat(
(df, pd.get_dummies(product.state_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df, pd.get_dummies(product.store_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df, pd.get_dummies(product.cat_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df, pd.get_dummies(product.dept_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df,
pd.get_dummies(
product.state_id.astype(str) + product.cat_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df,
pd.get_dummies(
product.state_id.astype(str) + product.dept_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df,
pd.get_dummies(
product.store_id.astype(str) + product.cat_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df,
pd.get_dummies(
product.store_id.astype(str) + product.dept_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df, pd.get_dummies(product.item_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat(
(df,
pd.get_dummies(
product.state_id.astype(str) + product.item_id.astype(str),
drop_first=False).astype('int8')),
axis=1)
df = pd.concat((df,
pd.DataFrame(np.identity(30490).astype(np.int8),
index=product.index,
columns=product["id"])),
axis=1)
df.index = product.id
df = df.T
df.to_pickle(jsn["WEIGHTS_DIR"] + "weight_size_change_format.pkl")
weight_scaled = weight2**2 / weight1
WEIGHT_SCALED_42840 = weight_scaled / weight_scaled[0]
WEIGHT_SCALED_30490 = np.dot(WEIGHT_SCALED_42840, df.values)
np.save(jsn["WEIGHTS_DIR"] + "WEIGHT_SCALED_42840.npy",
WEIGHT_SCALED_42840)
np.save(jsn["WEIGHTS_DIR"] + "WEIGHT_SCALED_30490.npy",
WEIGHT_SCALED_30490)
def weight_by_store():
################################# Make WEIGHT
#################################################################################
############## LOAD ORIGIN DATA
train_val_df_origin = pd.read_csv(jsn["TRAIN_DATA_PATH"])
sell_price_df = pd.read_csv(jsn["SELL_PRICES_PATH"])
calendar_df = pd.read_csv(jsn["CALENDAR_PATH"])
############# Make Weights
day_list = []
for i in range(1942, 1970):
day_list.append("d_{}".format(i))
for day in day_list:
train_val_df_origin[day] = np.nan
for category in CATEGORY_ID:
train_val_df = train_val_df_origin[
train_val_df_origin[f"{category_name}_id"] == category]
melt_sales = pd.melt(train_val_df,
id_vars=[
'id', 'item_id', 'dept_id', 'cat_id',
'store_id', 'state_id'
],
var_name="day",
value_name="sales")
calendar_df['date'] = pd.to_datetime(calendar_df['date'],
format='%Y-%m-%d')
melt_sales = melt_sales.merge(calendar_df, left_on="day", right_on="d")
melt_sales = pd.merge(melt_sales,
sell_price_df,
on=["store_id", "item_id", "wm_yr_wk"],
how='left')
melt_sales = melt_sales[["id", "date", "sales", "sell_price"]]
melt_sales = reduce_mem_usage(melt_sales)
############## Creat mat
# train_val_df = train_val_df[train_val_df["distribution"]=="poisson"]
NUM_ITEMS = train_val_df.shape[0]
product = train_val_df[[
'id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'
]].drop_duplicates()
# Weight_MAT
weight_mat = np.c_[
np.ones([NUM_ITEMS, 1]).astype(np.int8), # level 1
pd.get_dummies(product.state_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.store_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.cat_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.dept_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.cat_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.dept_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.store_id.astype(str) +
product.cat_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.store_id.astype(str) +
product.dept_id.astype(str),
drop_first=False).astype('int8').values,
pd.get_dummies(product.item_id.astype(str), drop_first=False
).astype('int8').values,
pd.get_dummies(product.state_id.astype(str) +
product.item_id.astype(str),
drop_first=False).astype('int8').values,
np.identity(NUM_ITEMS).astype(np.int8) #item :level 12
].T
np.save(jsn["WEIGHTS_DIR"] + f"weight_mat_{category}.npy", weight_mat)
weight_mat_csr = csr_matrix(weight_mat)
# del train_val_df, sell_price_df, calendar_df, weight_mat
# gc.collect()
############## Loss function weights are calculated and stored.
weight1, weight2 = weight_calc(melt_sales, product, weight_mat_csr,
category)
# del df, train_val_df; gc.collect()
# del melt_sales; gc.collect()
############## SAVE WEIGHT
np.save(jsn["WEIGHTS_DIR"] + f"weight1_{category}.npy", weight1)
np.save(jsn["WEIGHTS_DIR"] + f"weight2_{category}.npy", weight2)