def get_ICM_with_fields(reader: New_DataSplitter_leave_k_out): """ It returns all the ICM_train_all after applying feature engineering :param reader: data splitter :return: return ICM_train_all and feature fields """ URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() ICM_all_dict = reader.get_loaded_ICM_dict() ICM_all_dict.pop("ICM_all") ICM_all_dict = apply_feature_engineering_ICM(ICM_all_dict, URM_train, UCM_all_dict, ICM_names_to_count=[], UCM_names_to_list=[]) ICM_all_dict = apply_filtering_ICM( ICM_all_dict, ICM_name_to_filter_mapper={ "ICM_asset": lambda x: x < np.quantile(x, q=0.75) + 0.72 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)), "ICM_price": lambda x: x < np.quantile(x, q=0.75) + 4 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)) }) ICM_all_dict = apply_transformation_ICM(ICM_all_dict, ICM_name_to_transform_mapper={ "ICM_asset": lambda x: np.log1p(1 / x), "ICM_price": lambda x: np.log1p(1 / x), "ICM_item_pop": np.log1p }) ICM_all_dict = apply_discretization_ICM(ICM_all_dict, ICM_name_to_bins_mapper={ "ICM_asset": 200, "ICM_price": 200, "ICM_item_pop": 50 }) ICM_all = None item_feature_fields = None for idx, ICM_key_value in enumerate(ICM_all_dict.items()): ICM_name, ICM_object = ICM_key_value if idx == 0: ICM_all = ICM_object item_feature_fields = np.full(shape=ICM_object.shape[1], fill_value=idx) else: ICM_all = sps.hstack([ICM_all, ICM_object], format="csr") item_feature_fields = np.concatenate([ item_feature_fields, np.full(shape=ICM_object.shape[1], fill_value=idx) ]) return ICM_all, item_feature_fields
def get_ICM_train(reader: New_DataSplitter_leave_k_out): """ It returns all the ICM_train_all after applying feature engineering. This preprocessing is used on new_best_models file :param reader: data splitter :return: return ICM_train_all """ URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() ICM_all_dict = reader.get_loaded_ICM_dict() ICM_all_dict.pop("ICM_all") ICM_all_dict = apply_feature_engineering_ICM( ICM_all_dict, URM_train, UCM_all_dict, ICM_names_to_count=["ICM_sub_class"], UCM_names_to_list=["UCM_age"]) ICM_all_dict = apply_filtering_ICM( ICM_all_dict, ICM_name_to_filter_mapper={ "ICM_asset": lambda x: x < np.quantile(x, q=0.75) + 0.72 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)), "ICM_price": lambda x: x < np.quantile(x, q=0.75) + 4 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)) }) ICM_all_dict = apply_transformation_ICM(ICM_all_dict, ICM_name_to_transform_mapper={ "ICM_asset": lambda x: np.log1p(1 / x), "ICM_price": lambda x: np.log1p(1 / x), "ICM_item_pop": np.log1p, "ICM_sub_class_count": np.log1p, "ICM_age": lambda x: x**(1 / 2.5) }) ICM_all_dict = apply_discretization_ICM(ICM_all_dict, ICM_name_to_bins_mapper={ "ICM_asset": 200, "ICM_price": 200, "ICM_item_pop": 50, "ICM_sub_class_count": 50 }) ICM_all = build_ICM_all_from_dict(ICM_all_dict) return ICM_all
def get_UCM_train_cold(reader: New_DataSplitter_leave_k_out): """ It returns all the UCM_all after applying feature engineering. This preprocessing is used on new_best_models file :param reader: data splitter :return: return UCM_all """ URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() UCM_all_dict = apply_transformation_UCM( UCM_all_dict, UCM_name_to_transform_mapper={"UCM_user_act": np.log1p}) UCM_all_dict = apply_discretization_UCM( UCM_all_dict, UCM_name_to_bins_mapper={"UCM_user_act": 50}) UCM_all = build_UCM_all_from_dict(UCM_all_dict) return UCM_all
def get_UCM_with_fields(reader: New_DataSplitter_leave_k_out): """ It returns all the UCM_all after applying feature engineering :param reader: data splitter :return: return UCM_all """ URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() ICM_dict = reader.get_loaded_ICM_dict() UCM_all_dict = apply_feature_engineering_UCM( UCM_all_dict, URM_train, ICM_dict, ICM_names_to_UCM=["ICM_sub_class"]) # These are useful feature weighting for UserCBF_CF_Warm UCM_all_dict = apply_transformation_UCM(UCM_all_dict, UCM_name_to_transform_mapper={ "UCM_sub_class": lambda x: x / 2, "UCM_user_act": np.log1p }) UCM_all_dict = apply_discretization_UCM( UCM_all_dict, UCM_name_to_bins_mapper={"UCM_user_act": 50}) UCM_all = None user_feature_fields = None for idx, UCM_key_value in enumerate(UCM_all_dict.items()): UCM_name, UCM_object = UCM_key_value if idx == 0: UCM_all = UCM_object user_feature_fields = np.full(shape=UCM_object.shape[1], fill_value=idx) else: UCM_all = sps.hstack([UCM_all, UCM_object], format="csr") user_feature_fields = np.concatenate([ user_feature_fields, np.full(shape=UCM_object.shape[1], fill_value=idx) ]) return UCM_all, user_feature_fields
def get_UCM_train(reader: New_DataSplitter_leave_k_out): """ It returns all the UCM_all after applying feature engineering. This preprocessing is used on new_best_models file :param reader: data splitter :return: return UCM_all """ URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() ICM_dict = reader.get_loaded_ICM_dict() UCM_all_dict = apply_feature_engineering_UCM( UCM_all_dict, URM_train, ICM_dict, ICM_names_to_UCM=["ICM_sub_class"]) # These are useful feature weighting for UserCBF_CF_Warm UCM_all_dict = apply_transformation_UCM(UCM_all_dict, UCM_name_to_transform_mapper={ "UCM_sub_class": lambda x: x / 2, "UCM_user_act": np.log1p }) UCM_all_dict = apply_discretization_UCM( UCM_all_dict, UCM_name_to_bins_mapper={"UCM_user_act": 50}) UCM_all = build_UCM_all_from_dict(UCM_all_dict) return UCM_all
def get_UCM_train_new(reader: New_DataSplitter_leave_k_out): URM_train, _ = reader.get_holdout_split() UCM_all_dict = reader.get_loaded_UCM_dict() ICM_dict = reader.get_loaded_ICM_dict() # Preprocess ICM ICM_dict.pop("ICM_all") ICM_dict = apply_feature_engineering_ICM( ICM_dict, URM_train, UCM_all_dict, ICM_names_to_count=["ICM_sub_class"], UCM_names_to_list=["UCM_age"]) ICM_dict = apply_filtering_ICM( ICM_dict, ICM_name_to_filter_mapper={ "ICM_asset": lambda x: x < np.quantile(x, q=0.75) + 0.72 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)), "ICM_price": lambda x: x < np.quantile(x, q=0.75) + 4 * (np.quantile(x, q=0.75) - np.quantile(x, q=0.25)) }) ICM_dict = apply_transformation_ICM(ICM_dict, ICM_name_to_transform_mapper={ "ICM_asset": lambda x: np.log1p(1 / x), "ICM_price": lambda x: np.log1p(1 / x), "ICM_item_pop": np.log1p, "ICM_sub_class_count": np.log1p, "ICM_age": lambda x: x**(1 / 2.5) }) ICM_dict = apply_discretization_ICM(ICM_dict, ICM_name_to_bins_mapper={ "ICM_asset": 200, "ICM_price": 200, "ICM_item_pop": 50, "ICM_sub_class_count": 50 }) # Preprocess UCM UCM_all_dict = apply_feature_engineering_UCM( UCM_all_dict, URM_train, ICM_dict, ICM_names_to_UCM=["ICM_sub_class", "ICM_item_pop"]) UCM_all_dict = apply_feature_entropy_UCM( UCM_all_dict, UCM_names_to_entropy=["UCM_sub_class"]) # Apply useful transformation UCM_all_dict = apply_transformation_UCM( UCM_all_dict, UCM_name_to_transform_mapper={"UCM_user_act": np.log1p}) UCM_all_dict = apply_discretization_UCM(UCM_all_dict, UCM_name_to_bins_mapper={ "UCM_user_act": 50, "UCM_sub_class_entropy": 20 }) UCM_all = None user_feature_to_range_mapper = {} last_range = 0 for idx, UCM_key_value in enumerate(UCM_all_dict.items()): UCM_name, UCM_object = UCM_key_value if idx == 0: UCM_all = UCM_object else: UCM_all = sps.hstack([UCM_all, UCM_object], format="csr") user_feature_to_range_mapper[UCM_name] = (last_range, last_range + UCM_object.shape[1]) last_range = last_range + UCM_object.shape[1] return UCM_all, user_feature_to_range_mapper