def main(params: dict, output_dir: str): import mlflow print("start params={}".format(params)) logger = get_logger() df = pd.read_pickle( "../input/riiid-test-answer-prediction/train_merged.pickle") # df = pd.read_pickle("../input/riiid-test-answer-prediction/split10/train_0.pickle").sort_values(["user_id", "timestamp"]).reset_index(drop=True) if is_debug: df = df.head(500000) df["prior_question_had_explanation"] = df[ "prior_question_had_explanation"].fillna(-1) column_config = { ("content_id", "content_type_id"): { "type": "category" }, "user_answer": { "type": "leakage_feature" }, "answered_correctly": { "type": "leakage_feature" }, "part": { "type": "category" }, "prior_question_elapsed_time_bin300": { "type": "category" }, "duration_previous_content_bin300": { "type": "category" }, "prior_question_had_explanation": { "type": "category" }, "rating_diff_content_user_id": { "type": "numeric" } } with open(f"{output_dir}/transformer_param.json", "w") as f: json.dump(params, f) if is_make_feature_factory: # feature factory feature_factory_dict = {"user_id": {}} feature_factory_dict["user_id"][ "DurationPreviousContent"] = DurationPreviousContent( is_partial_fit=True) feature_factory_dict["user_id"][ "ElapsedTimeBinningEncoder"] = ElapsedTimeBinningEncoder() feature_factory_dict["user_id"][ "UserContentRateEncoder"] = UserContentRateEncoder( rate_func="elo", column="user_id") feature_factory_manager = FeatureFactoryManager( feature_factory_dict=feature_factory_dict, logger=logger, split_num=1, model_id=model_id, load_feature=not is_debug, save_feature=not is_debug) ff_for_transformer = FeatureFactoryForTransformer( column_config=column_config, dict_path="../feature_engineering/", sequence_length=params["max_seq"], logger=logger) ff_for_transformer.make_dict(df=df) df = df.sort_values(["user_id", "timestamp"]).reset_index(drop=True) feature_factory_manager.fit(df.iloc[:95000].copy()) w_df = feature_factory_manager.all_predict(df.iloc[:95000].copy()) ff_for_transformer.fit(w_df) for _, w_df in tqdm(df.iloc[95000:].groupby( ["user_id", "task_container_id"])): ww_df = feature_factory_manager.partial_predict( w_df.drop(["answered_correctly", "user_answer"], axis=1)) group = ff_for_transformer.partial_predict(ww_df) ww_df["answered_correctly"] = w_df["answered_correctly"] ww_df["user_answer"] = w_df["user_answer"] feature_factory_manager.fit(ww_df) ff_for_transformer.fit(ww_df) for dicts in feature_factory_manager.feature_factory_dict.values(): for factory in dicts.values(): factory.logger = None feature_factory_manager.logger = None with open(f"{output_dir}/feature_factory_manager.pickle", "wb") as f: pickle.dump(feature_factory_manager, f) ff_for_transformer.logger = None with open( f"{output_dir}/feature_factory_manager_for_transformer.pickle", "wb") as f: pickle.dump(ff_for_transformer, f)
def main(params: dict): import mlflow logger = get_logger() print("start params={}".format(params)) # df = pd.read_pickle("../input/riiid-test-answer-prediction/train_merged.pickle").head(30_000_000) df = pd.read_pickle( "../input/riiid-test-answer-prediction/split10/train_0.pickle" ).sort_values(["user_id", "timestamp"]).reset_index(drop=True) if is_debug: df = df.head(3000) df.to_csv("aaa.csv") df = df.sort_values(["user_id", "timestamp"]).reset_index(drop=True) df = df[[ "user_id", "content_id", "content_type_id", "part", "answered_correctly" ]] train_idx = [] val_idx = [] np.random.seed(0) # for _, w_df in df[df["content_type_id"] == 0].groupby("user_id"): for _, w_df in df.groupby("user_id"): if np.random.random() < 0.1: # all val val_idx.extend(w_df.index.tolist()) else: train_num = int(len(w_df) * 0.9) train_idx.extend(w_df[:train_num].index.tolist()) val_idx.extend(w_df[train_num:].index.tolist()) df["is_val"] = 0 df["is_val"].loc[val_idx] = 1 ff_for_transformer = FeatureFactoryForTransformer( column_config={ ("content_id", "content_type_id"): { "type": "category" }, "part": { "type": "category" } }, dict_path="../feature_engineering/", sequence_length=params["max_seq"], logger=logger) n_skill = len(ff_for_transformer.embbed_dict[("content_id", "content_type_id")]) group = ff_for_transformer.all_predict(df) dataset_val = SAKTDataset(group, is_test=True, n_skill=n_skill, max_seq=params["max_seq"]) dataloader_val = DataLoader(dataset_val, batch_size=1024, shuffle=False, num_workers=1) ## all_predict xs = [] target_ids = [] parts = [] labels = [] for d in tqdm(dataloader_val): xs.extend(d[0].to(device).long().data.cpu().numpy().tolist()) target_ids.extend(d[1].to(device).long().data.cpu().numpy().tolist()) parts.extend(d[2].to(device).long().data.cpu().numpy().tolist()) labels.extend(d[3].to(device).long().data.cpu().numpy().tolist()) print(xs[0]) print(target_ids[0]) print(parts[0]) print(labels[0]) ## partial_predict ff_for_transformer = FeatureFactoryForTransformer( column_config={ ("content_id", "content_type_id"): { "type": "category" }, "part": { "type": "category" } }, dict_path="../feature_engineering/", sequence_length=params["max_seq"], logger=logger) ff_for_transformer.fit(df.loc[train_idx]) xs_test = [] target_ids_test = [] parts_test = [] labels_test = [] for i in tqdm(range(len(df.loc[val_idx]))): w_df = df.loc[val_idx[i:i + 1]] group = ff_for_transformer.partial_predict(w_df.copy()) dataset_val = SAKTDataset(group, predict_mode=True, n_skill=n_skill, max_seq=params["max_seq"]) dataloader_val = DataLoader(dataset_val, batch_size=1024, shuffle=False, num_workers=1) for d in dataloader_val: xs_test.extend(d[0].to(device).long().data.cpu().numpy().tolist()) target_ids_test.extend( d[1].to(device).long().data.cpu().numpy().tolist()) parts_test.extend( d[2].to(device).long().data.cpu().numpy().tolist()) labels_test.extend( d[3].to(device).long().data.cpu().numpy().tolist()) ff_for_transformer.fit(w_df.copy()) pd.DataFrame(xs).to_csv("all_xs.csv") pd.DataFrame(xs_test).to_csv("partial_xs.csv") pd.DataFrame(labels).to_csv("all_labels.csv") pd.DataFrame(labels_test).to_csv("partial_labels.csv") df.loc[val_idx].to_csv("raw_data.csv") assert xs[0] == xs_test[0] assert xs == xs_test assert target_ids == target_ids_test assert parts == parts_test