def make_feature_factory_manager(split_num, model_id=None): logger = get_logger() feature_factory_dict = {} feature_factory_dict["tags"] = { "TagsSeparator": TagsSeparator(is_partial_fit=True) } for column in ["user_id", "content_id"]: is_partial_fit = (column == "content_id" or column == "user_id") if type(column) == str: feature_factory_dict[column] = { "TargetEncoder": TargetEncoder(column=column, is_partial_fit=is_partial_fit) } else: feature_factory_dict[column] = { "TargetEncoder": TargetEncoder(column=list(column), is_partial_fit=is_partial_fit) } feature_factory_dict["user_id"][ "ShiftDiffEncoderTimestamp"] = ShiftDiffEncoder(groupby="user_id", column="timestamp") for column in ["user_id", "content_id"]: feature_factory_dict[column][ f"MeanAggregatorPriorQuestionElapsedTimeby{column}"] = MeanAggregator( column=column, agg_column="prior_question_elapsed_time", remove_now=True) feature_factory_dict["content_id"][ "ContentLevelEncoder2UserId"] = ContentLevelEncoder( vs_column="user_id", is_partial_fit=True) feature_factory_dict["user_id"][ "MeanAggregatorContentLevel"] = MeanAggregator( column="user_id", agg_column="content_level_user_id", remove_now=False) feature_factory_dict["user_id"]["CountEncoder"] = CountEncoder( column="user_id", is_partial_fit=True) feature_factory_dict["user_id"][ "UserCountBinningEncoder"] = UserCountBinningEncoder( is_partial_fit=True) feature_factory_dict["user_count_bin"] = {} feature_factory_dict["user_count_bin"]["TargetEncoder"] = TargetEncoder( column="user_count_bin") feature_factory_dict[("user_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["user_id", "user_count_bin"]) } feature_factory_dict[("content_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["content_id", "user_count_bin"]) } feature_factory_dict["user_id"][ "CategoryLevelEncoderPart"] = CategoryLevelEncoder( groupby_column="user_id", agg_column="part", categories=[2, 5]) feature_factory_dict["prior_question_elapsed_time"] = { "PriorQuestionElapsedTimeBinningEncoder": PriorQuestionElapsedTimeBinningEncoder(is_partial_fit=True) } feature_factory_dict[("part", "prior_question_elapsed_time_bin")] = { "TargetEncoder": TargetEncoder(column=["part", "prior_question_elapsed_time_bin"]) } feature_factory_dict["user_id"]["PreviousAnswer2"] = PreviousAnswer2( groupby="user_id", column="content_id", is_debug=is_debug, model_id=model_id, n=1000) feature_factory_dict["user_id"][ "PreviousNAnsweredCorrectly"] = PreviousNAnsweredCorrectly( n=3, is_partial_fit=True) feature_factory_dict[f"previous_3_ans"] = { "TargetEncoder": TargetEncoder(column="previous_3_ans") } feature_factory_dict["user_id"][ "QuestionLectureTableEncoder2"] = QuestionLectureTableEncoder2( model_id=model_id, is_debug=is_debug, past_n=100) feature_factory_dict["user_id"][ "QuestionQuestionTableEncoder"] = QuestionQuestionTableEncoder( model_id=model_id, is_debug=is_debug, past_n=100) feature_factory_dict["user_id"][ "UserAnswerLevelEncoder"] = UserAnswerLevelEncoder(model_id=model_id, is_debug=is_debug, past_n=100) feature_factory_dict["user_id"][ "UserContentRateEncoder"] = UserContentRateEncoder(column="user_id", rate_func="elo", initial_rate=1500) feature_factory_dict[("user_id", "part")] = { "UserContentRateEncoder": UserContentRateEncoder(column=["user_id", "part"], rate_func="elo", initial_rate=1500) } feature_factory_dict["post"] = { "ContentIdTargetEncoderAggregator": TargetEncoderAggregator() } feature_factory_manager = FeatureFactoryManager( feature_factory_dict=feature_factory_dict, logger=logger, split_num=split_num, model_id=model_id, load_feature=not is_debug, save_feature=not is_debug) return feature_factory_manager
def make_feature_factory_manager(split_num, model_id=None): logger = get_logger() feature_factory_dict = {} feature_factory_dict["tags"] = { "TagsSeparator": TagsSeparator(is_partial_fit=True) } for column in [ "user_id", "content_id", "part", ("user_id", "prior_question_had_explanation"), ("user_id", "part"), ("content_id", "prior_question_had_explanation") ]: is_partial_fit = (column == "content_id" or column == "user_id") if type(column) == str: feature_factory_dict[column] = { "TargetEncoder": TargetEncoder(column=column, is_partial_fit=is_partial_fit) } else: feature_factory_dict[column] = { "TargetEncoder": TargetEncoder(column=list(column), is_partial_fit=is_partial_fit) } feature_factory_dict["user_id"][ "ShiftDiffEncoderTimestamp"] = ShiftDiffEncoder(groupby="user_id", column="timestamp") feature_factory_dict["user_id"][ "ShiftDiffEncoderContentId"] = ShiftDiffEncoder(groupby="user_id", column="content_id") for column in ["user_id", "content_id"]: feature_factory_dict[column][ f"MeanAggregatorPriorQuestionElapsedTimeby{column}"] = MeanAggregator( column=column, agg_column="prior_question_elapsed_time", remove_now=True) feature_factory_dict["user_id"][ "UserLevelEncoder2ContentId"] = UserLevelEncoder2( vs_column="content_id") feature_factory_dict["content_id"][ "ContentLevelEncoder2UserId"] = ContentLevelEncoder( vs_column="user_id", is_partial_fit=True) feature_factory_dict["user_id"][ "MeanAggregatorContentLevel"] = MeanAggregator( column="user_id", agg_column="content_level_user_id", remove_now=False) feature_factory_dict["user_id"]["CountEncoder"] = CountEncoder( column="user_id", is_partial_fit=True) feature_factory_dict["user_id"][ "UserCountBinningEncoder"] = UserCountBinningEncoder( is_partial_fit=True) feature_factory_dict["user_count_bin"] = {} feature_factory_dict["user_count_bin"]["TargetEncoder"] = TargetEncoder( column="user_count_bin") feature_factory_dict[("user_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["user_id", "user_count_bin"]) } feature_factory_dict[("content_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["content_id", "user_count_bin"]) } feature_factory_dict[( "prior_question_had_explanation", "user_count_bin")] = { "TargetEncoder": TargetEncoder( column=["prior_question_had_explanation", "user_count_bin"]) } feature_factory_dict["user_id"][ "CategoryLevelEncoderPart"] = CategoryLevelEncoder( groupby_column="user_id", agg_column="part", categories=[2, 5]) feature_factory_dict["user_id"][ "FirstColumnEncoderContentId"] = FirstColumnEncoder( column="content_id", astype="int16", is_partial_fit=True) feature_factory_dict["user_id"][ "FirstColumnEncoderPart"] = FirstColumnEncoder(column="part", astype="int8", is_partial_fit=True) for column in [ "user_id", "user_count_bin", "first_column_content_id", "first_column_part", ("user_id", "part") ]: if column not in feature_factory_dict: feature_factory_dict[column] = {} if type(column) == str: feature_factory_dict[column][ f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=column, agg_column="target_enc_content_id", remove_now=False) else: feature_factory_dict[column][ f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=list(column), agg_column="target_enc_content_id", remove_now=False) for column in [ "content_id", "part", "tags1", "tags2", "prior_question_had_explanation", ("content_id", "prior_question_had_explanation") ]: if column not in feature_factory_dict: feature_factory_dict[column] = {} if type(column) == str: feature_factory_dict[column][ f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=column, agg_column="target_enc_user_id", remove_now=False) else: feature_factory_dict[column][ f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=list(column), agg_column="target_enc_user_id", remove_now=False) feature_factory_dict["user_count_bin"]["CategoryLevelEncoderUserCountBin"] = \ CategoryLevelEncoder(groupby_column="user_id", agg_column="user_count_bin", categories=[0]) feature_factory_dict["prior_question_elapsed_time"] = { "PriorQuestionElapsedTimeBinningEncoder": PriorQuestionElapsedTimeBinningEncoder(is_partial_fit=True) } feature_factory_dict[("part", "prior_question_elapsed_time_bin")] = { "TargetEncoder": TargetEncoder(column=["part", "prior_question_elapsed_time_bin"]) } feature_factory_dict["user_id"]["PreviousAnswer2"] = PreviousAnswer2( groupby="user_id", column="content_id", is_debug=is_debug, model_id=model_id) feature_factory_manager = FeatureFactoryManager( feature_factory_dict=feature_factory_dict, logger=logger, split_num=split_num) return feature_factory_manager
def make_feature_factory_manager(split_num, model_id=None): logger = get_logger() feature_factory_dict = {} for column in ["user_id", "content_id", "part", ("user_id", "prior_question_had_explanation"), ("user_id", "part"), ("content_id", "prior_question_had_explanation")]: is_partial_fit = (column == "content_id" or column == "user_id") if type(column) == str: feature_factory_dict[column] = { "CountEncoder": CountEncoder(column=column, is_partial_fit=is_partial_fit), "TargetEncoder": TargetEncoder(column=column, is_partial_fit=is_partial_fit) } else: feature_factory_dict[column] = { "CountEncoder": CountEncoder(column=list(column), is_partial_fit=is_partial_fit), "TargetEncoder": TargetEncoder(column=list(column), is_partial_fit=is_partial_fit) } feature_factory_dict["user_id"]["ShiftDiffEncoderTimestamp"] = ShiftDiffEncoder(groupby="user_id", column="timestamp", is_partial_fit=True) for column in ["user_id", "content_id"]: feature_factory_dict[column][f"MeanAggregatorPriorQuestionElapsedTimeby{column}"] = MeanAggregator(column=column, agg_column="prior_question_elapsed_time", remove_now=True) feature_factory_dict["content_id"]["MeanAggregatorShiftDiffTimestamp"] = MeanAggregator(column="content_id", agg_column="shiftdiff_timestamp_by_user_id", remove_now=False) for column in ["target_enc_user_id", "prior_question_elapsed_time"]: feature_factory_dict["user_id"][f"MeanAggregatorContentIdUserAnswer{column}"] = MeanAggregator2(column=["content_id", "user_answer"], agg_column=column, remove_now=True) feature_factory_dict["user_id"]["UserLevelEncoder2ContentId"] = UserLevelEncoder2(vs_column="content_id") feature_factory_dict["content_id"]["ContentLevelEncoder2UserId"] = ContentLevelEncoder(vs_column="user_id", is_partial_fit=True) feature_factory_dict["user_id"]["MeanAggregatorContentLevel"] = MeanAggregator(column="user_id", agg_column="content_level_user_id", remove_now=False) feature_factory_dict["user_id"]["CountEncoder"] = CountEncoder(column="user_id", is_partial_fit=True) feature_factory_dict["user_id"]["UserCountBinningEncoder"] = UserCountBinningEncoder(is_partial_fit=True) feature_factory_dict["user_count_bin"] = {} feature_factory_dict["user_count_bin"]["TargetEncoder"] = TargetEncoder(column="user_count_bin") feature_factory_dict[("user_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["user_id", "user_count_bin"]) } feature_factory_dict[("content_id", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["content_id", "user_count_bin"]) } feature_factory_dict[("prior_question_had_explanation", "user_count_bin")] = { "TargetEncoder": TargetEncoder(column=["prior_question_had_explanation", "user_count_bin"]) } feature_factory_dict["user_id"]["CategoryLevelEncoderPart"] = CategoryLevelEncoder(groupby_column="user_id", agg_column="part", categories=[2, 5]) feature_factory_dict["user_id"]["FirstColumnEncoderContentId"] = FirstColumnEncoder(agg_column="content_id", astype="int16", is_partial_fit=True) feature_factory_dict["user_id"]["FirstColumnEncoderPart"] = FirstColumnEncoder(agg_column="part", astype="int8", is_partial_fit=True) for column in ["user_id", "user_count_bin", "first_column_content_id", "first_column_part", ("user_id", "part")]: if column not in feature_factory_dict: feature_factory_dict[column] = {} if type(column) == str: feature_factory_dict[column][f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=column, agg_column="target_enc_content_id", remove_now=False ) else: feature_factory_dict[column][f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=list(column), agg_column="target_enc_content_id", remove_now=False ) for column in [("content_id", "prior_question_had_explanation")]: if column not in feature_factory_dict: feature_factory_dict[column] = {} if type(column) == str: feature_factory_dict[column][f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=column, agg_column="target_enc_user_id", remove_now=False ) else: feature_factory_dict[column][f"MeanAggregatorTargetEncContentIdBy{column}"] = MeanAggregator( column=list(column), agg_column="target_enc_user_id", remove_now=False ) feature_factory_dict["prior_question_elapsed_time"] = { "PriorQuestionElapsedTimeBinningEncoder": PriorQuestionElapsedTimeBinningEncoder(is_partial_fit=True) } feature_factory_dict[("part", "prior_question_elapsed_time_bin")] = { "TargetEncoder": TargetEncoder(column=["part", "prior_question_elapsed_time_bin"]) } feature_factory_dict["user_id"]["PreviousAnswer2"] = PreviousAnswer2(groupby="user_id", column="content_id", is_debug=is_debug, model_id=model_id, n=1000) feature_factory_dict["user_id"]["PreviousNAnsweredCorrectly"] = PreviousNAnsweredCorrectly(n=3, is_partial_fit=True) feature_factory_dict[f"previous_3_ans"] = { "TargetEncoder": TargetEncoder(column="previous_3_ans") } feature_factory_dict["user_id"]["QuestionLectureTableEncoder2"] = QuestionLectureTableEncoder2(model_id=model_id, is_debug=is_debug, past_n=100) feature_factory_dict["post"] = { "ContentIdTargetEncoderAggregator": TargetEncoderAggregator() } feature_factory_manager = FeatureFactoryManager(feature_factory_dict=feature_factory_dict, logger=logger, split_num=split_num, model_id=model_id, load_feature=not is_debug, save_feature=not is_debug) return feature_factory_manager