def MLEM2_STAT(FILENAME, iter1, iter2): # rule induction fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules/' + 'rules_' + str( iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) # rule save if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # rules の数を求める num = len(rules) # 平均の長さを求める leng = mlem2.getMeanLength(rules) # 平均支持度を求める support = mlem2.getMeanSupport(rules) # ファイルにsave savepath = DIR_UCI + '/' + FILENAME + '/MLEM2_STAT.csv' with open(savepath, "a") as f: f.writelines( 'MLEM2_STAT,1,{FILENAME},{iter1},{iter2},{num},{leng},{support}'. format(FILENAME=FILENAME, iter1=iter1, iter2=iter2, num=num, leng=leng, support=support) + "\n") return (0)
def MLEM2_LERS(FILENAME, iter1, iter2): # rule induction and rule save fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules/' + 'rules_' + str( iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # test data setup decision_table_test, decision_class = getData(FILENAME, iter1, iter2, T="test") list_judgeNominal = getJudgeNominal(decision_table_test, FILENAME) # predict by LERS predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal) # 正答率を求める accuracy = accuracy_score(decision_class, predictions) # rules の数を求める num = len(rules) # 各クラスのrulesの数を求める num_class = strNumClassRules(rules) # 平均の長さを求める mean_length = mlem2.getMeanLength(rules) # 平均支持度と平均確信度を求める decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T="train") list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) mean_support, mean_conf = LERS.getSupportConfidenceRules( rules, decision_table_train, list_judgeNominal) # AccとRecallを求める acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal) # ファイルにsave savepath = DIR_UCI + '/' + FILENAME + '/fairness/00_normal/MLEM2_LERS.csv' with open(savepath, "a") as f: f.writelines( 'MLEM2_LERS,{FILENAME},{iter1},{iter2},{acc},{num},{num_class},{mean_length},{mean_support},{mean_conf},{acc_recall}' .format(FILENAME=FILENAME, iter1=iter1, iter2=iter2, acc=accuracy, num=num, num_class=num_class, mean_length=mean_length, mean_support=mean_support, mean_conf=mean_conf, acc_recall=strAccRecall(rules, acc_recall)) + "\n") return (0)
def MLEM2_RuleClusteringByConsistentExceptMRule_STAT(FILENAME, iter1, iter2, k, m): # rule induction fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules/' + 'rules_' + str( iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) # rule save if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # rule clustering filepath = DIR_UCI + '/' + FILENAME + '/' + FILENAME + '-train' + str( iter1) + '-' + str(iter2) + '.tsv' decision_table = mlem2.getDecisionTable(filepath) colnames = mlem2.getColNames(decision_table) filepath = DIR_UCI + '/' + FILENAME + '/' + FILENAME + '.nominal' list_nominal = mlem2.getNominalList(filepath) list_judgeNominal = mlem2.getJudgeNominal(decision_table, list_nominal) fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules_cluster_consistent_except_mrule/' + 'rules-' + str( k) + '_' + str(iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename ) else clustering.getRuleClusteringByConsistentExceptMRule( rules, colnames, list_judgeNominal, k=k, m=m) # rule save if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # rules の数を求める num = len(rules) # 平均の長さを求める leng = mlem2.getMeanLength(rules) # 平均支持度を求める support = mlem2.getMeanSupport(rules) # ファイルにsave savepath = DIR_UCI + '/' + FILENAME + '/MLEM2_RuleClusteringByConsistentExceptMRule_STAT.csv' with open(savepath, "a") as f: f.writelines( 'MLEM2_RuleClusteringByConsistentExceptMRule_STAT,{k},{FILENAME},{iter1},{iter2},{num},{leng},{support}' .format(FILENAME=FILENAME, k=k, iter1=iter1, iter2=iter2, num=num, leng=leng, support=support) + "\n") return (0)
def MLEM2_LERS(FILENAME, iter1, iter2): # rule induction and rule save fullpath_filename = DIR_UCI + "/" + FILENAME + "/rules/" + "rules_" + str(iter1) + "-" + str(iter2) + ".pkl" rules = ( mlem2.loadPickleRules(fullpath_filename) if os.path.isfile(fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) ) if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # test data setup decision_table_test, decision_class = getData(FILENAME, iter1, iter2, T="test") list_judgeNominal = getJudgeNominal(decision_table_test, FILENAME) # predict by LERS predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal) # 正答率を求める accuracy = accuracy_score(decision_class, predictions) # rules の数を求める num = len(rules) # 各クラスのrulesの数を求める num_class = strNumClassRules(rules) # 平均の長さを求める mean_length = mlem2.getMeanLength(rules) # 平均支持度と平均確信度を求める decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T="train") list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) mean_support, mean_conf = LERS.getSupportConfidenceRules(rules, decision_table_train, list_judgeNominal) # AccとRecallを求める acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal) # ファイルにsave savepath = DIR_UCI + "/" + FILENAME + "/fairness/00_normal/MLEM2_LERS.csv" with open(savepath, "a") as f: f.writelines( "MLEM2_LERS,{FILENAME},{iter1},{iter2},{acc},{num},{num_class},{mean_length},{mean_support},{mean_conf},{acc_recall}".format( FILENAME=FILENAME, iter1=iter1, iter2=iter2, acc=accuracy, num=num, num_class=num_class, mean_length=mean_length, mean_support=mean_support, mean_conf=mean_conf, acc_recall=strAccRecall(rules, acc_recall), ) + "\n" ) return 0
def MLEM2_OnlyK_STAT(FILENAME, iter1, iter2, k): print("START iter1 iter2 k : " + str(iter1) + "," + str(iter2) + "," + str(k)) # rule induction fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules/' + 'rules_' + str( iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) # rule save if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # only-k rule filter fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules_onlyK/' + 'rules-' + str( k) + '_' + str(iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else [r for r in rules if len(r.getSupport()) >= k] # rule save if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # rules の数を求める num = len(rules) # 平均の長さを求める leng = mlem2.getMeanLength(rules) # 平均支持度を求める support = mlem2.getMeanSupport(rules) # ファイルにsave savepath = DIR_UCI + '/' + FILENAME + '/MLEM2_OnlyK_STAT.csv' with open(savepath, "a") as f: f.writelines( 'MLEM2_OnlyK_STAT,{k},{FILENAME},{iter1},{iter2},{num},{leng},{support}' .format(FILENAME=FILENAME, k=k, iter1=iter1, iter2=iter2, num=num, leng=leng, support=support) + "\n") return (0)
def MLEM2_delEAlphaRule_LERS(FILENAME, iter1, iter2, DELFUN, CLASS, ATTRIBUTE_VALUE, alpha): print(datetime.now().strftime('%Y/%m/%d %H:%M:%S') + ' ' + FILENAME + ' ' + str(iter1) + ' ' + str(iter2) + ' ' + DELFUN.__name__ + ' ' + strAttributeValue(ATTRIBUTE_VALUE) + ' ' + str(alpha) + ' ' + "START") # rule induction and rule save fullpath_filename = DIR_UCI + '/' + FILENAME + '/rules/' + 'rules_' + str( iter1) + '-' + str(iter2) + '.pkl' rules = mlem2.loadPickleRules(fullpath_filename) if os.path.isfile( fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # train data setup decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T="train") list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) # alpha差別的なルールの基本条件削除 or ルールを削除 if CLASS != "ALL": rules_target = mlem2.getRulesClass(rules, CLASS) rules_nontarget = mlem2.getRulesClass(rules, CLASS, judge=False) for attr in ATTRIBUTE_VALUE: for e in ATTRIBUTE_VALUE[attr]: rules_target = DELFUN(rules_target, attr, e, decision_table_train, list_judgeNominal, alpha) rules_target.extend(rules_nontarget) rules = rules_target else: for attr in ATTRIBUTE_VALUE: for e in ATTRIBUTE_VALUE[attr]: rules = DELFUN(rules, attr, e, decision_table_train, list_judgeNominal, alpha) print(datetime.now().strftime('%Y/%m/%d %H:%M:%S') + ' ' + FILENAME + ' ' + str(iter1) + ' ' + str(iter2) + ' ' + DELFUN.__name__ + ' ' + strAttributeValue(ATTRIBUTE_VALUE) + ' ' + str(alpha) + ' ' + "RULES") # test data setup decision_table_test, decision_class = getData(FILENAME, iter1, iter2, T="test") list_judgeNominal = getJudgeNominal(decision_table_test, FILENAME) # predict by LERS predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal) # 正答率を求める accuracy = accuracy_score(decision_class, predictions) # rules の数を求める num = len(rules) # 各クラスのrulesの数を求める num_class = strNumClassRules(rules) # 平均の長さを求める mean_length = mlem2.getMeanLength(rules) # 平均支持度と平均確信度を求める list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) mean_support, mean_conf = LERS.getSupportConfidenceRules( rules, decision_table_train, list_judgeNominal) # AccとRecallを求める acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal) # ファイルにsave savepath = DIR_UCI + '/' + FILENAME + '/fairness/02_alpha_preserve/MLEM2_delEAlphaRule_LERS.csv' with open(savepath, "a") as f: f.writelines( 'MLEM2_delEAlphaRule_LERS,{DELFUN},{CLASS},{FILENAME},{ATTRIBUTE_VALUE},{alpha},{iter1},{iter2},{acc},{num},{num_class},{mean_length},{mean_support},{mean_conf},{acc_recall}' .format(DELFUN=DELFUN.__name__, CLASS=CLASS, FILENAME=FILENAME, ATTRIBUTE_VALUE=strAttributeValue(ATTRIBUTE_VALUE), alpha=alpha, iter1=iter1, iter2=iter2, acc=accuracy, num=num, num_class=num_class, mean_length=mean_length, mean_support=mean_support, mean_conf=mean_conf, acc_recall=strAccRecall(rules, acc_recall)) + "\n") print(datetime.now().strftime('%Y/%m/%d %H:%M:%S') + ' ' + FILENAME + ' ' + str(iter1) + ' ' + str(iter2) + ' ' + DELFUN.__name__ + ' ' + strAttributeValue(ATTRIBUTE_VALUE) + ' ' + str(alpha) + ' ' + "END") return (0)
# nominal data filepath = DIR_UCI+FILENAME+'/'+FILENAME+'.nominal' list_nominal = mlem2.getNominalList(filepath) list_judgeNominal = mlem2.getJudgeNominal(decision_table_test, list_nominal) # predict by LERS predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal) # 正答率を求める accuracy_score(decision_class, predictions) # rules の数を求める num = len(rules) # 平均の長さを求める mean_length = mlem2.getMeanLength(rules) # train data setup decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T = "train") list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) # 平均支持度と平均確信度を求める mean_support, mean_conf = LERS.getSupportConfidenceRules(rules, decision_table_train, list_judgeNominal) # AccとRecallを求める acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal) for i,c in enumerate(mlem2.getEstimatedClass(rules)): print(str(acc_recall[i][0])+","+str(acc_recall[i][1])) ###### 公正配慮のテスト # 基本条件を含むルールセット
def MLEM2_delEAlphaRule_LERS(FILENAME, iter1, iter2, DELFUN, CLASS, ATTRIBUTE_VALUE, alpha): print( datetime.now().strftime("%Y/%m/%d %H:%M:%S") + " " + FILENAME + " " + str(iter1) + " " + str(iter2) + " " + DELFUN.__name__ + " " + strAttributeValue(ATTRIBUTE_VALUE) + " " + str(alpha) + " " + "START" ) # rule induction and rule save fullpath_filename = DIR_UCI + "/" + FILENAME + "/rules/" + "rules_" + str(iter1) + "-" + str(iter2) + ".pkl" rules = ( mlem2.loadPickleRules(fullpath_filename) if os.path.isfile(fullpath_filename) else mlem2.getRulesByMLEM2(FILENAME, iter1, iter2) ) if not os.path.isfile(fullpath_filename): mlem2.savePickleRules(rules, fullpath_filename) # train data setup decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T="train") list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) # alpha差別的なルールの基本条件削除 or ルールを削除 if CLASS != "ALL": rules_target = mlem2.getRulesClass(rules, CLASS) rules_nontarget = mlem2.getRulesClass(rules, CLASS, judge=False) for attr in ATTRIBUTE_VALUE: for e in ATTRIBUTE_VALUE[attr]: rules_target = DELFUN(rules_target, attr, e, decision_table_train, list_judgeNominal, alpha) rules_target.extend(rules_nontarget) rules = rules_target else: for attr in ATTRIBUTE_VALUE: for e in ATTRIBUTE_VALUE[attr]: rules = DELFUN(rules, attr, e, decision_table_train, list_judgeNominal, alpha) print( datetime.now().strftime("%Y/%m/%d %H:%M:%S") + " " + FILENAME + " " + str(iter1) + " " + str(iter2) + " " + DELFUN.__name__ + " " + strAttributeValue(ATTRIBUTE_VALUE) + " " + str(alpha) + " " + "RULES" ) # test data setup decision_table_test, decision_class = getData(FILENAME, iter1, iter2, T="test") list_judgeNominal = getJudgeNominal(decision_table_test, FILENAME) # predict by LERS predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal) # 正答率を求める accuracy = accuracy_score(decision_class, predictions) # rules の数を求める num = len(rules) # 各クラスのrulesの数を求める num_class = strNumClassRules(rules) # 平均の長さを求める mean_length = mlem2.getMeanLength(rules) # 平均支持度と平均確信度を求める list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME) mean_support, mean_conf = LERS.getSupportConfidenceRules(rules, decision_table_train, list_judgeNominal) # AccとRecallを求める acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal) # ファイルにsave savepath = DIR_UCI + "/" + FILENAME + "/fairness/02_alpha_preserve/MLEM2_delEAlphaRule_LERS.csv" with open(savepath, "a") as f: f.writelines( "MLEM2_delEAlphaRule_LERS,{DELFUN},{CLASS},{FILENAME},{ATTRIBUTE_VALUE},{alpha},{iter1},{iter2},{acc},{num},{num_class},{mean_length},{mean_support},{mean_conf},{acc_recall}".format( DELFUN=DELFUN.__name__, CLASS=CLASS, FILENAME=FILENAME, ATTRIBUTE_VALUE=strAttributeValue(ATTRIBUTE_VALUE), alpha=alpha, iter1=iter1, iter2=iter2, acc=accuracy, num=num, num_class=num_class, mean_length=mean_length, mean_support=mean_support, mean_conf=mean_conf, acc_recall=strAccRecall(rules, acc_recall), ) + "\n" ) print( datetime.now().strftime("%Y/%m/%d %H:%M:%S") + " " + FILENAME + " " + str(iter1) + " " + str(iter2) + " " + DELFUN.__name__ + " " + strAttributeValue(ATTRIBUTE_VALUE) + " " + str(alpha) + " " + "END" ) return 0