def apply_mlp(self): if self.ml_pipeline.config.clf_mlp_flg: x_train = self.ml_pipeline.x_train y_train = self.ml_pipeline.y_train if self.ml_pipeline.config.clf_mlp_auto: cv_inner = StratifiedKFold(n_splits=5, shuffle=True, random_state=43) cv_outer = StratifiedKFold(n_splits=10, shuffle=True, random_state=43) grid_search_model = self.MLP_GridSearch() grid_search_model.cv = cv_inner grid_search_model.fit(x_train, y_train) chosen_model = grid_search_model.best_estimator_ scores = cross_val_score(grid_search_model, x_train, y_train, scoring='f1', cv=cv_outer, n_jobs=1) self.jlogger.info(str(chosen_model)) self.jlogger.info("MEAN F1 scores after nested CV {} ".format(mean(scores))) self.jlogger.info("Standard Deviation for F1 scores after nested CV {} ".format(std(scores))) else: manual_params = self.ml_pipeline.config.clf_svm_manual evalclf = Evaluation.Evaluation(self.ml_pipeline) evalclf.evaluate_and_save_results(chosen_model, "mlp") if self.ml_pipeline.config.clf_bagging_mlp: n = self.ml_pipeline.config.clf_bag_mlp_n evalclf.evaluate_bagging_model(chosen_model, n, "mlp_bagging")
def apply_et(self): if self.ml_pipeline.config.clf_et_flg: if self.ml_pipeline.config.clf_et_auto: x_train = self.ml_pipeline.x_train y_train = self.ml_pipeline.y_train cv_inner = StratifiedKFold(n_splits=5, shuffle=True, random_state=43) cv_outer = StratifiedKFold(n_splits=10, shuffle=True, random_state=43) # clf = ExtraTreesClassifier(n_estimators=200, random_state=42, max_depth=10, n_jobs=-1) grid_search_model = self.et_grid_search() grid_search_model.cv = cv_inner grid_search_model.fit(x_train, y_train) chosen_model = grid_search_model.best_estimator_ scores = cross_val_score(grid_search_model, x_train, y_train, scoring='f1', cv=cv_outer, n_jobs=1) self.jlogger.info(str(chosen_model)) self.jlogger.info("MEAN F1 scores after nested CV {} ".format(mean(scores))) self.jlogger.info("Standard Deviation for F1 scores after nested CV {} ".format(std(scores))) else: manual_params = self.ml_pipeline.config.clf_gbm_manual evalclf = Evaluation.Evaluation(self.ml_pipeline) evalclf.evaluate_and_save_results(chosen_model, "et") if self.ml_pipeline.config.clf_bagging_et: n = self.ml_pipeline.config.clf_bag_et_n evalclf.evaluate_bagging_model(chosen_model, n, "et_bagging")
def __init__(self, board, color): self.hashkeysEvalsSteps = [] self.evaluations = [] self.eval = Evaluation.Evaluation() (self.start_row, self.start_col, self.end_row, self.end_col) = self.eval.getStrongestGrid(board, color)
def RandomForest(self): #Balance Data r = 0.25 Handle_Data = Data_handling.DataHandling(self.X_trainval, self.y_trainval) X_trainval_bal, y_trainval_bal = Handle_Data.SMOT(r) # Standarize data scaler = StandardScaler().fit(X_trainval_bal) X_trainval_bal_transformed = scaler.transform(X_trainval_bal) X_test_transformed = scaler.transform(self.X_test) #Train Model n_estimators = [100, 200] max_features = ['auto', 'sqrt', 'log2'] min_samples_leaf = [1, 25, 50] best_e_num = 0 best_f_num = '' best_s_num = 0 best_score = 0 Eva = Evaluation.Evaluation() for i in n_estimators: for j in max_features: for k in min_samples_leaf: RFClassifier = RandomForestClassifier(n_estimators=i, max_features=j, min_samples_leaf=k) scorer = make_scorer(Eva.gini_score, needs_proba=True) score = cross_val_score(RFClassifier, X_trainval_bal_transformed, y_trainval_bal, cv=5, scoring=scorer) print(score) score = score.mean() print("When parameter n_estimators=", i, ", parameter max_features=", j, ", parameter min_samples_leaf=", k, ":\nMean score is", score) if score >= best_score: best_score = score best_e_num = i best_f_num = j best_s_num = k BestRFClassifier = RandomForestClassifier( n_estimators=best_e_num, max_features=best_f_num, min_samples_leaf=best_s_num).fit(X_trainval_bal_transformed, y_trainval_bal) #Get the best model # BestRFClassifier=RandomForestClassifier(n_estimators=200,max_features='sqrt',min_samples_leaf=1).fit(X_trainval_bal_transformed,y_trainval_bal) predict_proba = BestRFClassifier.predict_proba(X_test_transformed)[:, 1] self.gini = Eva.gini_score(self.y_test, predict_proba) return BestRFClassifier, self.gini
def testing_percentage(dataFrame, start, increment, finish, feature_vector): fold_quantity = 100 maxEnt_accuracy_test = [] svm_accuracy_test = [] naive_accuracy_test = [] maxEnt_precision_test = [] svm_precision_test = [] naive_precision_test = [] maxEnt_recall_test = [] svm_recall_test = [] naive_recall_test = [] x_axis = [] df_sliced = Utils.remove_concerns_under_quantity_threshold(dataFrame) for start in range(start, finish + increment, increment): x_axis.append(start) test_size = 1 - (start) / 100 evaluate = Eva.Evaluation( df_sliced, feature_vector, test_size, fold_quantity ) classifier_max_ent, metrics_max_ent = evaluate.evaluate_MaxEnt() classifier_svm, metrics_svm = evaluate.evaluate_SVM() classifier_naive, metrics_naive = evaluate.evaluate_Naive_Bayes() maxEnt_accuracy_test.append(metrics_max_ent.get_accuracy_score()) svm_accuracy_test.append(metrics_svm.get_accuracy_score()) naive_accuracy_test.append(metrics_naive.get_accuracy_score()) maxEnt_precision_test.append(metrics_max_ent.get_precision_score()) svm_precision_test.append(metrics_svm.get_precision_score()) naive_precision_test.append(metrics_naive.get_precision_score()) maxEnt_recall_test.append(metrics_max_ent.get_recall_score()) svm_recall_test.append(metrics_svm.get_recall_score()) naive_recall_test.append(metrics_naive.get_recall_score()) ##############ACCURACY######################## Graphs.plot_line_graph("Accuracy over over train size", x_axis, {'naive': naive_accuracy_test, 'maxEnt': maxEnt_accuracy_test, 'svm': svm_accuracy_test }, "Train size (%)", "Accuracy") ############################################## ##############PRECISION####################### Graphs.plot_line_graph("Macro avg. precision over train size", x_axis, {'naive': naive_precision_test, 'maxEnt': maxEnt_precision_test, 'svm': svm_precision_test }, "Train size (%)", "Macro.avg. precision", ) ##############RECALL######################### Graphs.plot_line_graph("Macro avg. recall over train size", x_axis, {'naive': naive_recall_test, 'maxEnt': maxEnt_recall_test, 'svm': svm_recall_test }, "Train size (%)", "Macro.avg. recall", )
def evaluate(self, config=None): if config is None: config = self._config["evaluation"] evaluation = Evaluation(self.iterators[config['iterator']], self.model, self.candidator, self.stats[config['stats']], sampling=config['sampling'] if 'sampling' in config else None, log_path=self.path + "/evaluation.txt", db=self.db, trained_mentions=self.trained_mentions, attn=config['attn'] if 'attn' in config else None) evaluation.evaluate()
def __init__(self, automated): self.database = Database.WebDB("data/cache/database.db") self.evaluator = Evaluation.Evaluation() if automated == True: self.automated = automated self.automatedQuery("nnn","nnn") self.automatedQuery("nnn","ltc") self.automatedQuery("ltc","nnn") self.automatedQuery("ltc","ltc") self.randomOrder = True self.automatedQuery("ltc","ltc") else: self.manualQuery()
def Logistic_Regression(self): #Balance Data r = 0.25 Handle_Data = Data_handling.DataHandling(self.X_trainval, self.y_trainval) X_trainval_bal, y_trainval_bal = Handle_Data.SMOT(r) #Standarize data scaler = StandardScaler().fit(X_trainval_bal) X_trainval_bal_transformed = scaler.transform(X_trainval_bal) X_test_transformed = scaler.transform(self.X_test) #Train Model penalty = ['l1', 'l2'] c = [0.01, 1.0] best_penalty = '' best_c = 0 best_score = 0 Eva = Evaluation.Evaluation() for i in penalty: for j in c: LogRegModel = LogisticRegression(penalty=i, C=j, solver='liblinear') scorer = make_scorer(Eva.gini_score, needs_proba=True) score = cross_val_score(LogRegModel, X_trainval_bal_transformed, y_trainval_bal, cv=5, scoring=scorer) print(score) score = score.mean() print("When parameter penalty=", i, ", parameter C=", j, ":\nMean score is", score) if score >= best_score: best_score = score best_penalty = i best_c = j BestLogRegModel = LogisticRegression(penalty=best_penalty, solver='liblinear', C=best_c).fit( X_trainval_bal_transformed, y_trainval_bal) #Get the best model using the best parameter we trained # BestLogRegModel=LogisticRegression(penalty='l2',solver='liblinear',C=0.01).fit(X_trainval_bal_transformed,y_trainval_bal) predict_proba = BestLogRegModel.predict_proba(X_test_transformed)[:, 1] self.gini = Eva.gini_score(self.y_test, predict_proba) return BestLogRegModel, self.gini
def automatedQuery(self, smartVarientDoc, smartVarientQuery): self.smartVarientDoc = smartVarientDoc self.smartVarientQuery = smartVarientQuery self.setIndexInstance() self.setIndex() list = self.database.listAllItems() #print("LIST: "+str(list)) if smartVarientQuery is "ltc": for item in list: self.ltcQueryAutomated(item[0]) elif smartVarientQuery is "nnn": for item in list: self.automatedNnnQuery(item[0]) self.evaluator.printFinalAdverages() self.evaluator = Evaluation.Evaluation()
def __init__(self, foreign_sentences, native_sentences, probabilities_path=''): self.foreign_sentences = foreign_sentences self.native_sentences = native_sentences self.foreign_words = set(chain(*foreign_sentences)) self.native_words = set(chain(*native_sentences)) self.translation_probs = {} # [n][f] self.pseudo_counts = {} # [n][f] self.foreign_counts = Counter() self.native_counts = Counter() self.evaluation = Evaluation()
def evaluate_kfold_splits(self, k, cv_data_splits): # Fetch all classifiers first if len(ALL_MODELS) == 0: self.get_all_classifiers() evaluation = Evaluation.Evaluation(self.ml_pipeline) for model_name, clf in ALL_MODELS.items(): # TODO check if can be moved to evaluation file res_list = [] iters = [] res_dict_keys = {} i = 0 for x_train, x_test, y_train, y_test in cv_data_splits: clf.fit(x_train, y_train) ypred = clf.predict(x_test) yproba = clf.predict_proba(x_test) res = evaluation.evaluate_model(ypred, y_test, yproba) res_list.append(list(res.values())) res_dict_keys = list(res.keys()) iters.append(i + 1) i += 1 self.jlogger.info( "Cross validation split {} has evaluation results {}". format(i + 1, res)) results = pd.DataFrame(res_list, columns=res_dict_keys) fld_path = os.path.join(*[ self.ml_pipeline.job_data['job_results_path'], self.fg_fld_name, RESULTS_FLD_NAME, str(k) + "Fold" ]) os.makedirs(fld_path, exist_ok=True) file_name = model_name + "_" + str(k) + "_fold.csv" cv_result_fp = os.path.join(fld_path, file_name) results.to_csv(cv_result_fp, float_format='%.4f')
def LDA(self): # load data df = pd.read_csv('data//train.csv') Train_data_transformed = df Y = Train_data_transformed["target"] X = Train_data_transformed.drop(['target'], axis=1) X_trainval, X_test, Y_trainval, Y_test = train_test_split( X, Y, random_state=0) # X_train, X_valid, Y_train, Y_valid = train_test_split(X_trainval, Y_trainval, random_state=0) # Standarize data scaler = StandardScaler().fit(X_trainval) X_trainval_transformed = scaler.transform(X_trainval) X_test_transformed = scaler.transform(X_test) # train LDA model Eva = Evaluation.Evaluation() best_score = 0 giniscore = 0 kfolds = 5 for C in [10, 20, 30, 40, 50]: Data_pca = PCA(n_components=C).fit(X_trainval_transformed) X_train_pca = Data_pca.transform(X_trainval_transformed) X_test_pca = Data_pca.transform(X_test_transformed) lda_model = LinearDiscriminantAnalysis().fit( X_train_pca, Y_trainval) prob = lda_model.predict_proba(X_test_pca)[:, 1] giniscore = Eva.gini_score(Y_test, prob) print("When n_components=", C, ":\nMean score is", giniscore) if giniscore > best_score: best_score = giniscore best_parameter = C #Get the best model using best parameter we chosen # Selected_PCA_model = PCA(n_components=50).fit(X_trainval_transformed) Selected_PCA_model = PCA( n_components=best_parameter).fit(X_trainval_transformed) X_train_pca_best = Selected_PCA_model.transform(X_trainval_transformed) X_test_pca_best = Selected_PCA_model.transform(X_test_transformed) LDA_model = LinearDiscriminantAnalysis().fit(X_train_pca_best, Y_trainval) self.gini = Eva.gini_score( Y_test, LDA_model.predict_proba(X_test_pca_best)[:, 1]) return LDA_model, self.gini
def apply_gnb(self): if self.ml_pipeline.config.clf_gnb_flg: x_train = self.ml_pipeline.x_train y_train = self.ml_pipeline.y_train if self.ml_pipeline.config.clf_gnb_auto: model = GaussianNB() chosen_model = model.fit(x_train, y_train) self.jlogger.info(str(chosen_model)) else: manual_params = self.ml_pipeline.config.clf_svm_manual evalclf = Evaluation.Evaluation(self.ml_pipeline) evalclf.evaluate_and_save_results(chosen_model, "gnb") if self.ml_pipeline.config.clf_bagging_gnb: n = self.ml_pipeline.config.clf_bag_gnb_n evalclf.evaluate_bagging_model(chosen_model, n, "gnb_bagging")
def apply_gbm(self): if self.ml_pipeline.config.clf_gbm_flg: if self.ml_pipeline.config.clf_gbm_auto: x_train = self.ml_pipeline.x_train y_train = self.ml_pipeline.y_train cv_inner = StratifiedKFold(n_splits=5, shuffle=True, random_state=43) cv_outer = StratifiedKFold(n_splits=10, shuffle=True, random_state=43) # clf = GradientBoostingClassifier(n_estimators=50, random_state=None, max_depth=2) grid_search_model = self.gbm_grid_search() grid_search_model.cv = cv_inner grid_search_model.fit(x_train, y_train) chosen_model = grid_search_model.best_estimator_ scores = cross_val_score(grid_search_model, x_train, y_train, scoring='f1', cv=cv_outer, n_jobs=1) self.jlogger.info(str(chosen_model)) self.jlogger.info("MEAN F1 scores after nested CV {} ".format(mean(scores))) self.jlogger.info("Standard Deviation for F1 scores after nested CV {} ".format(std(scores))) evalclf = Evaluation.Evaluation(self.ml_pipeline) evalclf.evaluate_and_save_results(chosen_model, "gbm") if self.ml_pipeline.config.clf_bagging_gbm: n = self.ml_pipeline.config.clf_bag_gbm_n evalclf.evaluate_bagging_model(chosen_model, n, "gbm_bagging")
def main(argv=None): # .........................Placeholders for input image and labels........................................................................ keep_prob = tf.placeholder(tf.float32, name="keep_probabilty") # Dropout probability image = tf.placeholder( tf.float32, shape=[None, None, None, 3], name="input_image" ) # Input image batch first dimension image number second dimension width third dimension height 4 dimension RGB # -------------------------Build Net---------------------------------------------------------------------------------------------- Net = BuildNetVgg16.BUILD_NET_VGG16( vgg16_npy_path=model_path) # Create class instance for the net Net.build(image, NUM_CLASSES, keep_prob ) # Build net and load intial weights (weights before training) # -------------------------Data reader for validation/testing images----------------------------------------------------------------------------------------------------------------------------- ValidReader = Data_Reader.Data_Reader(Image_Dir, BatchSize=1) #-------------------------Load Trained model if you dont have trained model see: Train.py----------------------------------------------------------------------------------------------------------------------------- sess = tf.Session() #Start Tensorflow session print("Setting up Saver...") saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(logs_dir) if ckpt and ckpt.model_checkpoint_path: # if train model exist restore it saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") else: print("ERROR NO TRAINED MODEL IN: " + ckpt.model_checkpoint_path + " See Train.py for creating train network ") sys.exit() #--------------------Create output directories for predicted label, one folder for each granulairy of label prediciton--------------------------------------------------------------------------------------------------------------------------------------------- if not os.path.exists(Pred_Dir): os.makedirs(Pred_Dir) if not os.path.exists(Pred_Dir + "/OverLay"): os.makedirs(Pred_Dir + "/OverLay") if not os.path.exists(Pred_Dir + "/Label"): os.makedirs(Pred_Dir + "/Label") print("Running Predictions:") print("Saving output to:" + Pred_Dir) #----------------------Go over all images and predict semantic segmentation in various of classes------------------------------------------------------------- fim = 0 print("Start Predicting " + str(ValidReader.NumFiles) + " images") while (ValidReader.itr < ValidReader.NumFiles): print(str(fim * 100.0 / ValidReader.NumFiles) + "%") fim += 1 # ..................................Load image....................................................................................... FileName = ValidReader.OrderedFiles[ ValidReader.itr] #Get input image name Images = ValidReader.ReadNextBatchClean() # load testing image # Predict annotation using net LabelPred = sess.run(Net.Pred, feed_dict={ image: Images, keep_prob: 1.0 }) Ground_true = cv2.resize( cv2.imread(Label_Ground_true_path + FileName)[:, :, 0], (LabelPred[0].shape[1], LabelPred[0].shape[0])).astype(np.int64) Mask_compare, report = Evaluation.Evaluation(LabelPred[0], Ground_true, NUM_CLASSES) print(report) #------------------------Save predicted labels overlay on images--------------------------------------------------------------------------------------------- misc.imsave(Pred_Dir + "/OverLay/ImagePredicted.png", Overlay.OverLayLabelOnImage(Images[0], LabelPred[0], w)) #Overlay label on image misc.imsave(Pred_Dir + "/OverLay/GroundTrue.png", Overlay.OverLayLabelOnImage(Images[0], Ground_true, w)) misc.imsave(Pred_Dir + "/OverLay/CompareResult.png", Overlay.OverLayLabelOnImage(Images[0], Mask_compare, w)) misc.imsave(Pred_Dir + "/Label/" + FileName[:-4] + ".png" + NameEnd, LabelPred[0].astype(np.uint8))
# SCORENET = torch.load(Args.args.model_name) NET = torch.load(Args.args.model_name) NET = NET.to(torch.device(Args.args.device)) with open(Args.args.model_name + '_trained.p', 'rb') as fp: # read input language referenced_id['trained'] = pickle.load(fp) if True: # closed test Evaluation print("Evaluating .... ") U.timeCheck('s', stime) print("closed test") trainloader = DataManager.get_trainloader(DataManager, 'train', Args.args.batch_size) referenced_id['closed'] = Evaluation.Evaluation(trainloader, NET, referenced_id['trained'], type='closed') if True: # Real test Evaluation print("Evaluating .... ") print("real test") trainloader = DataManager.get_trainloader(DataManager, 'test', Args.args.batch_size) referenced_id['tested'] = Evaluation.Evaluation(trainloader, NET, referenced_id['trained'], type='test') print("Done Evaluating !!!", end=' ') U.timeCheck('e', stime) '''
def main_loop(args): print(args) settings = Settings.Settings(args) history = History.History(settings) connection = Connection.Connection(settings, history) #if connection.failed: return -1 if connection.hard_stop: return -1 cropscoordinates = CropsCoordinates.CropsCoordinates(settings, history) videocapture = VideoCapture.VideoCapture(settings, history) evaluation = Evaluation.Evaluation(settings, connection, cropscoordinates, history) attentionmodel = AttentionModel.AttentionModel(settings, cropscoordinates, evaluation, history) postprocess = Postprocess.Postprocess(settings, history) renderer = Renderer.Renderer(settings, history) debugger = Debugger.Debugger(settings, cropscoordinates, evaluation) settings.save_settings() settings.set_debugger(debugger) for frame, next_frames, frame_number in videocapture.frame_generator_thread_loading( ): settings.frame_number = frame_number print("frame: ", frame[2]) for i in range(len(next_frames)): print("next_frames", i, ": ", next_frames[i][2], next_frames[i][0], next_frames[i][2:]) attention_coordinates = cropscoordinates.get_crops_coordinates( 'attention') #debugger.debug_coordinates_in_frame(attention_coordinates, frame[1],'attention') attention_evaluation = evaluation.evaluate_attention_with_precomputing( frame_number, attention_coordinates, frame, 'attention', next_frames) # attention_evaluation start in attention crops space (size of frame downscaled for attention evaluation # so that we can cut crops of 608x608 from it easily) projected_evaluation = cropscoordinates.project_evaluation_back( attention_evaluation, 'attention') #debugger.debug_evaluation_to_bboxes_after_reprojection(projected_evaluation, frame[1], 'attention', 'afterRepro') # projected_evaluation are now in original image space evaluation_coordinates = cropscoordinates.get_crops_coordinates( 'evaluation') # evaluation_coordinates are in evaluation space. (size of frame downscaled for regular evaluation # so that we can cut crops of 608x608 from it easily) #debugger.debug_coordinates_in_frame(evaluation_coordinates, frame[1], 'evaluation') active_coordinates = attentionmodel.get_active_crops_intersections( projected_evaluation, evaluation_coordinates, frame) #debugger.debug_coordinates_in_frame(active_coordinates, frame[1], 'evaluation', "__"+str(settings.frame_number)+'activeonly') if len(active_coordinates) == 0: print("Nothing left active - that's possibly ok, skip") renderer.render([], frame) history.report_skipped_final_evaluation(frame_number) continue final_evaluation = evaluation.evaluate(active_coordinates, frame, 'evaluation', frame_number) # evaluation are in evaluation space projected_final_evaluation = cropscoordinates.project_evaluation_back( final_evaluation, 'evaluation') # projected back to original space projected_active_coordinates = cropscoordinates.project_coordinates_back( active_coordinates, 'evaluation') processed_evaluations = postprocess.postprocess( projected_active_coordinates, projected_final_evaluation) #debugger.debug_evaluation_to_bboxes_after_reprojection(processed_evaluations, frame[1], 'finalpostprocessed'+frame[0][-8:-4]) renderer.render(processed_evaluations, frame) history.tick_loop(frame_number, True) history.save_whole_history_and_settings()
grad_ascent_model = test_lr.grad_ascent(feature_mat, label_mat, 0.001, 500) print("梯度下降法:") print(grad_ascent_model) print("*******************************") # 随机梯度上升法 origin_model = test_lr.stoc_grad_ascent(feature_mat, label_mat, 0.001) print("随机梯度上升法:") stoc_grad_ascent_model = [] for item in origin_model: new_list = [item] stoc_grad_ascent_model.append(new_list) print(stoc_grad_ascent_model) print("*******************************") print("小样本随机梯度上升法") mini_batch_model_origin = test_lr.mini_batch_grad_ascent( feature_mat, label_mat, 150) mini_batch_model = [] for item in mini_batch_model_origin.tolist()[0]: new_list = [item] mini_batch_model.append(new_list) print(mini_batch_model) print("*******************************") print("预测结果") prediction = test_lr.predict(feature_mat, mini_batch_model) test_ev = ev.Evaluation() confusion_matrix = test_ev.generate_confusion_matrix( np.array(label_mat["label"]).tolist(), prediction) print(confusion_matrix) sns.heatmap(confusion_matrix, annot=True) plt.show()
copy.deepcopy(seed_mioa_dict), mep_k_prod, mep_i_node, seed_set, mioa_dict[mep_k_prod][mep_i_node]) expected_inf = calculateExpectedInf(seed_exp_mioa_dict) ep_t = sum(expected_inf[k] * product_list[k][0] for k in range(num_product)) mg_t = round(ep_t - now_profit, 4) if r_flag: mg_t = safe_div(mg_t, sc) flag_t = seed_set_length if mg_t > 0: celf_item_t = (mg_t, mep_k_prod, mep_i_node, flag_t) heap.heappush_max(celf_heap, celf_item_t) distribution_type = 'm50e25' eva = Evaluation(graph_dict, product_list) iniW = IniWallet(dataset_name, product_name, distribution_type) wallet_dict = iniW.constructWalletDict() sample_pro_acc, sample_bud_acc = 0.0, 0.0 sample_sn_k_acc, sample_pnn_k_acc = [0.0 for _ in range(num_product) ], [0 for _ in range(num_product)] sample_pro_k_acc, sample_bud_k_acc = [0.0 for _ in range(num_product) ], [0.0 for _ in range(num_product)] for _ in range(eva_monte_carlo): pro_k_list, pnn_k_list = eva.getSeedSetProfit(seed_set, wallet_dict.copy()) sample_pro_k_acc = [ (pro_k + sample_pro_k) for pro_k, sample_pro_k in zip(pro_k_list, sample_pro_k_acc) ]
while now_budget < total_budget and mep_i_node != '-1': # print("addSeedIntoSeedSet") high_degree_set.remove(mep_i_node) seed_set, activated_node_set, current_k_budget, current_wallet_list, personal_prob_list = \ dnic.insertSeedIntoSeedSet(mep_k_prod, mep_i_node, seed_set, activated_node_set, current_wallet_list, personal_prob_list) bud_k_list[mep_k_prod] += round(current_k_budget, 4) now_budget += current_k_budget if len(high_degree_set) == 0: high_degree_set = sshd.getHighDegreeSet(degree_dict) expect_profit_list, high_degree_set, ban_set = sshd.calHighDegreeSeedProfit( high_degree_set, ban_set, now_budget, activated_node_set, current_wallet_list, personal_prob_list) mep_k_prod, mep_i_node = sshd.getMostValuableSeed(expect_profit_list) # print("result") eva = Evaluation(graph_dict, seed_cost_dict, product_list, pp_strategy, whether_infect_not_only_buying) now_profit, now_pro_k_list, now_num_k_an = eva.getSeedProfit( seed_set, wallet_list, [[1.0 for _ in range(num_node)] for _ in range(num_product)]) now_num_k_seed = [len(k) for k in seed_set] result.append([ round(now_profit, 4), round(now_budget, 4), now_num_k_seed, now_num_k_an, seed_set ]) avg_profit += now_profit avg_budget += now_budget for k in range(num_product): avg_num_k_seed[k] += now_num_k_seed[k] avg_num_k_an[k] += now_num_k_an[k] how_long = round(time.time() - start_time, 4)
K = env.nbArms policies = [ UCB(K, trunc), UCBV(K, trunc), klUCB(K, trunc), klUCB(K, klucb=klucbExp), KLempUCB(K, trunc) ] tsav = int_(linspace(100, horizon - 1, 200)) if graphic == 'yes': figure(1) k = 0 for policy in policies: ev = Evaluation(env, policy, nbRep, horizon, tsav) print ev.meanReward() print ev.meanNbDraws() meanRegret = ev.meanRegret() if graphic == 'yes': semilogx(1 + tsav, meanRegret, color=colors[k]) xlabel('Time') ylabel('Regret') k = k + 1 if graphic == 'yes': legend([policy.__class__.__name__ for policy in policies], loc=0) title('Average regret for various policies') show()
def run(self): self.e = ev.Evaluation(self.dataset, self.algo) self.score = self.e.getCrossValScores() self.meanscore = self.e.mean_scores_K_fold self.finishsignal.emit()
k=k, c_val=c_val, sigma=sigma, epochs=epochs, noise_dim=noise_dim, num_examples_to_generate=num_examples_to_generate) gan_instance.train_gan(train_images=desired_train_images) gan_instance.save_plot_fid() for k in k_values: for desired_digit in desired_digit_values: evaluator = gan_eval.Evaluation(desired_digit=desired_digit, alpha=alpha, beta=beta, gamma=gamma, k=k, c_val=c_val, sigma_values=sigma_values) evaluator.evaluate_gan_output() evaluator.plot_fid_and_confidence() fid_all_digits = pd.DataFrame() confidence_all_digits = pd.DataFrame() digits_colors = [ 'tab:blue', 'tab:red', 'tab:orange', 'tab:green', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan' ] for k in k_values:
print("########################################################") print("Shared items=", nSharedItems, ", Shared users=", nSharedUsers, ", iterations = ", iterations) print("########################################################") master = MasterRS(filename, chunkSize, itemCol, userCol, colNames, nSharedItems, nSharedUsers, iterations, colTypes) # finalData = master.consolidateData(filename) # finalData.to_csv(saveFile, sep=',') finalData = pd.read_csv(saveFile, index_col=0) finalData.columns = finalData.columns.astype(int) nRecom = len(set(finalData[itemCol])) nTest = len(set(finalData[userCol])) print("===== Begin partition =====") eval = Evaluation(finalData, itemCol, userCol, pRatios, testType) trainDF = eval.dp.trainDF validDF = eval.dp.validDF testDF = eval.dp.testDF newTrainDF, newTestDF = eval.dp.getNewTrainTestSet() userList = master.getUserList(newTrainDF) itemList = master.getItemList(newTrainDF) nCols = len(newTrainDF.columns) permList = np.random.permutation(userList) simMatrix = master.getCollaborativeSimMatrix(newTrainDF, simMetList[0]) recDictCF1 = master.getCFRecommenderDict(newTrainDF, simMatrix, nRecom)
ic = {} x = 0 for i in initial_clusters: ic.setdefault(x,[]) ic[x] = initial_clusters[i] x = x+1 features = "URDUPHONE_EDITDISTANCE_WORDID" weights = [1,1,1,1] st= time.time() sd = lexc.LexC() clusters = sd.make_clusters(ic,2,0.4,features,weights,prev_dict,next_dict) es = evaluation.Evaluation() tp,tr,f = es.evaluate(gold,clusters) print(tp) print(tr) print(f) filename = features + str(weights[0]) + "_" + str(weights[1]) + "_" + str(weights[2]) + "_" + str(weights[3]) file = open("Result/" + filename + "Evaluation_Results.txt","w") file.write("Precision:" + str(tp)) file.write("\n") file.write("Recall:" + str(tr)) file.write("\n") file.write("F Score:" + str(f)) print(time.time() -st) file.close()
def main(): Preprocess() Evaluation()
def unitEvaluation(lamIndex): systemParaDict['arrivalRate'] = lamIndex / 10 EvalObj = Evaluation(systemParaDict, PGNNParaDict) EvalObj.EvaConvergence()
# Does this input exists and is input a file (not a directory). if os.path.exists(input) or os.path.isfile(input): file = open(input, 'r') # open file for reading parser = Parser.Parser(file) # construct new parser object. (count, color, steps, board) = parser.parse() # Parse the file. file.close() if count == "1": print MoveGenerator.MoveGenerator.randSetup(color) else: hash = Hash.Hash() # Construct a new hash hash.calculateHashkey( board) # Calculate the hash key for this given board. eval = Evaluation.Evaluation() (start_row, start_col, end_row, end_col) = eval.getStrongestGrid(board, color) mv = MoveGenerator.MoveGenerator(count, color, board, hash) mv.MAX_STEPS = 3 mv.genMoves("", start_row, start_col, end_row, end_col) turns = mv.moveStepHashes bestMoveStrength = -sys.maxint - 1 bestStep = "" for turn in turns: newBoard = turn[0] strength = eval.evaluateBoard(newBoard, color, True) if strength > bestMoveStrength:
def execute_simulation(nbRep, horizon, pol_index, traffic, scenario, objective, base_dir='data', windows_size=0): # Set traffic volume if traffic == 'high_traffic': params = params_high else: params = params_low f = generate_ratio_function(params) process = NHPoissonProcess(f) # config = Not_Delayed_conf() #config = Delayed_DR_Conf_Random_poisson(1./150, 480) #config = Delayed_DR_ConfBatchPoisson(1./150, windows_time=2, trunc=480) # env = MABEventDelayed( # [Bernoulli(p, samples=nbRep * horizon) for p in [0.1, 0.05]], # config, process, exit_mode=0) config = get_config_reward(objective, pol_index, windows_size) arms = get_arms_distributions(scenario, objective) trunc_value = 1 if objective == 'conversion_rate': exit_mode = 0 else: exit_mode = 1 env = MABEventDelayed(arms, config, process, exit_mode) policies = get_policy(env.nbArms, trunc_value, pol_index) print 'policiy: ' + str(policies) # tsav = int_(linspace(100, horizon - 1, 200)) for wrap_pol in policies: ev = Evaluation(env, wrap_pol, nbRep, max_decisions) print ev.mean_reward() print ev.mean_nb_draws() prefix = '' means = ev.means regret = ev.regret() rewards = ev.get_rewards() if wrap_pol.__class__.__name__ == 'BlackBox': policy = wrap_pol.policy prefix = 'BlackBox_' else: policy = wrap_pol name_klucb = str(getattr(policy, "klucb", "none")) name_scale = str(getattr(policy, "scale", "none")) name_gap = str(getattr(policy, "gap", "none")) name_conf = str(getattr(policy, "confidence", "none")) base_name = base_dir + "/" + prefix + policy.__class__.__name__ + "_" + str( nbRep) \ + "-" + str(objective) + "_" + str(scenario) + "_wsize" + str(windows_size) tr = traffic # PR2: save(base_name + "_" + tr + "_" + "last_means_pr2", means['PR2']) save(base_name + "_" + tr + "_" + "last_regrets_pr2", ev.get_regrets('PR2')) save(base_name + "_" + tr + "_" + "last_rewards_pr2", ev.get_rewards('PR2')) save(base_name + "_" + tr + "_" + "num_events_pr2", ev.exit_points['PR2']) save(base_name + "_" + tr + "_" + "nbPulls_pr2", ev.nbPulls['PR2']) # All: save(base_name + "_" + tr + "_" + "last_means", means['all']) save(base_name + "_" + tr + "_" + "last_regrets", ev.get_regrets('all')) save(base_name + "_" + tr + "_" + "last_rewards", ev.get_rewards('all')) save(base_name + "_" + tr + "_" + "num_events", ev.exit_points['all']) save(base_name + "_" + tr + "_" + "nbPulls", ev.nbPulls['all']) # PR3: save(base_name + "_" + tr + "_" + "last_means_pr3", means['PR3']) save(base_name + "_" + tr + "_" + "last_regrets_pr3", ev.get_regrets('PR3')) save(base_name + "_" + tr + "_" + "last_rewards_pr3", ev.get_rewards('PR3')) save(base_name + "_" + tr + "_" + "num_events_pr3", ev.exit_points['PR3']) save(base_name + "_" + tr + "_" + "nbPulls_pr3", ev.nbPulls['PR3']) # sampling_beta: save(base_name + "_" + tr + "_" + "last_means_sampling_beta", means['sampling_beta']) save(base_name + "_" + tr + "_" + "last_regrets_sampling_beta", ev.get_regrets('sampling_beta')) save(base_name + "_" + tr + "_" + "last_rewards_sampling_beta", ev.get_rewards('sampling_beta')) save(base_name + "_" + tr + "_" + "num_events_sampling_beta", ev.exit_points['sampling_beta']) save(base_name + "_" + tr + "_" + "nbPulls_sampling_beta", ev.nbPulls['sampling_beta']) # sampling_beta_PR3: save(base_name + "_" + tr + "_" + "last_means_sampling_beta_PR3", means['sampling_beta_PR3']) save(base_name + "_" + tr + "_" + "last_regrets_sampling_beta_PR3", ev.get_regrets('sampling_beta_PR3')) save(base_name + "_" + tr + "_" + "last_rewards_sampling_beta_PR3", ev.get_rewards('sampling_beta_PR3')) save(base_name + "_" + tr + "_" + "num_events_sampling_beta_PR3", ev.exit_points['sampling_beta_PR3']) save(base_name + "_" + tr + "_" + "nbPulls_sampling_beta_PR3", ev.nbPulls['sampling_beta_PR3']) # empirical_bernstain: save(base_name + "_" + tr + "_" + "last_means_empirical_bernstain", means['empirical_bernstain']) save(base_name + "_" + tr + "_" + "last_regrets_empirical_bernstain", ev.get_regrets('empirical_bernstain')) save(base_name + "_" + tr + "_" + "last_rewards_empirical_bernstain", ev.get_rewards('empirical_bernstain')) save(base_name + "_" + tr + "_" + "num_events_empirical_bernstain", ev.exit_points['empirical_bernstain']) save(base_name + "_" + tr + "_" + "nbPulls_empirical_bernstain", ev.nbPulls['empirical_bernstain']) #regret: save(base_name + "_" + tr + "_" + "cum_regret_2", ev.cumRegret)
def Neural_Network(self): # Balance Data r = 0.25 Handle_Data = Data_handling.DataHandling(self.X_trainval, self.y_trainval) X_trainval_bal, y_trainval_bal = Handle_Data.SMOT(r) # Standarize data scaler = StandardScaler().fit(X_trainval_bal) X_trainval_bal_transformed = scaler.transform(X_trainval_bal) X_test_transformed = scaler.transform(self.X_test) # PCA Data_pca = PCA(n_components=200).fit(X_trainval_bal_transformed) X_trainval_bal_transformed = Data_pca.transform( X_trainval_bal_transformed) X_test_transformed = Data_pca.transform(X_test_transformed) activations = ['tanh', 'relu'] alphas = [0.6, 0.7] solvers = ['sgd', 'adam'] Eva = Evaluation.Evaluation() best_score = 0 # Get the best parameter, since this is a long-time work, so after we found the best parameter we need, we comment the following code. for acti in activations: for solver in solvers: MLPmodel = MLPClassifier(solver=solver, activation=acti, random_state=10, hidden_layer_sizes=[100, 10], alpha=0.5, max_iter=5000) scorer = make_scorer(Eva.gini_score, needs_proba=True) score = cross_val_score(MLPmodel, X_trainval_bal_transformed, y_trainval_bal, cv=5, scoring=scorer) # print(score) score = score.mean() print("When parameter activation=", acti, ", parameter solver=", solver, ":\nMean score is", score) if score >= best_score: best_score = score best_acti = acti best_solver = solver # Get the best model BestMLPModel = MLPClassifier(solver=best_solver, activation=best_acti, random_state=10, hidden_layer_sizes=[100, 10], alpha=0.5, max_iter=5000).fit( X_trainval_bal_transformed, y_trainval_bal) #Here we use the best parameter we found above to train the model # BestMLPModel = MLPClassifier(solver='sgd', activation='relu', random_state=10, # hidden_layer_sizes=[100, 10], # alpha=0.6, max_iter=5000).fit(X_trainval_bal_transformed, y_trainval_bal) predict_proba = BestMLPModel.predict_proba(X_test_transformed)[:, 1] self.gini = Eva.gini_score(self.y_test, predict_proba) return BestMLPModel, self.gini