def main(): k = 100 m = 30 threshold = 0.1 filename = "lena_petit.tif" slic = SLIC.SLIC(filename, k=k, m=m, threshold=threshold) np.save(filename[:-3] + 'npy', slic) savename = filename[:-4]+'_'+'slic_' + \ str(k)+'_'+str(m)+'_'+str(threshold)+filename[-4:] im = SLIC.show_segmentation(filename, savename, slic, show_im=True) slic = np.load(filename[:-3] + 'npy') slic_graph = Graph(filename) im_graph = slic_graph.generate_graph(slic) slic_graph.graph_save(filename[:-3] + 'pkl') # %% test post-processing new_graph = Graph() new_graph.graph_load(filename[:-3] + 'pkl') post_processing.distance_based_processing(new_graph) res = new_graph.translate_2_label_matrix() im_seg = SLIC.show_segmentation(filename, filename[:-4] + '_post_processing' + filename[-4:], res, show_im=True) im_seg = SLIC.show_segmentation(filename, filename[:-4] + '_post_processing_white' + filename[-4:], res, show_im=True, white=True) # # %% test fusion for i in range(54, 55, 5): fusion.fusion(new_graph, i) res = new_graph.translate_2_label_matrix() im_seg = SLIC.show_segmentation(filename, filename[:-4] + '_fusion_' + str(i) + filename[-4:], res, show_im=False, color=[255, 255, 255]) im_seg = SLIC.show_segmentation(filename, filename[:-4] + '_fusion_white_' + str(i) + filename[-4:], res, show_im=False, white=True)
def tri(self): # si la liste 'self.LIST' comporte 1 élément, on ne fait rien : if len(self.LIST) == 1: return self.LIST # si la liste 'self.LIST' comporte 2 éléments, # on inverse la liste si le premier élément est supérieur au deuxième élément : if len(self.LIST) == 2: if self.LIST[0] > self.LIST[1]: self.LIST = self.LIST[::-1] # inverse la liste return self.LIST # on initialise la variable 'self.LEFT', # 'self.LEFT.LIST' correspond à la PREMIÈRE moitié de la liste 'self.LIST' : self.LEFT = Node(self.LIST[0 : len(self.LIST) // 2]) # on initialise la variable 'self.RIGHT' # 'self.RIGHT.LIST' correspond à la DEUXIÈME moitié de la liste 'self.LIST' : self.RIGHT = Node(self.LIST[len(self.LIST) // 2 : len(self.LIST)]) # fusionne la liste 'self.LEFT.LIST' et 'self.RIGHT.LIST' : self.LIST = fusion(self.LEFT.tri(), self.RIGHT.tri()) return self.LIST
def main(): num_case = 0 CAM = KneeCAM(out_class=2) for num_loc in range(0, 23): CAM.Manager.options['slice_range'][0] = list(range(num_loc, num_loc + 1, 1)) CAM.Manager.options['slice_range'][1] = list(range(num_loc, num_loc + 1, 1)) """ Initialize """ text_name = 'out_' + CAM.Manager.options['network_choice'] + '_' + CAM.Manager.options['fusion_method'] + '_' \ + str(num_loc) + '.txt' model_ini = DefModel(Manager=CAM.Manager, zlen=len(CAM.Manager.options['slice_range'][0]), out_class=2) CAM.Manager.init_model(model_ini) CAM.prep_training(num_case, num_loc, text_name, training='training', cont=False) fusion(CAM.Manager, num_case, 512)
def launchChainSequential(PathTEST, tiles, pathTilesL8, pathTilesL5, pathTilesS2, pathNewProcessingChain, pathTilesFeat, configFeature, shapeRegion, field_Region, model, shapeData, dataField, pathConf, N, REARRANGE_PATH, MODE, REARRANGE_FLAG, CLASSIFMODE, NOMENCLATURE, COLORTABLE, RATIO, TRAIN_MODE): if PathTEST != "/" and os.path.exists(PathTEST): choice = "" while (choice != "yes") and (choice != "no") and (choice != "y") and ( choice != "n"): choice = raw_input( "the path " + PathTEST + " already exist, do you want to remove it ? yes or no : ") if (choice == "yes") or (choice == "y"): shutil.rmtree(PathTEST) else: print "Unsafe mode. Overwriting existing output folder." #sys.exit(-1) fieldEnv = "FID" # do not change pathModels = PathTEST + "/model" pathEnvelope = PathTEST + "/envelope" pathClassif = PathTEST + "/classif" pathTileRegion = PathTEST + "/shapeRegion" classifFinal = PathTEST + "/final" dataRegion = PathTEST + "/dataRegion" pathAppVal = PathTEST + "/dataAppVal" pathStats = PathTEST + "/stats" cmdPath = PathTEST + "/cmd" config_model = PathTEST + "/config_model" if not os.path.exists(PathTEST): os.mkdir(PathTEST) if not os.path.exists(pathModels): os.mkdir(pathModels) if not os.path.exists(pathEnvelope): os.mkdir(pathEnvelope) if not os.path.exists(pathClassif): os.mkdir(pathClassif) if not os.path.exists(config_model): os.mkdir(config_model) if not os.path.exists(pathTileRegion): os.mkdir(pathTileRegion) if not os.path.exists(classifFinal): os.mkdir(classifFinal) if not os.path.exists(dataRegion): os.mkdir(dataRegion) if not os.path.exists(pathAppVal): os.mkdir(pathAppVal) if not os.path.exists(pathStats): os.mkdir(pathStats) if not os.path.exists(cmdPath): os.mkdir(cmdPath) os.mkdir(cmdPath + "/stats") os.mkdir(cmdPath + "/train") os.mkdir(cmdPath + "/cla") os.mkdir(cmdPath + "/confusion") os.mkdir(cmdPath + "/features") os.mkdir(cmdPath + "/fusion") os.mkdir(cmdPath + "/splitShape") feat = GFD.CmdFeatures(PathTEST, tiles, pathNewProcessingChain, pathTilesL8, pathTilesL5, pathTilesS2, pathConf, pathTilesFeat, None) for i in range(len(feat)): print feat[i] os.system(feat[i]) # Création des enveloppes env.GenerateShapeTile(tiles, pathTilesFeat, pathEnvelope, None, configFeature) if MODE != "outside": area.generateRegionShape(MODE, pathEnvelope, model, shapeRegion, field_Region, configFeature, None) # Création des régions par tuiles RT.createRegionsByTiles(shapeRegion, field_Region, pathEnvelope, pathTileRegion, None) # pour tout les fichiers dans pathTileRegion regionTile = fu.FileSearch_AND(pathTileRegion, True, ".shp") # ///////////////////////////////////////////////////////////////////////////////////////// for path in regionTile: ExtDR.ExtractData(path, shapeData, dataRegion, pathTilesFeat, configFeature, None) # ///////////////////////////////////////////////////////////////////////////////////////// if REARRANGE_FLAG == 'True': RAM.generateRepartition(PathTEST, pathConf, shapeRegion, REARRANGE_PATH, dataField) # pour tout les shape file par tuiles présent dans dataRegion, créer un ensemble dapp et de val dataTile = fu.FileSearch_AND(dataRegion, True, ".shp") # ///////////////////////////////////////////////////////////////////////////////////////// for path in dataTile: RIST.RandomInSituByTile(path, dataField, N, pathAppVal, RATIO, pathConf, None) # ///////////////////////////////////////////////////////////////////////////////////////// if MODE == "outside" and CLASSIFMODE == "fusion": Allcmd = genCmdSplitS.genCmdSplitShape(pathConf) for cmd in Allcmd: print cmd os.system(cmd) if TRAIN_MODE == "points": trainShape = fu.FileSearch_AND(PathTEST + "/dataAppVal", True, ".shp", "learn") for shape in trainShape: print "" vs.generateSamples(shape, None, configFeature) VSM.vectorSamplesMerge(configFeature) # génération des fichiers de statistiques if not TRAIN_MODE == "points": AllCmd = MS.generateStatModel(pathAppVal, pathTilesFeat, pathStats, cmdPath + "/stats", None, configFeature) for cmd in AllCmd: print cmd print "" stat = cmd.split(' ')[-1] print "Checking if " + stat + " exists..." if not os.path.exists(stat): os.system(cmd) else: print "Keeping existing " + stat + "." # ///////////////////////////////////////////////////////////////////////////////////////// # génération des commandes pour lApp allCmd = LT.launchTraining(pathAppVal, pathConf, pathTilesFeat, dataField, pathStats, N, cmdPath + "/train", pathModels, None, None) # ///////////////////////////////////////////////////////////////////////////////////////// for cmd in allCmd: print cmd print "" os.system(cmd) # ///////////////////////////////////////////////////////////////////////////////////////// # génération des commandes pour la classification cmdClassif = LC.launchClassification(pathModels, pathConf, pathStats, pathTileRegion, pathTilesFeat, shapeRegion, field_Region, N, cmdPath + "/cla", pathClassif, None) # ///////////////////////////////////////////////////////////////////////////////////////// for cmd in cmdClassif: print cmd print "" os.system(cmd) # ///////////////////////////////////////////////////////////////////////////////////////// if CLASSIFMODE == "separate": # Mise en forme des classifications CS.ClassificationShaping(pathClassif, pathEnvelope, pathTilesFeat, fieldEnv, N, classifFinal, None, configFeature, COLORTABLE) # génération des commandes pour les matrices de confusions allCmd_conf = GCM.genConfMatrix(classifFinal, pathAppVal, N, dataField, cmdPath + "/confusion", configFeature, None) for cmd in allCmd_conf: print cmd os.system(cmd) confFus.confFusion(shapeData, dataField, classifFinal + "/TMP", classifFinal + "/TMP", classifFinal + "/TMP", configFeature) GR.genResults(classifFinal, NOMENCLATURE) elif CLASSIFMODE == "fusion" and MODE != "one_region": cmdFus = FUS.fusion(pathClassif, configFeature, None) for cmd in cmdFus: print cmd os.system(cmd) # gestion des nodata fusionFiles = fu.FileSearch_AND(pathClassif, True, "_FUSION_") for fusionpath in fusionFiles: ND.noData(PathTEST, fusionpath, field_Region, pathTilesFeat, shapeRegion, N, configFeature, None) # Mise en forme des classifications CS.ClassificationShaping(pathClassif, pathEnvelope, pathTilesFeat, fieldEnv, N, classifFinal, None, configFeature, COLORTABLE) # génération des commandes pour les matrices de confusions allCmd_conf = GCM.genConfMatrix(classifFinal, pathAppVal, N, dataField, cmdPath + "/confusion", configFeature, None) # ///////////////////////////////////////////////////////////////////////////////////////// for cmd in allCmd_conf: print cmd os.system(cmd) # ///////////////////////////////////////////////////////////////////////////////////////// confFus.confFusion(shapeData, dataField, classifFinal + "/TMP", classifFinal + "/TMP", classifFinal + "/TMP", configFeature) GR.genResults(classifFinal, NOMENCLATURE) elif CLASSIFMODE == "fusion" and MODE == "one_region": raise Exception( "You can't choose the 'one region' mode and use the fusion mode together" ) outStat = Config(file(pathConf)).chain.outputStatistics if outStat == "True": AllTiles = Config(file(pathConf)).chain.listTile AllTiles = AllTiles.split(" ") for currentTile in AllTiles: OutS.outStats(pathConf, currentTile, N, None) MOutS.mergeOutStats(pathConf)
for r in range(2, n_modalities + 1): # combinations of 2 or more modalities c = itertools.combinations(u_modalities, r=r) comb = itertools.chain(comb, c) if comb else c for c in comb: print(c) for d in u_datasets: npys2 = npys1[datasets == d] splits = npys2[:, 3] for s in u_splits: npys3 = npys2[splits == str(s)] modalities = npys3[:, 4] npy_paths = [] for m in c: n = npys3[modalities == m][0] npy_paths.append(os.path.join(n[0], n[1], n[2])) args = argparse.Namespace(d=d, m=method, npy_paths=npy_paths, s=s, settings=settings) best_weight, _, prec = fusion(args) print(d, s, " + ".join(list(c))) with open("fusion_fc.out", 'a') as f: f.write("{}\t{}\t{}\t{}\t{}\t{}\n".format( method, d, s, " + ".join(list(c)), best_weight, prec))
npy_dict = npy_dict[train_type] it = itertools.product(dataset, splits) with open(output, "w") as f: f.write("dataset\tsplit\tmodality\tprec\n") for d, s in it: key = "{}_s{}".format(d,s) npy_paths = [] for mod in modalities: npy_path = npy_path_fmt.format(npy_dict[key][mod], d, mod, s) if not os.path.isfile(npy_path): npy_paths = [] break npy_paths.append(npy_path) if len(npy_paths) >= 2: args2 = argparse.Namespace(d=d, m="individual", npy_paths=npy_paths, s=s, settings=settings) print(args2) _, _, prec = fusion(args2) with open(output, "a") as f: for mod, p in zip(modalities, prec): f.write("{}\t{}\t{}\t{:.04f}\n".format(d, s, mod, p))
with open(output, "w") as f: f.write("dataset\tsplit\tmethod\tcombination\tparam\tprec\n") for d, s, m, c in it: key = "{}_s{}".format(d, s) npy_paths = [] for mod in c: npy_path = npy_path_fmt.format(npy_dict[key][mod], d, mod, s) if not os.path.isfile(npy_path): npy_paths = [] break npy_paths.append(npy_path) if len(npy_paths) >= 2: args2 = argparse.Namespace(d=d, m=m, npy_paths=npy_paths, s=s, settings=settings) print(args2) ret = fusion(args2) if ret: param, _, prec = ret with open(output, "a") as f: f.write("{}\t{}\t{}\t{}\t{}\t{:.04f}\n".format( d, s, m, c, param, prec))
with timer("4. training the main model"): classifier = 'fusion' x_train = df_train.loc[:, col_feat].values # Return a Numpy representation of the DataFrame. x_valid = df_valid.loc[:, col_feat].values # \ is to change to another line after '=' finish_train, finish_valid = \ df_train['finish'].values, df_valid['finish'].values like_train, like_valid = \ df_train['like'].values, df_valid['like'].values print('features preparation done') y_train_finish, y_valid_finish = finish_train, finish_valid model_finish = fusion(x_train, y_train_finish) y_train_like, y_valid_like = like_train, like_valid model_like = fusion(x_train, y_train_like) y_pred_finish = model_finish.predict(x_valid) print('finish ROC ACC:', roc_auc_score(finish_valid, y_pred_finish)) print('finish confusion_matrix:', confusion_matrix(y_valid_finish, y_pred_finish)) print('finish accuracy_score:', accuracy_score(y_valid_finish, y_pred_finish)) #draw_roc(finish_valid, y_pred_finish,'Receiver operating characteristic Curve for finish') y_pred_like = model_like.predict(x_valid) print('like ROC ACC:', roc_auc_score(like_valid, y_pred_like)) print('like confusion_matrix:',
exit(1) else: input_BU = sys.argv[1] input_TD = sys.argv[2] groundtruth = sys.argv[3] # input_BU can stay as it is # Patterns from input_TD have to be extracted and mapped according to the groundtruth extract(input_TD,"extracted",groundtruth) print "Extraction and mapping done" # create fusion # fusion(input_BU,"extracted","fusion_result") fusion(input_BU,"extracted","fusion_result") print "Fusion of "+str(input_BU)+" and "+str(input_TD)+" is done" print # evaluation perl .pl ground input_BU 1 1 cmd = ["perl", "eval_metrics_v3.pl", groundtruth, input_BU,argument1,argument2] pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE) pipe.wait() cmd = ["perl", "results2texV2.pl", "evaluation_results/"+input_BU+".L1_"+argument1+"L2_"+argument2+".results.csv"] pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE) pipe.wait() print "Evaluation of "+str(input_BU)+ " done" print # evaluation perl .pl ground extracted 1 1 # cmd = ["perl", "eval_metrics.pl", groundtruth, "extracted","1","1"] cmd = ["perl", "eval_metrics_v3.pl", groundtruth, "extracted",argument1,argument2] pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE)
if __name__ == "__main__": if len(sys.argv) < 5: print "python start.py input_BU input_TD groundtruth threshold metric fusion_strategy mapping" exit(1) else: input_BU_orig = sys.argv[1] input_TD_orig = sys.argv[2] groundtruth = sys.argv[3] strategy = sys.argv[4] # 1 for simple union, 2 for top-down augmentation, 3 for bottom-up augmentation # mapping = sys.argv[5] # 0 for none, 1 for mapping top-down to bottom-up, 2 for reverse # threshold = sys.argv[6] # 1 (no threshold), 0.8 or 0.6 # Copy the top down and bottom up grammars for possible processing (to keep the originals unaffected) cmd = ["cp", input_TD_orig, "../extracted_TD"] pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) pipe.wait() input_TD = "extracted_TD" cmd = ["cp", input_BU_orig, "../extracted_BU"] pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) pipe.wait() input_BU = "extracted_BU" fusion("../" + input_TD, "../" + input_BU, "fusion_result." + strategy, strategy) print "Fusion of " + str(input_BU_orig) + " and " + str(input_TD_orig) + " is done" exit(1) # cmd = ["perl", "results2texV2.pl", "../evaluation_results/v_"+version+"/metric_"+metric+"/mapping_"+mapping+"/strategy_"+strategy+"/thresh_"+threshold+"/eval_fusion_result.L1_"+argument1+"L2_"+argument2+".results.csv"] # pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE) # print "\t\tEvaluation of the fusion fragments is done"