def main():
k = 100
m = 30
threshold = 0.1
filename = "lena_petit.tif"
slic = SLIC.SLIC(filename, k=k, m=m, threshold=threshold)
np.save(filename[:-3] + 'npy', slic)
savename = filename[:-4]+'_'+'slic_' + \
str(k)+'_'+str(m)+'_'+str(threshold)+filename[-4:]
im = SLIC.show_segmentation(filename, savename, slic, show_im=True)
slic = np.load(filename[:-3] + 'npy')
slic_graph = Graph(filename)
im_graph = slic_graph.generate_graph(slic)
slic_graph.graph_save(filename[:-3] + 'pkl')
# %% test post-processing
new_graph = Graph()
new_graph.graph_load(filename[:-3] + 'pkl')
post_processing.distance_based_processing(new_graph)
res = new_graph.translate_2_label_matrix()
im_seg = SLIC.show_segmentation(filename,
filename[:-4] + '_post_processing' +
filename[-4:],
res,
show_im=True)
im_seg = SLIC.show_segmentation(filename,
filename[:-4] + '_post_processing_white' +
filename[-4:],
res,
show_im=True,
white=True)
# # %% test fusion
for i in range(54, 55, 5):
fusion.fusion(new_graph, i)
res = new_graph.translate_2_label_matrix()
im_seg = SLIC.show_segmentation(filename,
filename[:-4] + '_fusion_' + str(i) +
filename[-4:],
res,
show_im=False,
color=[255, 255, 255])
im_seg = SLIC.show_segmentation(filename,
filename[:-4] + '_fusion_white_' +
str(i) + filename[-4:],
res,
show_im=False,
white=True)
def tri(self): # si la liste 'self.LIST' comporte 1 élément, on ne fait rien : if len(self.LIST) == 1: return self.LIST # si la liste 'self.LIST' comporte 2 éléments, # on inverse la liste si le premier élément est supérieur au deuxième élément : if len(self.LIST) == 2: if self.LIST[0] > self.LIST[1]: self.LIST = self.LIST[::-1] # inverse la liste return self.LIST # on initialise la variable 'self.LEFT', # 'self.LEFT.LIST' correspond à la PREMIÈRE moitié de la liste 'self.LIST' : self.LEFT = Node(self.LIST[0 : len(self.LIST) // 2]) # on initialise la variable 'self.RIGHT' # 'self.RIGHT.LIST' correspond à la DEUXIÈME moitié de la liste 'self.LIST' : self.RIGHT = Node(self.LIST[len(self.LIST) // 2 : len(self.LIST)]) # fusionne la liste 'self.LEFT.LIST' et 'self.RIGHT.LIST' : self.LIST = fusion(self.LEFT.tri(), self.RIGHT.tri()) return self.LIST
def main():
num_case = 0
CAM = KneeCAM(out_class=2)
for num_loc in range(0, 23):
CAM.Manager.options['slice_range'][0] = list(range(num_loc, num_loc + 1, 1))
CAM.Manager.options['slice_range'][1] = list(range(num_loc, num_loc + 1, 1))
""" Initialize """
text_name = 'out_' + CAM.Manager.options['network_choice'] + '_' + CAM.Manager.options['fusion_method'] + '_' \
+ str(num_loc) + '.txt'
model_ini = DefModel(Manager=CAM.Manager, zlen=len(CAM.Manager.options['slice_range'][0]), out_class=2)
CAM.Manager.init_model(model_ini)
CAM.prep_training(num_case, num_loc, text_name, training='training', cont=False)
fusion(CAM.Manager, num_case, 512)
def launchChainSequential(PathTEST, tiles, pathTilesL8, pathTilesL5,
pathTilesS2, pathNewProcessingChain, pathTilesFeat,
configFeature, shapeRegion, field_Region, model,
shapeData, dataField, pathConf, N, REARRANGE_PATH,
MODE, REARRANGE_FLAG, CLASSIFMODE, NOMENCLATURE,
COLORTABLE, RATIO, TRAIN_MODE):
if PathTEST != "/" and os.path.exists(PathTEST):
choice = ""
while (choice != "yes") and (choice != "no") and (choice != "y") and (
choice != "n"):
choice = raw_input(
"the path " + PathTEST +
" already exist, do you want to remove it ? yes or no : ")
if (choice == "yes") or (choice == "y"):
shutil.rmtree(PathTEST)
else:
print "Unsafe mode. Overwriting existing output folder."
#sys.exit(-1)
fieldEnv = "FID" # do not change
pathModels = PathTEST + "/model"
pathEnvelope = PathTEST + "/envelope"
pathClassif = PathTEST + "/classif"
pathTileRegion = PathTEST + "/shapeRegion"
classifFinal = PathTEST + "/final"
dataRegion = PathTEST + "/dataRegion"
pathAppVal = PathTEST + "/dataAppVal"
pathStats = PathTEST + "/stats"
cmdPath = PathTEST + "/cmd"
config_model = PathTEST + "/config_model"
if not os.path.exists(PathTEST):
os.mkdir(PathTEST)
if not os.path.exists(pathModels):
os.mkdir(pathModels)
if not os.path.exists(pathEnvelope):
os.mkdir(pathEnvelope)
if not os.path.exists(pathClassif):
os.mkdir(pathClassif)
if not os.path.exists(config_model):
os.mkdir(config_model)
if not os.path.exists(pathTileRegion):
os.mkdir(pathTileRegion)
if not os.path.exists(classifFinal):
os.mkdir(classifFinal)
if not os.path.exists(dataRegion):
os.mkdir(dataRegion)
if not os.path.exists(pathAppVal):
os.mkdir(pathAppVal)
if not os.path.exists(pathStats):
os.mkdir(pathStats)
if not os.path.exists(cmdPath):
os.mkdir(cmdPath)
os.mkdir(cmdPath + "/stats")
os.mkdir(cmdPath + "/train")
os.mkdir(cmdPath + "/cla")
os.mkdir(cmdPath + "/confusion")
os.mkdir(cmdPath + "/features")
os.mkdir(cmdPath + "/fusion")
os.mkdir(cmdPath + "/splitShape")
feat = GFD.CmdFeatures(PathTEST, tiles, pathNewProcessingChain,
pathTilesL8, pathTilesL5, pathTilesS2, pathConf,
pathTilesFeat, None)
for i in range(len(feat)):
print feat[i]
os.system(feat[i])
# Création des enveloppes
env.GenerateShapeTile(tiles, pathTilesFeat, pathEnvelope, None,
configFeature)
if MODE != "outside":
area.generateRegionShape(MODE, pathEnvelope, model, shapeRegion,
field_Region, configFeature, None)
# Création des régions par tuiles
RT.createRegionsByTiles(shapeRegion, field_Region, pathEnvelope,
pathTileRegion, None)
# pour tout les fichiers dans pathTileRegion
regionTile = fu.FileSearch_AND(pathTileRegion, True, ".shp")
# /////////////////////////////////////////////////////////////////////////////////////////
for path in regionTile:
ExtDR.ExtractData(path, shapeData, dataRegion, pathTilesFeat,
configFeature, None)
# /////////////////////////////////////////////////////////////////////////////////////////
if REARRANGE_FLAG == 'True':
RAM.generateRepartition(PathTEST, pathConf, shapeRegion,
REARRANGE_PATH, dataField)
# pour tout les shape file par tuiles présent dans dataRegion, créer un ensemble dapp et de val
dataTile = fu.FileSearch_AND(dataRegion, True, ".shp")
# /////////////////////////////////////////////////////////////////////////////////////////
for path in dataTile:
RIST.RandomInSituByTile(path, dataField, N, pathAppVal, RATIO,
pathConf, None)
# /////////////////////////////////////////////////////////////////////////////////////////
if MODE == "outside" and CLASSIFMODE == "fusion":
Allcmd = genCmdSplitS.genCmdSplitShape(pathConf)
for cmd in Allcmd:
print cmd
os.system(cmd)
if TRAIN_MODE == "points":
trainShape = fu.FileSearch_AND(PathTEST + "/dataAppVal", True, ".shp",
"learn")
for shape in trainShape:
print ""
vs.generateSamples(shape, None, configFeature)
VSM.vectorSamplesMerge(configFeature)
# génération des fichiers de statistiques
if not TRAIN_MODE == "points":
AllCmd = MS.generateStatModel(pathAppVal, pathTilesFeat, pathStats,
cmdPath + "/stats", None, configFeature)
for cmd in AllCmd:
print cmd
print ""
stat = cmd.split(' ')[-1]
print "Checking if " + stat + " exists..."
if not os.path.exists(stat):
os.system(cmd)
else:
print "Keeping existing " + stat + "."
# /////////////////////////////////////////////////////////////////////////////////////////
# génération des commandes pour lApp
allCmd = LT.launchTraining(pathAppVal, pathConf, pathTilesFeat, dataField,
pathStats, N, cmdPath + "/train", pathModels,
None, None)
# /////////////////////////////////////////////////////////////////////////////////////////
for cmd in allCmd:
print cmd
print ""
os.system(cmd)
# /////////////////////////////////////////////////////////////////////////////////////////
# génération des commandes pour la classification
cmdClassif = LC.launchClassification(pathModels, pathConf, pathStats,
pathTileRegion, pathTilesFeat,
shapeRegion, field_Region, N,
cmdPath + "/cla", pathClassif, None)
# /////////////////////////////////////////////////////////////////////////////////////////
for cmd in cmdClassif:
print cmd
print ""
os.system(cmd)
# /////////////////////////////////////////////////////////////////////////////////////////
if CLASSIFMODE == "separate":
# Mise en forme des classifications
CS.ClassificationShaping(pathClassif, pathEnvelope, pathTilesFeat,
fieldEnv, N, classifFinal, None,
configFeature, COLORTABLE)
# génération des commandes pour les matrices de confusions
allCmd_conf = GCM.genConfMatrix(classifFinal, pathAppVal, N, dataField,
cmdPath + "/confusion", configFeature,
None)
for cmd in allCmd_conf:
print cmd
os.system(cmd)
confFus.confFusion(shapeData, dataField, classifFinal + "/TMP",
classifFinal + "/TMP", classifFinal + "/TMP",
configFeature)
GR.genResults(classifFinal, NOMENCLATURE)
elif CLASSIFMODE == "fusion" and MODE != "one_region":
cmdFus = FUS.fusion(pathClassif, configFeature, None)
for cmd in cmdFus:
print cmd
os.system(cmd)
# gestion des nodata
fusionFiles = fu.FileSearch_AND(pathClassif, True, "_FUSION_")
for fusionpath in fusionFiles:
ND.noData(PathTEST, fusionpath, field_Region, pathTilesFeat,
shapeRegion, N, configFeature, None)
# Mise en forme des classifications
CS.ClassificationShaping(pathClassif, pathEnvelope, pathTilesFeat,
fieldEnv, N, classifFinal, None,
configFeature, COLORTABLE)
# génération des commandes pour les matrices de confusions
allCmd_conf = GCM.genConfMatrix(classifFinal, pathAppVal, N, dataField,
cmdPath + "/confusion", configFeature,
None)
# /////////////////////////////////////////////////////////////////////////////////////////
for cmd in allCmd_conf:
print cmd
os.system(cmd)
# /////////////////////////////////////////////////////////////////////////////////////////
confFus.confFusion(shapeData, dataField, classifFinal + "/TMP",
classifFinal + "/TMP", classifFinal + "/TMP",
configFeature)
GR.genResults(classifFinal, NOMENCLATURE)
elif CLASSIFMODE == "fusion" and MODE == "one_region":
raise Exception(
"You can't choose the 'one region' mode and use the fusion mode together"
)
outStat = Config(file(pathConf)).chain.outputStatistics
if outStat == "True":
AllTiles = Config(file(pathConf)).chain.listTile
AllTiles = AllTiles.split(" ")
for currentTile in AllTiles:
OutS.outStats(pathConf, currentTile, N, None)
MOutS.mergeOutStats(pathConf)
for r in range(2, n_modalities + 1): # combinations of 2 or more modalities
c = itertools.combinations(u_modalities, r=r)
comb = itertools.chain(comb, c) if comb else c
for c in comb:
print(c)
for d in u_datasets:
npys2 = npys1[datasets == d]
splits = npys2[:, 3]
for s in u_splits:
npys3 = npys2[splits == str(s)]
modalities = npys3[:, 4]
npy_paths = []
for m in c:
n = npys3[modalities == m][0]
npy_paths.append(os.path.join(n[0], n[1], n[2]))
args = argparse.Namespace(d=d,
m=method,
npy_paths=npy_paths,
s=s,
settings=settings)
best_weight, _, prec = fusion(args)
print(d, s, " + ".join(list(c)))
with open("fusion_fc.out", 'a') as f:
f.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(
method, d, s, " + ".join(list(c)), best_weight, prec))
npy_dict = npy_dict[train_type]
it = itertools.product(dataset, splits)
with open(output, "w") as f:
f.write("dataset\tsplit\tmodality\tprec\n")
for d, s in it:
key = "{}_s{}".format(d,s)
npy_paths = []
for mod in modalities:
npy_path = npy_path_fmt.format(npy_dict[key][mod], d, mod, s)
if not os.path.isfile(npy_path):
npy_paths = []
break
npy_paths.append(npy_path)
if len(npy_paths) >= 2:
args2 = argparse.Namespace(d=d, m="individual", npy_paths=npy_paths,
s=s, settings=settings)
print(args2)
_, _, prec = fusion(args2)
with open(output, "a") as f:
for mod, p in zip(modalities, prec):
f.write("{}\t{}\t{}\t{:.04f}\n".format(d, s, mod, p))
with open(output, "w") as f:
f.write("dataset\tsplit\tmethod\tcombination\tparam\tprec\n")
for d, s, m, c in it:
key = "{}_s{}".format(d, s)
npy_paths = []
for mod in c:
npy_path = npy_path_fmt.format(npy_dict[key][mod], d, mod, s)
if not os.path.isfile(npy_path):
npy_paths = []
break
npy_paths.append(npy_path)
if len(npy_paths) >= 2:
args2 = argparse.Namespace(d=d,
m=m,
npy_paths=npy_paths,
s=s,
settings=settings)
print(args2)
ret = fusion(args2)
if ret:
param, _, prec = ret
with open(output, "a") as f:
f.write("{}\t{}\t{}\t{}\t{}\t{:.04f}\n".format(
d, s, m, c, param, prec))
with timer("4. training the main model"):
classifier = 'fusion'
x_train = df_train.loc[:,
col_feat].values # Return a Numpy representation of the DataFrame.
x_valid = df_valid.loc[:, col_feat].values
# \ is to change to another line after '='
finish_train, finish_valid = \
df_train['finish'].values, df_valid['finish'].values
like_train, like_valid = \
df_train['like'].values, df_valid['like'].values
print('features preparation done')
y_train_finish, y_valid_finish = finish_train, finish_valid
model_finish = fusion(x_train, y_train_finish)
y_train_like, y_valid_like = like_train, like_valid
model_like = fusion(x_train, y_train_like)
y_pred_finish = model_finish.predict(x_valid)
print('finish ROC ACC:', roc_auc_score(finish_valid, y_pred_finish))
print('finish confusion_matrix:',
confusion_matrix(y_valid_finish, y_pred_finish))
print('finish accuracy_score:',
accuracy_score(y_valid_finish, y_pred_finish))
#draw_roc(finish_valid, y_pred_finish,'Receiver operating characteristic Curve for finish')
y_pred_like = model_like.predict(x_valid)
print('like ROC ACC:', roc_auc_score(like_valid, y_pred_like))
print('like confusion_matrix:',
exit(1)
else:
input_BU = sys.argv[1]
input_TD = sys.argv[2]
groundtruth = sys.argv[3]
# input_BU can stay as it is
# Patterns from input_TD have to be extracted and mapped according to the groundtruth
extract(input_TD,"extracted",groundtruth)
print "Extraction and mapping done"
# create fusion
# fusion(input_BU,"extracted","fusion_result")
fusion(input_BU,"extracted","fusion_result")
print "Fusion of "+str(input_BU)+" and "+str(input_TD)+" is done"
print
# evaluation perl .pl ground input_BU 1 1
cmd = ["perl", "eval_metrics_v3.pl", groundtruth, input_BU,argument1,argument2]
pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE)
pipe.wait()
cmd = ["perl", "results2texV2.pl", "evaluation_results/"+input_BU+".L1_"+argument1+"L2_"+argument2+".results.csv"]
pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE)
pipe.wait()
print "Evaluation of "+str(input_BU)+ " done"
print
# evaluation perl .pl ground extracted 1 1
# cmd = ["perl", "eval_metrics.pl", groundtruth, "extracted","1","1"]
cmd = ["perl", "eval_metrics_v3.pl", groundtruth, "extracted",argument1,argument2]
pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE)
if __name__ == "__main__":
if len(sys.argv) < 5:
print "python start.py input_BU input_TD groundtruth threshold metric fusion_strategy mapping"
exit(1)
else:
input_BU_orig = sys.argv[1]
input_TD_orig = sys.argv[2]
groundtruth = sys.argv[3]
strategy = sys.argv[4] # 1 for simple union, 2 for top-down augmentation, 3 for bottom-up augmentation
# mapping = sys.argv[5] # 0 for none, 1 for mapping top-down to bottom-up, 2 for reverse
# threshold = sys.argv[6] # 1 (no threshold), 0.8 or 0.6
# Copy the top down and bottom up grammars for possible processing (to keep the originals unaffected)
cmd = ["cp", input_TD_orig, "../extracted_TD"]
pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
pipe.wait()
input_TD = "extracted_TD"
cmd = ["cp", input_BU_orig, "../extracted_BU"]
pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
pipe.wait()
input_BU = "extracted_BU"
fusion("../" + input_TD, "../" + input_BU, "fusion_result." + strategy, strategy)
print "Fusion of " + str(input_BU_orig) + " and " + str(input_TD_orig) + " is done"
exit(1)
# cmd = ["perl", "results2texV2.pl", "../evaluation_results/v_"+version+"/metric_"+metric+"/mapping_"+mapping+"/strategy_"+strategy+"/thresh_"+threshold+"/eval_fusion_result.L1_"+argument1+"L2_"+argument2+".results.csv"]
# pipe = subprocess.Popen(cmd,stdout=subprocess.PIPE)
# print "\t\tEvaluation of the fusion fragments is done"
