def load_inter_probs(self, data, neighbors):
if os.path.exists('./output/probs.csv'):
return CSVReader.read_csv_as_float('./output/probs.csv')
else :
output = []
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
new_prob = []
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
for forest in self.lrforests:
new_prob.extend( forest.get_prob (vector) )
output += [new_prob]
os.remove('./output/probs.csv')
if not os.path.exists('./output'):
os.makedirs('./output')
f = open('./output/inter_probs.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
return CSVReader.read_csv_as_float('./output/layer_probs.csv')
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
test_data = []
label_col = len(data[0])-1
print "making test data"
print "inputting",vec_data_path
data = CSVReader.read_csv_as_float(vec_data_path)
for line in data:
prob = []
for forest in self.lrforests:
prob.extend( forest.get_prob (line[0:label_col]) )
test_data += [ prob ]
print "test data Complete"
output = self.forest.predict( test_data )
self.output_file(output, sp_map, num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_MLRFdata'+str(num)+'.png',0)
return Assessment.AssessmentByMIOU(img_a,img_b)
def create_forest(self):
start_time= time.time()
forest = RandomForestClassifier(n_estimators = 10, class_weight = 'balanced')
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data MLF : Layer",self.layer_num
for i in range(self.start_num ,self.end_num):
print "inputting",i,sp_vec_path_list[i]
#ある画像に対するスーパーピクセルのデータ集合
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
#ある画像に対するスーパーピクセルの近傍データ
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(sp_vec_path_list, sp_neighbors_list, self.layer_num, i)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
traininglabel+= [ data[j][label_col] ]
print "training data Complete"
#別のランダムフォレストとclassが一致しないとエラーを吐くので、
#0埋めした38ラベル分のtrainingdata , traininglabelセットを追加する
for i in range(38):
trainingdata += [ np.zeros_like(trainingdata[0])]
traininglabel+= [i]
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'MultiLayerForest Layer',self.layer_num,'Complete', time.time() - start_time
return forest
def load_probs(self, data):
if os.path.exists('./output/probs.csv'):
return CSVReader.read_csv_as_float('./output/probs.csv')
else :
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for forest in self.lrforests:
probs.extend( forest.get_prob (line[0:label_col]) )
output += [probs]
if not os.path.exists('./output'):
os.makedirs('./output')
f = open('./output/probs.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
return CSVReader.read_csv_as_float('./output/probs.csv')
def create_forest(self):
start_time = time.time()
forest = RandomForestRegressor(n_estimators = 10)
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "ILRF making training data Layer",self.layer,'Label',self.label
for i in range(self.start_num ,self.end_num):
#print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(data, i )
#print len(data),len(probs)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
#print j,neighbors[j]
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
if data[j][label_col] == self.label:
traininglabel +=[1]
else:
traininglabel +=[0]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print self.layer,"save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'InterLabelRondomForest layer',self.layer,'Label', self.label,'Complete' ,time.time() - start_time
return forest
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
neighbors = CSVReader.read_csv_as_int(sp_neighbors_path)
test_data = []
probs = self.load_probs_for_predict(data, neighbors, self.layer_num , num)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
test_data += [ vector ]
#print len(vector)
#if len(vector) != 190:
# raw_input(">>")
print "test data Complete"
#print len(test_data), len(test_data[0])
#self.show_detail()
output = self.forest.predict( test_data )
self.output_file(output, sp_map, num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_MLF'+str(self.start_num)+'-'+str(self.end_num-1)+'_layer'+str(self.layer_num)+'data'+str(num)+'.png',0)
return [jpg_path, label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
neighbors = CSVReader.read_csv_as_int(sp_neighbors_path)
test_data = []
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
test_data += [ vector ]
os.remove('./output/probs.csv')
print "test data Complete"
output = self.forest.predict( test_data )
self.output_file(output, sp_map, num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_CLRFdata'+str(num)+'.png',0)
return [label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def create_forest(self):
forest = RandomForestClassifier()
for i in range(38):
self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ]
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
traininglabel+= [ data[j][label_col] ]
os.remove('./output/probs.csv')
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
return forest
def data_deteil(self, sp_data_path ,i):
if not os.path.exists('./output/statistics'):
os.makedirs('./output/statistics')
f = open('./output/statistics/data'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
output = [0 for x in range(38)]
data = CSVReader.read_csv_as_float(sp_data_path)
label_col = len(data[0])-1
for line in data:
output[int(line[label_col])] += 1
print sp_data_path
for j in range( len(output) ):
print j,output[j]
writer.writerow([j,output[j]])
f.close()
def data_statistics(self):
if not os.path.exists('./output/statistics'):
os.makedirs('./output/statistics')
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m1000.csv",0)
for i in range(5000):
f = open('./output/statistics/data'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
output = [0 for x in range(38)]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0])-1
for line in data:
output[int(line[label_col])] += 1
print sp_vec_path_list[i]
for j in range( len(output) ):
print j,output[j]
writer.writerow([j,output[j]])
f.close()
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
print "making test data"
print "inputting",jpg_path
print "inputting",label_path
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+self.m+" "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
test_data = []
print "making test data"
#print "inputting",jpg_path
label_col = len(data[0])-1
for line in data:
test_data += [ line[0:label_col] ]
print "test data Complete"
output = self.forest.predict( test_data )
#self.output_file(output, num)
return output
def create_forest(self):
forest = RandomForestClassifier()
for i in range(38):
self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ]
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0])-1
for line in data:
prob = []
for lrf in self.lrforests:
prob.extend( lrf.get_prob (line[0:label_col]) )
#print "prob",prob
trainingdata += [ prob ]
traininglabel+= [ line[label_col] ]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
return forest
def create_forest(self):
start_time= time.time()
forest = RandomForestRegressor(n_estimators = 10)
#sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0)
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
count = 0
n_count = 0
print "making training data"
for i in range(self.start_num ,self.end_num):
#print i,"inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
#self.data_deteil(sp_vec_path_list[i],i)
label_col = len(data[0])-1
for line in data:
trainingdata += [ line[0:label_col] ]
if line[label_col] == self.label:
traininglabel +=[1]
else:
traininglabel +=[0]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'LabelRandomForest',self.label,'Complete',time.time() - start_time
return forest
def create_forest(self):
print "save to", self.forest_path
trainingdata = []
traininglabel = []
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0]) -1
for line in data:
#print line[0:256/int(self.cs)*4]
trainingdata += [ line[0:label_col] ]
#print int(line[label_col])
traininglabel+= [int(line[label_col])]
print "training data Complete"
#print traininglabel
self.forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(self.forest, self.forest_path+'/forest.bin')
return self.forest
def load_probs(self, data, data_num):
if self.layer == 0:
return CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs/layer'+str(self.layer-1)+'/'+str(data_num)+'.csv')
def load_probs_for_predict(self, data, neighbors, layer_num, num):
#(1)spDataをLRFを用いてprobs.csvに変換
if not os.path.exists('./output/probs_for_predict/base'):
os.makedirs('./output/probs_for_predict/base')
if not os.path.exists('./output/probs_for_predict/base/'+str(num)+'.csv'):
forests = []
for i in range(38):
forests += [ LRF(self.start_num, self.end_num-1, i) ]
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for f in forests:
probs.extend( f.get_prob (line[0:label_col]) )
output += [probs]
f = open('./output/probs_for_predict/base/'+str(num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
#(2)probs.csvをILRFを用いてlayer_numの回数分変換
for layer in range(layer_num):
if not os.path.exists('./output/probs_for_predict/layer'+str(layer)):
os.makedirs('./output/probs_for_predict/layer'+str(layer))
if not os.path.exists('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv'):
forests = []
for label in range(38):
forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ]
if layer == 0:
src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv')
else:
src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer-1)+'/'+str(num)+'.csv')
dist_probs = []
for i in range(len(src_probs)):
vector = []
vector.extend( src_probs[i] )
vector.extend( src_probs[neighbors[i][0]] )
vector.extend( src_probs[neighbors[i][1]] )
vector.extend( src_probs[neighbors[i][2]] )
vector.extend( src_probs[neighbors[i][3]] )
probs = []
for f in forests:
probs.extend( f.get_prob (vector) )
dist_probs += [probs]
f = open('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in dist_probs:
writer.writerow(line)
f.close()
if layer_num == 0:
return CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer_num-1)+'/'+str(num)+'.csv')
def load_probs(self, sp_vec_path_list, sp_neighbors_list, layer_num, img_num):
#(1)spDataをLRFを用いてprobs.csvに変換
forests = []
for i in range(38):
#print 'LRF',i
forests += [ LRF(self.start_num, self.end_num-1, i) ]
for i in range(self.start_num, self.end_num):
if not os.path.exists('./output/probs/base'):
os.makedirs('./output/probs/base')
if os.path.exists('./output/probs/base/'+str(i)+'.csv'): continue
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for f in forests:
probs.extend( f.get_prob (line[0:label_col]) )
output += [probs]
f = open('./output/probs/base/'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
#(2)probs.csvをILRFを用いてlayer_numの回数分変換
for layer in range(layer_num):
forests = []
for label in range(38):
#print 'ILRF',label
forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ]
for n in range(self.start_num, self.end_num):
data_num = n
if os.path.exists('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv'): continue
if layer == 0:
src_probs = CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv')
else:
src_probs = CSVReader.read_csv_as_float('./output/probs/layer'+str(layer-1)+'/'+str(data_num)+'.csv')
dist_probs = []
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[n])
for i in range(len(src_probs)):
vector = []
vector.extend( src_probs[i] )
vector.extend( src_probs[neighbors[i][0]] )
vector.extend( src_probs[neighbors[i][1]] )
vector.extend( src_probs[neighbors[i][2]] )
vector.extend( src_probs[neighbors[i][3]] )
probs = []
for f in forests:
probs.extend( f.get_prob (vector) )
dist_probs += [probs]
if not os.path.exists('./output/probs/layer'+str(layer)):
os.makedirs('./output/probs/layer'+str(layer))
f = open('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in dist_probs:
writer.writerow(line)
f.close()
if layer_num == 0:
return CSVReader.read_csv_as_float('./output/probs/base/'+str(img_num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs/layer'+str(layer_num-1)+'/'+str(img_num)+'.csv')