def load_inter_probs(self, data, neighbors):
if os.path.exists('./output/probs.csv'):
return CSVReader.read_csv_as_float('./output/probs.csv')
else :
output = []
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
new_prob = []
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
for forest in self.lrforests:
new_prob.extend( forest.get_prob (vector) )
output += [new_prob]
os.remove('./output/probs.csv')
if not os.path.exists('./output'):
os.makedirs('./output')
f = open('./output/inter_probs.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
return CSVReader.read_csv_as_float('./output/layer_probs.csv')
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
print "making test data"
print "inputting",jpg_path
jpg_img = cv2.imread(jpg_path)
label_img = cv2.imread(label_path,0)
size = jpg_img.shape
data = ImgToVec.get_vec_list( jpg_img, label_img, self.cut_size)
jpg_data = data.vec_list
print "test data Complete"
output = self.forest.predict( jpg_data )
self.output_file(output , size , num)
return output
def main():
BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer()
HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer()
fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME)
devices = CSVReader.bomFetch(components, BOM_FILENAME)
plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME)
raw_input(".plx File created! ")
def main():
BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer();
HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer()
fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME)
devices = CSVReader.bomFetch(components, BOM_FILENAME)
plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME)
raw_input('.plx File created! ')
def create_forest(self):
start_time= time.time()
forest = RandomForestClassifier(n_estimators = 10, class_weight = 'balanced')
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data MLF : Layer",self.layer_num
for i in range(self.start_num ,self.end_num):
print "inputting",i,sp_vec_path_list[i]
#ある画像に対するスーパーピクセルのデータ集合
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
#ある画像に対するスーパーピクセルの近傍データ
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(sp_vec_path_list, sp_neighbors_list, self.layer_num, i)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
traininglabel+= [ data[j][label_col] ]
print "training data Complete"
#別のランダムフォレストとclassが一致しないとエラーを吐くので、
#0埋めした38ラベル分のtrainingdata , traininglabelセットを追加する
for i in range(38):
trainingdata += [ np.zeros_like(trainingdata[0])]
traininglabel+= [i]
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'MultiLayerForest Layer',self.layer_num,'Complete', time.time() - start_time
return forest
def create_forest(self):
start_time = time.time()
forest = RandomForestRegressor(n_estimators = 10)
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "ILRF making training data Layer",self.layer,'Label',self.label
for i in range(self.start_num ,self.end_num):
#print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(data, i )
#print len(data),len(probs)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
#print j,neighbors[j]
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
if data[j][label_col] == self.label:
traininglabel +=[1]
else:
traininglabel +=[0]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print self.layer,"save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'InterLabelRondomForest layer',self.layer,'Label', self.label,'Complete' ,time.time() - start_time
return forest
def create_forest(self):
forest = RandomForestClassifier()
for i in range(38):
self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ]
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i])
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
#print "training vector",len(vector)
trainingdata += [ vector ]
traininglabel+= [ data[j][label_col] ]
os.remove('./output/probs.csv')
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
return forest
def main():
while True:
try:
BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer();
HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer()
fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME)
devices = CSVReader.bomFetch(components, BOM_FILENAME)
plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME)
break
except IOError:
print '\nERROR 002. Archivo no encontrado. La direccion puede '+\
'ser incorrecta\no la extension diferente a .csv\n'
raw_input('\nEl archivo .plx ha sido creado exitosamente! ASEGURESE DE'+\
' QUE EL NOMBRE\nDEL ARCHIVO SEA IGUAL AL DEL BOM')
def test_tree():
# using absolute path to prevent WORKDIR issue
data = CSVReader.read_csv("/Users/azhmakin/Documents/projects/own/university/intelligent-data-analysis/lab-3/random-forest/data/income.csv")
tree = DecisionTreeClassifier(random_features=True)
tree.fit(data)
print(
tree.predict(
[
39,
"State-gov",
"Bachelors",
13,
"Never-married",
"Adm-clerical",
"Not-in-family",
"White",
"Male",
2174,
0,
40,
"United-States",
]
)
)
def main():
while True:
try:
BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer()
HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer()
fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN,
CAD_FILENAME)
devices = CSVReader.bomFetch(components, BOM_FILENAME)
plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME)
break
except IOError:
print '\nERROR 002. Archivo no encontrado. La direccion puede '+\
'ser incorrecta\no la extension diferente a .csv\n'
raw_input('\nEl archivo .plx ha sido creado exitosamente! ASEGURESE DE'+\
' QUE EL NOMBRE\nDEL ARCHIVO SEA IGUAL AL DEL BOM')
def data_statistics(self):
if not os.path.exists('./output/statistics'):
os.makedirs('./output/statistics')
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m1000.csv",0)
for i in range(5000):
f = open('./output/statistics/data'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
output = [0 for x in range(38)]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0])-1
for line in data:
output[int(line[label_col])] += 1
print sp_vec_path_list[i]
for j in range( len(output) ):
print j,output[j]
writer.writerow([j,output[j]])
f.close()
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
print "making test data"
print "inputting",jpg_path
print "inputting",label_path
if os.path.exists(jpg_path+"_m"+self.m+"c"+self.cs+".csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+self.cs+" "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"c"+self.cs+".csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
test_data = []
label_col = len(data[0]) -1
for line in data:
test_data += [ line[0:label_col] ]
print "test data Complete"
output = []
output = self.forest.predict( test_data )
self.output_file(output, sp_map , num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_data'+str(num)+'.png',0)
return Assessment.AssessmentByMIOU(img_a,img_b)
def create_forest(self):
print "save to", self.forest_path
trainingdata = []
traininglabel = []
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = []
dep_path = []
label_path = []
for i in range(self.start_num, self.end_num):
jpg_path.append(src_jpg_path[2*i-1])
dep_path.append(src_dep_path[2*i-1])
label_path.append(src_label_path[2*i-1])
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,jpg_path[i]
jpg_img = cv2.imread(jpg_path[i])
label_img = cv2.imread(label_path[i],0)
depth_img = cv2.imread(dep_path[i],0)
data = ImgToVec.GetRGBDVecList( jpg_img, depth_img, label_img, self.cut_size)
trainingdata += data.vec_list
traininglabel += data.label_list
print "training data Complete"
self.forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(self.forest, self.forest_path+'/forest.bin')
return self.forest
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
print "making test data"
print "inputting",jpg_path
print "inputting",label_path
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+self.m+" "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
test_data = []
print "making test data"
#print "inputting",jpg_path
label_col = len(data[0])-1
for line in data:
test_data += [ line[0:label_col] ]
print "test data Complete"
output = self.forest.predict( test_data )
#self.output_file(output, num)
return output
def SaveData():
rows, fields = CSVReader.fileReader()
reqIndex = [0, 1, 4, 5, 6, 7] # for the requested fields
#creating dictionary from the retrieved data
for row in rows[:5]:
str = {}
for i in reqIndex:
str[fields[i]] = row[i]
rconn.hmset(row[0], str)
def test_tree():
data = CSVReader.read_csv("../data/income.csv")
tree = DecisionTreeClassifier(random_features=True)
tree.fit(data)
print(
tree.predict([
39, 'State-gov', 'Bachelors', 13, 'Never-married', 'Adm-clerical',
'Not-in-family', 'White', 'Male', 2174, 0, 40, 'United-States'
]))
def create_forest(self):
start_time= time.time()
forest = RandomForestRegressor(n_estimators = 10)
#sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0)
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
count = 0
n_count = 0
print "making training data"
for i in range(self.start_num ,self.end_num):
#print i,"inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
#self.data_deteil(sp_vec_path_list[i],i)
label_col = len(data[0])-1
for line in data:
trainingdata += [ line[0:label_col] ]
if line[label_col] == self.label:
traininglabel +=[1]
else:
traininglabel +=[0]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
print 'LabelRandomForest',self.label,'Complete',time.time() - start_time
return forest
def create_forest(self):
forest = RandomForestClassifier()
for i in range(38):
self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ]
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0)
sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
trainingdata = []
traininglabel = []
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0])-1
for line in data:
prob = []
for lrf in self.lrforests:
prob.extend( lrf.get_prob (line[0:label_col]) )
#print "prob",prob
trainingdata += [ prob ]
traininglabel+= [ line[label_col] ]
print "training data Complete"
forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(forest, self.forest_path+'/forest.bin')
return forest
def load_probs(self, data):
if os.path.exists('./output/probs.csv'):
return CSVReader.read_csv_as_float('./output/probs.csv')
else :
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for forest in self.lrforests:
probs.extend( forest.get_prob (line[0:label_col]) )
output += [probs]
if not os.path.exists('./output'):
os.makedirs('./output')
f = open('./output/probs.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
return CSVReader.read_csv_as_float('./output/probs.csv')
def main_action(self):
inDir = self.inputPicker.directory.get()
outDir = self.outputPicker.directory.get()
args = self.argumentsView.arguments.get()
cmd = self.executablePicker.fileName.get()
subprocess32.call(BenchCommandBuilder().buildCommand(
inDir, outDir, cmd, args))
array = CSVReader().read("res.csv")
self.plotCanvas.plot(array[0], array[1])
def main( argv ):
if len(argv) != 3:
return print( "ERROR: Usage \"python3 id3.py <training-set> <test-set> <model-file>\"" )
training_tup = CSVReader.readBooleanCSV( argv[ 0 ] )
global attributes; attributes = training_tup[ 0 ]
global data ; data = training_tup[ 1 ]
testing_tup = CSVReader.readBooleanCSV( argv[ 1 ] )
test_attributes = testing_tup[ 0 ]
test_data = testing_tup[ 1 ]
test_decisions = extractDecisions( test_data )
print( "Attributes" )
print( ', '.join( attributes ), "\n" )
tree = createDecisionTree( attributes, data )
predictions = [ getPrediction( tree.dataSetFromDecisions( convertRowToDict( row ) ) ) for row in test_data ]
print( "\nPrediction accuracy vs. testing data:", "{}%\n\n".format( 100 * compareDecisions( predictions, test_decisions ) ) )
tree.printTree( argv[2] )
def create_forest(self):
print "save to", self.forest_path
trainingdata = []
traininglabel = []
sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0)
if (len(sp_vec_path_list)<self.end_num):
print "end_num is larger than data length"
return -1
print "making training data"
for i in range(self.end_num - self.start_num):
print "inputting",i,sp_vec_path_list[i]
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
label_col = len(data[0]) -1
for line in data:
#print line[0:256/int(self.cs)*4]
trainingdata += [ line[0:label_col] ]
#print int(line[label_col])
traininglabel+= [int(line[label_col])]
print "training data Complete"
#print traininglabel
self.forest.fit(trainingdata , traininglabel )
# Save
print "save to", self.forest_path
if not os.path.exists(self.forest_path):
os.makedirs(self.forest_path)
joblib.dump(self.forest, self.forest_path+'/forest.bin')
return self.forest
def enter_screen(self):
self.player_list.clear_widgets()
self.selection_list.clear_widgets()
self.description_list.clear_widgets()
self.color_list.clear_widgets()
all_players = []
for resultFile in self.manager.i_file_results:
reader = CSVReader(resultFile.id)
newLine = reader.getNext()
while newLine != None:
all_players.append(reader.getItem("Name", newLine))
newLine = reader.getNext()
all_players = sorted(all_players, key=str.lower)
self.playerIdentList = []
index = 0
for p in all_players:
player = PlayerIdentification()
player.name = p
self.playerIdentList.append(player)
self.moveUp()
def data_deteil(self, sp_data_path ,i):
if not os.path.exists('./output/statistics'):
os.makedirs('./output/statistics')
f = open('./output/statistics/data'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
output = [0 for x in range(38)]
data = CSVReader.read_csv_as_float(sp_data_path)
label_col = len(data[0])-1
for line in data:
output[int(line[label_col])] += 1
print sp_data_path
for j in range( len(output) ):
print j,output[j]
writer.writerow([j,output[j]])
f.close()
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
neighbors = CSVReader.read_csv_as_int(sp_neighbors_path)
test_data = []
probs = self.load_probs_for_predict(data, neighbors, self.layer_num , num)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
test_data += [ vector ]
#print len(vector)
#if len(vector) != 190:
# raw_input(">>")
print "test data Complete"
#print len(test_data), len(test_data[0])
#self.show_detail()
output = self.forest.predict( test_data )
self.output_file(output, sp_map, num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_MLF'+str(self.start_num)+'-'+str(self.end_num-1)+'_layer'+str(self.layer_num)+'data'+str(num)+'.png',0)
return [jpg_path, label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def AssessmentByMIOU(img_a,img_b):
rate=1.0
LabelNum=38
PredictionLabelNum=0
Ic_Uc=0.0
width = img_a.shape[1]
height = img_a.shape[0]
GrandTruth=np.zeros(LabelNum,dtype=np.int32)
Prediction=np.zeros(LabelNum,dtype=np.int32)
Intersection=np.zeros(LabelNum,dtype=np.int32)
IntersectionRate=np.zeros(LabelNum,dtype=np.float32)
#ラベルも同時に出力したい場合
label = CSVReader.read_csv('seg37list.csv')
LabelList = ['other']
for row in label:
for i in range(len(row)):
LabelList.append(row[i])
#ここまで
for y in range(height):
for x in range(width):
GrandTruth[img_a[y,x]] += 1
Prediction[img_b[y,x]] += 1
if img_a[y,x] == img_b[y,x]:
Intersection[img_a[y,x]] += 1
print (GrandTruth)
print (Prediction)
print(Intersection)
for i in range(LabelNum):
if(GrandTruth[i]+Prediction[i]-Intersection[i]>0):
IntersectionRate[i]=1.0*Intersection[i]/(GrandTruth[i]+Prediction[i]-Intersection[i])
print(i,LabelList[i],IntersectionRate[i])
Ic_Uc += 1.0*IntersectionRate[i]
if(Prediction[i]>0):
PredictionLabelNum += 1
rate = Ic_Uc/PredictionLabelNum
'''
#ラベルごとIOU値の棒グラフ出力
X=np.arange(0,LabelNum,1)
plt.bar(X,IntersectionRate,width=0.7)
plt.xticks(X, LabelList) # 目盛りを数字からラベルに書き換える
plt.xticks(rotation=90) # 目盛りを傾ける
plt.show()
#ここまで
'''
return rate
def test_rf():
from sklearn.model_selection import train_test_split
data = CSVReader.read_csv("../data/income.csv")
train, test = train_test_split(data, test_size=0.3)
rf = RandomForestClassifier(nb_trees=60, nb_samples=3000, max_workers=4)
rf.fit(train)
errors = 0
features = [ft[:-1] for ft in test]
values = [ft[-1] for ft in test]
for feature, value in zip(features, values):
prediction = rf.predict(feature)
if prediction != value:
errors += 1
logging.info("Error rate: {}".format(errors / len(features) * 100))
def test_rf():
from sklearn.model_selection import train_test_split
# using absolute path to prevent WORKDIR issue
data = CSVReader.read_csv("/Users/azhmakin/Documents/projects/own/university/intelligent-data-analysis/lab-3/random-forest/data/income.csv")
train, test = train_test_split(data, test_size=0.3)
rf = RandomForestClassifier(nb_trees=60, nb_samples=3000, max_workers=4)
rf.fit(train)
errors = 0
features = [ft[:-1] for ft in test]
values = [ft[-1] for ft in test]
for feature, value in zip(features, values):
prediction = rf.predict(feature)
if prediction != value:
errors += 1
logging.info("Error rate: {}".format(errors / len(features) * 100))
def predict(self, num):
src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4)
src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3)
src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9)
jpg_path = src_jpg_path[2*num]
dep_path = src_dep_path[2*num]
label_path = src_label_path[2*num]
if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"):
print "exist",jpg_path
else:
cmd = u'"slic\SLICOSuperpixel.exe"'
os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight);
vec_data_path = jpg_path+"_m"+self.m+"spdata.csv"
sp_map_path = jpg_path+"_m"+self.m+"spmap.csv"
sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv"
data = CSVReader.read_csv_as_float(vec_data_path)
sp_map = CSVReader.read_csv_as_int(sp_map_path)
neighbors = CSVReader.read_csv_as_int(sp_neighbors_path)
test_data = []
probs = self.load_probs(data)
label_col = len(data[0])-1
for j in range(len(data)):
vector = []
vector.extend( probs[j] )
vector.extend( probs[neighbors[j][0]] )
vector.extend( probs[neighbors[j][1]] )
vector.extend( probs[neighbors[j][2]] )
vector.extend( probs[neighbors[j][3]] )
test_data += [ vector ]
os.remove('./output/probs.csv')
print "test data Complete"
output = self.forest.predict( test_data )
self.output_file(output, sp_map, num)
img_a = cv2.imread(label_path,0)
img_b = cv2.imread('output_CLRFdata'+str(num)+'.png',0)
return [label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def _loadNames(self): return CSVReader.getNameList()
from sklearn.model_selection import train_test_split
import CSVReader
import joblib
from sklearn.metrics import confusion_matrix
data = CSVReader.read_csv("2darray.csv")
#data = CSVReader.read_csv("new 1.csv")
features = [ft[:-1] for ft in data]
values = [ft[-1] for ft in data]
# Split dataset into training set and test set
X_train, X_test, y_train, y_test = train_test_split(features, values, test_size=0.3)
#Import Gaussian Naive Bayes model
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
#n_estimators = number of trees
#bootstrap
import time
start_time = time.time()
clf = RandomForestClassifier(n_estimators=50 ,max_depth=15, random_state=0)
clf.fit(X_train, y_train)
y_pred=clf.predict(X_test)
#print("--- %s seconds ---" % (time.time() - start_time))
joblib.dump(clf, 'random_forest.joblib', compress=9)
from sklearn import metrics
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
print("F1 Score: ", metrics.f1_score(y_test, y_pred))
def load_probs(self, sp_vec_path_list, sp_neighbors_list, layer_num, img_num):
#(1)spDataをLRFを用いてprobs.csvに変換
forests = []
for i in range(38):
#print 'LRF',i
forests += [ LRF(self.start_num, self.end_num-1, i) ]
for i in range(self.start_num, self.end_num):
if not os.path.exists('./output/probs/base'):
os.makedirs('./output/probs/base')
if os.path.exists('./output/probs/base/'+str(i)+'.csv'): continue
data = CSVReader.read_csv_as_float(sp_vec_path_list[i])
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for f in forests:
probs.extend( f.get_prob (line[0:label_col]) )
output += [probs]
f = open('./output/probs/base/'+str(i)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
#(2)probs.csvをILRFを用いてlayer_numの回数分変換
for layer in range(layer_num):
forests = []
for label in range(38):
#print 'ILRF',label
forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ]
for n in range(self.start_num, self.end_num):
data_num = n
if os.path.exists('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv'): continue
if layer == 0:
src_probs = CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv')
else:
src_probs = CSVReader.read_csv_as_float('./output/probs/layer'+str(layer-1)+'/'+str(data_num)+'.csv')
dist_probs = []
neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[n])
for i in range(len(src_probs)):
vector = []
vector.extend( src_probs[i] )
vector.extend( src_probs[neighbors[i][0]] )
vector.extend( src_probs[neighbors[i][1]] )
vector.extend( src_probs[neighbors[i][2]] )
vector.extend( src_probs[neighbors[i][3]] )
probs = []
for f in forests:
probs.extend( f.get_prob (vector) )
dist_probs += [probs]
if not os.path.exists('./output/probs/layer'+str(layer)):
os.makedirs('./output/probs/layer'+str(layer))
f = open('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in dist_probs:
writer.writerow(line)
f.close()
if layer_num == 0:
return CSVReader.read_csv_as_float('./output/probs/base/'+str(img_num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs/layer'+str(layer_num-1)+'/'+str(img_num)+'.csv')
from sklearn.model_selection import train_test_split
import CSVReader
from sklearn.metrics import confusion_matrix
from sklearn import svm
from sklearn.linear_model import LogisticRegression
data = CSVReader.read_csv("dataset_withfeatures.csv")
features = [ft[:-1] for ft in data]
values = [ft[-1] for ft in data]
# Split dataset into training set and test set
X_train, X_test, y_train, y_test = train_test_split(features, values, test_size=0.3)
#Import Gaussian Naive Bayes model
from sklearn.neighbors import KNeighborsClassifier
#clf = LogisticRegression(random_state=0).fit(X_train, y_train)
model = KNeighborsClassifier(n_neighbors=17)
model.fit(X_train,y_train)
y_pred = model.predict(X_test)
from sklearn import metrics
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
print("F1 Score: ", metrics.f1_score(y_test, y_pred))
tn, fp, fn, tp=(confusion_matrix(y_test, y_pred).ravel())
#print(tn,fp,fn,tp)
print("False Postive Rate: ",fp/(fp+tn))
print("False Negitve Rate: ",fn/(fn+tp))
print("True Negitive Rate: ",tn/(tn+fp))
def printTeamStandings(myCanvas, startTop, filepath, num_trophy_winners,
trophy_highlight):
myCanvas.setFont("Helvetica", 11 * POINT)
y = startTop
x = MARGIN_LEFT
teamResult = CSVReader(filepath)
columns = [
x, x + INCH * .5, x + INCH * 6.25, x + INCH * 7.00, x + INCH * 7.75,
x + INCH * 8.50, x + INCH * 9.25
]
ignore = 0
playerCount = 0
for i in range(0, len(teamResult.headers)):
if (teamResult.headers[i] not in IGNORE_TEAM):
myCanvas.drawString(columns[i - ignore], y, teamResult.headers[i])
else:
ignore += 1
y -= LINE
ignore = 0
num_trophies = 0
line = teamResult.getNext()
while (line != None):
if (y < MARGIN_BOTTOM):
y = MARGIN_TOP
myCanvas.showPage()
if (line[0] == ""):
playerCount += 1
else:
playerCount = 0
if (playerCount > 4):
line = teamResult.getNext()
continue
if (num_trophies < int(num_trophy_winners)) and (playerCount == 0):
num_trophies += 1
myCanvas.setStrokeColorRGB(trophy_highlight[0],
trophy_highlight[1],
trophy_highlight[2])
myCanvas.setFillColorRGB(trophy_highlight[0], trophy_highlight[1],
trophy_highlight[2])
myCanvas.rect(columns[0] - 3, y - 3, columns[3] - columns[0] - 2,
LINE + 1, 1, 1)
myCanvas.setStrokeColorRGB(0, 0, 0)
myCanvas.setFillColorRGB(0, 0, 0)
ignore = 0
for i in range(0, len(line)):
if (teamResult.headers[i] in IGNORE_TEAM):
ignore += 1
else:
myCanvas.drawString(columns[i - ignore], y, line[i])
y -= LINE
line = teamResult.getNext()
highlights = [trophy_highlight]
descriptions = ["Trophy Winners"]
if len(highlights) > 0:
draw_highlight_key(myCanvas, highlights, descriptions)
def printIndividual(myCanvas, left, top, filepath, num_trophy_winners,
trophy_highlight, player_identification):
player_ident = copy.copy(player_identification)
base_left = .20 * INCH
base_right = 10.30 * INCH
space = base_right - base_left
lengths = []
total_characters = 0
result = CSVReader(filepath)
buff = 3
for head in result.headers:
if head.lower() == "name".lower():
buff = 8
else:
buff = 3
length = result.getLongest(head) + buff
lengths.append(length)
total_characters += length
increment = math.floor(space / total_characters)
columns = [base_left]
for i in range(0, len(lengths) - 1):
columns.append(columns[i] + increment * lengths[i])
myCanvas.setFont("Helvetica", 12 * POINT)
y = top
for i in range(0, len(result.headers)):
myCanvas.drawString(columns[i], y, result.headers[i])
y -= LINE
highlights = []
descriptions = []
if int(num_trophy_winners) > 0:
highlights.append(trophy_highlight)
descriptions.append("Trophy Winners")
num_trophies = 0
next = result.getNext()
new_page = False
highlight_width = columns[6] - columns[0]
while (next != None):
if (y < (.5 * INCH)):
new_page = True
if (num_trophies < int(num_trophy_winners)) == True:
num_trophies += 1
highlight(myCanvas, columns[0], y, highlight_width,
trophy_highlight)
status = result.getItem("St", next)
name = result.getItem("Name", next)
if (status != "Out"):
if new_page == True:
draw_highlight_key(myCanvas, highlights, descriptions)
highlights = []
descriptions = []
y = top
myCanvas.showPage()
new_page = False
for player in player_ident:
if player.name == name:
if player.description in descriptions:
player.col = highlights[descriptions.index(
player.description)]
else:
highlights.append(player.col)
descriptions.append(player.description)
highlight(myCanvas, columns[0], y, highlight_width,
player.col)
player_ident.remove(player)
for i in range(0, len(next)):
myCanvas.drawString(columns[i], y, next[i])
y -= LINE
next = result.getNext()
if len(highlights) > 0:
draw_highlight_key(myCanvas, highlights, descriptions)
def _loadNames(self):
return CSVReader.getNameList()
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from sklearn.cluster import KMeans
from pandas import DataFrame
from algorithms import kmean as km
import CSVReader as cr
import DataNormalisation as dt
MAX_ITERATIONS = 100
filePathRusia = 'C:\\Users\\dprefac\\PycharmProjects\\netscan-master\\csv\\pythonScript_1000_telekom_VPNRusia.csv'
filePathLocal = 'C:\\Users\\dprefac\\PycharmProjects\\netscan-master\\csv\\bruteForceWithValidPassword100Attempts200Status.csv'
features = [9, 10, 13, 14]
csvReader = cr.CSVReader()
dataFromRusia = csvReader.load_csv_and_extract_feature(filePathRusia, features,
"RUSIA")
dataFromLocal = csvReader.load_csv_and_extract_feature(filePathLocal, features,
"LOCAL")
dataFromRusia = np.array(dataFromRusia)
dataFromLocal = np.array(dataFromLocal)
dataFromRusia = dt.DataNormalisation.eliminate_outlier_column(dataFromRusia, 0)
dataFromLocal = dt.DataNormalisation.eliminate_outlier_column(dataFromLocal, 0)
#
dataFromRusia = dt.DataNormalisation.eliminate_outlier_column(dataFromRusia, 1)
dataFromLocal = dt.DataNormalisation.eliminate_outlier_column(dataFromLocal, 1)
def load_probs(self, data, data_num):
if self.layer == 0:
return CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs/layer'+str(self.layer-1)+'/'+str(data_num)+'.csv')
# -*- coding: <UTF-8> -*-
import CSVReader
import WebScraper
import re
import matplotlib.pyplot as plt
import numpy
typeVsEXP = CSVReader.CSVWorker('/home/taidgh/Code/Python/Pokemon/typeVsEXP.csv', 'typeVsEXP.csv')
def listCleaner(list):
cString = re.sub('\[|\]| |\'', '', list)
return(cString.split(','))
def fileGen():
#Check if the file exsits, if it does, load type vs EXP else do the Web Scraping.
#This makes it so if the data is already retrieved the time can be saved.
if(typeVsEXP.isFile):
typeVsEXP.getRows([1,2])
#Load the data
else:
data = toCSV()
typeVsEXP.createNWD(data)
fileGen()
#Create the file, then load the data.
def CSVToDict():
optimization(num_iterations, epochs)
import os
path = ''
def csv_weights_write():
global path
path = os.path.join(os.getcwd(), 'Weights')
i = 1
while True:
if not os.path.exists(path):
os.mkdir(path)
break
if i >= 2:
path = path[:(-1 - len(str(i)))]
path += '_' + str(i)
i += 1
print("Weights are stored in " + path)
w = session.run(weights, feed_dict=whole_feed_dict)
for i in range(len(w)):
filename = os.path.join(path, 'w' + str(i + 1) + '.csv')
with open(filename, 'w') as fl:
csvwriter = csv.writer(fl)
csvwriter.writerows(w[i])
csv_weights_write()
param.print_equation(os.path.join(path, 'equation.txt'))
import CSVReader as csv
import CSVWriter
import MetaReader as meta
import DecisionTree as DT
########################################################################
####################### LOAD NECESSARY DATA ############################
########################################################################
# Load training set
training_set = csv.CSVReader(
raw_input("Please enter the full filepath of the TRAINING DATA below: \n"))
# training_set = csv.CSVReader("./data/btrain.csv")
training_header, training_data = training_set.readFile()
# Load meta data
meta_reader = meta.MetaReader(
raw_input("\nPlease enter the full filepath of the META DATA below: \n"))
# meta_reader = meta.MetaReader("./data/bmeta.csv")
meta_data = meta_reader.parseMetaData()
# Load validation set
validation_set = csv.CSVReader(
raw_input(
"\nPlease enter the full filepath of the VALIDATION DATA below: \n"))
# validation_set = csv.CSVReader("./data/bvalidate.csv")
validate_header, validate_data = validation_set.readFile()
# Load test set
test_set = csv.CSVReader(
raw_input("\nPlease enter the full filepath of the TEST DATA below: \n"))
try:
import CSVReader
import computation
except ModuleNotFoundError:
print("Cannot import file")
try:
colHeaders, rows = CSVReader.openFile("data.csv")
print(CSVReader.returnLocation(colHeaders, rows, -7, 2))
print(CSVReader.returnLocation(colHeaders, rows, -5, -2))
print(CSVReader.returnLocation(colHeaders, rows, 2, -1))
print(CSVReader.returnLocation(colHeaders, rows, -3, -3))
print(CSVReader.returnLocation(colHeaders, rows, 1, 3))
print(computation.selectOperation(colHeaders, rows, "price", "add"))
print(computation.selectOperation(colHeaders, rows, "price", "average"))
print(computation.selectOperation(colHeaders, rows, "id", "Min"))
print(computation.selectOperation(colHeaders, rows, "price", "Max"))
except AttributeError:
print("Function does not exists")
except NameError:
print("Given column does not exists")
def __init__(self, filename):
metaCSV = csv.CSVReader(filename)
self.attributes, self.types = metaCSV.readFile()
def load_probs_for_predict(self, data, neighbors, layer_num, num):
#(1)spDataをLRFを用いてprobs.csvに変換
if not os.path.exists('./output/probs_for_predict/base'):
os.makedirs('./output/probs_for_predict/base')
if not os.path.exists('./output/probs_for_predict/base/'+str(num)+'.csv'):
forests = []
for i in range(38):
forests += [ LRF(self.start_num, self.end_num-1, i) ]
output = []
label_col = len(data[0])-1
for line in data:
probs = []
for f in forests:
probs.extend( f.get_prob (line[0:label_col]) )
output += [probs]
f = open('./output/probs_for_predict/base/'+str(num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in output:
writer.writerow(line)
f.close()
#(2)probs.csvをILRFを用いてlayer_numの回数分変換
for layer in range(layer_num):
if not os.path.exists('./output/probs_for_predict/layer'+str(layer)):
os.makedirs('./output/probs_for_predict/layer'+str(layer))
if not os.path.exists('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv'):
forests = []
for label in range(38):
forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ]
if layer == 0:
src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv')
else:
src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer-1)+'/'+str(num)+'.csv')
dist_probs = []
for i in range(len(src_probs)):
vector = []
vector.extend( src_probs[i] )
vector.extend( src_probs[neighbors[i][0]] )
vector.extend( src_probs[neighbors[i][1]] )
vector.extend( src_probs[neighbors[i][2]] )
vector.extend( src_probs[neighbors[i][3]] )
probs = []
for f in forests:
probs.extend( f.get_prob (vector) )
dist_probs += [probs]
f = open('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv', 'w')
writer = csv.writer(f, lineterminator='\n')
for line in dist_probs:
writer.writerow(line)
f.close()
if layer_num == 0:
return CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv')
else:
return CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer_num-1)+'/'+str(num)+'.csv')
import CSVReader CSVReader.readQustionsCSV(CSVReader.readGroundTruthCSV()) #CSVReader.fillQuestionsGroundTruthAnsCSV()
