def load_inter_probs(self, data, neighbors): if os.path.exists('./output/probs.csv'): return CSVReader.read_csv_as_float('./output/probs.csv') else : output = [] probs = self.load_probs(data) label_col = len(data[0])-1 for j in range(len(data)): new_prob = [] vector = [] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) for forest in self.lrforests: new_prob.extend( forest.get_prob (vector) ) output += [new_prob] os.remove('./output/probs.csv') if not os.path.exists('./output'): os.makedirs('./output') f = open('./output/inter_probs.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in output: writer.writerow(line) f.close() return CSVReader.read_csv_as_float('./output/layer_probs.csv')
def predict(self, num): src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = src_jpg_path[2*num] dep_path = src_dep_path[2*num] label_path = src_label_path[2*num] print "making test data" print "inputting",jpg_path jpg_img = cv2.imread(jpg_path) label_img = cv2.imread(label_path,0) size = jpg_img.shape data = ImgToVec.get_vec_list( jpg_img, label_img, self.cut_size) jpg_data = data.vec_list print "test data Complete" output = self.forest.predict( jpg_data ) self.output_file(output , size , num) return output
def main(): BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer() HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer() fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME) devices = CSVReader.bomFetch(components, BOM_FILENAME) plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME) raw_input(".plx File created! ")
def main(): BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer(); HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer() fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME) devices = CSVReader.bomFetch(components, BOM_FILENAME) plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME) raw_input('.plx File created! ')
def create_forest(self): start_time= time.time() forest = RandomForestClassifier(n_estimators = 10, class_weight = 'balanced') sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0) sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 trainingdata = [] traininglabel = [] print "making training data MLF : Layer",self.layer_num for i in range(self.start_num ,self.end_num): print "inputting",i,sp_vec_path_list[i] #ある画像に対するスーパーピクセルのデータ集合 data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) #ある画像に対するスーパーピクセルの近傍データ neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i]) probs = self.load_probs(sp_vec_path_list, sp_neighbors_list, self.layer_num, i) label_col = len(data[0])-1 for j in range(len(data)): vector = [] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) #print "training vector",len(vector) trainingdata += [ vector ] traininglabel+= [ data[j][label_col] ] print "training data Complete" #別のランダムフォレストとclassが一致しないとエラーを吐くので、 #0埋めした38ラベル分のtrainingdata , traininglabelセットを追加する for i in range(38): trainingdata += [ np.zeros_like(trainingdata[0])] traininglabel+= [i] forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(forest, self.forest_path+'/forest.bin') print 'MultiLayerForest Layer',self.layer_num,'Complete', time.time() - start_time return forest
def create_forest(self): start_time = time.time() forest = RandomForestRegressor(n_estimators = 10) sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0) sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 trainingdata = [] traininglabel = [] print "ILRF making training data Layer",self.layer,'Label',self.label for i in range(self.start_num ,self.end_num): #print "inputting",i,sp_vec_path_list[i] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i]) probs = self.load_probs(data, i ) #print len(data),len(probs) label_col = len(data[0])-1 for j in range(len(data)): vector = [] #print j,neighbors[j] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) #print "training vector",len(vector) trainingdata += [ vector ] if data[j][label_col] == self.label: traininglabel +=[1] else: traininglabel +=[0] print "training data Complete" forest.fit(trainingdata , traininglabel ) # Save print self.layer,"save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(forest, self.forest_path+'/forest.bin') print 'InterLabelRondomForest layer',self.layer,'Label', self.label,'Complete' ,time.time() - start_time return forest
def create_forest(self): forest = RandomForestClassifier() for i in range(38): self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ] sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0) sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 trainingdata = [] traininglabel = [] print "making training data" for i in range(self.end_num - self.start_num): print "inputting",i,sp_vec_path_list[i] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[i]) probs = self.load_probs(data) label_col = len(data[0])-1 for j in range(len(data)): vector = [] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) #print "training vector",len(vector) trainingdata += [ vector ] traininglabel+= [ data[j][label_col] ] os.remove('./output/probs.csv') print "training data Complete" forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(forest, self.forest_path+'/forest.bin') return forest
def main(): while True: try: BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer(); HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer() fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME) devices = CSVReader.bomFetch(components, BOM_FILENAME) plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME) break except IOError: print '\nERROR 002. Archivo no encontrado. La direccion puede '+\ 'ser incorrecta\no la extension diferente a .csv\n' raw_input('\nEl archivo .plx ha sido creado exitosamente! ASEGURESE DE'+\ ' QUE EL NOMBRE\nDEL ARCHIVO SEA IGUAL AL DEL BOM')
def test_tree(): # using absolute path to prevent WORKDIR issue data = CSVReader.read_csv("/Users/azhmakin/Documents/projects/own/university/intelligent-data-analysis/lab-3/random-forest/data/income.csv") tree = DecisionTreeClassifier(random_features=True) tree.fit(data) print( tree.predict( [ 39, "State-gov", "Bachelors", 13, "Never-married", "Adm-clerical", "Not-in-family", "White", "Male", 2174, 0, 40, "United-States", ] ) )
def main(): while True: try: BOM_FILENAME, CAD_FILENAME, PLX_FILENAME = fileNamer() HEADER, BOARD_ORIGIN, BOARD_WIDTH = varInitializer() fiducials, components = CSVReader.cadFetch(BOARD_ORIGIN, CAD_FILENAME) devices = CSVReader.bomFetch(components, BOM_FILENAME) plxGenerator(HEADER, BOARD_WIDTH, fiducials, devices, PLX_FILENAME) break except IOError: print '\nERROR 002. Archivo no encontrado. La direccion puede '+\ 'ser incorrecta\no la extension diferente a .csv\n' raw_input('\nEl archivo .plx ha sido creado exitosamente! ASEGURESE DE'+\ ' QUE EL NOMBRE\nDEL ARCHIVO SEA IGUAL AL DEL BOM')
def data_statistics(self): if not os.path.exists('./output/statistics'): os.makedirs('./output/statistics') sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m1000.csv",0) for i in range(5000): f = open('./output/statistics/data'+str(i)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') output = [0 for x in range(38)] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) label_col = len(data[0])-1 for line in data: output[int(line[label_col])] += 1 print sp_vec_path_list[i] for j in range( len(output) ): print j,output[j] writer.writerow([j,output[j]]) f.close()
def predict(self, num): src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = src_jpg_path[2*num] dep_path = src_dep_path[2*num] label_path = src_label_path[2*num] print "making test data" print "inputting",jpg_path print "inputting",label_path if os.path.exists(jpg_path+"_m"+self.m+"c"+self.cs+".csv"): print "exist",jpg_path else: cmd = u'"slic\SLICOSuperpixel.exe"' os.system(cmd+" "+ self.m + " "+self.cs+" "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight); vec_data_path = jpg_path+"_m"+self.m+"c"+self.cs+".csv" sp_map_path = jpg_path+"_m"+self.m+"spmap.csv" data = CSVReader.read_csv_as_float(vec_data_path) sp_map = CSVReader.read_csv_as_int(sp_map_path) test_data = [] label_col = len(data[0]) -1 for line in data: test_data += [ line[0:label_col] ] print "test data Complete" output = [] output = self.forest.predict( test_data ) self.output_file(output, sp_map , num) img_a = cv2.imread(label_path,0) img_b = cv2.imread('output_data'+str(num)+'.png',0) return Assessment.AssessmentByMIOU(img_a,img_b)
def create_forest(self): print "save to", self.forest_path trainingdata = [] traininglabel = [] src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = [] dep_path = [] label_path = [] for i in range(self.start_num, self.end_num): jpg_path.append(src_jpg_path[2*i-1]) dep_path.append(src_dep_path[2*i-1]) label_path.append(src_label_path[2*i-1]) print "making training data" for i in range(self.end_num - self.start_num): print "inputting",i,jpg_path[i] jpg_img = cv2.imread(jpg_path[i]) label_img = cv2.imread(label_path[i],0) depth_img = cv2.imread(dep_path[i],0) data = ImgToVec.GetRGBDVecList( jpg_img, depth_img, label_img, self.cut_size) trainingdata += data.vec_list traininglabel += data.label_list print "training data Complete" self.forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(self.forest, self.forest_path+'/forest.bin') return self.forest
def predict(self, num): src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = src_jpg_path[2*num] dep_path = src_dep_path[2*num] label_path = src_label_path[2*num] print "making test data" print "inputting",jpg_path print "inputting",label_path if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"): print "exist",jpg_path else: cmd = u'"slic\SLICOSuperpixel.exe"' os.system(cmd+" "+self.m+" "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight); vec_data_path = jpg_path+"_m"+self.m+"spdata.csv" sp_map_path = jpg_path+"_m"+self.m+"spmap.csv" data = CSVReader.read_csv_as_float(vec_data_path) sp_map = CSVReader.read_csv_as_int(sp_map_path) test_data = [] print "making test data" #print "inputting",jpg_path label_col = len(data[0])-1 for line in data: test_data += [ line[0:label_col] ] print "test data Complete" output = self.forest.predict( test_data ) #self.output_file(output, num) return output
def SaveData(): rows, fields = CSVReader.fileReader() reqIndex = [0, 1, 4, 5, 6, 7] # for the requested fields #creating dictionary from the retrieved data for row in rows[:5]: str = {} for i in reqIndex: str[fields[i]] = row[i] rconn.hmset(row[0], str)
def test_tree(): data = CSVReader.read_csv("../data/income.csv") tree = DecisionTreeClassifier(random_features=True) tree.fit(data) print( tree.predict([ 39, 'State-gov', 'Bachelors', 13, 'Never-married', 'Adm-clerical', 'Not-in-family', 'White', 'Male', 2174, 0, 40, 'United-States' ]))
def create_forest(self): start_time= time.time() forest = RandomForestRegressor(n_estimators = 10) #sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0) sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 trainingdata = [] traininglabel = [] count = 0 n_count = 0 print "making training data" for i in range(self.start_num ,self.end_num): #print i,"inputting",i,sp_vec_path_list[i] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) #self.data_deteil(sp_vec_path_list[i],i) label_col = len(data[0])-1 for line in data: trainingdata += [ line[0:label_col] ] if line[label_col] == self.label: traininglabel +=[1] else: traininglabel +=[0] print "training data Complete" forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(forest, self.forest_path+'/forest.bin') print 'LabelRandomForest',self.label,'Complete',time.time() - start_time return forest
def create_forest(self): forest = RandomForestClassifier() for i in range(38): self.lrforests += [ LRF(self.start_num, self.end_num-1, i) ] sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",0) sp_neighbors_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+".csv",2) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 trainingdata = [] traininglabel = [] print "making training data" for i in range(self.end_num - self.start_num): print "inputting",i,sp_vec_path_list[i] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) label_col = len(data[0])-1 for line in data: prob = [] for lrf in self.lrforests: prob.extend( lrf.get_prob (line[0:label_col]) ) #print "prob",prob trainingdata += [ prob ] traininglabel+= [ line[label_col] ] print "training data Complete" forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(forest, self.forest_path+'/forest.bin') return forest
def load_probs(self, data): if os.path.exists('./output/probs.csv'): return CSVReader.read_csv_as_float('./output/probs.csv') else : output = [] label_col = len(data[0])-1 for line in data: probs = [] for forest in self.lrforests: probs.extend( forest.get_prob (line[0:label_col]) ) output += [probs] if not os.path.exists('./output'): os.makedirs('./output') f = open('./output/probs.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in output: writer.writerow(line) f.close() return CSVReader.read_csv_as_float('./output/probs.csv')
def main_action(self): inDir = self.inputPicker.directory.get() outDir = self.outputPicker.directory.get() args = self.argumentsView.arguments.get() cmd = self.executablePicker.fileName.get() subprocess32.call(BenchCommandBuilder().buildCommand( inDir, outDir, cmd, args)) array = CSVReader().read("res.csv") self.plotCanvas.plot(array[0], array[1])
def main( argv ): if len(argv) != 3: return print( "ERROR: Usage \"python3 id3.py <training-set> <test-set> <model-file>\"" ) training_tup = CSVReader.readBooleanCSV( argv[ 0 ] ) global attributes; attributes = training_tup[ 0 ] global data ; data = training_tup[ 1 ] testing_tup = CSVReader.readBooleanCSV( argv[ 1 ] ) test_attributes = testing_tup[ 0 ] test_data = testing_tup[ 1 ] test_decisions = extractDecisions( test_data ) print( "Attributes" ) print( ', '.join( attributes ), "\n" ) tree = createDecisionTree( attributes, data ) predictions = [ getPrediction( tree.dataSetFromDecisions( convertRowToDict( row ) ) ) for row in test_data ] print( "\nPrediction accuracy vs. testing data:", "{}%\n\n".format( 100 * compareDecisions( predictions, test_decisions ) ) ) tree.printTree( argv[2] )
def create_forest(self): print "save to", self.forest_path trainingdata = [] traininglabel = [] sp_vec_path_list = CSVReader.get_path_list2("./output/csvpath/spdata_path_m"+self.m+"cs"+self.cs+"w"+self.s_weight+".csv",0) if (len(sp_vec_path_list)<self.end_num): print "end_num is larger than data length" return -1 print "making training data" for i in range(self.end_num - self.start_num): print "inputting",i,sp_vec_path_list[i] data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) label_col = len(data[0]) -1 for line in data: #print line[0:256/int(self.cs)*4] trainingdata += [ line[0:label_col] ] #print int(line[label_col]) traininglabel+= [int(line[label_col])] print "training data Complete" #print traininglabel self.forest.fit(trainingdata , traininglabel ) # Save print "save to", self.forest_path if not os.path.exists(self.forest_path): os.makedirs(self.forest_path) joblib.dump(self.forest, self.forest_path+'/forest.bin') return self.forest
def enter_screen(self): self.player_list.clear_widgets() self.selection_list.clear_widgets() self.description_list.clear_widgets() self.color_list.clear_widgets() all_players = [] for resultFile in self.manager.i_file_results: reader = CSVReader(resultFile.id) newLine = reader.getNext() while newLine != None: all_players.append(reader.getItem("Name", newLine)) newLine = reader.getNext() all_players = sorted(all_players, key=str.lower) self.playerIdentList = [] index = 0 for p in all_players: player = PlayerIdentification() player.name = p self.playerIdentList.append(player) self.moveUp()
def data_deteil(self, sp_data_path ,i): if not os.path.exists('./output/statistics'): os.makedirs('./output/statistics') f = open('./output/statistics/data'+str(i)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') output = [0 for x in range(38)] data = CSVReader.read_csv_as_float(sp_data_path) label_col = len(data[0])-1 for line in data: output[int(line[label_col])] += 1 print sp_data_path for j in range( len(output) ): print j,output[j] writer.writerow([j,output[j]]) f.close()
def predict(self, num): src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = src_jpg_path[2*num] dep_path = src_dep_path[2*num] label_path = src_label_path[2*num] if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"): print "exist",jpg_path else: cmd = u'"slic\SLICOSuperpixel.exe"' os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight); vec_data_path = jpg_path+"_m"+self.m+"spdata.csv" sp_map_path = jpg_path+"_m"+self.m+"spmap.csv" sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv" data = CSVReader.read_csv_as_float(vec_data_path) sp_map = CSVReader.read_csv_as_int(sp_map_path) neighbors = CSVReader.read_csv_as_int(sp_neighbors_path) test_data = [] probs = self.load_probs_for_predict(data, neighbors, self.layer_num , num) label_col = len(data[0])-1 for j in range(len(data)): vector = [] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) test_data += [ vector ] #print len(vector) #if len(vector) != 190: # raw_input(">>") print "test data Complete" #print len(test_data), len(test_data[0]) #self.show_detail() output = self.forest.predict( test_data ) self.output_file(output, sp_map, num) img_a = cv2.imread(label_path,0) img_b = cv2.imread('output_MLF'+str(self.start_num)+'-'+str(self.end_num-1)+'_layer'+str(self.layer_num)+'data'+str(num)+'.png',0) return [jpg_path, label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def AssessmentByMIOU(img_a,img_b): rate=1.0 LabelNum=38 PredictionLabelNum=0 Ic_Uc=0.0 width = img_a.shape[1] height = img_a.shape[0] GrandTruth=np.zeros(LabelNum,dtype=np.int32) Prediction=np.zeros(LabelNum,dtype=np.int32) Intersection=np.zeros(LabelNum,dtype=np.int32) IntersectionRate=np.zeros(LabelNum,dtype=np.float32) #ラベルも同時に出力したい場合 label = CSVReader.read_csv('seg37list.csv') LabelList = ['other'] for row in label: for i in range(len(row)): LabelList.append(row[i]) #ここまで for y in range(height): for x in range(width): GrandTruth[img_a[y,x]] += 1 Prediction[img_b[y,x]] += 1 if img_a[y,x] == img_b[y,x]: Intersection[img_a[y,x]] += 1 print (GrandTruth) print (Prediction) print(Intersection) for i in range(LabelNum): if(GrandTruth[i]+Prediction[i]-Intersection[i]>0): IntersectionRate[i]=1.0*Intersection[i]/(GrandTruth[i]+Prediction[i]-Intersection[i]) print(i,LabelList[i],IntersectionRate[i]) Ic_Uc += 1.0*IntersectionRate[i] if(Prediction[i]>0): PredictionLabelNum += 1 rate = Ic_Uc/PredictionLabelNum ''' #ラベルごとIOU値の棒グラフ出力 X=np.arange(0,LabelNum,1) plt.bar(X,IntersectionRate,width=0.7) plt.xticks(X, LabelList) # 目盛りを数字からラベルに書き換える plt.xticks(rotation=90) # 目盛りを傾ける plt.show() #ここまで ''' return rate
def test_rf(): from sklearn.model_selection import train_test_split data = CSVReader.read_csv("../data/income.csv") train, test = train_test_split(data, test_size=0.3) rf = RandomForestClassifier(nb_trees=60, nb_samples=3000, max_workers=4) rf.fit(train) errors = 0 features = [ft[:-1] for ft in test] values = [ft[-1] for ft in test] for feature, value in zip(features, values): prediction = rf.predict(feature) if prediction != value: errors += 1 logging.info("Error rate: {}".format(errors / len(features) * 100))
def test_rf(): from sklearn.model_selection import train_test_split # using absolute path to prevent WORKDIR issue data = CSVReader.read_csv("/Users/azhmakin/Documents/projects/own/university/intelligent-data-analysis/lab-3/random-forest/data/income.csv") train, test = train_test_split(data, test_size=0.3) rf = RandomForestClassifier(nb_trees=60, nb_samples=3000, max_workers=4) rf.fit(train) errors = 0 features = [ft[:-1] for ft in test] values = [ft[-1] for ft in test] for feature, value in zip(features, values): prediction = rf.predict(feature) if prediction != value: errors += 1 logging.info("Error rate: {}".format(errors / len(features) * 100))
def predict(self, num): src_jpg_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',4) src_dep_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',3) src_label_path = CSVReader.get_path_list2('SUNRGBDMeta_reduced.csv',9) jpg_path = src_jpg_path[2*num] dep_path = src_dep_path[2*num] label_path = src_label_path[2*num] if os.path.exists(jpg_path+"_m"+self.m+"spdata.csv"): print "exist",jpg_path else: cmd = u'"slic\SLICOSuperpixel.exe"' os.system(cmd+" "+ self.m + " "+jpg_path+" "+dep_path+" "+label_path+" "+ self.s_weight); vec_data_path = jpg_path+"_m"+self.m+"spdata.csv" sp_map_path = jpg_path+"_m"+self.m+"spmap.csv" sp_neighbors_path = jpg_path+"_m"+self.m+"neighbors.csv" data = CSVReader.read_csv_as_float(vec_data_path) sp_map = CSVReader.read_csv_as_int(sp_map_path) neighbors = CSVReader.read_csv_as_int(sp_neighbors_path) test_data = [] probs = self.load_probs(data) label_col = len(data[0])-1 for j in range(len(data)): vector = [] vector.extend( probs[j] ) vector.extend( probs[neighbors[j][0]] ) vector.extend( probs[neighbors[j][1]] ) vector.extend( probs[neighbors[j][2]] ) vector.extend( probs[neighbors[j][3]] ) test_data += [ vector ] os.remove('./output/probs.csv') print "test data Complete" output = self.forest.predict( test_data ) self.output_file(output, sp_map, num) img_a = cv2.imread(label_path,0) img_b = cv2.imread('output_CLRFdata'+str(num)+'.png',0) return [label_path ,Assessment.AssessmentByMIOU(img_a,img_b)]
def _loadNames(self): return CSVReader.getNameList()
from sklearn.model_selection import train_test_split import CSVReader import joblib from sklearn.metrics import confusion_matrix data = CSVReader.read_csv("2darray.csv") #data = CSVReader.read_csv("new 1.csv") features = [ft[:-1] for ft in data] values = [ft[-1] for ft in data] # Split dataset into training set and test set X_train, X_test, y_train, y_test = train_test_split(features, values, test_size=0.3) #Import Gaussian Naive Bayes model from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.datasets import make_classification #n_estimators = number of trees #bootstrap import time start_time = time.time() clf = RandomForestClassifier(n_estimators=50 ,max_depth=15, random_state=0) clf.fit(X_train, y_train) y_pred=clf.predict(X_test) #print("--- %s seconds ---" % (time.time() - start_time)) joblib.dump(clf, 'random_forest.joblib', compress=9) from sklearn import metrics # Model Accuracy, how often is the classifier correct? print("Accuracy:",metrics.accuracy_score(y_test, y_pred)) print("F1 Score: ", metrics.f1_score(y_test, y_pred))
def load_probs(self, sp_vec_path_list, sp_neighbors_list, layer_num, img_num): #(1)spDataをLRFを用いてprobs.csvに変換 forests = [] for i in range(38): #print 'LRF',i forests += [ LRF(self.start_num, self.end_num-1, i) ] for i in range(self.start_num, self.end_num): if not os.path.exists('./output/probs/base'): os.makedirs('./output/probs/base') if os.path.exists('./output/probs/base/'+str(i)+'.csv'): continue data = CSVReader.read_csv_as_float(sp_vec_path_list[i]) output = [] label_col = len(data[0])-1 for line in data: probs = [] for f in forests: probs.extend( f.get_prob (line[0:label_col]) ) output += [probs] f = open('./output/probs/base/'+str(i)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in output: writer.writerow(line) f.close() #(2)probs.csvをILRFを用いてlayer_numの回数分変換 for layer in range(layer_num): forests = [] for label in range(38): #print 'ILRF',label forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ] for n in range(self.start_num, self.end_num): data_num = n if os.path.exists('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv'): continue if layer == 0: src_probs = CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv') else: src_probs = CSVReader.read_csv_as_float('./output/probs/layer'+str(layer-1)+'/'+str(data_num)+'.csv') dist_probs = [] neighbors = CSVReader.read_csv_as_int(sp_neighbors_list[n]) for i in range(len(src_probs)): vector = [] vector.extend( src_probs[i] ) vector.extend( src_probs[neighbors[i][0]] ) vector.extend( src_probs[neighbors[i][1]] ) vector.extend( src_probs[neighbors[i][2]] ) vector.extend( src_probs[neighbors[i][3]] ) probs = [] for f in forests: probs.extend( f.get_prob (vector) ) dist_probs += [probs] if not os.path.exists('./output/probs/layer'+str(layer)): os.makedirs('./output/probs/layer'+str(layer)) f = open('./output/probs/layer'+str(layer)+'/'+str(data_num)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in dist_probs: writer.writerow(line) f.close() if layer_num == 0: return CSVReader.read_csv_as_float('./output/probs/base/'+str(img_num)+'.csv') else: return CSVReader.read_csv_as_float('./output/probs/layer'+str(layer_num-1)+'/'+str(img_num)+'.csv')
from sklearn.model_selection import train_test_split import CSVReader from sklearn.metrics import confusion_matrix from sklearn import svm from sklearn.linear_model import LogisticRegression data = CSVReader.read_csv("dataset_withfeatures.csv") features = [ft[:-1] for ft in data] values = [ft[-1] for ft in data] # Split dataset into training set and test set X_train, X_test, y_train, y_test = train_test_split(features, values, test_size=0.3) #Import Gaussian Naive Bayes model from sklearn.neighbors import KNeighborsClassifier #clf = LogisticRegression(random_state=0).fit(X_train, y_train) model = KNeighborsClassifier(n_neighbors=17) model.fit(X_train,y_train) y_pred = model.predict(X_test) from sklearn import metrics # Model Accuracy, how often is the classifier correct? print("Accuracy:",metrics.accuracy_score(y_test, y_pred)) print("F1 Score: ", metrics.f1_score(y_test, y_pred)) tn, fp, fn, tp=(confusion_matrix(y_test, y_pred).ravel()) #print(tn,fp,fn,tp) print("False Postive Rate: ",fp/(fp+tn)) print("False Negitve Rate: ",fn/(fn+tp)) print("True Negitive Rate: ",tn/(tn+fp))
def printTeamStandings(myCanvas, startTop, filepath, num_trophy_winners, trophy_highlight): myCanvas.setFont("Helvetica", 11 * POINT) y = startTop x = MARGIN_LEFT teamResult = CSVReader(filepath) columns = [ x, x + INCH * .5, x + INCH * 6.25, x + INCH * 7.00, x + INCH * 7.75, x + INCH * 8.50, x + INCH * 9.25 ] ignore = 0 playerCount = 0 for i in range(0, len(teamResult.headers)): if (teamResult.headers[i] not in IGNORE_TEAM): myCanvas.drawString(columns[i - ignore], y, teamResult.headers[i]) else: ignore += 1 y -= LINE ignore = 0 num_trophies = 0 line = teamResult.getNext() while (line != None): if (y < MARGIN_BOTTOM): y = MARGIN_TOP myCanvas.showPage() if (line[0] == ""): playerCount += 1 else: playerCount = 0 if (playerCount > 4): line = teamResult.getNext() continue if (num_trophies < int(num_trophy_winners)) and (playerCount == 0): num_trophies += 1 myCanvas.setStrokeColorRGB(trophy_highlight[0], trophy_highlight[1], trophy_highlight[2]) myCanvas.setFillColorRGB(trophy_highlight[0], trophy_highlight[1], trophy_highlight[2]) myCanvas.rect(columns[0] - 3, y - 3, columns[3] - columns[0] - 2, LINE + 1, 1, 1) myCanvas.setStrokeColorRGB(0, 0, 0) myCanvas.setFillColorRGB(0, 0, 0) ignore = 0 for i in range(0, len(line)): if (teamResult.headers[i] in IGNORE_TEAM): ignore += 1 else: myCanvas.drawString(columns[i - ignore], y, line[i]) y -= LINE line = teamResult.getNext() highlights = [trophy_highlight] descriptions = ["Trophy Winners"] if len(highlights) > 0: draw_highlight_key(myCanvas, highlights, descriptions)
def printIndividual(myCanvas, left, top, filepath, num_trophy_winners, trophy_highlight, player_identification): player_ident = copy.copy(player_identification) base_left = .20 * INCH base_right = 10.30 * INCH space = base_right - base_left lengths = [] total_characters = 0 result = CSVReader(filepath) buff = 3 for head in result.headers: if head.lower() == "name".lower(): buff = 8 else: buff = 3 length = result.getLongest(head) + buff lengths.append(length) total_characters += length increment = math.floor(space / total_characters) columns = [base_left] for i in range(0, len(lengths) - 1): columns.append(columns[i] + increment * lengths[i]) myCanvas.setFont("Helvetica", 12 * POINT) y = top for i in range(0, len(result.headers)): myCanvas.drawString(columns[i], y, result.headers[i]) y -= LINE highlights = [] descriptions = [] if int(num_trophy_winners) > 0: highlights.append(trophy_highlight) descriptions.append("Trophy Winners") num_trophies = 0 next = result.getNext() new_page = False highlight_width = columns[6] - columns[0] while (next != None): if (y < (.5 * INCH)): new_page = True if (num_trophies < int(num_trophy_winners)) == True: num_trophies += 1 highlight(myCanvas, columns[0], y, highlight_width, trophy_highlight) status = result.getItem("St", next) name = result.getItem("Name", next) if (status != "Out"): if new_page == True: draw_highlight_key(myCanvas, highlights, descriptions) highlights = [] descriptions = [] y = top myCanvas.showPage() new_page = False for player in player_ident: if player.name == name: if player.description in descriptions: player.col = highlights[descriptions.index( player.description)] else: highlights.append(player.col) descriptions.append(player.description) highlight(myCanvas, columns[0], y, highlight_width, player.col) player_ident.remove(player) for i in range(0, len(next)): myCanvas.drawString(columns[i], y, next[i]) y -= LINE next = result.getNext() if len(highlights) > 0: draw_highlight_key(myCanvas, highlights, descriptions)
import numpy as np from mpl_toolkits.mplot3d import Axes3D from sklearn.cluster import KMeans from pandas import DataFrame from algorithms import kmean as km import CSVReader as cr import DataNormalisation as dt MAX_ITERATIONS = 100 filePathRusia = 'C:\\Users\\dprefac\\PycharmProjects\\netscan-master\\csv\\pythonScript_1000_telekom_VPNRusia.csv' filePathLocal = 'C:\\Users\\dprefac\\PycharmProjects\\netscan-master\\csv\\bruteForceWithValidPassword100Attempts200Status.csv' features = [9, 10, 13, 14] csvReader = cr.CSVReader() dataFromRusia = csvReader.load_csv_and_extract_feature(filePathRusia, features, "RUSIA") dataFromLocal = csvReader.load_csv_and_extract_feature(filePathLocal, features, "LOCAL") dataFromRusia = np.array(dataFromRusia) dataFromLocal = np.array(dataFromLocal) dataFromRusia = dt.DataNormalisation.eliminate_outlier_column(dataFromRusia, 0) dataFromLocal = dt.DataNormalisation.eliminate_outlier_column(dataFromLocal, 0) # dataFromRusia = dt.DataNormalisation.eliminate_outlier_column(dataFromRusia, 1) dataFromLocal = dt.DataNormalisation.eliminate_outlier_column(dataFromLocal, 1)
def load_probs(self, data, data_num): if self.layer == 0: return CSVReader.read_csv_as_float('./output/probs/base/'+str(data_num)+'.csv') else: return CSVReader.read_csv_as_float('./output/probs/layer'+str(self.layer-1)+'/'+str(data_num)+'.csv')
# -*- coding: <UTF-8> -*- import CSVReader import WebScraper import re import matplotlib.pyplot as plt import numpy typeVsEXP = CSVReader.CSVWorker('/home/taidgh/Code/Python/Pokemon/typeVsEXP.csv', 'typeVsEXP.csv') def listCleaner(list): cString = re.sub('\[|\]| |\'', '', list) return(cString.split(',')) def fileGen(): #Check if the file exsits, if it does, load type vs EXP else do the Web Scraping. #This makes it so if the data is already retrieved the time can be saved. if(typeVsEXP.isFile): typeVsEXP.getRows([1,2]) #Load the data else: data = toCSV() typeVsEXP.createNWD(data) fileGen() #Create the file, then load the data. def CSVToDict():
optimization(num_iterations, epochs) import os path = '' def csv_weights_write(): global path path = os.path.join(os.getcwd(), 'Weights') i = 1 while True: if not os.path.exists(path): os.mkdir(path) break if i >= 2: path = path[:(-1 - len(str(i)))] path += '_' + str(i) i += 1 print("Weights are stored in " + path) w = session.run(weights, feed_dict=whole_feed_dict) for i in range(len(w)): filename = os.path.join(path, 'w' + str(i + 1) + '.csv') with open(filename, 'w') as fl: csvwriter = csv.writer(fl) csvwriter.writerows(w[i]) csv_weights_write() param.print_equation(os.path.join(path, 'equation.txt'))
import CSVReader as csv import CSVWriter import MetaReader as meta import DecisionTree as DT ######################################################################## ####################### LOAD NECESSARY DATA ############################ ######################################################################## # Load training set training_set = csv.CSVReader( raw_input("Please enter the full filepath of the TRAINING DATA below: \n")) # training_set = csv.CSVReader("./data/btrain.csv") training_header, training_data = training_set.readFile() # Load meta data meta_reader = meta.MetaReader( raw_input("\nPlease enter the full filepath of the META DATA below: \n")) # meta_reader = meta.MetaReader("./data/bmeta.csv") meta_data = meta_reader.parseMetaData() # Load validation set validation_set = csv.CSVReader( raw_input( "\nPlease enter the full filepath of the VALIDATION DATA below: \n")) # validation_set = csv.CSVReader("./data/bvalidate.csv") validate_header, validate_data = validation_set.readFile() # Load test set test_set = csv.CSVReader( raw_input("\nPlease enter the full filepath of the TEST DATA below: \n"))
try: import CSVReader import computation except ModuleNotFoundError: print("Cannot import file") try: colHeaders, rows = CSVReader.openFile("data.csv") print(CSVReader.returnLocation(colHeaders, rows, -7, 2)) print(CSVReader.returnLocation(colHeaders, rows, -5, -2)) print(CSVReader.returnLocation(colHeaders, rows, 2, -1)) print(CSVReader.returnLocation(colHeaders, rows, -3, -3)) print(CSVReader.returnLocation(colHeaders, rows, 1, 3)) print(computation.selectOperation(colHeaders, rows, "price", "add")) print(computation.selectOperation(colHeaders, rows, "price", "average")) print(computation.selectOperation(colHeaders, rows, "id", "Min")) print(computation.selectOperation(colHeaders, rows, "price", "Max")) except AttributeError: print("Function does not exists") except NameError: print("Given column does not exists")
def __init__(self, filename): metaCSV = csv.CSVReader(filename) self.attributes, self.types = metaCSV.readFile()
def load_probs_for_predict(self, data, neighbors, layer_num, num): #(1)spDataをLRFを用いてprobs.csvに変換 if not os.path.exists('./output/probs_for_predict/base'): os.makedirs('./output/probs_for_predict/base') if not os.path.exists('./output/probs_for_predict/base/'+str(num)+'.csv'): forests = [] for i in range(38): forests += [ LRF(self.start_num, self.end_num-1, i) ] output = [] label_col = len(data[0])-1 for line in data: probs = [] for f in forests: probs.extend( f.get_prob (line[0:label_col]) ) output += [probs] f = open('./output/probs_for_predict/base/'+str(num)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in output: writer.writerow(line) f.close() #(2)probs.csvをILRFを用いてlayer_numの回数分変換 for layer in range(layer_num): if not os.path.exists('./output/probs_for_predict/layer'+str(layer)): os.makedirs('./output/probs_for_predict/layer'+str(layer)) if not os.path.exists('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv'): forests = [] for label in range(38): forests += [ ILRF(self.start_num, self.end_num-1, layer, label) ] if layer == 0: src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv') else: src_probs = CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer-1)+'/'+str(num)+'.csv') dist_probs = [] for i in range(len(src_probs)): vector = [] vector.extend( src_probs[i] ) vector.extend( src_probs[neighbors[i][0]] ) vector.extend( src_probs[neighbors[i][1]] ) vector.extend( src_probs[neighbors[i][2]] ) vector.extend( src_probs[neighbors[i][3]] ) probs = [] for f in forests: probs.extend( f.get_prob (vector) ) dist_probs += [probs] f = open('./output/probs_for_predict/layer'+str(layer)+'/'+str(num)+'.csv', 'w') writer = csv.writer(f, lineterminator='\n') for line in dist_probs: writer.writerow(line) f.close() if layer_num == 0: return CSVReader.read_csv_as_float('./output/probs_for_predict/base/'+str(num)+'.csv') else: return CSVReader.read_csv_as_float('./output/probs_for_predict/layer'+str(layer_num-1)+'/'+str(num)+'.csv')
import CSVReader CSVReader.readQustionsCSV(CSVReader.readGroundTruthCSV()) #CSVReader.fillQuestionsGroundTruthAnsCSV()