def main(): try: instanceSize = 10 step = 1 numOfClasses = 100 filename = 'detector_7_no_5_angle_2.jpg' mode = 'PRO' image_files, bubble_num, bubble_regions = getinfo() data_set = ds.read_data_sets( instanceSize = instanceSize, stride = step, instanceMode = 'test', labelMode = mode, imageName = filename, dtype = tf.float32, plot_show = 0, label_mutiplier = 1.0) print data_set.labels.shape[-1] print 'Sum of labels: ' + str(np.sum(data_set.labels)) print 'Number bubbles: ' + str(bubble_num[image_files.index(filename)]) except KeyboardInterrupt: print "Shutdown requested... exiting" except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
def plotresultfromfile(ImagePatchWidth = 100, ImagePatchStep = 10, trainBatchNum = 10000, trainBatchSize = 2000, label_mutiplier = 1.0, hidden_units = [200, 400, 200], trainSteps = 200, optimizer = 'Adagrad', #"SGD", "Adam", "Adagrad" learning_rate = 0.01, label_mode = 'PRO', clip_gradients = 5.0, config = None, verbose = 1, dropout = None): """ load data from file and show the data """ image_files, bubble_num, bubble_regions = getinfo() directory = gv.__DIR__ + gv.dp__tuningPar_dir filename = ( str(ImagePatchWidth) + '_' + str(ImagePatchStep) + '_' + str(trainBatchNum) + '_' + str(trainBatchSize) + '_' + str(label_mutiplier) + '_' + str(hidden_units) + '_' + str(trainSteps) + '_' + optimizer + '_' + str(learning_rate) + '_' + label_mode + '_' + str(clip_gradients) + '_' + str(config) + '_' + str(verbose) + '_' + str(dropout) + '.dat') result = np.fromfile(directory + filename) slope, intercept, r_value, p_value, std_err = linregress( bubble_num, result.T) yfit = slope * bubble_num + intercept fig, ax = plt.subplots(1,1) ax.set_title('Linearity of DNN Regressor') ax.set_xlabel('Bubble number counted manually') ax.set_ylabel('Bubble number labeled by DNN') ax.scatter(bubble_num, result, color = 'blue', label = 'DNN result') ax.plot(bubble_num, yfit, color = 'red', label = 'linear fit') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") text_x = np.amax(bubble_num)*0.6 text_y = np.amax(result)*0.2 text = "R_SQUARED: %.5f\nSTD_ERR: %.5f" % (r_value**2, std_err) ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y)) plt.show()
def labellinearity(patch_size, stride, labelMode = 'PRO', label_mutiplier = 1, progress_show = 1, plot_show = 1): gv.ds_show_filename = False image_files, bubble_num, bubble_regions = getinfo() bubble_num_afterlabel = np.zeros(len(image_files)) probar = progress.progress(0, len(image_files)) for i, image in enumerate(image_files): probar.setCurrentIteration(i+1) probar.setInfo(prefix_info = 'dataset linearity ...', suffix_info = image) probar.printProgress() image_ds = ds.read_data_sets( instanceSize = patch_size, stride = stride, instanceMode = 'test', labelMode = labelMode, imageName = image, label_mutiplier = label_mutiplier) labels = image_ds.labels bubble_num_afterlabel[i] = np.sum(labels) slope, intercept, r_value, p_value, std_err = (linregress(bubble_num, bubble_num_afterlabel)) if(plot_show == 1): yfit = slope * bubble_num + intercept fig, ax = plt.subplots(1,1) ax.set_title('Linearity of Labeling Methods') ax.set_xlabel('Bubble number counted manually') ax.set_ylabel('Bubble number after labeling') ax.scatter(bubble_num, bubble_num_afterlabel, color = 'blue', label = 'bubble number') ax.plot(bubble_num, yfit, color = 'red', label = 'linear regression') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") text_x = np.amax(bubble_num)*0.6 text_y = np.amax(bubble_num_afterlabel)*0.1 text = "r_squared: %.5f\nstd_err: %.5f" % (r_value**2, std_err) ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y)) plt.show() return ([[slope, intercept, r_value, p_value, std_err], bubble_num, bubble_num_afterlabel])
def testall(classifier, ImagePatchWidth = 20, ImagePatchStep = 4, label_mutiplier = 1.0, labelMode = 'PRO', image_show = 0, # show labeling result of each image save_image = True, # save each labeled image save_result = True): # save labeling result and accuracy """ labeling all avilable images """ # get all image information image_files, bubble_num, bubble_regions = getinfo() # files to be saved result_filename = gv.dp__result_filename accuracy_filename = gv.dp__accuracy_filename # initialize benchmark variables labeling_result = np.zeros((len(image_files),1)) correct = np.zeros(4) accuracy = np.zeros(4) totalnum_instances = 0 for i, image_file in enumerate(image_files): # labeling each image bubbleNum, correctNumber, totalInstanceNumber = test( classifier, image_file, ImagePatchWidth = ImagePatchWidth, ImagePatchStep = ImagePatchStep, labelMode = labelMode, label_mutiplier = label_mutiplier, plot_show = image_show, save_image = save_image) labeling_result[i] = bubbleNum correct = np.add(correct, correctNumber) totalnum_instances += totalInstanceNumber # calculate total labeling accuracy accuracy = np.true_divide(correct, totalnum_instances) # save data if(gv.dp_test_save_data and save_result): directory = gv.__dir__ + gv.dp__tuningPar_dir if not os.path.exists(directory): os.makedirs(directory) accuracy.tofile(directory + accuracy_filename, sep = " ") labeling_result.tofile(result_filename, sep = " ") return [labeling_result, accuracy]
def main(): try: data_filename = gv.cu__result_filename image_files, bubble_num, bubble_regions = getinfo() if not os.path.isfile(data_filename): number = linearity(image_files) number = np.loadtxt(data_filename) number = np.reshape(number, (len(number)/2,2)) fig, ax = plt.subplots(1,1) ax.scatter(bubble_num, number[:, 0], c = 'r') k, b = np.polyfit(bubble_num, number[:, 0], 1) ax.plot(bubble_num, bubble_num*k+b, 'r', linewidth = 2) ax.scatter(bubble_num, number[:, 1], c = 'b') k, b = np.polyfit(bubble_num, number[:, 1], 1) ax.plot(bubble_num, bubble_num*k+b, 'b', linewidth = 2) plt.show() except KeyboardInterrupt: print "Shutdown requested... exiting" except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
def gaussianDatatoFile(instanceSize, stride, labelMode): """ Calculation of Gaussian value of every point is time consuming. For accelarate the trainning and test process, this function generate Gaussian data for each image and save to file. When training and testing, only need to load data from file. """ image_files, bubble_num, bubble_regions = getinfo() PROGRESS = progress.progress(0, len(image_files)) for i, imageFilename in enumerate(image_files): PROGRESS.setCurrentIteration(i+1) PROGRESS.setInfo(prefix_info = 'Gaussian data generating ...', suffix_info = imageFilename) PROGRESS.printProgress() filename = gv.__DIR__ + gv.__TrainImageDir__ + imageFilename imageData = io.imread(filename) positiveLabels = bubble_regions[image_files.index(imageFilename)] [m, n, c] = imageData.shape Y = np.arange(0, (m-instanceSize + 1), stride) X = np.arange(0, (n-instanceSize + 1), stride) instances, labels = patchlabel(Y,X,positiveLabels, patchsize = instanceSize, stride = stride, mode = labelMode) directory = gv.__DIR__ + gv.dp__Gaussia_data_dir if not os.path.exists(directory): os.makedirs(directory) filename = (directory + str(instanceSize) + '_' + str(stride) + '_' + labelMode + imageFilename[0:-3]) + 'npy' np.save(filename, np.array(labels))
def performance(ImagePatchWidth = 100, ImagePatchStep = 10, trainBatchNum = 10000, trainBatchSize = 2000, label_mutiplier = 1.0, hidden_units = [200, 400, 200], trainSteps = 200, optimizer = 'Adagrad', #"SGD", "Adam", "Adagrad" learning_rate = 0.01, label_mode = 'PRO', clip_gradients = 5.0, config = None, verbose = 1, dropout = None, plot_show = 1, save_result = True): """ performance benchmark through train nand labeling """ # information of all aviliable images image_files, bubble_num, bubble_regions = getinfo() ds.gaussianDatatoFile(ImagePatchWidth, ImagePatchStep, label_mode) # train DNN classifier, _ = train( batchNum = trainBatchNum, batchSize = trainBatchSize, ImagePatchWidth = ImagePatchWidth, ImagePatchStep = ImagePatchStep, label_mutiplier = label_mutiplier, labelMode = label_mode, hidden_units = hidden_units, steps = trainSteps, optimizer = optimizer, learning_rate = learning_rate, clip_gradients = clip_gradients, config = config, verbose = verbose, dropout = dropout) # labeling all images result, accuracy = testall( classifier, ImagePatchWidth = ImagePatchWidth, ImagePatchStep = ImagePatchStep, labelMode = label_mode, label_mutiplier = label_mutiplier, image_show = 0, save_image = False, save_result = False) # Save in this function, no need to save twice # linear regression slope, intercept, r_value, p_value, std_err = linregress( bubble_num, result.T) status = np.append( np.array([slope, intercept, r_value, p_value, std_err]), accuracy) # save data if(save_result): directory = gv.__DIR__ + gv.dp__tuningPar_dir if not os.path.exists(directory): os.makedirs(directory) filename = ( str(ImagePatchWidth) + '_' + str(ImagePatchStep) + '_' + str(trainBatchNum) + '_' + str(trainBatchSize) + '_' + str(label_mutiplier) + '_' + str(hidden_units) + '_' + str(trainSteps) + '_' + optimizer + '_' + str(learning_rate) + '_' + label_mode + '_' + str(clip_gradients) + '_' + str(config) + '_' + str(verbose) + '_' + str(dropout) + '.dat') result.tofile(directory + filename) # show linearity if(plot_show == 1): print accuracy yfit = slope * bubble_num + intercept fig, ax = plt.subplots(1,1) ax.set_title('Linearity of DNN Regressor') ax.set_xlabel('Bubble number counted manually') ax.set_ylabel('Bubble number labeled by DNN') ax.scatter(bubble_num, result, color = 'blue', label = 'DNN result') ax.plot(bubble_num, yfit, color = 'red', label = 'linear fit') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") text_x = np.amax(bubble_num)*0.6 text_y = np.amax(result)*0.1 text = "R_SQUARED: %.5f\nSTD_ERR: %.5f" % (r_value**2, std_err) ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y)) plt.show() return [result, status]
def main(): try: echos_tune = 10 batchNum = 2000 batchSize = 2000 ImagePatchWidth = 40 ImagePatchStep = 1 labelMode = "NUM" label_mutiplier = 1.0 hidden_units = [200, 400, 200] steps = 200 optimizer = "Adagrad" # "SGD", "Adam", "Adagrad" learning_rate = 0.1 clip_gradients = 5.0 config = None verbose = 1 dropout = None filename = "detector_3_no_5_angle_2.jpg" image_files, bubble_num, bubble_regions = getinfo() benchmark_tune = np.zeros((echos_tune, 2)) # generate data: # ds.gaussianDatatoFile(ImagePatchWidth, ImagePatchStep, labelMode) classifier, trainDataset = dl.train( batchNum=0, batchSize=1, ImagePatchWidth=ImagePatchWidth, ImagePatchStep=ImagePatchStep, labelMode=labelMode, hidden_units=hidden_units, steps=steps, optimizer=optimizer, learning_rate=learning_rate, clip_gradients=clip_gradients, config=config, verbose=verbose, dropout=dropout, ) PROGRESS = progress.progress(0, echos_tune) for i in range(echos_tune): PROGRESS.setCurrentIteration(i + 1) PROGRESS.setInfo(prefix_info="Batch Number Tune ...", suffix_info=str(i + 1) + "/" + str(echos_tune)) classifier = dl.continuetrain( classifier=classifier, trainDataset=trainDataset, batchNum=batchNum, batchSize=batchSize ) result, accuracy = dl.testall( classifier, ImagePatchWidth=ImagePatchWidth, ImagePatchStep=ImagePatchStep, label_mutiplier=label_mutiplier, labelMode=labelMode, image_show=0, # show labeling result of each image save_image=False, # save each labeled image save_result=False, ) slope, intercept, r_value, p_value, std_err = linregress(bubble_num, result.T) benchmark_tune[i, 0] = r_value benchmark_tune[i, 1] = std_err PROGRESS.printProgress() print "\n" filename = gv.__DIR__ + gv.dp__tuningPar_dir + "batchnumberresult.npy" np.save(filename, np.array(benchmark_tune)) fig, ax = plt.subplots(1) ax.plot(np.arange(echos_tune), benchmark_tune[:, 0], color="r", label="r_value") ax.plot(np.arange(echos_tune), benchmark_tune[:, 1], color="b", label="std_err") handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") plt.show() except KeyboardInterrupt: print "Shutdown requested... exiting" except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
def generateInstancesNN(instanceSize, stride, filenameList, mode, plot_show = 1, label_mutiplier = 1.0): allInstances = np.array([]) allLabels = np.array([]) allImages = [] xlen = 0 ylen = 0 image_files, bubble_num, bubble_regions = getinfo() for i, imageFilename in enumerate(filenameList): if(gv.ds_show_filename): print ('Generating instances from [' + imageFilename + ']... { patch width: ' + str(instanceSize) + ' stride: ' + str(stride) + ' }') filename = gv.__DIR__ + gv.__TrainImageDir__ + imageFilename imageData = io.imread(filename) positiveLabels = bubble_regions[image_files.index(imageFilename)] [m, n, c] = imageData.shape Y = np.arange(0, (m-instanceSize + 1), stride) X = np.arange(0, (n-instanceSize + 1), stride) ylen = len(Y) xlen = len(X) fname = (gv.__DIR__ + gv.dp__Gaussia_data_dir + str(instanceSize) + '_' + str(stride) + '_' + mode + imageFilename[0:-3] + 'npy') if(os.path.isfile(fname)): instances, labels = gaussianDataFromFile(fname, X, Y) else: instances, labels = patchlabel( Y, X, positiveLabels,patchsize = instanceSize, stride = stride, mode = mode) labels = labels*label_mutiplier instances = np.append(instances, i*np.ones((instances.shape[0], 1)), axis = 1) allImages.append(imageData) # append current instance data to all instance data if (allInstances.size == 0): allInstances = instances allLabels = labels else: allInstances = np.append(allInstances, instances, axis=0) allLabels = np.append(allLabels, labels, axis=0) if(plot_show == 1): image_show = np.reshape(labels, (ylen, xlen)) fig, ax = plt.subplots(2) ax[0].imshow(imageData) ax[0].set_title('Original Image') img = ax[1].imshow(np.sqrt(image_show)) ax[1].set_title('Labels') plt.colorbar(img) plt.show() return [allInstances, allLabels, ylen, xlen, allImages]