def linearity(image_filenames, data_filename = gv.cu__result_filename): result = np.zeros((len(image_filenames), 2)) center_region = [-400,400,-800,600] probar = progress.progress(0, len(image_filenames)) for iteration, filename in enumerate(image_filenames): probar.setCurrentIteration(iteration+1) probar.setInfo( prefix_info = 'Curvature linearity ...', suffix_info = filename) probar.printProgress() refname = filename[0:23] + '_background.jpg' image = io.imread(gv.__DIR__ + gv.__TrainImageDir__ + filename) OY = int(image.shape[0]/2) OX = int(image.shape[1]/2) margin = [max(OY + center_region[0], 0), \ min(OY + center_region[1], image.shape[0]), \ max(OX + center_region[2], 0), \ min(OX + center_region[3], image.shape[1])] circles = curvature.count_bubble(filename, refname, plot_show = 0) index_in_box = np.all( [np.all([circles[:, 0] <= margin[1], circles[:, 0] >= margin[0]], \ axis = 0), \ np.all([circles[:, 1] <= margin[3], circles[:, 1] >= margin[2]], \ axis = 0)], axis = 0) circles_in_box = circles[index_in_box] result[iteration, 0] = len(circles) result[iteration, 1] = len(circles_in_box) result.tofile(data_filename, sep=" ") return result
def main(): try: n = 100 test = progress.progress(0, n) for i in range(n): test.setCurrentIteration(i + 1) test.setInfo(prefix_info="Progress Bar Example ... ", suffix_info="Iteration " + str(i) + "/" + str(n)) test.printProgress() time.sleep(0.1) except KeyboardInterrupt: print "Shutdown requested... exiting" except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
def labellinearity(patch_size, stride, labelMode = 'PRO', label_mutiplier = 1, progress_show = 1, plot_show = 1): gv.ds_show_filename = False image_files, bubble_num, bubble_regions = getinfo() bubble_num_afterlabel = np.zeros(len(image_files)) probar = progress.progress(0, len(image_files)) for i, image in enumerate(image_files): probar.setCurrentIteration(i+1) probar.setInfo(prefix_info = 'dataset linearity ...', suffix_info = image) probar.printProgress() image_ds = ds.read_data_sets( instanceSize = patch_size, stride = stride, instanceMode = 'test', labelMode = labelMode, imageName = image, label_mutiplier = label_mutiplier) labels = image_ds.labels bubble_num_afterlabel[i] = np.sum(labels) slope, intercept, r_value, p_value, std_err = (linregress(bubble_num, bubble_num_afterlabel)) if(plot_show == 1): yfit = slope * bubble_num + intercept fig, ax = plt.subplots(1,1) ax.set_title('Linearity of Labeling Methods') ax.set_xlabel('Bubble number counted manually') ax.set_ylabel('Bubble number after labeling') ax.scatter(bubble_num, bubble_num_afterlabel, color = 'blue', label = 'bubble number') ax.plot(bubble_num, yfit, color = 'red', label = 'linear regression') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") text_x = np.amax(bubble_num)*0.6 text_y = np.amax(bubble_num_afterlabel)*0.1 text = "r_squared: %.5f\nstd_err: %.5f" % (r_value**2, std_err) ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y)) plt.show() return ([[slope, intercept, r_value, p_value, std_err], bubble_num, bubble_num_afterlabel])
def continuetrain(classifier, trainDataset, batchNum = 500, batchSize = 200): """Continue Train deep neural-network. """ print ('Training deep learning neural network ...') # train the DNNRegressor on generated data set probar = progress.progress(0, batchNum) gv.log_write = False for i in range(batchNum): probar.setCurrentIteration(i+1) probar.setInfo( prefix_info = 'Training ...', suffix_info = 'Batch: ' + str(i+1) + '/' + str(batchNum)) probar.printProgress() images, labels = trainDataset.next_batch(batchSize) if(gv.log_write): classifier.fit(images, labels, logdir = gv.__DIR__ + gv.tensorflow_log_dir) else: classifier.fit(images, labels) return classifier
def gaussianDatatoFile(instanceSize, stride, labelMode): """ Calculation of Gaussian value of every point is time consuming. For accelarate the trainning and test process, this function generate Gaussian data for each image and save to file. When training and testing, only need to load data from file. """ image_files, bubble_num, bubble_regions = getinfo() PROGRESS = progress.progress(0, len(image_files)) for i, imageFilename in enumerate(image_files): PROGRESS.setCurrentIteration(i+1) PROGRESS.setInfo(prefix_info = 'Gaussian data generating ...', suffix_info = imageFilename) PROGRESS.printProgress() filename = gv.__DIR__ + gv.__TrainImageDir__ + imageFilename imageData = io.imread(filename) positiveLabels = bubble_regions[image_files.index(imageFilename)] [m, n, c] = imageData.shape Y = np.arange(0, (m-instanceSize + 1), stride) X = np.arange(0, (n-instanceSize + 1), stride) instances, labels = patchlabel(Y,X,positiveLabels, patchsize = instanceSize, stride = stride, mode = labelMode) directory = gv.__DIR__ + gv.dp__Gaussia_data_dir if not os.path.exists(directory): os.makedirs(directory) filename = (directory + str(instanceSize) + '_' + str(stride) + '_' + labelMode + imageFilename[0:-3]) + 'npy' np.save(filename, np.array(labels))
def train(batchNum = 500, batchSize = 200, ImagePatchWidth = 20, ImagePatchStep = 4, labelMode = 'PRO', label_mutiplier = 1.0, hidden_units = [200, 400, 200], steps = 200, optimizer = 'Adagrad', #"SGD", "Adam", "Adagrad" learning_rate = 0.001, clip_gradients = 5.0, config = None, verbose = 1, dropout = None): """Train deep neural-network. Parameters: hidden_units: List of hidden units per layer. batch_size: Mini batch size. steps: Number of steps to run over data. optimizer: Optimizer name (or class), for example "SGD", "Adam", "Adagrad". learning_rate: If this is constant float value, no decay function is used. Instead, a customized decay function can be passed that accepts global_step as parameter and returns a Tensor. e.g. exponential decay function: def exp_decay(global_step): return tf.train.exponential_decay( learning_rate=0.1, global_step, decay_steps=2, decay_rate=0.001) continue_training: when continue_training is True, once initialized model will be continuely trained on every call of fit. config: RunConfig object that controls the configurations of the session, e.g. num_cores, gpu_memory_fraction, etc. verbose: Controls the verbosity, possible values: 0: the algorithm and debug information is muted. 1: trainer prints the progress. 2: log device placement is printed. dropout: When not None, the probability we will drop out a given coordinate. """ print ('Training deep learning neural network ...') # generate data set trainDataset = ds.read_data_sets( instanceSize = ImagePatchWidth, stride = ImagePatchStep, instanceMode = 'train', labelMode = labelMode, label_mutiplier = label_mutiplier) # deep neural-network regression class (DNNRegressor) classifier = skflow.TensorFlowDNNRegressor( hidden_units = hidden_units, batch_size = 32, steps = steps, optimizer = optimizer, #"SGD", "Adam", "Adagrad" learning_rate = learning_rate, continue_training = True, clip_gradients = clip_gradients, config = config, verbose = verbose, dropout = dropout) # train the DNNRegressor on generated data set probar = progress.progress(0, batchNum) gv.log_write = False for i in range(batchNum): probar.setCurrentIteration(i+1) probar.setInfo( prefix_info = 'Training ...', suffix_info = 'Batch: ' + str(i+1) + '/' + str(batchNum)) probar.printProgress() images, labels = trainDataset.next_batch(batchSize) if(gv.log_write): classifier.fit(images, labels, logdir = gv.__DIR__ + gv.tensorflow_log_dir) else: classifier.fit(images, labels) return classifier, trainDataset
def test(classifier, filename, ImagePatchWidth = 20, ImagePatchStep = 4, labelMode = 'PRO', label_mutiplier = 1.0, plot_show = 1, save_image = True): """ label image with trained deep neural-network """ if(labelMode == 'NUM' and ImagePatchStep < ImagePatchWidth): ImagePatchStep = ImagePatchWidth # generate test data testDS = ds.read_data_sets( instanceSize = ImagePatchWidth, stride = ImagePatchStep, instanceMode = 'test', labelMode = labelMode, imageName = filename, label_mutiplier = label_mutiplier) # decide batch number and batch size according to memory requirement memory_limit = gv.MEM_LIM batch_size = np.floor(memory_limit / (ImagePatchWidth**2*3) / 4 / 3) batch_num = int(np.ceil( np.true_divide(testDS.xlength * testDS.ylength, batch_size))) # labeling _y = testDS.labels # correct labels y = np.zeros((testDS.num_examples,1)) # label results PROGRESS = progress.progress(0, batch_num) for j in range(batch_num): PROGRESS.setCurrentIteration(j+1) PROGRESS.setInfo(prefix_info = 'Labeling ... ', suffix_info = filename) PROGRESS.printProgress() start = testDS.index_in_epoch if (start + batch_size > testDS.num_examples) : end = testDS.num_examples else: end = start + batch_size batch_images, _ = testDS.next_batch(end-start) y[int(start):int(end)] = classifier.predict( batch_images, batch_size = 1024) # benchmark correctNumber = np.array([ np.sum(y == _y), # all correct number np.sum(np.all([y == _y, _y == 0], axis = 0)), # correct negtive np.sum(np.all([y == _y, _y > 0], axis = 0)), # correct positive np.sum(np.absolute(np.subtract(y, _y)))]) # distance totalInstanceNumber = _y.size # number of instances # save image image_data = np.reshape(y, (testDS.ylength, testDS.xlength)) if(gv.dp_image_save and save_image): img_save = image_data - np.amin(image_data) img_save = img_save / np.amax(img_save) io.imsave(gv.__DIR__ + gv.dp__image_dir + filename, img_save) # show image if(plot_show == 1): fig, ax = plt.subplots(1,2) ax[0].set_title('Original Image') img = io.imread(gv.__DIR__ + gv.__TrainImageDir__ + filename) ax[0].imshow(img) ax[1].set_title('Labeling Result') ax[1].imshow(image_data) plt.show() return [np.sum(y), correctNumber, totalInstanceNumber]
def main(): try: echos_tune = 10 batchNum = 2000 batchSize = 2000 ImagePatchWidth = 40 ImagePatchStep = 1 labelMode = "NUM" label_mutiplier = 1.0 hidden_units = [200, 400, 200] steps = 200 optimizer = "Adagrad" # "SGD", "Adam", "Adagrad" learning_rate = 0.1 clip_gradients = 5.0 config = None verbose = 1 dropout = None filename = "detector_3_no_5_angle_2.jpg" image_files, bubble_num, bubble_regions = getinfo() benchmark_tune = np.zeros((echos_tune, 2)) # generate data: # ds.gaussianDatatoFile(ImagePatchWidth, ImagePatchStep, labelMode) classifier, trainDataset = dl.train( batchNum=0, batchSize=1, ImagePatchWidth=ImagePatchWidth, ImagePatchStep=ImagePatchStep, labelMode=labelMode, hidden_units=hidden_units, steps=steps, optimizer=optimizer, learning_rate=learning_rate, clip_gradients=clip_gradients, config=config, verbose=verbose, dropout=dropout, ) PROGRESS = progress.progress(0, echos_tune) for i in range(echos_tune): PROGRESS.setCurrentIteration(i + 1) PROGRESS.setInfo(prefix_info="Batch Number Tune ...", suffix_info=str(i + 1) + "/" + str(echos_tune)) classifier = dl.continuetrain( classifier=classifier, trainDataset=trainDataset, batchNum=batchNum, batchSize=batchSize ) result, accuracy = dl.testall( classifier, ImagePatchWidth=ImagePatchWidth, ImagePatchStep=ImagePatchStep, label_mutiplier=label_mutiplier, labelMode=labelMode, image_show=0, # show labeling result of each image save_image=False, # save each labeled image save_result=False, ) slope, intercept, r_value, p_value, std_err = linregress(bubble_num, result.T) benchmark_tune[i, 0] = r_value benchmark_tune[i, 1] = std_err PROGRESS.printProgress() print "\n" filename = gv.__DIR__ + gv.dp__tuningPar_dir + "batchnumberresult.npy" np.save(filename, np.array(benchmark_tune)) fig, ax = plt.subplots(1) ax.plot(np.arange(echos_tune), benchmark_tune[:, 0], color="r", label="r_value") ax.plot(np.arange(echos_tune), benchmark_tune[:, 1], color="b", label="std_err") handles, labels = ax.get_legend_handles_labels() ax.legend(handles, labels, loc="upper left") plt.show() except KeyboardInterrupt: print "Shutdown requested... exiting" except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
def batch_process(pars = [0, 630, 50, 40, 0.5, 0.5], label = False): f_read = open(gv.__DIR__ + gv.pp__image_dir + 'positiveInstances.dat', 'r') f_write = open(gv.__DIR__+ gv.pp__result_dir+ 'positiveInstances.dat', 'w') num_lines = sum(1 for line in f_read) f_read.seek(0) probar = progress.progress(0, num_lines) for i, line in enumerate(f_read): line_list = line.split() image_file = line_list[0] ref_file = image_file[0:15] + '_background.jpg' ref_save_name = image_file[0:23] + '_background.jpg' probar.setCurrentIteration(i+1) probar.setInfo(prefix_info = 'Preprocessing ...', suffix_info = image_file) probar.printProgress() image = io.imread(gv.__DIR__ + gv.pp__image_dir + image_file) ref = io.imread(gv.__DIR__ + gv.pp__image_dir + ref_file) [interest_image, margin, angle] = interest_region(image, plot_image = pars[0], lid_thickness = pars[1], bot_thickness = pars[2], side_thickness = pars[3], th_x = pars[4], th_y = pars[5]) # generate background ref = rotate(ref, angle) ref = ref[int(margin[0]):int(margin[1]), int(margin[2]):int(margin[3])] # rotation of labeling box origin_y = (image.shape[0]-1)/2 origin_x = (image.shape[1]-1)/2 angle = -angle/180*3 rotation_matrix = [[np.cos(angle), -np.sin(angle)],\ [np.sin(angle), np.cos(angle)]] #print 'rotation: ' + str(rotation_matrix) num= 0 regions = [] #print margin #print interest_image.shape for i in range(int(line_list[1])): x1 = int(line_list[4*i+2]) y1 = int(line_list[4*i+3]) x2 = x1 + int(line_list[4*i+4]) y2 = y1 + int(line_list[4*i+5]) rotated_box = np.dot(rotation_matrix, \ np.array([[x1-origin_x, x2-origin_x], \ [y1-origin_y, y2-origin_y]])) [[x1, x2], [y1, y2]] = np.add(rotated_box.astype(int), np.array([[origin_x, origin_x],[origin_y, origin_y]])) if (x1 >= margin[2] and x1 <= margin[3] and\ y1 >= margin[0] and y1 <= margin[1]) or\ (x2 >= margin[2] and x2 <= margin[3] and\ y2 >= margin[0] and y2 <= margin[1]) : new_x1 = int(min(max(x1 - margin[2], 0), margin[3]-margin[2])) new_y1 = int(min(max(y1 - margin[0], 0), margin[1]-margin[0])) new_x2 = int(min(max(x2 - margin[2], 0), margin[3]-margin[2])) new_y2 = int(min(max(y2 - margin[0], 0), margin[1]-margin[0])) minmium_size = 5 if(new_x2 - new_x1 >= minmium_size and \ new_y2 - new_y1 >= minmium_size): regions.append(new_x1) regions.append(new_y1) regions.append(new_x2 - new_x1) regions.append(new_y2 - new_y1) num = num + 1 if(label): interest_image[new_y1:new_y2, new_x1:new_x2, 0] = 1 io.imsave(gv.__DIR__+ gv.pp__result_dir + ref_save_name, ref) io.imsave(gv.__DIR__+ gv.pp__result_dir + image_file, interest_image) f_write.write(image_file) f_write.write(' ' + str(num)) for data in regions: f_write.write(' ' + str(data)) f_write.write('\n') f_read.close() f_write.close()