def collect_results(X_tr, y_tr, X_te, y_te, W, b, convergence_results, success_rate_results, epoch): loss_tr = compute_loss(X_tr, y_tr, W, b) loss_te = compute_loss(X_te, y_te, W, b) convergence_results[epoch] = loss_tr, loss_te success_rate_tr = get_success_rate(X_tr, y_tr, W, b) success_rate_te = get_success_rate(X_te, y_te, W, b) success_rate_results[epoch] = success_rate_tr, success_rate_te
def loss_gradient_tests(m=100, n=30, l=5, num_of_tests=10): results = np.zeros( (EPSILON_ITERATIONS, LOSS_VARS_NUM * GRADIENT_TESTS_NUM)) for t in range(num_of_tests): X, y, W, b = get_random_params(m, n, l) loss = compute_loss(X, y, W, b) grad_at_W = compute_loss_gradient_by_W(X, y, W, b) grad_by_X = compute_loss_gradient_by_X(X, y, W, b) grad_by_b = compute_loss_gradient_by_b(X, y, W, b) dW = np.random.rand(*W.shape) dX = np.random.rand(*X.shape) db = np.random.rand(*b.shape) epsilon = 1 for i in range(EPSILON_ITERATIONS): epsilon *= 0.5 Wnxt = get_next_step(W, epsilon, dW) Xnxt = get_next_step(X, epsilon, dX) bnxt = get_next_step(b, epsilon, db) loss_at_Wnxt = compute_loss(X, y, Wnxt, b) loss_at_Xnxt = compute_loss(Xnxt, y, W, b) loss_at_bnxt = compute_loss(X, y, W, bnxt) W_grad_test1, W_grad_test2 = compute_tests(loss, loss_at_Wnxt, grad_at_W, epsilon, dW) X_grad_test1, X_grad_test2 = compute_tests(loss, loss_at_Xnxt, grad_by_X, epsilon, dX) b_grad_test1, b_grad_test2 = compute_tests(loss, loss_at_bnxt, grad_by_b, epsilon, db) results[ i, :] += W_grad_test1, W_grad_test2, X_grad_test1, X_grad_test2, b_grad_test1, b_grad_test2 results /= num_of_tests a = results[2:, ] b = results[1:-1, :] dec_factors = b / a plot_gradient_decrease_factors(dec_factors[:, :2], 'W') plot_gradient_decrease_factors(dec_factors[:, 2:4], 'X') plot_gradient_decrease_factors(dec_factors[:, 4:], 'b')
def nn_gradient_tests(m=100, n=30, l=5, num_of_tests=10): results = np.zeros( (EPSILON_ITERATIONS, LOSS_VARS_NUM * GRADIENT_TESTS_NUM)) for t in range(num_of_tests): X, y, _, _ = get_random_params(m, n, l) nn = Net(m, l, [n, n + 1, n + 2]) Xs = nn.forward_pass(X) W, b = nn.get_output_layer_weights() f = compute_loss(Xs[-1], y, W, b) Ws, bs = nn.get_layers_weights() W_grads, b_grads = nn.back_prop(Xs, y) dWs = [np.random.rand(*W.shape) for W in Ws] dbs = [np.random.rand(*b.shape) for b in bs] epsilon = 1 for i in range(EPSILON_ITERATIONS): epsilon *= 0.5 nxt_Ws = get_next_steps(Ws, epsilon, dWs) nxt_bs = get_next_steps(bs, epsilon, dbs) f_at_Wnxts = get_next_f(nn, X, y, nxt_Ws, bs) f_at_bnxts = get_next_f(nn, X, y, Ws, nxt_bs) W_grad_test1, W_grad_test2 = compute_tests(f, f_at_Wnxts, W_grads, epsilon, dWs) b_grad_test1, b_grad_test2 = compute_tests(f, f_at_bnxts, b_grads, epsilon, dbs) results[ i, :] += W_grad_test1, W_grad_test2, b_grad_test1, b_grad_test2 results /= num_of_tests a = results[2:, ] b = results[1:-1, :] dec_factors = b / a plot_gradient_decrease_factors(dec_factors[:, :2], 'W') plot_gradient_decrease_factors(dec_factors[:, 2:], 'b')
# Reshape images for visualization x_reshape = tf.reshape(X, [-1, Input_shape, Input_shape, 1]) tf.summary.image("input", x_reshape) # STEP 2: Building the graph ####################################################################################### # Building the graph # Generate output tensor targets for filtered bounding boxes. scale1, scale2, scale3 = YOLOv3(X, len(classes_data)).feature_extractor() scale_total = [scale1, scale2, scale3] with tf.name_scope("Loss_and_Detect"): # Label y_predict = [Y1, Y2, Y3] # Calculate loss loss = compute_loss(scale_total, y_predict, anchors, len(classes_data), print_loss=False) # loss_print = compute_loss(scale_total, y_predict, anchors, len(classes_data), print_loss=False) tf.summary.scalar("Loss", loss) with tf.name_scope("Optimizer"): # optimizer # for VOC: lr:0.0001, decay:0.0003 with RMSProOp after 60 epochs # learning_rate = tf.placeholder(tf.float32, shape=[1], name='lr') decay = 0.0003 # optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss) optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize( loss, global_step=global_step) # optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=decay).minimize(loss) # optimizer = tf.train.MomentumOptimizer(learning_rate, 0.01).minimize(loss) # STEP 3: Build the evaluation step ################################################################################
X = tf.placeholder(tf.float32, [None, 128, 128, 3]) # expected outputs Y1 = tf.placeholder(tf.float32, shape=[None, Input_shape/32, Input_shape/32, 3, (5+num_classes)]) Y2 = tf.placeholder(tf.float32, shape=[None, Input_shape/16, Input_shape/16, 3, (5+num_classes)]) Y3 = tf.placeholder(tf.float32, shape=[None, Input_shape/8, Input_shape/8, 3, (5+num_classes)]) # Generate output tensor targets for filtered bounding boxes. scale1, scale2, scale3 = YOLOv3(X, num_classes).feature_extractor() scale_total = [scale1, scale2, scale3] # detect boxes, scores, classes = predict(scale_total, anchors, num_classes, input_shape) # Label Y_ = [Y1, Y2, Y3] # Calculate loss loss = compute_loss(scale_total, Y_, anchors, num_classes, print_loss=True) # Optimizer optimizer = tf.train.AdamOptimizer(lr).minimize(loss) #, global_step=global_step) ################################################################################################################## # Init for saving models. They will be saved into a directory named 'checkpoints'. # Only the last checkpoint is kept. if not os.path.exists("checkpoints"): os.mkdir("checkpoints") saver = tf.train.Saver() # for display: init the progress bar DISPLAY_FREQ = 50 _50_BATCHES = DISPLAY_FREQ * batch_size progress = Progress(DISPLAY_FREQ, size=111+2, msg="Training on next "+str(DISPLAY_FREQ)+" batches")
scale_total.append(scale1) scale_total.append(scale2) scale_total.append(scale3) with tf.name_scope("Loss"): # predict boxes, score, classes boxes, scores, classes = predict(scale_total, anchors, len(classes_data), S, score_threshold=threshold, iou_threshold=ignore_thresh) # Label y_true = preprocess_true_boxes(Y, 416, anchors, len(classes_data)) # Calculate loss loss = compute_loss(scale_total, y_true, anchors, len(classes_data)) with tf.name_scope("Optimizer"): # optimizer learning_rate = 0.001 decay = 0.0005 optimizer = tf.train.RMSPropOptimizer(learning_rate, decay).minimize(loss) # STEP 3: Build the evaluation step ################################################################################ # with tf.name_scope("Accuracy"): # # Model evaluation # accuracy = 1 # # STEP 4: Merge all summaries for Tensorboard generation ########################################################### # create a saver saver = tf.train.Saver(tf.global_variables()) # Build the summary operation based on the TF collection of Summaries summary_op = tf.summary.merge_all()
# for 3D Y1 = tf.placeholder(tf.float32, shape=[None, Input_height/32, Input_width/32, 3, (5+5+len(classes_data))], name='target_S1') Y2 = tf.placeholder(tf.float32, shape=[None, Input_height/16, Input_width/16, 3, (5+5+len(classes_data))], name='target_S2') Y3 = tf.placeholder(tf.float32, shape=[None, Input_height/8, Input_width/8, 3, (5+5+len(classes_data))], name='target_S3') # STEP 2: Building the graph ####################################################################################### # Building the graph # Generate output tensor targets for filtered bounding boxes. scale1, scale2, scale3 = YOLOv3(X, len(classes_data)).feature_extractor() y_pred = [scale1, scale2, scale3] with tf.name_scope("Loss_and_Detect"): # Label y_gt = [Y1, Y2, Y3] # Calculate loss loss = compute_loss(y_pred, y_gt, anchors, len(classes_data), print_loss=False) tf.summary.scalar("Loss", loss) with tf.name_scope("Optimizer"): optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_step) # STEP 3: Merge all summaries for Tensorboard generation ########################################################### # create a saver saver = tf.train.Saver() # Build the summary operation based on the TF collection of Summaries # summary_op = tf.summary.merge_all() # STEP 4: Train the model, and write summaries ##################################################################### # The Graph to be launched (described above) config = tf.ConfigProto() config.gpu_options.allow_growth = True
def get_next_f(nn, X, y, Ws, bs): nn.set_layers_weights(Ws, bs) Xs = nn.forward_pass(X) W, b = nn.get_output_layer_weights() nxt_f = compute_loss(Xs[-1], y, W, b) return nxt_f