def __init__(self, json_model_path, weights_path, target_size=(200, 200), crop_size=(150, 150), imgs_rootpath="../models"): self.pub = rospy.Publisher("cnn_predictions", CNN_out, queue_size=5) self.feedthrough_sub = rospy.Subscriber("state_change", Bool, self.callback_feedthrough, queue_size=1) self.land_sub = rospy.Subscriber("land", Empty, self.callback_land, queue_size=1) self.use_network_out = True self.imgs_rootpath = imgs_rootpath # Set keras utils K.set_learning_phase(TEST_PHASE) # Load json and create model model = utils.jsonToModel(json_model_path) # Load weights model.load_weights(weights_path) print("Loaded model from {}".format(weights_path)) model.compile(loss='mse', optimizer='sgd') self.model = model self.target_size = target_size self.crop_size = crop_size
def _main(): K.set_learning_phase(0) # input_dict = {"fist" : "000000000.jpg", # "l" : "000000003.jpg", # "ok" : "000000001.jpg", # "palm" : "000000001.jpg", # "pointer" : "000000001.jpg", # "thumb down" : "000000000.jpg", # "thumb up" : "000000003.jpg"} # Load json and create model json_model_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.json_model_fname) model = utils.jsonToModel(json_model_path) # Load weights weights_load_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.weights_fname) try: model.load_weights(weights_load_path) print("Loaded model from {}".format(weights_load_path)) except: print("Impossible to find weight path. Returning untrained model") #print(model.summary()) # Compile model model.compile(loss='categorical_crossentropy', optimizer='adam') experiments = glob.glob(FLAGS.test_dir + '/*') for exp in experiments: img_path = os.path.join(exp, 'images') images = glob.glob(img_path + '/*') for i, input_path in enumerate(sorted(images)): img_name = '{0:09d}.jpg'.format(i) output_path = os.path.join(FLAGS.experiment_rootdir, 'activations') if not os.path.exists(output_path): os.makedirs(output_path) img_out_name = os.path.join(output_path, img_name) visualize_class_attention_map(input_path, model, img_out_name)
from keras.preprocessing.image import ImageDataGenerator from keras.utils.generic_utils import Progbar from keras.models import model_from_json import utils import img_utils from common_flags import FLAGS test_datagen = utils.DroneDataGenerator(rescale=1. / 255) y = img_utils.load_img("data/testing/HMB_3/images/1479425597710017397.jpg", grayscale=True, crop_size=(200, 200), target_size=(320, 240)) x = test_datagen.random_transform(y, seed=None) x = test_datagen.standardize(x) batch_x = np.zeros((1, ) + (200, 200, 1), dtype=K.floatx()) batch_x[0] = x json_model_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.json_model_fname) model = utils.jsonToModel(json_model_path) # Load weights weights_load_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.weights_fname) try: model.load_weights(weights_load_path) print("Loaded model from {}".format(weights_load_path)) except: print("Impossible to find weight path. Returning untrained model") outs = model.predict(batch_x) print(outs)
def _main(): # Set testing mode (dropout/batchnormalization) K.set_learning_phase(TEST_PHASE) # Output dimension (empty place probability) output_dim = 1 # Generate testing data test_datagen = data_utils.DataGenerator(rescale=1. / 255) # Iterator object containing testing data to be generated batch by batch test_generator = test_datagen.flow_from_directory( FLAGS.test_dir, output_dim, shuffle=False, img_mode=FLAGS.img_mode, target_size=(FLAGS.img_height, FLAGS.img_width), batch_size=FLAGS.batch_size) # Load json and create model json_model_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.json_model_fname) model = utils.jsonToModel(json_model_path) # Load weights weights_load_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.weights_fname) try: model.load_weights(weights_load_path) print("Loaded model from {}".format(weights_load_path)) except: print("Impossible to find weight path. Returning untrained model") # Compile model model.compile(loss='categorical_crossentropy', optimizer='adam') # Get predictions and ground truth n_samples = test_generator.samples nb_batches = int(np.ceil(n_samples / FLAGS.batch_size)) probs_per_class, ground_truth = utils.compute_predictions_and_gt( model, test_generator, nb_batches, verbose=1) # Predicted probabilities pred_probs = np.max(probs_per_class, axis=-1) # Prediced labels pred_labels = np.argmax(probs_per_class, axis=-1) # Real labels (ground truth) real_labels = np.argmax(ground_truth, axis=-1) # Evaluate predictions: Average accuracy and highest errors print("-----------------------------------------------") print("Evalutaion:") evaluation = evaluate_classification(pred_probs, pred_labels, real_labels) print("-----------------------------------------------") # Save evaluation utils.write_to_file( evaluation, os.path.join(FLAGS.experiment_rootdir, 'test_results.json')) # Save predicted and real steerings as a dictionary labels_dict = { 'pred_labels': pred_labels.tolist(), 'real_labels': real_labels.tolist() } utils.write_to_file( labels_dict, os.path.join(FLAGS.experiment_rootdir, 'predicted_and_real_labels.json')) # Visualize confusion matrix utils.plot_confusion_matrix(FLAGS.experiment_rootdir, real_labels, pred_labels, CLASSES, normalize=True)
def main(): json_model_path = '/model/UAVPatrolNet_model_struct.json' weights_path = FLAGS.model_dir pics_path = FLAGS.test_dir # Input image dimensions img_width, img_height = FLAGS.img_width, FLAGS.img_height # Cropped image dimensions crop_img_width, crop_img_height = FLAGS.crop_img_width, FLAGS.crop_img_height target_size = (img_height,img_width) crop_size = (crop_img_height,crop_img_width) # Set keras utils K.set_learning_phase(TEST_PHASE) # Load json and create model model = utils.jsonToModel(json_model_path) # model = cnn_models.resnet8_MDN(crop_img_width,crop_img_height,3,1) #model = merge_models.merge_model() # Load weights model.load_weights(weights_path,by_name=True) #model.compile(loss='mse', optimizer='sgd') model.compile(loss='mse', optimizer='adam') print("json_model_path: {}".format(json_model_path)) print("Loaded model from {}".format(weights_path)) #cv2.namedWindow("img", 0); #cv2.resizeWindow("img", 640, 360); #cv2.namedWindow("crop",0); #cv2.resizeWindow("crop", 400,160); pic_list = os.listdir(pics_path) pic_list.sort() counter = 0 roll_speed = 0. try: while True: time_start=time.time() pic = pic_list[int(counter)] counter = counter + 1 print(pic) img_origi = cv2.imread(os.path.join(pics_path, pic), cv2.IMREAD_COLOR) img_origi = cv2.resize(img_origi, (640, 360)) # run predict if FLAGS.img_mode == 'grayscale': img = cv2.cvtColor(img_origi, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (target_size[1], target_size[0])) else: img = cv2.resize(img_origi, (target_size[1], target_size[0])) img = central_image_crop(img, crop_size[0], crop_size[1]) if FLAGS.img_mode == 'grayscale': img = img.reshape((img.shape[0], img.shape[1], 1)) cv_image = np.asarray(img, dtype=np.float32) * np.float32(1.0/255.0) # print(cv_image) outs = model.predict_on_batch(cv_image[None]) #print(len(outs[0])) parameter, translation = outs[0][0], outs[1][0] #print("steer = {}, translation = {}".format(parameter,translation)) y_pred = np.reshape(parameter, [-1, 6]) out_mu, out_pi = np.split(y_pred, 2, axis=1) #print(out_pi) pi = sum_exp(out_pi, 1) pi = np.split(pi, 3, axis=1) # component_splits = [1, 1, 1] mus = np.split(out_mu, 3, axis=1) out_sigma = np.array([[0.1, 0.1, 0.1]], dtype='float32') sigs = np.split(out_sigma, 3, axis=1) x = np.linspace(-1, 1, 100) y = np.array([]) for x_ in x: y = np.append(y, gaussian(sigs, mus, pi, x_)) possible_direct = [] possible_roll_speed = [] start = 0 continue_flag = 0 sum_y = 0 sum_x = 0 for x_, y_ in zip(x, y): # print(point) if(y_ > 0.3): if(continue_flag == 0): continue_flag = 1 start = x_ sum_y = sum_y + y_ sum_x = sum_x + 1 y_ = (img_origi.shape[0] - y_ * 200 - 80).astype(np.int32) x_ = ((x_ + 1) / 2 * img_origi.shape[1]).astype(np.int32) x_ = img_origi.shape[1] - x_ cv2.circle(img_origi, (x_, int(y_/2)+150), 3, (0, 255, 0), 4) else: if(continue_flag == 1): continue_flag = 0 possible_direct.append((x_ + start)/2) possible_roll_speed.append((sum_y/sum_x - 1.)/2) sum_y = 0 sum_x = 0 y_ = (img_origi.shape[0] - y_ * 200 - 80).astype(np.int32) x_ = ((x_ + 1) / 2 * img_origi.shape[1]).astype(np.int32) x_ = img_origi.shape[1] - x_ cv2.circle(img_origi, (x_, int(y_/2)+150), 1, (255, 0, 255), 4) # print("====Map_direct = {} ====".format(map_direct)) map_direct = 0 min_direct_diff = 180 steer = 0. roll_speed_ = 0 count = 0 for possible_direct_ in possible_direct: # print(possible_direct_) cv2.line(img_origi, (int(img_origi.shape[1] / 2), img_origi.shape[0] - 50), (int(img_origi.shape[1] / 2 - math.tan(possible_direct_ * 3.14 / 2) * 100), img_origi.shape[0] - 150), (0, 255, 0), 3) diff = abs(-possible_direct_*90 - map_direct) if(diff<min_direct_diff): min_direct_diff = diff steer = possible_direct_ roll_speed_ = possible_roll_speed[count] count = count + 1 cv2.line(img_origi, (int(img_origi.shape[1] / 2), img_origi.shape[0] - 50), (int(img_origi.shape[1] / 2 - math.tan(steer * 3.14 / 2) * 100), img_origi.shape[0] - 150),(0, 255, 255), 3) # map_direct = map_direct/90 # seq = "ab"+'%f'%(map_direct*400)+',%f'%(0*200) roll_speed = roll_speed*0.9 + roll_speed_*0.1 # cv2.line(img_origi, (int(img_origi.shape[1]/2),img_origi.shape[0]), (int(img_origi.shape[1]/2),50), (0,255,0), 1) cv2.line(img_origi, (int(img_origi.shape[1]/2),img_origi.shape[0]-50), (int((translation+1)/2*img_origi.shape[1]), img_origi.shape[0] - 50), (255,255,0), 8) cv2.imshow("img", img_origi) cv2.imshow('crop',img) #cv2.imwrite(pics_path + '_save /our' + pic, img_origi) time_end=time.time() print('totally cost',time_end-time_start) cv2.waitKey(0) except KeyboardInterrupt: print("calling to end")
def main(): FLAGS(sys.argv) json_model_path = 'model/UAVPatrolNet_model_struct.json' weights_path = FLAGS.model_dir # Set keras utils # Input image dimensions img_width, img_height = FLAGS.img_width, FLAGS.img_height # Cropped image dimensions crop_img_width, crop_img_height = FLAGS.crop_img_width, FLAGS.crop_img_height target_size = (img_height, img_width) crop_size = (crop_img_height, crop_img_width) K.set_learning_phase(TEST_PHASE) # Load json and create model model = utils.jsonToModel(json_model_path) model = cnn_models.resnet8_MDN(crop_img_width, crop_img_height, 1, 1) # Load weights model.load_weights(weights_path, by_name=True) # model.compile(loss='mse', optimizer='sgd') model.compile(loss='mse', optimizer='adam') print("json_model_path: {}".format(json_model_path)) print("Loaded model from {}".format(weights_path)) # print("[INFO]") # model.summary() cv2.namedWindow("img", 0) cv2.resizeWindow("img", 960, 540) dataset_dir = FLAGS.test_dir for dirs in os.listdir(dataset_dir): foldername = dirs if (foldername[0] != 't' and foldername[len(foldername) - 1] != 'p'): print(dirs) pics_path = dataset_dir + foldername + '/images' # sys.path[0] + '/pics' dirct_label_exist = 1 trans_label_exist = 1 direct_label_path = pics_path + '/../direction_n_filted.txt' trans_label_path = pics_path + '/../../' + 'translation' + foldername + '/translation.txt' try: direct_label = np.loadtxt(direct_label_path, usecols=0) except OSError as e: dirct_label_exist = 0 print('No direction labels.') try: trans_label = np.loadtxt(trans_label_path, usecols=0) except OSError as e: trans_label_exist = 0 print('No translation labels.') dril2_set = [] tral2_set = [] l2_set = [] avg_l2 = 0 sd_l2 = 0 correct_dirct_num = 0 correct_trans_num = 0 count = 0 pic_list = os.listdir(pics_path) pic_list.sort() for count, pic in enumerate(pic_list): # select pic # for file in pic_list: # print("{0}, {1}".format(count, file)) # count = count + 1 # pic_index = input("Input the number of the pic:") #pic = pic_list[int(pic_index)] print(pic) img_origi = cv2.imread(os.path.join(pics_path, pic), cv2.IMREAD_COLOR) #img_origi = cv2.resize(img_origi, (640, 360)) # run predict if FLAGS.img_mode == 'grayscale': img = cv2.cvtColor(img_origi, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (target_size[1], target_size[0])) else: img = cv2.resize(img_origi, (target_size[1], target_size[0])) img = central_image_crop(img, crop_size[0], crop_size[1]) if FLAGS.img_mode == 'grayscale': img = img.reshape((img.shape[0], img.shape[1], 1)) cv_image = np.asarray(img, dtype=np.float32) * np.float32( 1.0 / 255.0) # print(cv_image) outs = model.predict_on_batch(cv_image[None]) # print(len(outs[0])) parameter, translation = outs[0][0], outs[1][0] # print("steer = {}, translation = {}".format(parameter,translation)) y_pred = np.reshape(parameter, [-1, 6]) out_mu, out_pi = np.split(y_pred, 2, axis=1) # print(out_pi) #out_pi = np.array([[0.3333, 0.3333, 0.3333]]) pi = sum_exp(out_pi, 1) pi = np.split(pi, 3, axis=1) # component_splits = [1, 1, 1] mus = np.split(out_mu, 3, axis=1) out_sigma = np.array([[0.05, 0.05, 0.05]], dtype='float32') sigs = np.split(out_sigma, 3, axis=1) x = np.linspace(-1, 1, 100) y = np.array([]) for x_ in x: y = np.append(y, gaussian(sigs, mus, pi, x_)) possible_direct = [] start = 0 continue_flag = 0 for x_, y_ in zip(x, y): # print(point) if (y_ > 0.6): if (continue_flag == 0): continue_flag = 1 start = x_ y_ = (img_origi.shape[0] - y_ * 200 - 80).astype( np.int32) x_ = ((x_ + 1) / 2 * img_origi.shape[1]).astype( np.int32) x_ = img_origi.shape[1] - x_ cv2.circle(img_origi, (x_, y_), 3, (0, 255, 0), 4) else: if (continue_flag == 1): continue_flag = 0 possible_direct.append((x_ + start) / 2) y_ = (img_origi.shape[0] - y_ * 200 - 80).astype( np.int32) x_ = ((x_ + 1) / 2 * img_origi.shape[1]).astype( np.int32) x_ = img_origi.shape[1] - x_ cv2.circle(img_origi, (x_, y_), 1, (255, 0, 255), 4) # cat = tfd.Categorical(logits=out_pi) # coll = [tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale) for loc, scale # in zip(mus, sigs)] # mixture = tfd.Mixture(cat=cat, components=coll) # with tf.Session() as sess: # xx = tf.expand_dims(tf.linspace(-1., 1., int(1e2)), 1) # yy = mixture.prob(xx).eval() # xx = tf.cast(((xx+1)/2*img_origi.shape[1]), dtype=tf.int32).eval() # yy = tf.cast(img_origi.shape[0]-yy*200-80, dtype=tf.int32).eval() # for point in zip(xx, yy): # # print(point) # cv2.circle(img_origi, point, 1, (0, 0, 255), 4) # # plt.plot(x, mixture.prob(x).eval()); # # plt.savefig("abc.png") # if (dirct_label_exist): steer = direct_label[count] print('direction label: {}'.format(steer)) cv2.line(img_origi, (int( img_origi.shape[1] / 2), img_origi.shape[0] - 150), (int(img_origi.shape[1] / 2 - math.tan(steer * 3.14 / 2) * 100), img_origi.shape[0] - 180), (255, 0, 0), 3) steer_x = ((steer + 1) / 2 * img_origi.shape[1]).astype( np.int32) steer_x = img_origi.shape[1] - steer_x steer_y = gaussian(sigs, mus, pi, steer) steer_y = (img_origi.shape[0] - steer_y * 200 - 80).astype( np.int32) steer_x_left = ((steer + 0.1 + 1) / 2 * img_origi.shape[1]).astype(np.int32) steer_x_left = img_origi.shape[1] - steer_x_left steer_y_left = gaussian(sigs, mus, pi, steer + 0.1) steer_y_left = (img_origi.shape[0] - steer_y_left * 200 - 80).astype(np.int32) steer_x_right = ((steer - 0.1 + 1) / 2 * img_origi.shape[1]).astype(np.int32) steer_x_right = img_origi.shape[1] - steer_x_right steer_y_right = gaussian(sigs, mus, pi, steer - 0.1) steer_y_right = (img_origi.shape[0] - steer_y_right * 200 - 80).astype(np.int32) # print('x:{}, y:{}'.format(steer_x, steer_y)) if (steer_y < 2 * img_origi.shape[0] / 3): cv2.circle(img_origi, (steer_x, steer_y), 6, (255, 0, 0), 6) cv2.circle(img_origi, (steer_x_left, steer_y_left), 6, (255, 0, 0), 6) cv2.circle(img_origi, (steer_x_right, steer_y_right), 6, (255, 0, 0), 6) # correct_dirct_num = correct_dirct_num+1 else: cv2.circle(img_origi, (steer_x, steer_y), 3, (0, 0, 255), 6) cv2.circle(img_origi, (steer_x_left, steer_y_left), 3, (0, 0, 255), 6) cv2.circle(img_origi, (steer_x_right, steer_y_right), 3, (0, 0, 255), 6) # cv2.line(img_origi, (int(steer), img_origi.shape[0] - 150), # (int(steer), 50), (255, 0, 0), 4) # pics in /translation*/images are not the same as those pics in direction dataset. # (if they are same, following code can be used) # computing l2 loss direct_l2_min = 180 * 180 direct_diff_min = 2 trans_l2 = 0 if (trans_label_exist): trans = trans_label[count] # random #translation = random.randint(0,10000)/5000 - 1 tral2_set.append(trans - translation) trans_l2 = (trans - translation)**2 if (math.fabs(translation - trans) < 0.2): correct_trans_num = correct_trans_num + 1 print('translation label: {}'.format(trans)) cv2.line(img_origi, (int( img_origi.shape[1] / 2), img_origi.shape[0] - 150), (int((trans + 1) / 2 * img_origi.shape[1]), img_origi.shape[0] - 150), (255, 0, 0), 8) cv2.line(img_origi, (int(img_origi.shape[1] / 2), img_origi.shape[0]), (int(img_origi.shape[1] / 2), 50), (0, 255, 0), 1) cv2.line( img_origi, (int(img_origi.shape[1] / 2), img_origi.shape[0] - 150), (int((translation + 1) / 2 * img_origi.shape[1]), img_origi.shape[0] - 150), (0, 255, 0), 8) for possible_direct_ in possible_direct: if (dirct_label_exist): steer = direct_label[count] #possible_direct_ = random.randint(0,10000)/5000 - 1 l2_direct = (steer - possible_direct_)**2 if (l2_direct < direct_l2_min): direct_l2_min = l2_direct if abs(steer - possible_direct_) < abs(direct_diff_min): direct_diff_min = (steer - possible_direct_) if (abs(steer - possible_direct_) < (2 / (180 / 15))): correct_dirct_num = correct_dirct_num + 1 print("yes") break print("predicted: {}".format(possible_direct_)) cv2.line(img_origi, (int( img_origi.shape[1] / 2), img_origi.shape[0] - 150), (int(img_origi.shape[1] / 2 - math.tan(possible_direct_ * 3.14 / 2) * 100), img_origi.shape[0] - 250), (0, 255, 0), 3) l2 = (trans_l2 + direct_l2_min)**0.5 l2_set.append(l2) dril2_set.append(direct_diff_min) avg_l2 = avg_l2 + l2 #print(direct_diff_min,direct_l2_min ** 0.5) cv2.imshow("img", img_origi) cv2.imshow('crop', img) cv2.waitKey(1) print('==================================') print('==================================') print('==================================') print('direct_accuracy = {}'.format(correct_dirct_num / len(pic_list))) print('trans_accuracy = {}'.format(correct_trans_num / len(pic_list))) # print('avg_l2 = {}'.format(avg_l2/len(pic_list))) # print('avg_l2 = {}'.format(np.mean(l2_set))) print('direct_SD_l2 = {}'.format(np.std(dril2_set, ddof=0))) print('trans_SD_l2 = {}'.format(np.std(tral2_set, ddof=0)))
def _main(): # Set testing mode (dropout/batchnormalization) K.set_learning_phase(TEST_PHASE) # Generate testing data test_datagen = utils.DroneDataGenerator(rescale=1. / 255) test_generator = test_datagen.flow_from_directory( FLAGS.test_dir, shuffle=False, color_mode=FLAGS.img_mode, target_size=(FLAGS.img_width, FLAGS.img_height), crop_size=(FLAGS.crop_img_height, FLAGS.crop_img_width), batch_size=FLAGS.batch_size) # Load json and create model json_model_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.json_model_fname) model = utils.jsonToModel(json_model_path) # Load weights weights_load_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.weights_fname) try: model.load_weights(weights_load_path) print("Loaded model from {}".format(weights_load_path)) except: print("Impossible to find weight path. Returning untrained model") # Compile model model.compile(loss='mse', optimizer='adam') # Get predictions and ground truth n_samples = test_generator.samples nb_batches = int(np.ceil(n_samples / FLAGS.batch_size)) predictions, ground_truth, t = utils.compute_predictions_and_gt( model, test_generator, nb_batches, verbose=1) # Param t. t=1 steering, t=0 collision t_mask = t == 1 # ************************* Steering evaluation *************************** # Predicted and real steerings pred_steerings = predictions[t_mask, 0] real_steerings = ground_truth[t_mask, 0] # Compute random and constant baselines for steerings random_steerings = random_regression_baseline(real_steerings) constant_steerings = constant_baseline(real_steerings) # Create dictionary with filenames dict_fname = { 'test_regression.json': pred_steerings, 'random_regression.json': random_steerings, 'constant_regression.json': constant_steerings } # Evaluate predictions: EVA, residuals, and highest errors print('direction:') for fname, pred in dict_fname.items(): abs_fname = os.path.join(FLAGS.experiment_rootdir, fname) evaluate_regression(pred, real_steerings, abs_fname) # Write predicted and real steerings dict_test = { 'pred_steerings': pred_steerings.tolist(), 'real_steerings': real_steerings.tolist() } utils.write_to_file( dict_test, os.path.join(FLAGS.experiment_rootdir, 'predicted_and_real_steerings.json')) # ************************* collision(translation) evaluation *************************** # Predicted and real labels pred_prob = predictions[~t_mask, 1] real_labels = ground_truth[~t_mask, 1] # Compute random and constant baselines for steerings random_labels = random_regression_baseline(real_labels) constant_labels = constant_baseline(real_labels) # Create dictionary with filenames dict_fname = { 'translation-test_regression.json': pred_prob, 'translation-random_regression.json': random_labels, 'translation-constant_regression.json': constant_labels } # Evaluate predictions: EVA, residuals, and highest errors print('translation:') for fname, pred in dict_fname.items(): abs_fname = os.path.join(FLAGS.experiment_rootdir, fname) evaluate_regression(pred, real_labels, abs_fname) # Write predicted and real steerings dict_test = { 'pred_labels': pred_prob.tolist(), 'real_probs': real_labels.tolist() } utils.write_to_file( dict_test, os.path.join(FLAGS.experiment_rootdir, 'predicted_and_real_labels.json'))