def test_list_file(config, list_file, weights, model_file): labels_file = config['model']['labels'] data_dir = config['test']['data_dir'] input_width = config['model']['input_width'] input_height = config['model']['input_height'] # builder = ModelBuilder(config) preprocess_input = builder.preprocess_input image_size = (input_width, input_height) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph, config=tf_config) ret_dict = {} error_list = [] name_list = [] keras.backend.set_session(train_sess) with train_graph.as_default(): if model_file is not None: model = load_model(model_file, compile=False) elif weights: model = builder.build_model() model.load_weights(weights) else: print('using base model') model = builder.build_model() model.summary() with open(list_file, 'r') as f: filelist = f.readlines() conf_thresh = 0.6 conf_num = 0 num = 0 n_correct = 0 for line in filelist: line = line.rstrip('\n') img_path = os.path.join(data_dir, line) # load dataset img = cv2.imread(img_path) img = cv2.resize(img, (input_width, input_height)) # img = img_pad(img, input_width, input_height) img = img[:, :, ::-1] x_pred = np.array([img]).astype('float32') x_pred = np.array([img]) / 255.0 # x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32') x_pred = preprocess_input(x_pred) y_pred = model.predict(x_pred) ret_dict[img_path] = y_pred[0] np.save('./anchor.npy', ret_dict)
def predict(config, input_image, weights): input_width = config['model']['input_width'] input_height = config['model']['input_height'] class_num = config['model']['class_num'] builder = ModelBuilder(config) model = builder.build_model(training=False) model.summary() if weights is not None: model.load_weights(weights) img = cv2.imread(input_image) image_h, image_w, _ = img.shape print(image_h, image_w) img = cv2.resize(img, (input_width, input_height)) print(img.shape) img = img[:, :, ::-1] x_pred = np.array([img]).astype('float32') x_pred /= 255. print(x_pred.shape) y_outs = model(x_pred) y_outs = [a[0].numpy() for a in y_outs] print(y_outs[0].shape) print(y_outs[1].shape) print(y_outs[2].shape) # 画框设置不同的颜色 hsv_tuples = [(x / class_num, 1., 1.) for x in range(class_num)] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) boxes, scores, classes, colors = yolo_decodeout(y_outs, image_h, image_w, colors, input_w=input_width, input_h=input_height, nms_thresh=0.25, class_thresh=0.25, obj_thresh=0.5) for box in boxes: print(box.xmin, box.ymin, box.xmax, box.ymax) print(scores) print(classes) print(len(classes))
def train(config): input_width = config['model']['input_width'] input_height = config['model']['input_height'] label_file = config['model']['labels'] model_name = config['model']['name'] train_data_dir = config['train']['data_dir'] train_file_list = config['train']['file_list'] pretrained_weights = config['train']['pretrained_weights'] batch_size = config['train']['batch_size'] learning_rate = config['train']['learning_rate'] nb_epochs = config['train']['nb_epochs'] start_epoch = config['train']['start_epoch'] train_base = config['train']['train_base'] valid_data_dir = config['valid']['data_dir'] valid_file_list = config['valid']['file_list'] builder = ModelBuilder(config) filepath = os.path.join('', train_file_list) train_gen = builder.build_datagen(filepath) # train_gen.save_labels(label_file) # trainDataGen, train_steps_per_epoch = train_gen.from_frame(directory=train_data_dir) # filepath = os.path.join(valid_data_dir, valid_file_list) # valid_gen = builder.build_datagen(filepath, with_aug=False) # validDataGen, valid_steps_per_epoch = valid_gen.from_frame(directory=valid_data_dir) # define checkpoint dataset_name = model_name dirname = 'ckpt-' + dataset_name if not os.path.exists(dirname): os.makedirs(dirname) timestr = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") filepath = os.path.join( dirname, 'weights-%s-%s-{epoch:02d}-{loss:.5f}.hdf5' % (model_name, timestr)) checkpoint = ModelCheckpoint( filepath=filepath, monitor='loss', # acc outperforms loss verbose=1, save_best_only=True, save_weights_only=True, period=1) # define logs for tensorboard tensorboard = TensorBoard(log_dir='logs', histogram_freq=0) wgtdir = 'weights' if not os.path.exists(wgtdir): os.makedirs(wgtdir) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph, config=tf_config) tf.keras.backend.set_session(train_sess) with train_graph.as_default(): model = builder.build_model() model.compile( optimizer=tf.train.AdamOptimizer(learning_rate=learning_rate), loss=triple_loss, metrics=[p_loss, n_loss]) model.summary() # Load weight of unfinish training model(optional) if pretrained_weights != '': model.load_weights(pretrained_weights) model.fit_generator( generator=train_gen, # validation_data = validDataGen, initial_epoch=start_epoch, epochs=nb_epochs, callbacks=[checkpoint, tensorboard], use_multiprocessing=False, workers=16) model_file = '%s_%s.h5' % (model_name, timestr) model.save(model_file) print('save model to %s' % (model_file))
def layer_visualize(config, img_file, weights, model_file, layer_name): labels_file = config['model']['labels'] input_width = config['model']['input_width'] input_height = config['model']['input_height'] test_data_dir = config['test']['data_dir'] # load labels print('load label file', labels_file) label_dict = np.load(labels_file).item() class_num = len(label_dict) print('class num:', class_num) print(label_dict) # builder = ModelBuilder(config) preprocess_input = builder.preprocess_input visdir = 'visual' if not os.path.exists(visdir): os.makedirs(visdir) laydir = os.path.join(visdir, layer_name) if not os.path.exists(laydir): os.makedirs(laydir) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph, config=tf_config) keras.backend.set_session(train_sess) with train_graph.as_default(): if model_file is not None: cls = load_model(model_file, compile=False) elif weights: cls = builder.build_model() cls.load_weights(weights) else: print('either weights or model file should be specified.') model = build_model(cls, layer_name) img_path = os.path.join(test_data_dir, img_file) print(img_path) img = cv2.imread(img_path) img = cv2.resize(img, (input_width, input_height)) # img = img_pad(img, input_width, input_height) img = img[:, :, ::-1] x_pred = np.array([img]).astype('float32') # x_pred = np.array([img])/255.0 # x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32') x_pred = preprocess_input(x_pred) y_pred = model.predict(x_pred) _, _, _, n = y_pred.shape for i in range(n): filename = './%s/%i.png' % (laydir, i) print(filename) plt.imsave(filename, y_pred[0][:, :, i], cmap='viridis') #gray
def test_file(config, position, src_image_file, tgt_image_file, weights, model_file): labels_file = config['model']['labels'] data_dir = config['test']['data_dir'] input_width = config['model']['input_width'] input_height = config['model']['input_height'] # builder = ModelBuilder(config) preprocess_input = builder.preprocess_input image_size = (input_width,input_height) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph,config=tf_config) keras.backend.set_session(train_sess) with train_graph.as_default(): if model_file is not None: model = load_model(model_file, compile=False) elif weights: model = builder.build_model() model.load_weights(weights) else: print('using base model') model = builder.build_model() model.summary() x0,y0,w,h = tuple(position) img_path = os.path.join(data_dir, src_image_file) src = cv2.imread(img_path) src = src[:,:,[2,1,0]] # to RGB anchor = src[y0:y0+h, x0:x0+w] anchor = cv2.resize(anchor, (input_width, input_height)) x_pred = np.array([anchor]).astype('float32') # x_pred = np.expand_dims(x_pred, axis=0) # x_pred = preprocess_input(x_pred) # print(x_pred) # x_pred = x_pred/255.0 x_pred = x_pred/127.5 x_pred = x_pred-1. anchor_vector = model.predict(x_pred)[0] print(img_path,x0,y0,w,h) print(anchor_vector) for v in anchor_vector: print(v) img_path = os.path.join(data_dir, tgt_image_file) tgt = cv2.imread(img_path) tgt = tgt[:,:,[2,1,0]] # to RGB x_pred = np.array(tgt).astype('float32') # x_pred = preprocess_input(x_pred) # x_pred = x_pred/255.0 x_pred = x_pred/127.5 x_pred = x_pred-1. ratio = 1.0 max_score = 0.0 min_dist = 100000000000.0 x,y = 0,0 win_size = (w,h) i = 0 try: print(anchor.shape, x_pred.shape) ww,hh=704,int(np.floor(576*0.8)) x_pred=x_pred[57:hh+57,0:ww] it = region_search(anchor,x_pred,int(70*ratio),0.0) # print('1111') while True: # print('222') x1,y1,mv_size,seg = next(it) i = i+1 # print('333') seg = np.expand_dims(seg, axis=0) tgt_vector = model.predict(seg)[0] dist = euclidean_distance(anchor_vector, tgt_vector) if x0==x1 and y0==y1+57: print(x0,y0,mv_size,dist) if dist < min_dist: min_dist = dist x,y,win_size = x1,y1,mv_size except StopIteration: print('region search done ', i) pass x,y,win_size = int(x/ratio),int(y/ratio)+57,(int(win_size[0]/ratio),int(win_size[1]/ratio)) print(x,y,win_size,min_dist)
def test_list_file(config, list_file, weights, model_file): labels_file = config['model']['labels'] data_dir = config['test']['data_dir'] input_width = config['model']['input_width'] input_height = config['model']['input_height'] # load labels print('load label file', labels_file) label_dict = np.load(labels_file).item() class_num = len(label_dict) print('class num:', class_num) print(label_dict) # builder = ModelBuilder(config) preprocess_input = builder.preprocess_input image_size = (input_width,input_height) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph,config=tf_config) error_list = [] name_list = [] keras.backend.set_session(train_sess) with train_graph.as_default(): if model_file is not None: cls = load_model(model_file, compile=False) elif weights: cls = builder.build_model() cls.load_weights(weights) else: print('either weights or model file should be specified.') with open(os.path.join(data_dir,list_file), 'r') as f: filelist = f.readlines() conf_thresh = 0.6 conf_num = 0 num = 0 n_correct = 0 for line in filelist: line = line.rstrip('\n') img_path = os.path.join(data_dir, line.split(' ')[0]) y_true = line.split(' ')[1] # load dataset img = cv2.imread(img_path) img = cv2.resize(img, (input_width, input_height)) # img = img_pad(img, input_width, input_height) img = img[:,:,::-1] x_pred = np.array([img]).astype('float32') # x_pred = np.array([img])/255.0 # x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32') x_pred = preprocess_input(x_pred) y_pred = cls.predict(x_pred) # print(img_path) # print(y_pred) # if (y_pred[0][0] > 0.99): # y_index = 0 # else: # y_index = 1 y_index = np.argmax(y_pred[0]) confidence = y_pred[0][y_index] num = num + 1 if confidence > conf_thresh: conf_num = conf_num + 1 if y_true == label_dict[y_index]: n_correct = n_correct + 1 else: print('%s,%f,%s vs %s' % (img_path, confidence, y_true, label_dict[y_index])) error_list.append(img_path) name_list.append(label_dict[y_index]) # print(y_pred) print('confidence:%f' % (conf_thresh)) print('correct/conf_num/total: %d/%d/%d' % (n_correct,conf_num,num)) print('precision: %f,%f' % (n_correct/num, n_correct/conf_num)) print('totoal error:', len(error_list)) print('error_list = [') for i,error_path in enumerate(error_list): print('\'%s\', # %d' % (error_path,i)) print(']') print('name_list = [') for name in name_list: print('\'%s\',' % (name)) print(']')
def test_file(config, position, src_image_file, tgt_image_file, weights, model_file): labels_file = config['model']['labels'] data_dir = config['test']['data_dir'] input_width = config['model']['input_width'] input_height = config['model']['input_height'] # builder = ModelBuilder(config) preprocess_input = builder.preprocess_input image_size = (input_width, input_height) # train train_graph = tf.Graph() train_sess = tf.Session(graph=train_graph, config=tf_config) keras.backend.set_session(train_sess) with train_graph.as_default(): if model_file is not None: model = load_model(model_file, compile=False) elif weights: model = builder.build_model() model.load_weights(weights) else: print('using base model') model = builder.build_model() # model.summary() img_path = os.path.join(data_dir, src_image_file) src = cv2.imread(img_path) src = src[:, :, [2, 1, 0]] # to RGB src = np.array(src).astype('float32') src = src / 127.5 src = src - 1. img_path = os.path.join(data_dir, tgt_image_file) tgt = cv2.imread(img_path) tgt = tgt[:, :, [2, 1, 0]] # to RGB tgt = np.array(tgt).astype('float32') tgt = tgt / 127.5 tgt = tgt - 1. out_img = cv2.imread(img_path) ratio = 1.0 i = 0 try: win_size = (112, 112) tgt_size = (704, int(np.floor(576 * 0.8))) it = slide_window(win_size, tgt_size, int(100 * ratio), 0.0) while True: x1, y1, mv_size = next(it) i = i + 1 seg = src[y1 + 57:y1 + 57 + win_size[1], x1:x1 + win_size[0]] seg = np.expand_dims(seg, axis=0) src_vector = model.predict(seg)[0] seg = tgt[y1 + 57:y1 + 57 + win_size[1], x1:x1 + win_size[0]] seg = np.expand_dims(seg, axis=0) tgt_vector = model.predict(seg)[0] dist = euclidean_distance(src_vector, tgt_vector) print(i, x1, y1 + 57, dist) if dist < 0.15: cv2.rectangle(out_img, (x1, y1 + 57), (x1 + mv_size[0], y1 + 57 + mv_size[1]), (0, 255, 0), 2) # score = mr.mutual_info_score(src_vector, tgt_vector) # print(i,x1,y1+57,score) # if score > 4.: # cv2.rectangle(out_img, (x1, y1+57), (x1+mv_size[0], y1+57+mv_size[1]), (0, 255, 0), 2) except StopIteration: print('region search done ', i) pass cv2.imwrite('img_grid.jpg', out_img)
def train(config): input_width = config['model']['input_width'] input_height = config['model']['input_height'] input_depth = config['model']['input_depth'] label_file = config['model']['labels'] model_name = config['model']['name'] class_num = config['model']['class_num'] train_data_dir = config['train']['data_dir'] train_file_list = config['train']['file_list'] pretrained_weights = config['train']['pretrained_weights'] batch_size = config['train']['batch_size'] learning_rate = config['train']['learning_rate'] nb_epochs = config['train']['nb_epochs'] start_epoch = config['train']['start_epoch'] train_base = config['train']['train_base'] valid_data_dir = config['valid']['data_dir'] valid_file_list = config['valid']['file_list'] builder = ModelBuilder(config) monitorStr = 'accuracy' filepath = train_file_list train_gen = builder.build_train_datagen(filepath) trainDs = tf.data.Dataset.from_generator( lambda: train_gen, output_types=(tf.float32, tf.float32), output_shapes=([batch_size,input_depth,input_width,input_height,3], [batch_size,class_num]) ) trainDs = trainDs.shuffle(10).repeat() options = tf.data.Options() options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA trainDs = trainDs.with_options(options) train_steps_per_epoch = train_gen.steps_per_epoch validDs = None valid_steps_per_epoch = None if valid_file_list is not None and valid_file_list != '': filepath = valid_file_list valid_gen = builder.build_valid_datagen(filepath) validDs = tf.data.Dataset.from_generator( lambda: valid_gen, output_types=(tf.float32, tf.float32), output_shapes=([batch_size,input_depth,input_width,input_height,3], [batch_size,class_num]) ) validDs = validDs.shuffle(4).repeat() options = tf.data.Options() options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA validDs = validDs.with_options(options) valid_steps_per_epoch = valid_gen.steps_per_epoch monitorStr = 'val_accuracy' # define checkpoint dirname = 'ckpt-' + model_name if not os.path.exists(dirname): os.makedirs(dirname) timestr = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") filepath = os.path.join(dirname, 'weights-%s-%s-{epoch:02d}-{%s:.2f}.hdf5' %(model_name, timestr, monitorStr)) checkpoint = ModelCheckpoint(filepath=filepath, monitor=monitorStr, # acc outperforms loss verbose=1, save_best_only=True, save_weights_only=True, period=5) # define logs for tensorboard tensorboard = TensorBoard(log_dir='logs', histogram_freq=0) wgtdir = 'weights' if not os.path.exists(wgtdir): os.makedirs(wgtdir) # train # tf2.5 strategy = tf.distribute.MirroredStrategy() print("Number of devices: {}".format(strategy.num_replicas_in_sync)) # Open a strategy scope. with strategy.scope(): model = builder.build_model() # tf2.5 model.compile(optimizer=tf.optimizers.Adam(learning_rate=learning_rate), loss='categorical_crossentropy',metrics=['accuracy']) model.summary() # Load weight of unfinish training model(optional) if pretrained_weights != '': model.load_weights(pretrained_weights) model.fit(trainDs, batch_size = batch_size, steps_per_epoch=train_steps_per_epoch, validation_data = validDs, validation_steps=valid_steps_per_epoch, initial_epoch=start_epoch, epochs=nb_epochs, callbacks=[checkpoint,tensorboard], use_multiprocessing=True, workers=16) model_file = '%s_%s.h5' % (model_name,timestr) model.save(model_file) print('save model to %s' % (model_file))
def webcam(config, weight_path, stream_path, output_path): input_width = config['model']['input_width'] input_height = config['model']['input_height'] input_depth = config['model']['input_depth'] label_file = config['model']['labels'] model_name = config['model']['name'] train_data_dir = config['train']['data_dir'] train_file_list = config['train']['file_list'] pretrained_weights = config['train']['pretrained_weights'] batch_size = config['train']['batch_size'] learning_rate = config['train']['learning_rate'] nb_epochs = config['train']['nb_epochs'] start_epoch = config['train']['start_epoch'] train_base = config['train']['train_base'] valid_data_dir = config['valid']['data_dir'] valid_file_list = config['valid']['file_list'] builder = ModelBuilder(config) # train graph = tf.Graph() sess = tf.Session(graph=graph, config=tf_config) tf.keras.backend.set_session(sess) with graph.as_default(): model = builder.build_model() model.load_weights(weight_path) ### Define empty sliding window frame_window = np.empty( (0, input_width, input_height, 3)) # seq, dim0, dim1, channel ### State Machine Define RUN_STATE = 0 WAIT_STATE = 1 SET_NEW_ACTION_STATE = 2 state = RUN_STATE # previous_action = -1 # no action text_show = 'no action' # Class label define class_text = ['debris', 'rockfail', 'rain'] # class_text = [ # '1 Horizontal arm wave', # '2 High arm wave', # '3 Two hand wave', # '4 Catch Cap', # '5 High throw', # '6 Draw X', # '7 Draw Tick', # '8 Toss Paper', # '9 Forward Kick', # '10 Side Kick', # '11 Take Umbrella', # '12 Bend', # '13 Hand Clap', # '14 Walk', # '15 Phone Call', # '16 Drink', # '17 Sit down', # '18 Stand up'] # class_text = [ # '1 Horizontal arm wave', # '2 High arm wave', # '3 Two hand wave', # '4 Catch Cap', # # '5 High throw', # # '6 Draw X', # # '7 Draw Tick', # # '8 Toss Paper', # # '9 Forward Kick', # # '10 Side Kick', # # '11 Take Umbrella', # # '12 Bend', # '13 Hand Clap', # '14 Walk', # # '15 Phone Call', # # '16 Drink', # # '17 Sit down', # # '18 Stand up' # ] cap = cv2.VideoCapture(stream_path) if output_path is not None: sz = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) fps = 25 fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') vout = cv2.VideoWriter() # vout.open('output.mp4',fourcc,fps,(768,576),True) # 4:3 # vout.open(output_path,fourcc,fps,(800,800),True) vout.open(output_path, fourcc, fps, sz, True) start_time = time.time() while (cap.isOpened()): ret, frame = cap.read() if ret == True: # frame = cv2.resize(frame, (1024,576)) # frame = frame[0:576,128:128+768] # 4:3 # frame = frame[100:900,400:1200] new_f = cv2.resize(frame, (input_width, input_height)) new_f_rs = np.reshape(new_f, (1, *new_f.shape)) frame_window = np.append(frame_window, new_f_rs, axis=0) ### if sliding window is full(8 frames), start action recognition if frame_window.shape[0] >= input_depth: ### Predict action from model input_0 = frame_window.reshape(1, *frame_window.shape) with graph.as_default(): output = model.predict(input_0)[0] predict_ind = np.argmax(output) ### Check noise of action if output[predict_ind] < 0.70: new_action = -1 # no action(noise) else: new_action = predict_ind # action detect ### Use State Machine to delete noise between action(just for stability) ### RUN_STATE: normal state, change to wait state when action is changed if state == RUN_STATE: if new_action != previous_action: # action change state = WAIT_STATE start_time = time.time() else: if previous_action == -1: # or previous_action == 5: text_show = 'no action' else: text_show = "{: <10} {:.2f} ".format( class_text[previous_action], output[previous_action]) print(text_show) ### WAIT_STATE: wait 0.5 second when action from prediction is change to fillout noise elif state == WAIT_STATE: dif_time = time.time() - start_time if dif_time > 0.5: # wait 0.5 second state = RUN_STATE previous_action = new_action ### put text to image font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(frame, text_show, (10, 50), font, 0.8, (0, 255, 0), 2, cv2.LINE_AA) ### shift sliding window frame_window = frame_window[1:input_depth] if output_path is not None: vout.write(frame) else: ## To show dif RGB image vis = np.concatenate( (new_f, frame_window_new[0, n_sequence - 1]), axis=0) cv2.imshow('Frame', vis) cv2.imshow('Frame', frame) ### To show FPS # end_time = time.time() # diff_time =end_time - start_time # print("FPS:",1/diff_time) # start_time = end_time # Press Q on keyboard to exit if cv2.waitKey(25) & 0xFF == ord('q'): break else: break vout.release() cap.release()