def test_list_file(config, list_file, weights, model_file):
labels_file = config['model']['labels']
data_dir = config['test']['data_dir']
input_width = config['model']['input_width']
input_height = config['model']['input_height']
#
builder = ModelBuilder(config)
preprocess_input = builder.preprocess_input
image_size = (input_width, input_height)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph, config=tf_config)
ret_dict = {}
error_list = []
name_list = []
keras.backend.set_session(train_sess)
with train_graph.as_default():
if model_file is not None:
model = load_model(model_file, compile=False)
elif weights:
model = builder.build_model()
model.load_weights(weights)
else:
print('using base model')
model = builder.build_model()
model.summary()
with open(list_file, 'r') as f:
filelist = f.readlines()
conf_thresh = 0.6
conf_num = 0
num = 0
n_correct = 0
for line in filelist:
line = line.rstrip('\n')
img_path = os.path.join(data_dir, line)
# load dataset
img = cv2.imread(img_path)
img = cv2.resize(img, (input_width, input_height))
# img = img_pad(img, input_width, input_height)
img = img[:, :, ::-1]
x_pred = np.array([img]).astype('float32')
x_pred = np.array([img]) / 255.0
# x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32')
x_pred = preprocess_input(x_pred)
y_pred = model.predict(x_pred)
ret_dict[img_path] = y_pred[0]
np.save('./anchor.npy', ret_dict)
def predict(config, input_image, weights):
input_width = config['model']['input_width']
input_height = config['model']['input_height']
class_num = config['model']['class_num']
builder = ModelBuilder(config)
model = builder.build_model(training=False)
model.summary()
if weights is not None:
model.load_weights(weights)
img = cv2.imread(input_image)
image_h, image_w, _ = img.shape
print(image_h, image_w)
img = cv2.resize(img, (input_width, input_height))
print(img.shape)
img = img[:, :, ::-1]
x_pred = np.array([img]).astype('float32')
x_pred /= 255.
print(x_pred.shape)
y_outs = model(x_pred)
y_outs = [a[0].numpy() for a in y_outs]
print(y_outs[0].shape)
print(y_outs[1].shape)
print(y_outs[2].shape)
# 画框设置不同的颜色
hsv_tuples = [(x / class_num, 1., 1.) for x in range(class_num)]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
boxes, scores, classes, colors = yolo_decodeout(y_outs,
image_h,
image_w,
colors,
input_w=input_width,
input_h=input_height,
nms_thresh=0.25,
class_thresh=0.25,
obj_thresh=0.5)
for box in boxes:
print(box.xmin, box.ymin, box.xmax, box.ymax)
print(scores)
print(classes)
print(len(classes))
def train(config):
input_width = config['model']['input_width']
input_height = config['model']['input_height']
label_file = config['model']['labels']
model_name = config['model']['name']
train_data_dir = config['train']['data_dir']
train_file_list = config['train']['file_list']
pretrained_weights = config['train']['pretrained_weights']
batch_size = config['train']['batch_size']
learning_rate = config['train']['learning_rate']
nb_epochs = config['train']['nb_epochs']
start_epoch = config['train']['start_epoch']
train_base = config['train']['train_base']
valid_data_dir = config['valid']['data_dir']
valid_file_list = config['valid']['file_list']
builder = ModelBuilder(config)
filepath = os.path.join('', train_file_list)
train_gen = builder.build_datagen(filepath)
# train_gen.save_labels(label_file)
# trainDataGen, train_steps_per_epoch = train_gen.from_frame(directory=train_data_dir)
# filepath = os.path.join(valid_data_dir, valid_file_list)
# valid_gen = builder.build_datagen(filepath, with_aug=False)
# validDataGen, valid_steps_per_epoch = valid_gen.from_frame(directory=valid_data_dir)
# define checkpoint
dataset_name = model_name
dirname = 'ckpt-' + dataset_name
if not os.path.exists(dirname):
os.makedirs(dirname)
timestr = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
filepath = os.path.join(
dirname,
'weights-%s-%s-{epoch:02d}-{loss:.5f}.hdf5' % (model_name, timestr))
checkpoint = ModelCheckpoint(
filepath=filepath,
monitor='loss', # acc outperforms loss
verbose=1,
save_best_only=True,
save_weights_only=True,
period=1)
# define logs for tensorboard
tensorboard = TensorBoard(log_dir='logs', histogram_freq=0)
wgtdir = 'weights'
if not os.path.exists(wgtdir):
os.makedirs(wgtdir)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph, config=tf_config)
tf.keras.backend.set_session(train_sess)
with train_graph.as_default():
model = builder.build_model()
model.compile(
optimizer=tf.train.AdamOptimizer(learning_rate=learning_rate),
loss=triple_loss,
metrics=[p_loss, n_loss])
model.summary()
# Load weight of unfinish training model(optional)
if pretrained_weights != '':
model.load_weights(pretrained_weights)
model.fit_generator(
generator=train_gen,
# validation_data = validDataGen,
initial_epoch=start_epoch,
epochs=nb_epochs,
callbacks=[checkpoint, tensorboard],
use_multiprocessing=False,
workers=16)
model_file = '%s_%s.h5' % (model_name, timestr)
model.save(model_file)
print('save model to %s' % (model_file))
def layer_visualize(config, img_file, weights, model_file, layer_name):
labels_file = config['model']['labels']
input_width = config['model']['input_width']
input_height = config['model']['input_height']
test_data_dir = config['test']['data_dir']
# load labels
print('load label file', labels_file)
label_dict = np.load(labels_file).item()
class_num = len(label_dict)
print('class num:', class_num)
print(label_dict)
#
builder = ModelBuilder(config)
preprocess_input = builder.preprocess_input
visdir = 'visual'
if not os.path.exists(visdir):
os.makedirs(visdir)
laydir = os.path.join(visdir, layer_name)
if not os.path.exists(laydir):
os.makedirs(laydir)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph, config=tf_config)
keras.backend.set_session(train_sess)
with train_graph.as_default():
if model_file is not None:
cls = load_model(model_file, compile=False)
elif weights:
cls = builder.build_model()
cls.load_weights(weights)
else:
print('either weights or model file should be specified.')
model = build_model(cls, layer_name)
img_path = os.path.join(test_data_dir, img_file)
print(img_path)
img = cv2.imread(img_path)
img = cv2.resize(img, (input_width, input_height))
# img = img_pad(img, input_width, input_height)
img = img[:, :, ::-1]
x_pred = np.array([img]).astype('float32')
# x_pred = np.array([img])/255.0
# x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32')
x_pred = preprocess_input(x_pred)
y_pred = model.predict(x_pred)
_, _, _, n = y_pred.shape
for i in range(n):
filename = './%s/%i.png' % (laydir, i)
print(filename)
plt.imsave(filename, y_pred[0][:, :, i], cmap='viridis') #gray
def test_file(config, position, src_image_file, tgt_image_file, weights, model_file):
labels_file = config['model']['labels']
data_dir = config['test']['data_dir']
input_width = config['model']['input_width']
input_height = config['model']['input_height']
#
builder = ModelBuilder(config)
preprocess_input = builder.preprocess_input
image_size = (input_width,input_height)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph,config=tf_config)
keras.backend.set_session(train_sess)
with train_graph.as_default():
if model_file is not None:
model = load_model(model_file, compile=False)
elif weights:
model = builder.build_model()
model.load_weights(weights)
else:
print('using base model')
model = builder.build_model()
model.summary()
x0,y0,w,h = tuple(position)
img_path = os.path.join(data_dir, src_image_file)
src = cv2.imread(img_path)
src = src[:,:,[2,1,0]] # to RGB
anchor = src[y0:y0+h, x0:x0+w]
anchor = cv2.resize(anchor, (input_width, input_height))
x_pred = np.array([anchor]).astype('float32')
# x_pred = np.expand_dims(x_pred, axis=0)
# x_pred = preprocess_input(x_pred)
# print(x_pred)
# x_pred = x_pred/255.0
x_pred = x_pred/127.5
x_pred = x_pred-1.
anchor_vector = model.predict(x_pred)[0]
print(img_path,x0,y0,w,h)
print(anchor_vector)
for v in anchor_vector:
print(v)
img_path = os.path.join(data_dir, tgt_image_file)
tgt = cv2.imread(img_path)
tgt = tgt[:,:,[2,1,0]] # to RGB
x_pred = np.array(tgt).astype('float32')
# x_pred = preprocess_input(x_pred)
# x_pred = x_pred/255.0
x_pred = x_pred/127.5
x_pred = x_pred-1.
ratio = 1.0
max_score = 0.0
min_dist = 100000000000.0
x,y = 0,0
win_size = (w,h)
i = 0
try:
print(anchor.shape, x_pred.shape)
ww,hh=704,int(np.floor(576*0.8))
x_pred=x_pred[57:hh+57,0:ww]
it = region_search(anchor,x_pred,int(70*ratio),0.0)
# print('1111')
while True:
# print('222')
x1,y1,mv_size,seg = next(it)
i = i+1
# print('333')
seg = np.expand_dims(seg, axis=0)
tgt_vector = model.predict(seg)[0]
dist = euclidean_distance(anchor_vector, tgt_vector)
if x0==x1 and y0==y1+57:
print(x0,y0,mv_size,dist)
if dist < min_dist:
min_dist = dist
x,y,win_size = x1,y1,mv_size
except StopIteration:
print('region search done ', i)
pass
x,y,win_size = int(x/ratio),int(y/ratio)+57,(int(win_size[0]/ratio),int(win_size[1]/ratio))
print(x,y,win_size,min_dist)
def test_list_file(config, list_file, weights, model_file):
labels_file = config['model']['labels']
data_dir = config['test']['data_dir']
input_width = config['model']['input_width']
input_height = config['model']['input_height']
# load labels
print('load label file', labels_file)
label_dict = np.load(labels_file).item()
class_num = len(label_dict)
print('class num:', class_num)
print(label_dict)
#
builder = ModelBuilder(config)
preprocess_input = builder.preprocess_input
image_size = (input_width,input_height)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph,config=tf_config)
error_list = []
name_list = []
keras.backend.set_session(train_sess)
with train_graph.as_default():
if model_file is not None:
cls = load_model(model_file, compile=False)
elif weights:
cls = builder.build_model()
cls.load_weights(weights)
else:
print('either weights or model file should be specified.')
with open(os.path.join(data_dir,list_file), 'r') as f:
filelist = f.readlines()
conf_thresh = 0.6
conf_num = 0
num = 0
n_correct = 0
for line in filelist:
line = line.rstrip('\n')
img_path = os.path.join(data_dir, line.split(' ')[0])
y_true = line.split(' ')[1]
# load dataset
img = cv2.imread(img_path)
img = cv2.resize(img, (input_width, input_height))
# img = img_pad(img, input_width, input_height)
img = img[:,:,::-1]
x_pred = np.array([img]).astype('float32')
# x_pred = np.array([img])/255.0
# x_pred = np.expand_dims(img_to_array(load_img(img_path, target_size=image_size)), axis=0).astype('float32')
x_pred = preprocess_input(x_pred)
y_pred = cls.predict(x_pred)
# print(img_path)
# print(y_pred)
# if (y_pred[0][0] > 0.99):
# y_index = 0
# else:
# y_index = 1
y_index = np.argmax(y_pred[0])
confidence = y_pred[0][y_index]
num = num + 1
if confidence > conf_thresh:
conf_num = conf_num + 1
if y_true == label_dict[y_index]:
n_correct = n_correct + 1
else:
print('%s,%f,%s vs %s' % (img_path, confidence, y_true, label_dict[y_index]))
error_list.append(img_path)
name_list.append(label_dict[y_index])
# print(y_pred)
print('confidence:%f' % (conf_thresh))
print('correct/conf_num/total: %d/%d/%d' % (n_correct,conf_num,num))
print('precision: %f,%f' % (n_correct/num, n_correct/conf_num))
print('totoal error:', len(error_list))
print('error_list = [')
for i,error_path in enumerate(error_list):
print('\'%s\', # %d' % (error_path,i))
print(']')
print('name_list = [')
for name in name_list:
print('\'%s\',' % (name))
print(']')
def test_file(config, position, src_image_file, tgt_image_file, weights,
model_file):
labels_file = config['model']['labels']
data_dir = config['test']['data_dir']
input_width = config['model']['input_width']
input_height = config['model']['input_height']
#
builder = ModelBuilder(config)
preprocess_input = builder.preprocess_input
image_size = (input_width, input_height)
# train
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph, config=tf_config)
keras.backend.set_session(train_sess)
with train_graph.as_default():
if model_file is not None:
model = load_model(model_file, compile=False)
elif weights:
model = builder.build_model()
model.load_weights(weights)
else:
print('using base model')
model = builder.build_model()
# model.summary()
img_path = os.path.join(data_dir, src_image_file)
src = cv2.imread(img_path)
src = src[:, :, [2, 1, 0]] # to RGB
src = np.array(src).astype('float32')
src = src / 127.5
src = src - 1.
img_path = os.path.join(data_dir, tgt_image_file)
tgt = cv2.imread(img_path)
tgt = tgt[:, :, [2, 1, 0]] # to RGB
tgt = np.array(tgt).astype('float32')
tgt = tgt / 127.5
tgt = tgt - 1.
out_img = cv2.imread(img_path)
ratio = 1.0
i = 0
try:
win_size = (112, 112)
tgt_size = (704, int(np.floor(576 * 0.8)))
it = slide_window(win_size, tgt_size, int(100 * ratio), 0.0)
while True:
x1, y1, mv_size = next(it)
i = i + 1
seg = src[y1 + 57:y1 + 57 + win_size[1], x1:x1 + win_size[0]]
seg = np.expand_dims(seg, axis=0)
src_vector = model.predict(seg)[0]
seg = tgt[y1 + 57:y1 + 57 + win_size[1], x1:x1 + win_size[0]]
seg = np.expand_dims(seg, axis=0)
tgt_vector = model.predict(seg)[0]
dist = euclidean_distance(src_vector, tgt_vector)
print(i, x1, y1 + 57, dist)
if dist < 0.15:
cv2.rectangle(out_img, (x1, y1 + 57),
(x1 + mv_size[0], y1 + 57 + mv_size[1]),
(0, 255, 0), 2)
# score = mr.mutual_info_score(src_vector, tgt_vector)
# print(i,x1,y1+57,score)
# if score > 4.:
# cv2.rectangle(out_img, (x1, y1+57), (x1+mv_size[0], y1+57+mv_size[1]), (0, 255, 0), 2)
except StopIteration:
print('region search done ', i)
pass
cv2.imwrite('img_grid.jpg', out_img)
def train(config):
input_width = config['model']['input_width']
input_height = config['model']['input_height']
input_depth = config['model']['input_depth']
label_file = config['model']['labels']
model_name = config['model']['name']
class_num = config['model']['class_num']
train_data_dir = config['train']['data_dir']
train_file_list = config['train']['file_list']
pretrained_weights = config['train']['pretrained_weights']
batch_size = config['train']['batch_size']
learning_rate = config['train']['learning_rate']
nb_epochs = config['train']['nb_epochs']
start_epoch = config['train']['start_epoch']
train_base = config['train']['train_base']
valid_data_dir = config['valid']['data_dir']
valid_file_list = config['valid']['file_list']
builder = ModelBuilder(config)
monitorStr = 'accuracy'
filepath = train_file_list
train_gen = builder.build_train_datagen(filepath)
trainDs = tf.data.Dataset.from_generator(
lambda: train_gen,
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size,input_depth,input_width,input_height,3], [batch_size,class_num])
)
trainDs = trainDs.shuffle(10).repeat()
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA
trainDs = trainDs.with_options(options)
train_steps_per_epoch = train_gen.steps_per_epoch
validDs = None
valid_steps_per_epoch = None
if valid_file_list is not None and valid_file_list != '':
filepath = valid_file_list
valid_gen = builder.build_valid_datagen(filepath)
validDs = tf.data.Dataset.from_generator(
lambda: valid_gen,
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size,input_depth,input_width,input_height,3], [batch_size,class_num])
)
validDs = validDs.shuffle(4).repeat()
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA
validDs = validDs.with_options(options)
valid_steps_per_epoch = valid_gen.steps_per_epoch
monitorStr = 'val_accuracy'
# define checkpoint
dirname = 'ckpt-' + model_name
if not os.path.exists(dirname):
os.makedirs(dirname)
timestr = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
filepath = os.path.join(dirname, 'weights-%s-%s-{epoch:02d}-{%s:.2f}.hdf5' %(model_name, timestr, monitorStr))
checkpoint = ModelCheckpoint(filepath=filepath,
monitor=monitorStr, # acc outperforms loss
verbose=1,
save_best_only=True,
save_weights_only=True,
period=5)
# define logs for tensorboard
tensorboard = TensorBoard(log_dir='logs', histogram_freq=0)
wgtdir = 'weights'
if not os.path.exists(wgtdir):
os.makedirs(wgtdir)
# train
# tf2.5
strategy = tf.distribute.MirroredStrategy()
print("Number of devices: {}".format(strategy.num_replicas_in_sync))
# Open a strategy scope.
with strategy.scope():
model = builder.build_model()
# tf2.5
model.compile(optimizer=tf.optimizers.Adam(learning_rate=learning_rate),
loss='categorical_crossentropy',metrics=['accuracy'])
model.summary()
# Load weight of unfinish training model(optional)
if pretrained_weights != '':
model.load_weights(pretrained_weights)
model.fit(trainDs,
batch_size = batch_size,
steps_per_epoch=train_steps_per_epoch,
validation_data = validDs,
validation_steps=valid_steps_per_epoch,
initial_epoch=start_epoch,
epochs=nb_epochs,
callbacks=[checkpoint,tensorboard],
use_multiprocessing=True,
workers=16)
model_file = '%s_%s.h5' % (model_name,timestr)
model.save(model_file)
print('save model to %s' % (model_file))
def webcam(config, weight_path, stream_path, output_path):
input_width = config['model']['input_width']
input_height = config['model']['input_height']
input_depth = config['model']['input_depth']
label_file = config['model']['labels']
model_name = config['model']['name']
train_data_dir = config['train']['data_dir']
train_file_list = config['train']['file_list']
pretrained_weights = config['train']['pretrained_weights']
batch_size = config['train']['batch_size']
learning_rate = config['train']['learning_rate']
nb_epochs = config['train']['nb_epochs']
start_epoch = config['train']['start_epoch']
train_base = config['train']['train_base']
valid_data_dir = config['valid']['data_dir']
valid_file_list = config['valid']['file_list']
builder = ModelBuilder(config)
# train
graph = tf.Graph()
sess = tf.Session(graph=graph, config=tf_config)
tf.keras.backend.set_session(sess)
with graph.as_default():
model = builder.build_model()
model.load_weights(weight_path)
### Define empty sliding window
frame_window = np.empty(
(0, input_width, input_height, 3)) # seq, dim0, dim1, channel
### State Machine Define
RUN_STATE = 0
WAIT_STATE = 1
SET_NEW_ACTION_STATE = 2
state = RUN_STATE #
previous_action = -1 # no action
text_show = 'no action'
# Class label define
class_text = ['debris', 'rockfail', 'rain']
# class_text = [
# '1 Horizontal arm wave',
# '2 High arm wave',
# '3 Two hand wave',
# '4 Catch Cap',
# '5 High throw',
# '6 Draw X',
# '7 Draw Tick',
# '8 Toss Paper',
# '9 Forward Kick',
# '10 Side Kick',
# '11 Take Umbrella',
# '12 Bend',
# '13 Hand Clap',
# '14 Walk',
# '15 Phone Call',
# '16 Drink',
# '17 Sit down',
# '18 Stand up']
# class_text = [
# '1 Horizontal arm wave',
# '2 High arm wave',
# '3 Two hand wave',
# '4 Catch Cap',
# # '5 High throw',
# # '6 Draw X',
# # '7 Draw Tick',
# # '8 Toss Paper',
# # '9 Forward Kick',
# # '10 Side Kick',
# # '11 Take Umbrella',
# # '12 Bend',
# '13 Hand Clap',
# '14 Walk',
# # '15 Phone Call',
# # '16 Drink',
# # '17 Sit down',
# # '18 Stand up'
# ]
cap = cv2.VideoCapture(stream_path)
if output_path is not None:
sz = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = 25
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
vout = cv2.VideoWriter()
# vout.open('output.mp4',fourcc,fps,(768,576),True) # 4:3
# vout.open(output_path,fourcc,fps,(800,800),True)
vout.open(output_path, fourcc, fps, sz, True)
start_time = time.time()
while (cap.isOpened()):
ret, frame = cap.read()
if ret == True:
# frame = cv2.resize(frame, (1024,576))
# frame = frame[0:576,128:128+768] # 4:3
# frame = frame[100:900,400:1200]
new_f = cv2.resize(frame, (input_width, input_height))
new_f_rs = np.reshape(new_f, (1, *new_f.shape))
frame_window = np.append(frame_window, new_f_rs, axis=0)
### if sliding window is full(8 frames), start action recognition
if frame_window.shape[0] >= input_depth:
### Predict action from model
input_0 = frame_window.reshape(1, *frame_window.shape)
with graph.as_default():
output = model.predict(input_0)[0]
predict_ind = np.argmax(output)
### Check noise of action
if output[predict_ind] < 0.70:
new_action = -1 # no action(noise)
else:
new_action = predict_ind # action detect
### Use State Machine to delete noise between action(just for stability)
### RUN_STATE: normal state, change to wait state when action is changed
if state == RUN_STATE:
if new_action != previous_action: # action change
state = WAIT_STATE
start_time = time.time()
else:
if previous_action == -1: # or previous_action == 5:
text_show = 'no action'
else:
text_show = "{: <10} {:.2f} ".format(
class_text[previous_action],
output[previous_action])
print(text_show)
### WAIT_STATE: wait 0.5 second when action from prediction is change to fillout noise
elif state == WAIT_STATE:
dif_time = time.time() - start_time
if dif_time > 0.5: # wait 0.5 second
state = RUN_STATE
previous_action = new_action
### put text to image
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(frame, text_show, (10, 50), font, 0.8, (0, 255, 0),
2, cv2.LINE_AA)
### shift sliding window
frame_window = frame_window[1:input_depth]
if output_path is not None:
vout.write(frame)
else:
## To show dif RGB image
vis = np.concatenate(
(new_f, frame_window_new[0, n_sequence - 1]), axis=0)
cv2.imshow('Frame', vis)
cv2.imshow('Frame', frame)
### To show FPS
# end_time = time.time()
# diff_time =end_time - start_time
# print("FPS:",1/diff_time)
# start_time = end_time
# Press Q on keyboard to exit
if cv2.waitKey(25) & 0xFF == ord('q'):
break
else:
break
vout.release()
cap.release()