def validate(data_type,
model,
seq_length=40,
saved_model=None,
class_limit=None,
image_shape=None):
batch_size = 32
# Get the data and process it.
if image_shape is None:
data = DataSet(seq_length=seq_length, class_limit=class_limit)
else:
data = DataSet(seq_length=seq_length,
class_limit=class_limit,
image_shape=image_shape)
val_generator = data.frame_generator(batch_size, 'test', data_type)
# Get the model.
rm = ResearchModels(len(data.classes), model, seq_length, saved_model)
# Evaluate!
results = rm.model.evaluate_generator(generator=val_generator,
val_samples=3200)
print(results)
print(rm.model.metrics_names)
def init_embedding(self):
with tf.device('/cpu:0'), tf.name_scope("embedding"):
train_data = DataSet(Config.train_data_source)
self.W, _ = train_data.onehot_dic_build()
self.x_image = tf.nn.embedding_lookup(self.W, self.input_x)
self.x_flat = tf.expand_dims(
self.x_image,
-1) # batch_size * sequence_length * embedding_size * -1
def validate(data_type,
model,
seq_length=50,
saved_model=None,
class_limit=None,
image_shape=None,
train_test='test'):
# batch_size = 32
batch_size = 1
# Get the data and process it.
if image_shape is None:
data = DataSet(seq_length=seq_length, class_limit=class_limit)
else:
data = DataSet(seq_length=seq_length,
class_limit=class_limit,
image_shape=image_shape)
# _, test = data.split_train_test()
# size = len(test)
# val_generator = data.frame_generator(batch_size, 'test', data_type)
# Get the model.
rm = ResearchModels(len(data.classes), model, seq_length, saved_model)
rm.model.layers.pop()
rm.model.outputs = [rm.model.layers[-2].output]
rm.model.output_layers = [rm.model.layers[-2]]
rm.model.layers[-2].outbound_nodes = []
# X = rm.layers[-1].output
# self.model.layers.pop() # two pops to get to pool layer
# self.model.outputs = [self.model.layers[-1].output]
X, y = data.get_data_train_test(data_type, train_test)
size = len(X)
# Evaluate!
# results = rm.model.evaluate_generator(
# generator=val_generator,
# val_samples=3200)
#
# print(results)
# print(rm.model.metrics_names)
# results = rm.model.predict_generator(
# generator=val_generator,
# val_samples=size,
# # val_samples=3200,
# verbose=1)
results = rm.model.predict(
X,
# val_samples=size,
# val_samples=3200,
verbose=1)
print(results.shape)
return (results, y)
def extract_full_features(weights, seq_length = 40):
# Set defaults.
class_limit = None # Number of classes to extract. Can be 1-101 or None for all.
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit, check_dir='data/check')
# get the model.
# model = Extractor()
# model = Extractor(weights="data/checkpoints/inception.009-0.29.hdf5")
model = Extractor(weights)
# Loop through data.
print(data.data)
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join('data', 'sequences_test', video[2] + '-' + str(seq_length) + \
'-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(video)
# Now downsample to just the ones we need.
# frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# print(path)
output_dir = os.path.join('data', 'sequences_test')
if not (os.path.exists(output_dir)):
# create the directory you want to save to
os.mkdir(output_dir)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
pbar.close()
def main(nb_images=5):
"""Spot-check `nb_images` images."""
data = DataSet()
model = load_model('data/checkpoints/inception.035-0.17.hdf5')
# Get all our test images.
images = glob.glob(os.path.join('data', 'test', '**', '*.jpg'))
nb_images = 500
for _ in range(nb_images):
print('-' * 80)
# Get a random row.
sample = random.randint(0, len(images) - 1)
image = images[sample]
# Turn the image into an array.
print(image)
image_arr = process_image(image, (299, 299, 3))
image_arr = np.expand_dims(image_arr, axis=0)
# Predict.
predictions = model.predict(image_arr)
# Show how much we think it's each one.
label_predictions = {}
for i, label in enumerate(data.classes):
label_predictions[label] = predictions[0][i]
sorted_lps = sorted(label_predictions.items(),
key=operator.itemgetter(1),
reverse=True)
print(label_predictions)
print(sorted_lps)
for i, class_prediction in enumerate(sorted_lps):
# Just get the top five.
# if i > 4:
# break
print("%s: %.2f" % (class_prediction[0], class_prediction[1]))
i += 1
def main(videos=5):
seq_length = 50
# weights_path = get_file('lstm-features.hdf5', 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu')
# url = 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu'
# urllib.request.urlretrieve(url, 'data/checkpoints/lstm-features.hdf5')
# model = load_model('data/checkpoints/lstm-features.009-0.454.hdf5')
# model = load_model('data/checkpoints/lstm-features.004-0.614.hdf5')
cnn_model = 'data/checkpoints/inception.hdf5'
# lstm_model = 'data/checkpoints/lstm-features.086-0.895.hdf5'
# lstm_model = 'data/checkpoints/lstm-features.051-0.902.hdf5'
lstm_model = 'data/checkpoints/lstm-features.017-0.849.hdf5'
extract_files(folders=['check'])
extract_full_features(weights=cnn_model, seq_length=seq_length)
model = load_model(lstm_model)
output_json = {}
output_json["smoking"] = []
test_dir = "check"
data = DataSet(seq_length=seq_length, check_dir='check')
random.shuffle(data.data)
# model = load_model('data/checkpoints/inception.057-1.16.hdf5')
for video in data.data:
X, y = [], []
sequences = data.get_extracted_sequence('features', video)
total = sequences.shape[0]
frames = np.arange(total)
frame_pred = np.ones(total)
frame_pred_sum = np.zeros(total)
frame_pred_count = np.zeros(total)
# print("Size : " + str(total))
frame_pred_prob = np.empty(total)
frame_pred_prob[:] = np.nan
# X.append(sequence)
y.append(data.get_class_one_hot(video[1]))
print(y)
print("video: " + video[2])
skip_clips = 50 #100
skip_frames = 2 #6
start_frame = 0
end_frame = skip_frames * seq_length
# end_frame = 250
print("Number of frames: ", total)
label_predictions = {}
if end_frame > sequences.shape[0]:
sequences = data.rescale_list(sequences, seq_length)
X = []
X.append(sequences)
predictions = model.predict(np.array(X), batch_size=1)
label_predictions = {}
for i, label in enumerate(data.classes):
# print(predictions)
label_predictions[label] = predictions[0][i]
# print(label_predictions)
# if label_predictions["smoking"] <= 0.5:
# frame_pred[start_frame:total] = 0
for i in range(start_frame, total):
frame_pred_sum[i] += label_predictions["smoking"]
frame_pred_count[i] += 1
# else:
# frame_pred[start_frame:total] = -1
# frame_pred_prob[start_frame:total] = str(label_predictions["smoking"])
else:
while end_frame <= sequences.shape[0]:
X = []
x = []
for i in range(start_frame, end_frame, skip_frames):
x.append(sequences[i, :])
X.append(x)
# print("video: " + video[2] + " start frame: " + str(start_frame) + " end frame: " + str(end_frame))
# X.append(sequences[start_frame: end_frame,:])
# sequence = sequence.reshape(1, 3, 3)
predictions = model.predict(np.array(X), batch_size=1)
label_predictions = {}
for i, label in enumerate(data.classes):
# print(predictions)
label_predictions[label] = predictions[0][i]
# print(label_predictions)
# if label_predictions["smoking"] <= 0.5:
# frame_pred[start_frame:end_frame] = 0
for i in range(start_frame, end_frame):
frame_pred_sum[i] += label_predictions["smoking"]
frame_pred_count[i] += 1
# else:
# frame_pred[start_frame:end_frame] = 0
# frame_pred_prob[start_frame:end_frame] = str(label_predictions["smoking"])
start_frame += skip_clips
end_frame += skip_clips
for i in range(start_frame, min(sequences.shape[0],
end_frame - 1)):
frame_pred_sum[i] += label_predictions["smoking"]
frame_pred_count[i] += 1
#
# for i in range(start_frame, min(sequences.shape[0], end_frame-1)):
# # frame_pred_prob.append(str(label_predictions["smoking"]))
# frame_pred_prob[i] = str(label_predictions["smoking"])
# if label_predictions["smoking"] <= 0.5:
# frame_pred[i] = 0
# print(frame_pred)
for i in range(0, total):
frame_pred_prob[i] = frame_pred_sum[i] / frame_pred_count[i]
if frame_pred_prob[i] < 0.5:
frame_pred[i] = 0
plt.title("Smoking action detection")
plt.xlabel("Frame")
plt.ylabel("Smoking action present")
plt.plot(frames, frame_pred)
output_path = os.path.join('data', 'out', video[2] + '.png')
print("Saving output labels to: ", output_path)
plt.savefig(output_path)
plt.close()
# plt.show()
# plt.figure()
output_json["smoking"] = list(zip(frames.tolist(), frame_pred_prob))
y = json.dumps(output_json)
# with open('frameLabel.json', 'w') as outfile:
# json.dump(y, outfile)
output_path = os.path.join('data', 'out', video[2] + '.json')
print(y)
with open(output_path, 'w') as outfile:
json.dump(output_json, outfile)
print('Output JSON saved under {}'.format(output_path))
label_video(video, frame_pred, frame_pred_prob)
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU
# config.log_device_placement = True # to log device placement (on which device the operation ran)
# sess = tf.Session(config=config)
# set_session(sess)
import tensorflow as tf
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)
# Set defaults.
seq_length = 50
class_limit = None # Number of classes to extract. Can be 1-101 or None for all.
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit)
model = Extractor(weights="data/checkpoints/inception.hdf5")
# get the model.
# model = Extractor(weights="data/checkpoints/inception.035-0.17.hdf5")
# model = Extractor(weights="data/checkpoints/inception.009-0.29.hdf5")
# Loop through data.
# print(data.data)
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
def train(data_type, seq_length, model, saved_model=None,
class_limit=None, image_shape=None,
load_to_memory=False, batch_size=32, nb_epoch=100):
# Helper: Save the model.
checkpointer = ModelCheckpoint(
filepath=os.path.join('data', 'checkpoints', model + '-' + data_type + \
'.{epoch:03d}-{val_accuracy:.3f}.hdf5'),
monitor='val_accuracy',
verbose=1,
save_best_only=True)
# Helper: TensorBoard
tb = TensorBoard(log_dir=os.path.join('data', 'logs', model))
# Helper: Stop when we stop learning.
early_stopper = EarlyStopping(patience=50, monitor='val_accuracy')
# Helper: Save results.
timestamp = time.time()
csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \
str(timestamp) + '.log'))
# Get the data and process it.
if image_shape is None:
data = DataSet(
seq_length=seq_length,
class_limit=class_limit
)
else:
data = DataSet(
seq_length=seq_length,
class_limit=class_limit,
image_shape=image_shape
)
# Get samples per epoch.
# Multiply by 0.7 to attempt to guess how much of data.data is the train set.
steps_per_epoch = (len(data.data) * 0.7) // batch_size
if load_to_memory:
# Get data.
X, y = data.get_all_sequences_in_memory('train', data_type)
X_test, y_test = data.get_all_sequences_in_memory('test', data_type)
else:
# Get generators.
generator = data.frame_generator(batch_size, 'train', data_type)
val_generator = data.frame_generator(batch_size, 'test', data_type)
# Get the model.
rm = ResearchModels(len(data.classes), model, seq_length, saved_model)
# Fit!
if load_to_memory:
# Use standard fit.
rm.model.fit(
X,
y,
batch_size=batch_size,
validation_data=(X_test, y_test),
verbose=1,
# callbacks=[tb, early_stopper, csv_logger],
callbacks=[early_stopper, csv_logger, checkpointer],
epochs=nb_epoch)
else:
# Use fit generator.
rm.model.fit_generator(
generator=generator,
steps_per_epoch=steps_per_epoch,
epochs=nb_epoch,
verbose=1,
# callbacks=[tb, early_stopper, csv_logger, checkpointer],
callbacks=[ early_stopper, checkpointer],
validation_data=val_generator,
validation_steps=40,
workers=4)
def main(unused_argv):
train_data_path = FLAGS.train_data_path
val_data_path = FLAGS.validate_data_path
# load train data
train_data = DataSet(train_data_path)
dev_data = DataSet(val_data_path)
train_data.dataset_process()
dev_data.dataset_process()
with tf.Graph().as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
sess = tf.Session(config = session_conf)
with sess.as_default():
cnn = CharCNN(
l0 = Config.l0,
num_classes = Config.nums_classes,
conv_layers = Config.model.conv_layers,
fc_layers = Config.model.fc_layers,
l2_reg_lambda = 0
)
global_step = tf.Variable(0, name = 'global_step', trainable = False)
optimizer = tf.train.AdamOptimizer(Config.model.learning_rate)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram("{}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
timestamp = str(int(time.time()))
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)
# Train summaries
train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
# Dev summaries
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables())
# Initialize all variables
sess.run(tf.global_variables_initializer())
def train_step(x_batch, y_batch):
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: Config.model.dropout_keep_prob
}
_, step, summaries, loss, accuracy = sess.run(
[train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def dev_step(x_batch, y_batch, writer=None):
"""
Evaluates model on a dev set
"""
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, summaries, loss, accuracy = sess.run(
[global_step, dev_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
if writer:
writer.add_summary(summaries, step)
print "初始化完毕，开始训练"
for i in range(Config.training.epoches):
batch_train = train_data.next_batch()
# 训练模型
train_step(batch_train[0], batch_train[1])
current_step = tf.train.global_step(sess, global_step)
# train_step.run(feed_dict={x: batch_train[0], y_actual: batch_train[1], keep_prob: 0.5})
# 对结果进行记录
if current_step % Config.training.evaluate_every == 0:
print("\nEvaluation:")
dev_step(dev_data.doc_image, dev_data.label_image, writer=dev_summary_writer)
print("")
if current_step % Config.training.checkpoint_every == 0:
path = saver.save(sess, checkpoint_prefix, global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))