def __init__(self,
latent_dim=100,
length=16,
width=120,
height=160,
channels=3,
c3d_weights=None):
self.image_path = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Imgs/RoadAccidents/'
self.n_classes = 2
self.latent_dim = latent_dim
self.length = length
self.width = width
self.height = height
self.channels = channels
self.c3d_weights = c3d_weights
self.shape = (self.width, self.height, self.channels)
self.disc_optimizer = Adam(lr=1e-6, decay=0.00005)
self.gen_optimizer = Adam(lr=0.0006, decay=0.00005)
#init the c3d model
self.c3d_model = c3d_model.get_model()
if self.c3d_weights == None:
raise Exception('weights is requited!')
try:
self.c3d_model.load_weights(self.c3d_weights)
except OSError:
print(
"the pretrained weights doesn't exist, please use <-h> to check usage"
)
exit()
convLayers = [
'conv1', 'conv2', 'conv3a', 'conv3b', 'conv4a', 'conv4b', 'conv5a',
'conv5b'
]
for layer in convLayers:
self.c3d_model.get_layer(layer).trainable = False
self.add_outputs(1)
#fixed c3d (conv1 - pool5), extracting real features
self.fixed_c3d = Model(
inputs=self.c3d_model.input,
outputs=self.c3d_model.get_layer('flatten_1').output)
#discriminator,
self.D = self.__discriminator()
self.D.compile(loss='categorical_crossentropy',
optimizer=self.disc_optimizer,
metrics=['accuracy'])
#generator
self.G = self.__generator()
# self.G.compile(loss='', optimizer=self.optimizer)
self.GAN = self.__stacked_generator_discriminator()
self.GAN.compile(loss=self.loss_matching, optimizer=self.gen_optimizer)
self.c3d_model.summary()
self.fixed_c3d.summary()
self.G.summary()
self.D.summary()
self.GAN.summary()
def main():
windows_length = 16
model = c3d_model.get_model(2)
model.summary()
weights_dir = './weight'
# model_weight_filename = os.path.join(weights_dir, 'sports1M_weights_tf.h5')
gan = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/gan_f1/weights/c3d_TC_GAN_1_outputs_it200.hdf5'
tc = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/thesis/adam_temporal_f1/weights/weights.03.hdf5'
c3d = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/thesis/adam_c3d_c3d_as_211/weights/weights.04.hdf5'
model.load_weights(
'/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/adam_c3d_final_1_retrain_march21/weights/weights.09-1.660.hdf5'
) #, by_name = True, skip_mismatch=True, reshape=True)
# model.load_weights('results/weights_c3d.h5')
# read video
crime = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Videos/Arson/Arson009_x264.mp4'
fight = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Videos/Fighting/Fighting047_x264.mp4'
accident = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Videos/RoadAccidents_test/RoadAccidents002_x264.mp4'
cap = cv2.VideoCapture(accident)
clip = []
counter = 0
while True:
ret, frame = cap.read()
counter += 1
# if counter < 2300:
# continue
if ret:
#tmp = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
clip.append(cv2.resize(frame, (160, 120)))
if len(clip) == 16:
inputs = np.array(clip).astype(np.float32)
# inputs = np_utils.normalize(inputs)
# inputs = inputs[:,8:120,30:142,:]
inputs /= 255.
# inputs = np.expand_dims(inputs, axis=0)
# inputs = np.transpose(inputs, (0, 2, 3, 1, 4))
input_window = []
input_window.append(inputs)
input_window = np.array(input_window).astype(np.float32)
print('----')
print(input_window.shape)
pred = model.predict(input_window)
print(pred)
label = np.argmax(pred[0])
print('#############' + str(label))
cv2.putText(frame, class_index[label], (20, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 1)
cv2.putText(frame, "prob: %.4f" % pred[0][label], (20, 40),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 1)
clip.pop(0)
cv2.imshow('result', frame)
# cv2.imwrite( "/media/lq/C13E-1ED0/dataset/c3d_ucf/img_{}.jpg".format(counter), frame );
# time.sleep(0.1)
cv2.waitKey(10)
else:
print('err')
break
cap.release()
cv2.destroyAllWindows()
def run(self):
print("predict task running")
# Loading the model
print('Loading model')
model = c3d_model.get_model()
print('Compiling model')
print('Compiling done!')
print('laoding weight-------{}'.format(weights_file))
model.load_weights(weights_file)
print("wight <{}> loaded.".format(weights_file))
print('Starting extracting features')
while not (_stop_all_generators.is_set()
and data_gen_queue.empty()):
generator_output = None
while True:
# print("----{} {} {}".format(_stop_all_extractors.is_set(),data_gen_queue.empty(),data_gen_queue.qsize()))
if not data_gen_queue.empty():
generator_output = data_gen_queue.get()
if not generator_output:
continue
break
else:
time.sleep(wait_time)
video_id, X = generator_output
print("predicting video :{}".format(video_id))
if X is None:
print('Could not be read the video {}'.format(video_id))
continue
tic = time.time()
prediction = []
'''1 window pre batch'''
# for i in range(len(X)-length+1):
# inputs = X[i:i+length]
# inputs = inputs.astype(np.float32)
# inputs /=255.
# inputs = np.expand_dims(inputs, axis=0)
# Y = model.predict_on_batch(inputs)
# prediction.append(Y)
''' > 1 window '''
X /= 255.
indexes = list(range(len(X) - length + 1))
batch_index = indexes[::batch_size]
for index in batch_index[:-1]:
inputs = []
for j in range(batch_size):
window = X[index + j:index + j + length]
inputs.append(window)
inputs = np.array(inputs)
Y = model.predict_on_batch(inputs)
prediction.extend(Y)
last_batch_index = batch_index[-1]
inputs = []
for index in range(last_batch_index, len(X) - length + 1):
window = X[index:index + length]
inputs.append(window)
inputs = np.array(inputs)
Y = model.predict_on_batch(inputs)
prediction.extend(Y)
toc = time.time()
data_save_queue.put((video_id, prediction))
print("video :{} -done! {:.2f} second(s)".format(
video_id, toc - tic))
print('prediction task stopped')
def main(id):
img_path = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Imgs/RoadAccidents/'
weights_dir = '/home/lq/Documents/Thesis/Thesis/weight'
# model_weight_filename = os.path.join(weights_dir, 'sports1M_weights_tf.h5')
model_weight_filename = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/thesis/adam_c3d_pre_train/weights/weights.04.hdf5'
N_classes = 2
batch_size = 24
epochs = 16
input_shape = (16, 120, 160, 3)
windows_length = 16
model = c3d_model.get_model()
# Setting the Learning rate multipliers
LR_mult_dict = {}
LR_mult_dict['conv1'] = 1
LR_mult_dict['conv2'] = 1
LR_mult_dict['conv3a'] = 1
LR_mult_dict['conv3b'] = 1
LR_mult_dict['conv4a'] = 1
LR_mult_dict['conv4b'] = 1
LR_mult_dict['conv5a'] = 1
LR_mult_dict['conv5b'] = 1
LR_mult_dict['fc6'] = 1
LR_mult_dict['fc7'] = 1
LR_mult_dict['fc8'] = 5
# Setting up optimizer
base_lr = 0.000001
adam = Adam(lr=base_lr, decay=0.00005, multipliers=LR_mult_dict)
# sgd = SGD(lr=base_lr, decay=0.00005, multipliers=LR_mult_dict)
opt = adam
convLayers = [
'conv1', 'conv2', 'conv3a', 'conv3b', 'conv4a', 'conv4b', 'conv5a',
'conv5b'
]
for layer in convLayers:
model.get_layer(layer).trainable = False
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.load_weights(model_weight_filename
) #, by_name = True, skip_mismatch=True, reshape=True)
######################
# for layer in model.layers:
# weights = layer.get_weights()
# print(layer)
# l = model.get_layer(name='fc8')
# list = l.get_weights()
# for l in list:
# print(l.shape)
# print(l[10])
# return 0
#####################
model.summary()
from dataUtil import load_train_data, load_val_data
train_AS_windows, train_A_windows, train_BG_windows = load_train_data(
) # load train data
N_train_samples = len(train_AS_windows) * 2
N_train_iterations = N_train_samples // batch_size
val_AS_windows, val_A_windows, val_BG_windows = load_val_data(
) # load val data
N_val_samples = len(val_A_windows) * 2 + len(
val_AS_windows
) # len(val_AS_windows) + int(0.15*len(val_A_windows))+ int(0.15*len(val_BG_windows))
# N_val_samples = len(val_AS_windows) << 1
N_val_iterations = N_val_samples // batch_size
# ####################################
print("--#train AS windows: " + str(len(train_AS_windows)) +
" #train A windows: " + str(len(train_A_windows)) +
" #train BG windows: " + str(len(train_BG_windows)))
print("-N_val_samples:" + str(N_val_samples) + "\n--#val AS windows: " +
str(len(val_AS_windows)) + " #val A windows: " +
str(len(val_A_windows)) + " #val BG windows: " +
str(len(val_BG_windows)))
# ##################################
result_dir = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/adam_c3d_{}/'.format(
id)
best_weight_dir = result_dir + 'weights'
best_weight_name = best_weight_dir + '/weights.{epoch:02d}-{val_loss:.3f}.hdf5'
if not os.path.isdir(best_weight_dir):
os.makedirs(best_weight_dir)
if not os.path.exists(best_weight_dir):
os.makedirs(result_dir)
desp = result_dir + 'desp.txt'
with open(desp, 'w') as f:
f.write(
'batch size: {}\nbase_lr: {} \ntrain_samples:{} \nval_samples:{}\n '
.format(batch_size, base_lr, N_train_samples, N_val_samples))
f.write('init_weiht: {}'.format(model_weight_filename))
f.write('batch_generator: {}'.format(train_batch_generator))
# callbacks
csv_logger = CSVLogger(result_dir + '/log.csv', separator=',')
checkpointer = ModelCheckpoint(filepath=best_weight_name,
verbose=1,
save_best_only=False,
save_weights_only=True)
# NAME = "THUMOS-{}".format(int(time.time()))
log_dir = os.path.join(result_dir, 'log')
tbCallBack = callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=0,
write_graph=True,
write_images=True)
train_generator = train_batch_generator(train_AS_windows, train_A_windows,
train_BG_windows, windows_length,
batch_size, N_train_iterations,
N_classes, img_path)
val_generator = val_batch_generator(val_AS_windows, val_A_windows,
val_BG_windows, windows_length,
batch_size, N_val_iterations,
N_classes, img_path)
history = model.fit_generator(
train_generator,
steps_per_epoch=N_train_iterations,
epochs=epochs,
callbacks=[csv_logger, tbCallBack, checkpointer],
validation_data=val_generator,
validation_steps=N_val_iterations,
verbose=1)
plot_history(history, result_dir)
def main():
input_shape = (16,112,112,3)
windows_length = 16
model = c3d_model.get_model()
model.summary()
weights_dir = './weight'
model_weight_filename = os.path.join(weights_dir, 'sports1M_weights_tf.h5')
c3d = '/media/lq/C13E-1ED0/dataset/THUMOS/result/adam_2epoches/best_weight/weights.hdf5'
c3d_TC_weights ='/home/lq/Documents/Thesis/Thesis/results/adam_temporal_8/weights/weights.02-2.111.hdf5'
c3d_TC_GAN_weights = '/home/lq/Documents/Thesis/Thesis/results/gan_9/weights/c3d_TC_GAN_21_outputs_it1500.hdf5'
model.load_weights(c3d)
# model.load_weights('results/weights_c3d.h5')
# read video
train_baseball = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000690.mp4'
train_basketball = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000901.mp4' #this video used as train data
train_billiards ='/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000055.mp4'
val_baseball = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000687.mp4'
val_basketball = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000907.mp4'
val_billiards = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000057.mp4'
val_golf = '/media/lq/C13E-1ED0/dataset/THUMOS/validation/video_validation_0000282.mp4'
test_firsbeeCatch= '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0000413.mp4'
test_baseball = '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0000664.mp4'
test_basketball= '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0000179.mp4'
test_billiards = '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0001146.mp4'
bike = '/home/lq/Documents/Thesis/C3D-keras-master/videos/v_Biking_g05_c02.avi'
golf = '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0000238.mp4'
basketball = '/media/lq/C13E-1ED0/dataset/THUMOS/tmptest/video_test_0000179.mp4'
CleanandJerk ='/media/lq/C13E-1ED0/dataset/THUMOS/111/video_test_0000698.mp4'
large = '/media/lq/C13E-1ED0/dataset/THUMOS/test_large/video_test_0000793.mp4'
test_cleanandjerk = '/media/lq/C13E-1ED0/dataset/THUMOS/test/video_test_0001307.mp4'
video = test_basketball
cap = cv2.VideoCapture(test_cleanandjerk)
clip = []
counter = 0
while True:
ret, frame = cap.read()
counter += 1
# if counter < 8250:
# continue
if ret:
#tmp = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
clip.append(cv2.resize(frame, (171, 128))[8:120,30:142,:])
if len(clip) == 16:
inputs = np.array(clip).astype(np.float32)
# inputs = np_utils.normalize(inputs)
# inputs = inputs[:,8:120,30:142,:]
inputs /= 255.
# inputs = np.expand_dims(inputs, axis=0)
# inputs = np.transpose(inputs, (0, 2, 3, 1, 4))
input_window = []
input_window.append(inputs)
input_window = np.array(input_window).astype(np.float32)
print('----')
print(input_window.shape)
pred = model.predict(input_window)
print(pred)
label = np.argmax(pred[0])
print('#############' + str(label))
text = class_index[label]
color = (0, 0, 255) if not text == 'background' else (0,0,0)
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale=0.6
(text_width, text_height) = cv2.getTextSize(text, font, fontScale=font_scale, thickness=1)[0]
text_offset_x = 20
text_offset_y = 20
# set the rectangle background to white
rectangle_bgr = (255, 255, 255)
box_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width+2, text_offset_y - text_height -5))
cv2.rectangle(frame, box_coords[0], box_coords[1], rectangle_bgr, cv2.FILLED)
cv2.putText(frame, class_index[label], (20, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.6,
color, 1)
text2 = "prob: %.4f" % pred[0][label]
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale=0.6
(text_width, text_height) = cv2.getTextSize(text2, font, fontScale=font_scale, thickness=1)[0]
text_offset_x = 20
text_offset_y = 40
# set the rectangle background to white
rectangle_bgr = (255, 255, 255)
box_coords = ((text_offset_x, text_offset_y+2), (text_offset_x + text_width +2, text_offset_y - text_height -5))
cv2.rectangle(frame, box_coords[0], box_coords[1], rectangle_bgr, cv2.FILLED)
cv2.putText(frame, text2, (20, 40),
cv2.FONT_HERSHEY_SIMPLEX, 0.6,
color, 1)
clip.pop(0)
cv2.imshow('result', frame)
# cv2.imwrite( "/media/lq/C13E-1ED0/dataset/clean_example/img_{}.jpg".format(counter), frame );
# time.sleep(0.2)
cv2.waitKey(10)
else:
break
cap.release()
cv2.destroyAllWindows()