def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = self.path_dataset + ID
print(path_file)
cap = cv2.VideoCapture(path_file)
print(cap.get(5))
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X[i, ] = self.sequence_augment(X[i, ]) / 255.0
else:
X[i, ] = X[i, ] / 255.0
if self.option == 'RGBdiff':
X[i, ] = calculateRGBdiff(X[i, ])
Y[i] = self.labels[ID]
cap.release()
return X, Y
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
Y = np.empty((self.batch_size), dtype=int)
D = np.empty((self.batch_size, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
S = np.empty((self.batch_size, self.n_sequence,
*self.dim)) # X : (n_samples, n_sequence, *dim)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = os.path.join(self.path_dataset, ID + '.mp4')
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X[i, ] = self.sequence_augment(X[i, ]) # apply the same rule
else:
X[i, ] = X[i, ]
n = len(X[i, ])
# always get the last image
D[i, :, :, :] = X[i, n - 1, :, :, :]
# convert to gray image and normalize to 0~1
for k in range(n):
gray = X[i, k, :, :]
gray = np.array(gray, dtype=np.float32)
gray = cv2.cvtColor(gray, cv2.COLOR_BGR2GRAY)
S[i, k, :, :] = gray / 255.0
D[i, :, :, :] = D[i, :, :, :] / 255.0
if self.option == 'RGBdiff':
S[i, ] = calculateRGBdiff(S[i, ])
Y[i] = self.labels[ID]
cap.release()
S = np.rollaxis(S, 1, 4)
return D, S[:, :, :, 1:self.n_sequence], Y # remove first image
def __data_generation2(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
D = np.empty((self.batch_size, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
X = np.empty((self.batch_size, self.n_sequence,
*self.dim)) # X : (n_samples, n_sequence, *dim)
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = self.path_dataset + ID + '.mp4'
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
new_image = None
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
gray = cv2.cvtColor(new_image, cv2.COLOR_BGR2GRAY)
X[i, j, :, :] = gray
D[i, :, :, :] = new_image
if self.type_gen == 'train':
# X[i,] = self.sequence_augment(X[i,])/255.0
X[i, ] = X[i, ] / 255.0
D[i, ] = D[i, ] / 255.0
else:
X[i, ] = X[i, ] / 255.0
D[i, ] = D[i, ] / 255.0
if self.option == 'RGBdiff':
X[i, ] = calculateRGBdiff(X[i, ])
# X = np.absolute(X)
# X[i,] = np.delete(X[i,], 0, axis=0) # remove first image
Y[i] = self.labels[ID]
cap.release()
X = np.rollaxis(X, 1, 4)
return D, X[:, :, :, 1:self.n_sequence], Y # remove first image
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X1 = np.empty((self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, timestep, *dim, n_channels)
X2 = np.empty((self.batch_size, self.n_sequence, self.n_joint*3))
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file (2 batch)
path_video = self.path_dataset + ID + '.mp4'
cap = cv2.VideoCapture(path_video)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # get how many frames this video have ,-1 because some bug
index_sampling = self.get_sampling_frame(length_file, path_video) # get sampling index
# Get RGB sequence
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
# frame = self.get_crop_img(frame)
new_image = cv2.resize(frame, self.dim)
# new_image = frame
# new_image = new_image/255.0
X1[i,j,:,:,:] = new_image
if self.type_gen =='train':
X1[i,] = self.sequence_augment(X1[i,])/255.0*2-1
else:
X1[i,] = X1[i,]/255.0*2-1
if self.option == 'RGBdiff':
# print("dddddddddddd")
X1[i,] = calculateRGBdiff(X1[i,], 0)
# Get label
Y[i] = self.labels[ID]
cap.release()
X = X1
return X,Y
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
Y = np.zeros((self.batch_size, self.class_num))
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = os.path.join(self.path_dataset, ID)
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
# print(path_file,length_file)
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
if ret is True:
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
else:
print('read file ', path_file, 'error', length_file, n_pic)
if self.type_gen == 'train':
X[i, ] = self.sequence_augment(X[i, ]) # apply the same rule
else:
X[i, ] = X[i, ]
if self.option == 'RGBdiff':
X[i, ] = calculateRGBdiff(X[i, ])
Y[i][self.labels[ID] - 1] = 1.0
cap.release()
return X, Y
SET_NEW_ACTION_STATE = 2
state = RUN_STATE #
previous_action = -1 # no action
text_show = 'no action'
class_text = ['violent', 'non-violent']
cap = cv2.VideoCapture('newfi83.avi')
while True:
ret, frame = cap.read()
frame = cv2.resize(frame, dim)
frame_new = frame / 255.0
frame_new_rs = np.reshape(frame_new, (1, *frame_new.shape))
frame_window = np.append(frame_window, frame_new_rs, axis=0)
if frame_window.shape[0] >= n_sequence:
frame_window_dif = calculateRGBdiff(frame_window.copy())
frame_window_new = frame_window_dif.reshape(1, *frame_window_dif.shape)
# print(frame_window_new.dtype)
### Predict action from model
output = model.predict(frame_window_new)[0]
predict_ind = np.argmax(output)
### Check noise of action
if output[predict_ind] < 0.55:
new_action = -1 # no action(noise)
else:
new_action = predict_ind # action detect
### Use State Machine to delete noise between action(just for stability)
### RUN_STATE: normal state, change to wait state when action is changed
if state == RUN_STATE:
new_y1 = max(
int(new_mid_y - new_length_y - new_length_y * 0.25), 0)
new_y2 = min(
int(new_mid_y + new_length_y + new_length_y * 0.2),
height - 1)
crop_image = frame[new_y1:new_y2, new_x1:new_x2]
# crop_image = frame[y1:y2, x1:x2] #extract_image[i]
new_f0 = cv2.resize(crop_image, dim)
new_f0 = new_f0 / 255.0
new_f = np.reshape(new_f0, (1, *new_f0.shape))
frame_window[i] = np.append(frame_window[i], new_f, axis=0)
# print(frame_window[i].shape)
if frame_window[i].shape[0] >= n_sequence:
frame_window_dif = calculateRGBdiff(
frame_window[i].copy(), 0)
frame_window_new = frame_window_dif.reshape(
1, *frame_window_dif.shape)
# frame_window_new = frame_window[i].reshape(1, *frame_window[i].shape)
result = model.predict(frame_window_new)
output = result[0]
predict_ind = np.argmax(output)
## Noise remove
if output[predict_ind] < 0.30:
new_action = -1 # no action(noise)
else:
new_action = predict_ind # action detect
### Use State Machine to delete noise between action(just for stability)
### RUN_STATE: normal state, change to wait state when action is changed
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X1 = np.empty(
(self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, timestep, *dim, n_channels)
X2 = np.empty((self.batch_size, self.n_sequence, self.n_joint * 3))
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file (2 batch)
path_video = self.path_dataset + ID + '.mp4'
# print(path_video)
path_skeleton = self.path_dataset + ID + '.npy'
# print(path_video)
if self.type_model == '2stream' or self.type_model == 'rgb':
cap = cv2.VideoCapture(path_video)
length_file = int(
cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have ,-1 because some bug
if self.type_model == '2stream' or self.type_model == 'skeleton':
skeleton_data = np.load(path_skeleton)
length_file = skeleton_data.shape[0]
index_sampling = self.get_sampling_frame(
length_file, path_video) # get sampling index
# print(index_sampling)
if self.type_model == '2stream' or self.type_model == 'rgb':
# Get RGB sequence
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES,
n_pic) # jump to that index
ret, frame = cap.read()
# frame = self.get_crop_img(frame)
new_image = cv2.resize(frame, self.dim)
# new_image = frame
# new_image = new_image/255.0
X1[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X1[i, ] = self.sequence_augment(X1[i, ]) / 255.0 * 2 - 1
else:
X1[i, ] = X1[i, ] / 255.0 * 2 - 1
# cv2.imshow('imgae',X1[i,0])
# cv2.waitKey(2000)
if self.option == 'RGBdiff':
# print("dddddddddddd")
X1[i, ] = calculateRGBdiff(X1[i, ], 0)
if self.type_model == '2stream' or self.type_model == 'skeleton':
# Get skeleton sequence
skeleton_data = skeleton_data[index_sampling]
skeleton_data = skeleton_data[:, :, self.select_joint]
skeleton_data = skeleton_data.reshape(self.n_sequence,
self.n_joint * 3)
X2[i] = skeleton_data
# Get label
Y[i] = self.labels[ID]
if self.type_model == '2stream' or self.type_model == 'rgb':
cap.release()
# for i_frame in range(self.n_sequence):
# cv2.imshow('Frame', X1[i,i_frame])
# cv2.waitKey(1000)
if self.type_model == 'rgb':
X = X1
elif self.type_model == 'skeleton':
X = X2
elif self.type_model == '2stream':
X = [X1, X2]
return X, Y