def __data_generation(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization X = np.empty((self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, *dim, n_channels) Y = np.empty((self.batch_size), dtype=int) for i, ID in enumerate(list_IDs_temp): # ID is name of file path_file = self.path_dataset + ID print(path_file) cap = cv2.VideoCapture(path_file) print(cap.get(5)) length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) ) # get how many frames this video have index_sampling = self.get_sampling_frame( length_file) # get sampling index for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() new_image = cv2.resize(frame, self.dim) X[i, j, :, :, :] = new_image if self.type_gen == 'train': X[i, ] = self.sequence_augment(X[i, ]) / 255.0 else: X[i, ] = X[i, ] / 255.0 if self.option == 'RGBdiff': X[i, ] = calculateRGBdiff(X[i, ]) Y[i] = self.labels[ID] cap.release() return X, Y
def __data_generation(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization X = np.empty((self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, *dim, n_channels) Y = np.empty((self.batch_size), dtype=int) D = np.empty((self.batch_size, *self.dim, self.n_channels)) # X : (n_samples, *dim, n_channels) S = np.empty((self.batch_size, self.n_sequence, *self.dim)) # X : (n_samples, n_sequence, *dim) for i, ID in enumerate(list_IDs_temp): # ID is name of file path_file = os.path.join(self.path_dataset, ID + '.mp4') cap = cv2.VideoCapture(path_file) length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) ) # get how many frames this video have index_sampling = self.get_sampling_frame( length_file) # get sampling index for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() new_image = cv2.resize(frame, self.dim) X[i, j, :, :, :] = new_image if self.type_gen == 'train': X[i, ] = self.sequence_augment(X[i, ]) # apply the same rule else: X[i, ] = X[i, ] n = len(X[i, ]) # always get the last image D[i, :, :, :] = X[i, n - 1, :, :, :] # convert to gray image and normalize to 0~1 for k in range(n): gray = X[i, k, :, :] gray = np.array(gray, dtype=np.float32) gray = cv2.cvtColor(gray, cv2.COLOR_BGR2GRAY) S[i, k, :, :] = gray / 255.0 D[i, :, :, :] = D[i, :, :, :] / 255.0 if self.option == 'RGBdiff': S[i, ] = calculateRGBdiff(S[i, ]) Y[i] = self.labels[ID] cap.release() S = np.rollaxis(S, 1, 4) return D, S[:, :, :, 1:self.n_sequence], Y # remove first image
def __data_generation2(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization D = np.empty((self.batch_size, *self.dim, self.n_channels)) # X : (n_samples, *dim, n_channels) X = np.empty((self.batch_size, self.n_sequence, *self.dim)) # X : (n_samples, n_sequence, *dim) Y = np.empty((self.batch_size), dtype=int) for i, ID in enumerate(list_IDs_temp): # ID is name of file path_file = self.path_dataset + ID + '.mp4' cap = cv2.VideoCapture(path_file) length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) ) # get how many frames this video have index_sampling = self.get_sampling_frame( length_file) # get sampling index new_image = None for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() new_image = cv2.resize(frame, self.dim) gray = cv2.cvtColor(new_image, cv2.COLOR_BGR2GRAY) X[i, j, :, :] = gray D[i, :, :, :] = new_image if self.type_gen == 'train': # X[i,] = self.sequence_augment(X[i,])/255.0 X[i, ] = X[i, ] / 255.0 D[i, ] = D[i, ] / 255.0 else: X[i, ] = X[i, ] / 255.0 D[i, ] = D[i, ] / 255.0 if self.option == 'RGBdiff': X[i, ] = calculateRGBdiff(X[i, ]) # X = np.absolute(X) # X[i,] = np.delete(X[i,], 0, axis=0) # remove first image Y[i] = self.labels[ID] cap.release() X = np.rollaxis(X, 1, 4) return D, X[:, :, :, 1:self.n_sequence], Y # remove first image
def __data_generation(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization X1 = np.empty((self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, timestep, *dim, n_channels) X2 = np.empty((self.batch_size, self.n_sequence, self.n_joint*3)) Y = np.empty((self.batch_size), dtype=int) for i, ID in enumerate(list_IDs_temp): # ID is name of file (2 batch) path_video = self.path_dataset + ID + '.mp4' cap = cv2.VideoCapture(path_video) length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # get how many frames this video have ,-1 because some bug index_sampling = self.get_sampling_frame(length_file, path_video) # get sampling index # Get RGB sequence for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() # frame = self.get_crop_img(frame) new_image = cv2.resize(frame, self.dim) # new_image = frame # new_image = new_image/255.0 X1[i,j,:,:,:] = new_image if self.type_gen =='train': X1[i,] = self.sequence_augment(X1[i,])/255.0*2-1 else: X1[i,] = X1[i,]/255.0*2-1 if self.option == 'RGBdiff': # print("dddddddddddd") X1[i,] = calculateRGBdiff(X1[i,], 0) # Get label Y[i] = self.labels[ID] cap.release() X = X1 return X,Y
def __data_generation(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization X = np.empty((self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, *dim, n_channels) Y = np.zeros((self.batch_size, self.class_num)) for i, ID in enumerate(list_IDs_temp): # ID is name of file path_file = os.path.join(self.path_dataset, ID) cap = cv2.VideoCapture(path_file) length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) ) # get how many frames this video have # print(path_file,length_file) index_sampling = self.get_sampling_frame( length_file) # get sampling index for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() if ret is True: new_image = cv2.resize(frame, self.dim) X[i, j, :, :, :] = new_image else: print('read file ', path_file, 'error', length_file, n_pic) if self.type_gen == 'train': X[i, ] = self.sequence_augment(X[i, ]) # apply the same rule else: X[i, ] = X[i, ] if self.option == 'RGBdiff': X[i, ] = calculateRGBdiff(X[i, ]) Y[i][self.labels[ID] - 1] = 1.0 cap.release() return X, Y
SET_NEW_ACTION_STATE = 2 state = RUN_STATE # previous_action = -1 # no action text_show = 'no action' class_text = ['violent', 'non-violent'] cap = cv2.VideoCapture('newfi83.avi') while True: ret, frame = cap.read() frame = cv2.resize(frame, dim) frame_new = frame / 255.0 frame_new_rs = np.reshape(frame_new, (1, *frame_new.shape)) frame_window = np.append(frame_window, frame_new_rs, axis=0) if frame_window.shape[0] >= n_sequence: frame_window_dif = calculateRGBdiff(frame_window.copy()) frame_window_new = frame_window_dif.reshape(1, *frame_window_dif.shape) # print(frame_window_new.dtype) ### Predict action from model output = model.predict(frame_window_new)[0] predict_ind = np.argmax(output) ### Check noise of action if output[predict_ind] < 0.55: new_action = -1 # no action(noise) else: new_action = predict_ind # action detect ### Use State Machine to delete noise between action(just for stability) ### RUN_STATE: normal state, change to wait state when action is changed if state == RUN_STATE:
new_y1 = max( int(new_mid_y - new_length_y - new_length_y * 0.25), 0) new_y2 = min( int(new_mid_y + new_length_y + new_length_y * 0.2), height - 1) crop_image = frame[new_y1:new_y2, new_x1:new_x2] # crop_image = frame[y1:y2, x1:x2] #extract_image[i] new_f0 = cv2.resize(crop_image, dim) new_f0 = new_f0 / 255.0 new_f = np.reshape(new_f0, (1, *new_f0.shape)) frame_window[i] = np.append(frame_window[i], new_f, axis=0) # print(frame_window[i].shape) if frame_window[i].shape[0] >= n_sequence: frame_window_dif = calculateRGBdiff( frame_window[i].copy(), 0) frame_window_new = frame_window_dif.reshape( 1, *frame_window_dif.shape) # frame_window_new = frame_window[i].reshape(1, *frame_window[i].shape) result = model.predict(frame_window_new) output = result[0] predict_ind = np.argmax(output) ## Noise remove if output[predict_ind] < 0.30: new_action = -1 # no action(noise) else: new_action = predict_ind # action detect ### Use State Machine to delete noise between action(just for stability) ### RUN_STATE: normal state, change to wait state when action is changed
def __data_generation(self, list_IDs_temp): 'Generates data containing batch_size samples' # Initialization X1 = np.empty( (self.batch_size, self.n_sequence, *self.dim, self.n_channels)) # X : (n_samples, timestep, *dim, n_channels) X2 = np.empty((self.batch_size, self.n_sequence, self.n_joint * 3)) Y = np.empty((self.batch_size), dtype=int) for i, ID in enumerate(list_IDs_temp): # ID is name of file (2 batch) path_video = self.path_dataset + ID + '.mp4' # print(path_video) path_skeleton = self.path_dataset + ID + '.npy' # print(path_video) if self.type_model == '2stream' or self.type_model == 'rgb': cap = cv2.VideoCapture(path_video) length_file = int( cap.get(cv2.CAP_PROP_FRAME_COUNT) ) # get how many frames this video have ,-1 because some bug if self.type_model == '2stream' or self.type_model == 'skeleton': skeleton_data = np.load(path_skeleton) length_file = skeleton_data.shape[0] index_sampling = self.get_sampling_frame( length_file, path_video) # get sampling index # print(index_sampling) if self.type_model == '2stream' or self.type_model == 'rgb': # Get RGB sequence for j, n_pic in enumerate(index_sampling): cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index ret, frame = cap.read() # frame = self.get_crop_img(frame) new_image = cv2.resize(frame, self.dim) # new_image = frame # new_image = new_image/255.0 X1[i, j, :, :, :] = new_image if self.type_gen == 'train': X1[i, ] = self.sequence_augment(X1[i, ]) / 255.0 * 2 - 1 else: X1[i, ] = X1[i, ] / 255.0 * 2 - 1 # cv2.imshow('imgae',X1[i,0]) # cv2.waitKey(2000) if self.option == 'RGBdiff': # print("dddddddddddd") X1[i, ] = calculateRGBdiff(X1[i, ], 0) if self.type_model == '2stream' or self.type_model == 'skeleton': # Get skeleton sequence skeleton_data = skeleton_data[index_sampling] skeleton_data = skeleton_data[:, :, self.select_joint] skeleton_data = skeleton_data.reshape(self.n_sequence, self.n_joint * 3) X2[i] = skeleton_data # Get label Y[i] = self.labels[ID] if self.type_model == '2stream' or self.type_model == 'rgb': cap.release() # for i_frame in range(self.n_sequence): # cv2.imshow('Frame', X1[i,i_frame]) # cv2.waitKey(1000) if self.type_model == 'rgb': X = X1 elif self.type_model == 'skeleton': X = X2 elif self.type_model == '2stream': X = [X1, X2] return X, Y