def __define_timeception_model(): """ Define Timeception classifier. """ # some configurations for the model classification_type = config.cfg.MODEL.CLASSIFICATION_TYPE solver_name = config.cfg.SOLVER.NAME solver_lr = config.cfg.SOLVER.LR adam_epsilon = config.cfg.SOLVER.ADAM_EPSILON n_tc_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS backbone_name = config.cfg.MODEL.BACKBONE_CNN feature_name = config.cfg.MODEL.BACKBONE_FEATURE n_tc_layers = config.cfg.MODEL.N_TC_LAYERS n_classes = config.cfg.MODEL.N_CLASSES is_dilated = config.cfg.MODEL.MULTISCALE_TYPE n_channels_in, channel_h, channel_w = utils.get_model_feat_maps_info(backbone_name, feature_name) n_groups = int(n_channels_in / 128.0) # optimizer and loss for either multi-label "ml" or single-label "sl" classification if classification_type == 'ml': loss = keras_utils.LOSSES[3] output_activation = keras_utils.ACTIVATIONS[2] metric_function = keras_utils.map_charades else: loss = keras_utils.LOSSES[0] output_activation = keras_utils.ACTIVATIONS[3] metric_function = keras_utils.METRICS[0] # define the optimizer optimizer = SGD(lr=0.01) if solver_name == 'sgd' else Adam(lr=solver_lr, epsilon=adam_epsilon) # input layer input_shape = (n_tc_timesteps, channel_h, channel_w, n_channels_in) # (T, H, W, C) tensor_input = Input(shape=input_shape, name='input') # (T, H, W, C) # define timeception layers, as a standalone module timeception_module = timeception.Timeception(n_channels_in, n_tc_layers, n_groups, is_dilated=is_dilated) tensor = timeception_module(tensor_input) # (T, H, W, C) # but if you fancy, you can define timeception layers as a series of layers # tensor = timeception.timeception_layers(tensor_input, n_tc_layers, n_groups, is_dilated=is_dilated) # (T, H, W, C) # max-pool over space-time tensor = MaxLayer(axis=(1, 2, 3), name='maxpool_t_s')(tensor) # dense layers for classification tensor = Dropout(0.5)(tensor) tensor = Dense(512)(tensor) tensor = BatchNormalization()(tensor) tensor = LeakyReLU(alpha=0.2)(tensor) tensor = Dropout(0.25)(tensor) tensor = Dense(n_classes)(tensor) tensor_output = Activation(output_activation)(tensor) # define the model model = Model(inputs=tensor_input, outputs=tensor_output) model.compile(loss=loss, optimizer=optimizer, metrics=[metric_function]) return model
def __init__(self): super(Model, self).__init__() # some configurations for the model n_tc_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS #32 backbone_name = config.cfg.MODEL.BACKBONE_CNN #'i3d_pytorch_charades_rgb' feature_name = config.cfg.MODEL.BACKBONE_FEATURE #'mixed_5c' n_tc_layers = config.cfg.MODEL.N_TC_LAYERS #2 n_classes = config.cfg.MODEL.N_CLASSES #157 is_dilated = config.cfg.MODEL.MULTISCALE_TYPE #采用的多核策略类型'ks' OutputActivation = Sigmoid if config.cfg.MODEL.CLASSIFICATION_TYPE == 'ml' else LogSoftmax n_channels_in, channel_h, channel_w = utils.get_model_feat_maps_info( backbone_name, feature_name) #1024, 7, 7 n_groups = int(n_channels_in / 128.0) #8 input_shape = (None, n_channels_in, n_tc_timesteps, channel_h, channel_w ) # (None, C, T, H, W),(None, 1024, 32, 7, 7)其中T是自己设定的 self._input_shape = input_shape #(None, 1024, 32, 7, 7) # define 4 layers of timeception self.timeception = timeception_pytorch.Timeception( input_shape, n_tc_layers, n_groups, is_dilated) # get number of output channels after timeception n_channels_in = self.timeception.n_channels_out # define layers for classifier self.do1 = Dropout(0.5) self.l1 = Linear(n_channels_in, 512) self.bn1 = BatchNorm1d(512) self.ac1 = LeakyReLU(0.2) self.do2 = Dropout(0.25) self.l2 = Linear(512, n_classes) self.ac2 = OutputActivation()
def __define_data_generator(is_training): """ Define data generator. """ # get some configs for the training n_classes = config.cfg.MODEL.N_CLASSES dataset_name = config.cfg.DATASET_NAME backbone_model_name = config.cfg.MODEL.BACKBONE_CNN backbone_feature_name = config.cfg.MODEL.BACKBONE_FEATURE n_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS batch_size_tr = config.cfg.TRAIN.BATCH_SIZE batch_size_te = config.cfg.TEST.BATCH_SIZE batch_size = batch_size_tr if is_training else batch_size_te # size and name of feature feature_name = 'features_%s_%s_%sf' % (backbone_model_name, backbone_feature_name, n_timesteps) c, h, w = utils.get_model_feat_maps_info(backbone_model_name, backbone_feature_name) feature_dim = (n_timesteps, h, w, c) # data generators params = {'batch_size': batch_size, 'n_classes': n_classes, 'feature_name': feature_name, 'feature_dim': feature_dim, 'is_shuffle': True, 'is_training': is_training} data_generator_class = data_utils_keras.KERAS_DATA_GENERATORS_DICT[dataset_name] data_generator = data_generator_class(**params) return data_generator
def __init__(self): super(Model, self).__init__() # some configurations for the model n_tc_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS # 输入到timeception的timesteps backbone_name = config.cfg.MODEL.BACKBONE_CNN feature_name = config.cfg.MODEL.BACKBONE_FEATURE n_tc_layers = config.cfg.MODEL.N_TC_LAYERS # timeception的层数 n_classes = config.cfg.MODEL.N_CLASSES # 157 is_dilated = config.cfg.MODEL.MULTISCALE_TYPE # 多尺度的类型(空洞卷积/多尺度卷积核) OutputActivation = Sigmoid if config.cfg.MODEL.CLASSIFICATION_TYPE == 'ml' else LogSoftmax n_channels_in, channel_h, channel_w = utils.get_model_feat_maps_info( backbone_name, feature_name) n_groups = int(n_channels_in / 128.0) # groups的数量 # timeception层的输入shape input_shape = (None, n_channels_in, n_tc_timesteps, channel_h, channel_w) # (C, T, H, W) self._input_shape = input_shape # define 4 layers of timeception self.timeception = timeception_pytorch.Timeception( input_shape, n_tc_layers, n_groups, is_dilated) # (C, T, H, W) # get number of output channels after timeception n_channels_in = self.timeception.n_channels_out # define layers for classifier self.do1 = Dropout(0.5) self.l1 = Linear(n_channels_in, 512) self.bn1 = BatchNorm1d(512) self.ac1 = LeakyReLU(0.2) #negative_slope:控制负斜率的角度,默认等于0.01 self.do2 = Dropout(0.25) self.l2 = Linear(512, n_classes) self.ac2 = OutputActivation()
def __define_loader(is_training): """ Define data loader. """ # get some configs for the training,配置数据加载的参数 n_classes = config.cfg.MODEL.N_CLASSES #157 dataset_name = config.cfg.DATASET_NAME #charades backbone_model_name = config.cfg.MODEL.BACKBONE_CNN #i3d_pytorch_charades_rgb backbone_feature_name = config.cfg.MODEL.BACKBONE_FEATURE #mixed_5c n_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS #32 n_workers = config.cfg.TRAIN.N_WORKERS #读取数据的线程数 batch_size_tr = config.cfg.TRAIN.BATCH_SIZE #32 batch_size_te = config.cfg.TEST.BATCH_SIZE #64 batch_size = batch_size_tr if is_training else batch_size_te # size and name of feature feature_name = 'features_%s_%s_%sf' % ( backbone_model_name, backbone_feature_name, n_timesteps ) #'features _i3d_pytorch_charades_rgb_ mixed_5c_32f' c, h, w = utils.get_model_feat_maps_info(backbone_model_name, backbone_feature_name) #features_i3d_pytorch_charades_rgb,mixed_5c,||获得模型对应的feature_map的大小细节:c,h,w = 1024, 7, 7 feature_dim = (c, n_timesteps, h, w ) #特征的维度:1024, 32, 7, 7,其中的n_timesteps是自己设定的 # data generators params = { 'batch_size': batch_size, 'n_classes': n_classes, 'feature_name': feature_name, 'feature_dim': feature_dim, 'is_training': is_training } dataset_class = data_utils_pytorch.PYTORCH_DATASETS_DICT[ dataset_name] #core.data_utils_pytorch.DatasetCharades dataset = dataset_class(**params) n_samples = dataset.n_samples #7811 1814 n_batches = dataset.n_batches #245 37 data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=n_workers, shuffle=True) return data_loader, n_samples, n_batches
def __define_loader(is_training): """ Define data loader. """ # get some configs for the training n_classes = config.cfg.MODEL.N_CLASSES dataset_name = config.cfg.DATASET_NAME backbone_model_name = config.cfg.MODEL.BACKBONE_CNN backbone_feature_name = config.cfg.MODEL.BACKBONE_FEATURE n_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS n_workers = config.cfg.TRAIN.N_WORKERS batch_size_tr = config.cfg.TRAIN.BATCH_SIZE batch_size_te = config.cfg.TEST.BATCH_SIZE batch_size = batch_size_tr if is_training else batch_size_te # size and name of feature feature_name = 'features_%s_%s_%sf' % (backbone_model_name, backbone_feature_name, n_timesteps) c, h, w = utils.get_model_feat_maps_info(backbone_model_name, backbone_feature_name) feature_dim = (c, n_timesteps, h, w) # data generators params = { 'batch_size': batch_size, 'n_classes': n_classes, 'feature_name': feature_name, 'feature_dim': feature_dim, 'is_training': is_training } dataset_class = data_utils.PYTORCH_DATASETS_DICT[dataset_name] dataset = dataset_class(**params) n_samples = dataset.n_samples n_batches = dataset.n_batches data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=n_workers, shuffle=True) return data_loader, n_samples, n_batches
def __define_loader(is_training): """ Define data loader. """ # get some configs for the training n_classes = config.cfg.MODEL.N_CLASSES # charades数据集157类 dataset_name = config.cfg.DATASET_NAME # charades backbone_model_name = config.cfg.MODEL.BACKBONE_CNN # 'i3d_pytorch_charades_rgb'# which backbone cnn is used backbone_feature_name = config.cfg.MODEL.BACKBONE_FEATURE #'mixed_5c' # type of feature output from backbone cnn n_timesteps = config.cfg.MODEL.N_TC_TIMESTEPS #32 # how many timesteps expected as input to the timeception layers n_workers = config.cfg.TRAIN.N_WORKERS batch_size_tr = config.cfg.TRAIN.BATCH_SIZE #32 # batch size for training batch_size_te = config.cfg.TEST.BATCH_SIZE #64 batch_size = batch_size_tr if is_training else batch_size_te # size and name of feature feature_name = 'features_%s_%s_%sf' % (backbone_model_name, backbone_feature_name, n_timesteps) c, h, w = utils.get_model_feat_maps_info( backbone_model_name, backbone_feature_name) # 获取backbone模型的输出维度 feature_dim = (c, n_timesteps, h, w) # data generators params = { 'batch_size': batch_size, 'n_classes': n_classes, 'feature_name': feature_name, 'feature_dim': feature_dim, 'is_training': is_training } dataset_class = data_utils.PYTORCH_DATASETS_DICT[dataset_name] dataset = dataset_class(**params) n_samples = dataset.n_samples n_batches = dataset.n_batches data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=n_workers, shuffle=True) return data_loader, n_samples, n_batches
def train_model_on_pickled_features(): """ Train model. """ annotation_type = 'noun' annot_path = Pth( 'EPIC-Kitchens/annotation/annot_video_level_many_shots.pkl') (y_tr, y_te), n_classes = __load_annotation(annot_path, annotation_type) model_type = 'i3d_rgb' feature_type = 'mixed_5c' n_nodes = 128 n_timesteps = 64 n_frames_per_segment = 8 n_frames_per_video = n_timesteps * n_frames_per_segment batch_size_tr = 20 batch_size_te = 30 n_epochs = 500 epoch_offset = 0 model_name = 'classifier_%s' % (utils.timestamp()) model_root_path = Pth('EPIC-Kitchens/models') features_path = Pth( 'EPIC-Kitchens/features/features_i3d_mixed_5c_%d_frames.h5', (n_frames_per_video, )) nodes_path = Pth('EPIC-Kitchens/features_centroids/features_random_%d.pkl', (n_nodes, )) n_channels, side_dim = utils.get_model_feat_maps_info( model_type, feature_type) input_shape = (None, n_timesteps, side_dim, side_dim, n_channels) nodes = utils.pkl_load(nodes_path) print('--- start time') print(datetime.datetime.now()) # building the model print('... building model %s' % (model_name)) t1 = time.time() model = __load_model_videograph(nodes, n_classes, input_shape) t2 = time.time() duration = t2 - t1 print(model.summary(line_length=130, positions=None, print_fn=None)) print('... model built, duration (sec): %d' % (duration)) # load data print('... loading data: %s' % (features_path)) t1 = time.time() # features are extracting using datasets.Epic_Kitchens.i3d_keras_epic_kitchens() # we use out-of-box i3d (pre-trained on kinetics, NOT fine-tuned on epic-kitchens) with last conv feature 7*7*1024 'mixed_5c' # to get a better performance, you need to write code to randomly sample new frames and extract their features every new epoch # please use this function to random sampling, instead of uniform sampling: Epic_Kitchens.__random_sample_frames_per_video_for_i3d() # then extract their features, as done in: Epic_Kitchens._901_extract_features_i3d() # then train on the extracted features. Please do so in every epoch. It's computationally heavy, but you cannot avoid random sampling to get better results. # Even better results if you replace I3D with a 2D/3D CNN that's previously fine-tuned on Epic-Kitchens (x_tr, x_te) = utils.h5_load_multi(features_path, ['x_tr', 'x_te']) t2 = time.time() duration = t2 - t1 print('... data loaded: %d' % (duration)) n_tr = len(x_tr) n_te = len(x_te) n_batch_tr = utils.calc_num_batches(n_tr, batch_size_tr) n_batch_te = utils.calc_num_batches(n_te, batch_size_te) print('... [tr]: n, n_batch, batch_size: %d, %d, %d' % (n_tr, n_batch_tr, batch_size_tr)) print('... [te]: n, n_batch, batch_size: %d, %d, %d' % (n_te, n_batch_te, batch_size_te)) print(x_tr.shape) print(x_te.shape) print(y_tr.shape) print(y_te.shape) save_callback = keras_utils.ModelSaveCallback(model, model_name, epoch_offset, model_root_path) score_callback = keras_utils.MapScoreCallback(model, None, None, x_te, y_te, batch_size_te, n_classes) model_callbacks = [save_callback, score_callback] model.fit(x_tr, y_tr, epochs=n_epochs, batch_size=batch_size_tr, validation_split=0.0, validation_data=(x_te, y_te), shuffle=True, callbacks=model_callbacks, verbose=2) print('--- finish time') print(datetime.datetime.now())
def train_model_videograph(): """ Train model. """ annotation_type = 'noun' annot_path = Pth( 'EPIC-Kitchens/annotations/annot_video_level_many_shots.pkl') (y_tr, y_te), n_classes = __load_annotation(annot_path, annotation_type) model_type = 'i3d_rgb' feature_type = 'mixed_5c' n_nodes = 128 n_timesteps = 64 n_frames_per_segment = 8 n_frames_per_video = n_timesteps * n_frames_per_segment batch_size_tr = 20 batch_size_te = 30 n_epochs = 500 epoch_offset = 0 model_name = 'classifier_%s' % (utils.timestamp()) model_root_path = Pth('EPIC-Kitchens/models') nodes_path = Pth('EPIC-Kitchens/features/nodes_random_%d.pkl', (n_nodes, )) features_path = Pth( 'EPIC-Kitchens/features/features_i3d_mixed_5c_%d_frames.h5', (n_frames_per_video, )) n_channels, side_dim = utils.get_model_feat_maps_info( model_type, feature_type) input_shape = (None, n_timesteps, side_dim, side_dim, n_channels) # either load nodes, or generate them on the fly, but remeber to save them, as you need them in test time # nodes = utils.pkl_load(nodes_path) nodes = utils.generate_centroids(n_nodes, n_channels) print('--- start time') print(datetime.datetime.now()) # building the model print('... building model %s' % (model_name)) t1 = time.time() model = __load_model_videograph(nodes, n_classes, input_shape) t2 = time.time() duration = t2 - t1 print(model.summary(line_length=130, positions=None, print_fn=None)) print('... model built, duration (sec): %d' % (duration)) # load data print('... loading data: %s' % (features_path)) t1 = time.time() # features are extracting using datasets.epic_kitchens.i3d_keras_epic_kitchens() # we use out-of-box i3d (pre-trained on kinetics, NOT fine-tuned on epic-kitchens) with last conv feature 7*7*1024 'mixed_5c' (x_tr, x_te) = utils.h5_load_multi(features_path, ['x_tr', 'x_te']) t2 = time.time() duration = t2 - t1 print('... data loaded: %d' % (duration)) n_tr = len(x_tr) n_te = len(x_te) n_batch_tr = utils.calc_num_batches(n_tr, batch_size_tr) n_batch_te = utils.calc_num_batches(n_te, batch_size_te) print('... [tr]: n, n_batch, batch_size: %d, %d, %d' % (n_tr, n_batch_tr, batch_size_tr)) print('... [te]: n, n_batch, batch_size: %d, %d, %d' % (n_te, n_batch_te, batch_size_te)) print(x_tr.shape) print(x_te.shape) print(y_tr.shape) print(y_te.shape) save_callback = keras_utils.ModelSaveCallback(model, model_name, epoch_offset, model_root_path) score_callback = keras_utils.MapScoreCallback(model, None, None, x_te, y_te, batch_size_te, n_classes) model_callbacks = [save_callback, score_callback] model.fit(x_tr, y_tr, epochs=n_epochs, batch_size=batch_size_tr, validation_split=0.0, validation_data=(x_te, y_te), shuffle=True, callbacks=model_callbacks, verbose=2) print('--- finish time') print(datetime.datetime.now())
def train_model_on_pickled_features(): """ Train model. """ model_type = 'i3d_rgb' feature_type = 'mixed_5c' n_centroids = 128 n_timesteps = 64 is_spatial_pooling = True is_resume_training = False batch_size_tr = 12 batch_size_te = 30 n_epochs = 100 n_classes = N_CLASSES n_gpus = 1 model_name = 'classifier_%s' % (utils.timestamp()) model_weight_path = '' features_path = Pth('Breakfast/features/features_i3d_mixed_5c_%d_frames_max_pool.h5', (n_timesteps * 8,)) if is_spatial_pooling else Pth('Breakfast/features/features_i3d_mixed_5c_%d_frames.h5', (n_timesteps * 8,)) centroids_path = Pth('Breakfast/features_centroids/features_random_%d_centroids.pkl', (n_centroids,)) gt_actions_path = Pth('Breakfast/annotation/gt_unit_actions.pkl') (_, y_tr), (_, y_te) = utils.pkl_load(gt_actions_path) centroids = utils.pkl_load(centroids_path) n_feat_maps, feat_map_side_dim = utils.get_model_feat_maps_info(model_type, feature_type) feat_map_side_dim = 1 if is_spatial_pooling else feat_map_side_dim input_shape = (None, n_timesteps, feat_map_side_dim, feat_map_side_dim, n_feat_maps) print ('--- start time') print (datetime.datetime.now()) # building the model print('... building model %s' % (model_name)) t1 = time.time() root_model, model = __load_model_mlp_classifier_video_graph(centroids, n_classes, input_shape, n_gpus=n_gpus, is_load_weights=is_resume_training, weight_path=model_weight_path) t2 = time.time() duration = t2 - t1 print (root_model.summary(line_length=130, positions=None, print_fn=None)) print ('... model built, duration (sec): %d' % (duration)) # load data print ('... loading data: %s' % (features_path)) print ('... centroids: %s' % (centroids_path)) t1 = time.time() (x_tr, x_te) = utils.h5_load_multi(features_path, ['x_tr', 'x_te']) t2 = time.time() duration = t2 - t1 print ('... data loaded: %d' % (duration)) print(x_tr.shape) print(y_tr.shape) print(x_te.shape) print(y_te.shape) n_tr = len(x_tr) n_te = len(x_te) n_batch_tr = utils.calc_num_batches(n_tr, batch_size_tr) n_batch_te = utils.calc_num_batches(n_te, batch_size_te) print ('... [tr]: n, n_batch, batch_size, n_gpus: %d, %d, %d, %d' % (n_tr, n_batch_tr, batch_size_tr, n_gpus)) print ('... [te]: n, n_batch, batch_size, n_gpus: %d, %d, %d, %d' % (n_te, n_batch_te, batch_size_te, n_gpus)) score_callback = ScoreCallback(model, None, None, x_te, y_te, batch_size_te) callbacks = [score_callback] model.fit(x_tr, y_tr, epochs=n_epochs, batch_size=batch_size_tr, validation_split=0.0, validation_data=(x_te, y_te), shuffle=True, callbacks=callbacks, verbose=2) print ('--- finish time') print (datetime.datetime.now())