def __init__(self, sample_size, num_seq=8, seq_len=5, pred_step=3, network='resnet50'): super(DPC_RNN_Infer_Pred_Error, self).__init__() torch.cuda.manual_seed(233) print('Using DPC-RNN model') self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.pred_step = pred_step self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) print('final feature map has size %dx%d' % (self.last_size, self.last_size)) self.backbone, self.param = select_resnet(network, track_running_stats=False) self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0) ) self.mask = None self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred)
def __init__(self, sample_size, num_seq, seq_len, network='resnet18', dropout=0.5, num_class=101): ''' Original DPC, according to diagram in appendix No future prediction network involved num_class: If integer => single output layer; if list => multiple output layers (for example verb + noun) ''' super(LC_present, self).__init__() torch.cuda.manual_seed(666) # very innocent number self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.num_class = num_class print('=> Using RNN + FC model with num_class:', num_class) print('=> Use 2D-3D %s!' % network) self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) track_running_stats = True self.backbone, self.param = select_resnet( network, track_running_stats=track_running_stats) self.param['num_layers'] = 1 self.param['hidden_size'] = self.param['feature_size'] print('=> using ConvRNN, kernel_size = 1') self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) self._initialize_weights(self.agg) self.final_bn = nn.BatchNorm1d(self.param['feature_size']) self.final_bn.weight.data.fill_(1) self.final_bn.bias.data.zero_() # Initializer final FC layer(s), one or multiple, depending on class configuration if isinstance(num_class, int): # Single self.multi_output = False self.final_fc = nn.Sequential(nn.Dropout(dropout), nn.Linear(self.param['feature_size'], self.num_class)) self._initialize_weights(self.final_fc) elif isinstance(num_class, list): # Multi self.multi_output = True self.final_fc = [] for cur_num_cls in num_class: cur_fc = nn.Sequential(nn.Dropout(dropout), nn.Linear(self.param['feature_size'], cur_num_cls)) self._initialize_weights(cur_fc) self.final_fc.append(cur_fc) # IMPORTANT, otherwise pytorch won't register self.final_fc = nn.ModuleList(self.final_fc) else: raise ValueError( 'num_class is of unknown type (expected int or list of ints)')
def __init__(self, args): super(DpcRnn, self).__init__() torch.cuda.manual_seed(233) print('Using DPC-RNN model for mode: {}'.format(args["mode"])) self.num_seq = args["num_seq"] self.seq_len = args["seq_len"] self.pred_step = args["pred_step"] self.sample_size = args["img_dim"] self.last_duration = int(math.ceil(self.seq_len / 4)) self.last_size = int(math.ceil(self.sample_size / 32)) print('final feature map has size %dx%d' % (self.last_size, self.last_size)) self.mode = args["mode"] self.in_channels = get_num_channels(self.mode) self.l2_norm = args["l2_norm"] track_running_stats = True print("Track running stats: {}".format(track_running_stats)) self.backbone, self.param = select_resnet( args["net"], track_running_stats=track_running_stats, in_channels=self.in_channels ) # params for GRU self.param['num_layers'] = 1 self.param['hidden_size'] = self.param['feature_size'] # param for current model self.final_feature_size = self.param["feature_size"] # self.final_feature_size = self.param['hidden_size'] * (self.last_size ** 2) self.total_feature_size = self.param['hidden_size'] * (self.last_size ** 2) self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0) ) self.compiled_features = self.get_modality_feature_extractor() self.interModeDotHandler = su.InterModeDotHandler(self.last_size) self.cosSimHandler = su.CosSimHandler() self.mask = None # self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred)
def __init__(self, sample_size, num_seq, seq_len, in_channels, network='resnet18', dropout=0.5, num_class=101): super(LC, self).__init__() torch.cuda.manual_seed(666) self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.num_class = num_class self.in_channels = in_channels print('=> Using RNN + FC model with ic:', self.in_channels) print('=> Use 2D-3D %s!' % network) self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) track_running_stats = True self.backbone, self.param = \ select_resnet(network, track_running_stats=track_running_stats, in_channels=self.in_channels) self.param['num_layers'] = 1 self.param['hidden_size'] = self.param['feature_size'] print('=> using ConvRNN, kernel_size = 1') self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) self._initialize_weights(self.agg) self.final_bn = nn.BatchNorm1d(self.param['feature_size']) self.final_bn.weight.data.fill_(1) self.final_bn.bias.data.zero_() self.num_classes = num_class self.dropout = dropout self.hidden_size = 128 self.final_fc = nn.Sequential( nn.Dropout(self.dropout), nn.Linear(self.param['feature_size'], self.num_classes), ) self._initialize_weights(self.final_fc)
def __init__(self, img_dim, network='resnet50', num_layers_in_fc_layers=1024, dropout=0.5): super(Audio_RNN, self).__init__() self.__nFeatures__ = 24 self.__nChs__ = 32 self.__midChs__ = 32 self.netcnnaud = nn.Sequential( nn.Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.BatchNorm2d(64), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1)), nn.Conv2d(64, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.BatchNorm2d(192), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=(3, 3), stride=(1, 2)), nn.Conv2d(192, 384, kernel_size=(3, 3), padding=(1, 1)), nn.BatchNorm2d(384), nn.ReLU(inplace=True), nn.Conv2d(384, 256, kernel_size=(3, 3), padding=(1, 1)), nn.BatchNorm2d(256), nn.ReLU(inplace=True), nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)), nn.BatchNorm2d(256), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=(3, 3), stride=(2, 2)), nn.Conv2d(256, 512, kernel_size=(5, 4), padding=(0, 0)), nn.BatchNorm2d(512), nn.ReLU(), ) self.netfcaud = nn.Sequential( nn.Linear(512 * 21, 4096), nn.BatchNorm1d(4096), nn.ReLU(), nn.Linear(4096, num_layers_in_fc_layers), ) self.netcnnlip, self.param = select_resnet(network, track_running_stats=False) self.last_duration = int(math.ceil(30 / 4)) self.last_size = int(math.ceil(img_dim / 32)) self.netfclip = nn.Sequential( nn.Linear( self.param['feature_size'] * self.last_size * self.last_size, 4096), nn.BatchNorm1d(4096), nn.ReLU(), nn.Linear(4096, num_layers_in_fc_layers), ) self.final_bn_lip = nn.BatchNorm1d(num_layers_in_fc_layers) self.final_bn_lip.weight.data.fill_(1) self.final_bn_lip.bias.data.zero_() self.final_fc_lip = nn.Sequential( nn.Dropout(dropout), nn.Linear(num_layers_in_fc_layers, 2)) self._initialize_weights(self.final_fc_lip) self.final_bn_aud = nn.BatchNorm1d(num_layers_in_fc_layers) self.final_bn_aud.weight.data.fill_(1) self.final_bn_aud.bias.data.zero_() self.final_fc_aud = nn.Sequential( nn.Dropout(dropout), nn.Linear(num_layers_in_fc_layers, 2)) self._initialize_weights(self.final_fc_aud) self._initialize_weights(self.netcnnaud) self._initialize_weights(self.netfcaud) self._initialize_weights(self.netfclip)
def __init__(self, sample_size, num_seq=8, seq_len=5, pred_step=3, network='resnet50', distance='dot', poincare_c=1.0, poincare_ball_dim=256): super(DPC_RNN, self).__init__() # to reproduce the experiments torch.cuda.manual_seed(233) print('Using DPC-RNN model') # number of dimensions in the image self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.distance = distance # how many futures to predict self.pred_step = pred_step # 2 if seq_len is 5 if network == 'resnet8' or network == 'resnet10': self.last_duration = int(math.ceil(seq_len / 2)) else: self.last_duration = int(math.ceil(seq_len / 4)) # 4 if size of the image is 128 # change for toy experiment #self.last_size = 1 self.last_size = int(math.ceil(sample_size / 32)) print('final feature map has size %dx%d' % (self.last_size, self.last_size)) # f - choose an appropriate feature extractor. In this case, a resent self.backbone, self.param = select_resnet( network, track_running_stats=False, distance=self.distance) #print (self.param) self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) # two layered network \phi self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0) ) # what does mask do ? self.mask = None self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred) # exponential map self.tp = hypnn.ToPoincare(c=1.0, train_x=True, train_c=True, ball_dim=self.param['feature_size'])
def __init__(self, sample_size, num_seq, seq_len, pred_step, network='resnet18', dropout=0.5, num_class=101): ''' modified from dpc-eval model. Different from the original model, pred function is run pred_step number of times just like pretext training. The last context variable is used to project to the action space num_class: If integer => single output layer; if list => multiple output layers (for example verb + noun) ''' super(LC_future_DPC, self).__init__() torch.cuda.manual_seed(666) # size of the image 128x128 self.sample_size = sample_size # num_seq = 5 and seq_len = 8 self.num_seq = num_seq self.seq_len = seq_len self.pred_step = pred_step self.num_class = num_class print('=> Using RNN + FC model ') print('=> Use 2D-3D %s!' % network) # dimensions of the output self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) track_running_stats = True # f network (= extract representation given video) self.backbone, self.param = select_resnet( network, track_running_stats=track_running_stats) print('feature_size:', self.param['feature_size']) self.param['num_layers'] = 1 self.param['hidden_size'] = self.param['feature_size'] # g-network (= aggregate into present context) print('=> using ConvRNN, kernel_size = 1') self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) # two layered network \phi (= predict future given context) self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0)) self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred) # not in the training network self.final_bn = nn.BatchNorm1d(self.param['feature_size']) self.final_bn.weight.data.fill_(1) self.final_bn.bias.data.zero_() # Initializer final FC layer(s), one or multiple, depending on class configuration if isinstance(num_class, int): # Single self.multi_output = False self.final_fc = nn.Sequential( nn.Dropout(dropout), nn.Linear(self.param['feature_size'], self.num_class)) self._initialize_weights(self.final_fc) elif isinstance(num_class, list): # Multi, for predicting noun and verb simultaneously self.multi_output = True self.final_fc = [] for cur_num_cls in num_class: cur_fc = nn.Sequential( nn.Dropout(dropout), nn.Linear(self.param['feature_size'], cur_num_cls)) self._initialize_weights(cur_fc) self.final_fc.append(cur_fc) # IMPORTANT, otherwise pytorch won't register self.final_fc = nn.ModuleList(self.final_fc) else: raise ValueError( 'num_class is of unknown type (expected int or list of ints)')
def __init__(self, img_dim, num_seq=8, seq_len=5, pred_step=3, network='resnet50', latent_size=8, kernel_size=1, rnn_dropout=0.1, action_cls_head=None, cls_dropout=0.5, num_class=101, time_indep=False): super(DPC_VRNN, self).__init__() torch.cuda.manual_seed(233) print('Using DPC-VRNN model, feature extractor: ' + network + ', latent size: ' + str(latent_size) + ', kernel size: ' + str(kernel_size)) self.img_dim = img_dim self.num_seq = num_seq self.seq_len = seq_len self.pred_step = pred_step self.latent_size = latent_size self.kernel_size = kernel_size self.rnn_dropout = rnn_dropout self.action_cls_head = action_cls_head self.cls_dropout = cls_dropout self.num_class = num_class self.time_indep = time_indep if action_cls_head is not None: print('Action classification head(s) enabled for: ' + action_cls_head) if time_indep: print( 'Temporal independence => ablation study enabled => VRNN is now like CVAE' ) if network == 'resnet8' or network == 'resnet10': # 3 if seq_len is 5 self.last_duration = int(math.ceil(seq_len / 2)) else: # 2 if seq_len is 5 self.last_duration = int(math.ceil(seq_len / 4)) # 4 if size of the image is 128 self.spatial_size = int(math.ceil(img_dim / 32)) # Feature extractor f self.feat_backbone, self.param = select_resnet( network, track_running_stats=False) # Context aggregator g and projection function phi governed by random z self.input_size = self.param['feature_size'] self.state_size = self.param['feature_size'] self.param[ 'input_size'] = self.input_size # for input w_t and output ^w_t self.param['state_size'] = self.state_size # for context c_t self.param[ 'latent_size'] = self.latent_size # for probabilistic latent vector z_t self.param['kernel_size'] = self.kernel_size # for all conv layers self.param['spatial_size'] = self.spatial_size # H, W self.param['rnn_dropout'] = self.rnn_dropout self.vrnn_backbone = My_VRNN_Conv_GRU( input_size=self.param['input_size'], state_size=self.param['state_size'], latent_size=self.param['latent_size'], spatial_size=self.param['spatial_size'], kernel_size=self.param['kernel_size'], dropout=self.param['rnn_dropout'], time_indep=self.time_indep) self.sim_div_dropout = nn.Dropout( p=self.rnn_dropout, inplace=False) # for diversity measurements self.mask = None self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.vrnn_backbone) if action_cls_head is not None: # Initializer final FC layer(s), one or multiple, depending on class configuration if isinstance(num_class, int): num_class = [num_class] # singleton to simplify code assert (isinstance(num_class, list)) self.num_class = num_class self.final_bn = nn.BatchNorm1d(self.param['state_size']) self.final_bn.weight.data.fill_(1) self.final_bn.bias.data.zero_() self.final_fc = [] for cur_num_cls in num_class: cur_fc = nn.Sequential( nn.Dropout(cls_dropout), nn.Linear(self.param['state_size'], cur_num_cls)) self._initialize_weights(cur_fc) self.final_fc.append(cur_fc) self.final_fc = nn.ModuleList( self.final_fc) # IMPORTANT, otherwise pytorch won't register
def __init__(self, sample_size, num_seq=8, seq_len=5, pred_step=3, network='resnet10', distance='L2', distance_type='uncertain', positive_vs_negative='same', radius_type='linear', radius_which='pred'): super(DPC_RNN, self).__init__() # to reproduce the experiments torch.cuda.manual_seed(233) print('[model_3d.py] Using DPC-RNN model') # number of dimensions in the image self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.distance = distance self.distance_type = distance_type self.positive_vs_negative = positive_vs_negative self.radius_which = radius_which self.radius_type = radius_type print('[model_3d.py] Using distance metric : ', self.distance) print('[model_3d.py] Using distance type : ', self.distance_type) print('[model_3d.py] Treating positive and negative instances as : ', self.positive_vs_negative) print('[model_3d.py] Using radius type : ', self.radius_type) # how many futures to predict self.pred_step = pred_step # what is sample size ? # 2 if seq_len is 5 if network == 'resnet8' or network == 'resnet10': self.last_duration = int(math.ceil(seq_len / 2)) else: self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) # print('final feature map has size %dx%d' % # (self.last_size, self.last_size)) # f - choose an appropriate feature extractor. In this case, a resent self.backbone, self.param = select_resnet( network, track_running_stats=False, distance_type=self.distance_type, radius_type=self.radius_type) #print (self.param) # number of layers in GRU self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers'], radius_type=self.radius_type) # two layered network \phi self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0)) if self.radius_type == 'log' and self.distance_type == 'uncertain': print('[model_3d.py] Using log as radius_type') self.activation = exp_activation() # what does mask do ? self.mask = None self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred)
def __init__(self, sample_size, num_seq=8, seq_len=5, pred_step=3, network='monkeynet'): super(DPC_Plus, self).__init__() # torch.cuda.manual_seed(233) #233 print('Using DPC-RNN model') self.sample_size = sample_size self.num_seq = num_seq self.seq_len = seq_len self.pred_step = pred_step if network == 'vgg' or network == 'mousenet' or network == 'simmousenet' or network == 'monkeynet': self.last_duration = seq_len else: self.last_duration = int(math.ceil(seq_len / 4)) if network == 'resnet0': self.last_size = int(math.ceil(sample_size / 8)) #8 self.pool_size = 1 elif network == 'mousenet': self.last_size = 16 self.pool_size = 2 # (2 for all readout, 4 for VISp5 readout) elif network == 'simmousenet': self.last_size = 16 self.pool_size = 1 elif network == 'monkeynet': self.last_size = 16 self.pool_size = 1 else: self.last_size = int(math.ceil(sample_size / 32)) self.pool_size = 1 print('final feature map has size %dx%d' % (self.last_size, self.last_size)) if network == 'mousenet': self.backbone, self.param = select_mousenet() elif network == 'simmousenet': self.backbone, self.param = select_simmousenet(hp) elif network == 'monkeynet': self.backbone, self.param = select_monkeynet() else: self.backbone, self.param = select_resnet( network, track_running_stats=False) self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0)) self.mask = None self.relu = nn.ReLU(inplace=False) self.linear1_1 = nn.Linear(self.backbone.path1.resblocks_out_channels, 64) self.linear1_2 = nn.Linear(self.backbone.path1.resblocks_out_channels, 64) self.linear2_1 = nn.Linear(64, 2) self.linear2_2 = nn.Linear(64, 1) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred)
def __init__(self, sample_size, num_seq=5, seq_len=5, network='resnet18', distance_type='uncertain', feature_type='F', pred_steps=1, pool='avg', radius_location='Phi'): super(model_visualize, self).__init__() torch.cuda.manual_seed(666) # size of the image 128x128 self.sample_size = sample_size self.distance_type = distance_type self.feature_type = feature_type self.network = network self.pool = pool self.pred_steps = pred_steps self.radius_location = radius_location # num_seq = 5 and seq_len = 8 self.num_seq = num_seq self.seq_len = seq_len if self.feature_type == 'F': print('[model_visualize.py] Using <<F>> mapping ') elif self.feature_type == 'G': print('[model_visualize.py] Using <<F+G>> mapping ') elif self.feature_type == 'Phi': print('[model_visualize.py] Using <<F+G+Phi>> mapping ') print('[model_visualize.py] Use 2D-3D %s!' % network) # dimensions of the output if self.network == 'resnet8' or self.network == 'resnet10': self.last_duration = int(math.ceil(seq_len / 2)) else: self.last_duration = int(math.ceil(seq_len / 4)) self.last_size = int(math.ceil(sample_size / 32)) track_running_stats = True # f network print('[model_visualize.py] Using distance type : <<', self.distance_type, ' >>') # f - choose an appropriate feature extractor. In this case, a resent if self.radius_location == 'Phi': self.backbone, self.param = select_resnet( network, track_running_stats=False, distance_type='certain') elif self.radius_location == 'F': self.backbone, self.param = select_resnet( network, track_running_stats=False, distance_type=self.distance_type) self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) # two layered network \phi if self.radius_location == 'Phi': if self.distance_type == 'certain': output_size = self.param['feature_size'] elif self.distance_type == 'uncertain': output_size = self.param['feature_size'] + 1 self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], output_size, kernel_size=1, padding=0)) elif self.radius_location == 'F': self.network_pred = nn.Sequential( nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv2d(self.param['feature_size'], self.param['feature_size'], kernel_size=1, padding=0)) self.avg_pool = nn.AvgPool3d((1, self.last_size, self.last_size), stride=1) self.max_pool = nn.MaxPool3d((1, self.last_size, self.last_size), stride=1) # what does mask do ? self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred)
def __init__(self, img_dim, num_seq=8, seq_len=5, pred_step=3, network='resnet50', cvae_arch='fc', action_cls_head=False, dropout=0.5, num_class=101): super(DPC_CVAE, self).__init__() # to reproduce the experiments torch.cuda.manual_seed(233) print('Using DPC-CVAE model ' + network + ' ' + cvae_arch) # number of dimensions in the image self.img_dim = img_dim self.num_seq = num_seq self.seq_len = seq_len self.action_cls_head = action_cls_head if action_cls_head: print('Action classification head(s) enabled with final FC') # if force_encode_train: # print('::: WARNING ::: Gaussian parameter encoding will take place during TRAIN, which might inflate accuracy!') # if force_encode_eval: # print('::: WARNING ::: Gaussian parameter encoding will take place during EVAL, which will inflate accuracy!') # how many futures to predict self.pred_step = pred_step # 2 if seq_len is 5 if network == 'resnet8' or network == 'resnet10': self.last_duration = int(math.ceil(seq_len / 2)) else: self.last_duration = int(math.ceil(seq_len / 4)) # 4 if size of the image is 128 self.last_size = int(math.ceil(img_dim / 32)) self.spatial_size = self.last_size print('final feature map has size %dx%d' % (self.last_size, self.last_size)) # f - choose an appropriate feature extractor. In this case, a resent self.backbone, self.param = select_resnet(network, track_running_stats=False) #print (self.param) self.param['num_layers'] = 1 # param for GRU self.param['hidden_size'] = self.param['feature_size'] # param for GRU # Converts input (video block representation) + old hidden state to new hidden state self.agg = ConvGRU(input_size=self.param['feature_size'], hidden_size=self.param['hidden_size'], kernel_size=1, num_layers=self.param['num_layers']) # two layered network \phi # Replaced with CVAE # self.network_pred = nn.Sequential( # nn.Conv2d(self.param['feature_size'], # self.param['feature_size'], kernel_size=1, padding=0), # nn.ReLU(inplace=True), # nn.Conv2d(self.param['feature_size'], # self.param['feature_size'], kernel_size=1, padding=0) # ) if cvae_arch == 'fc': print('Using CVAE class: My_CVAE_FC') self.network_pred_cvae = My_CVAE_FC( self.param['feature_size'] * self.last_size * self.last_size, self.param['feature_size'] * self.last_size * self.last_size, latent_size=256, hidden_size=1024) elif cvae_arch == 'conv' or cvae_arch == 'conv_a': # Conv 1x1 version A print( 'Using CVAE class: My_CVAE_Conv1x1_A (latent=64x4x4, hidden=128x4x4)' ) self.network_pred_cvae = My_CVAE_Conv1x1( self.param['feature_size'], self.param['feature_size'], latent_size=64, hidden_size=128) elif cvae_arch == 'conv_b': # Conv 1x1 version B (smaller latent dimension) print( 'Using CVAE class: My_CVAE_Conv1x1_B (latent=16x4x4, hidden=128x4x4)' ) self.network_pred_cvae = My_CVAE_Conv1x1( self.param['feature_size'], self.param['feature_size'], latent_size=16, hidden_size=128) elif cvae_arch == 'conv_c': # Conv 1x1 version C (even smaller latent dimension) print( 'Using CVAE class: My_CVAE_Conv1x1_C (latent=4x4x4, hidden=128x4x4)' ) self.network_pred_cvae = My_CVAE_Conv1x1( self.param['feature_size'], self.param['feature_size'], latent_size=4, hidden_size=128) elif cvae_arch == 'conv_d': # Conv + FC version D (global latent space) print( 'Using CVAE class: My_CVAE_ConvFC (latent=8x1x1, hidden=256x4x4)' ) self.network_pred_cvae = My_CVAE_ConvFC( self.param['feature_size'], self.param['feature_size'], latent_size=8, hidden_size=256, spatial_size=self.last_size) elif cvae_arch == 'conv_e': # Conv + FC version E (global latent space, size 16) print( 'Using CVAE class: My_CVAE_ConvFC (latent=16x1x1, hidden=256x4x4)' ) self.network_pred_cvae = My_CVAE_ConvFC( self.param['feature_size'], self.param['feature_size'], latent_size=16, hidden_size=256, spatial_size=self.last_size) elif cvae_arch == 'vrnn_a': print('Using VRNN class: My_VRNN_A') else: raise Exception('CVAE architecture not recognized: ' + cvae_arch) self.mask = None self.relu = nn.ReLU(inplace=False) self._initialize_weights(self.agg) self._initialize_weights(self.network_pred_cvae) self.action_cls_head = action_cls_head if action_cls_head: # See eval/model_3d_lc.py if isinstance(num_class, int): num_class = [num_class] # singleton to simplify code assert (isinstance(num_class, list)) self.num_class = num_class self.final_bn = nn.BatchNorm1d(self.param['feature_size']) self.final_bn.weight.data.fill_(1) self.final_bn.bias.data.zero_() self.final_fc = [] for cur_num_cls in num_class: cur_fc = nn.Sequential( nn.Dropout(dropout), nn.Linear(self.param['feature_size'], cur_num_cls)) self._initialize_weights(cur_fc) self.final_fc.append(cur_fc) self.final_fc = nn.ModuleList( self.final_fc) # IMPORTANT, otherwise pytorch won't register