def __init__( self, dataset, image_modality, feature_extractor, task_classifier, critic, method, **base_params, ): super(DANNTrainerVideo, self).__init__(dataset, feature_extractor, task_classifier, critic, method, **base_params) self.image_modality = image_modality self.rgb, self.flow = get_image_modality(self.image_modality) self.rgb_feat = self.feat["rgb"] self.flow_feat = self.feat["flow"]
def __init__( self, dataset, image_modality, feature_extractor, task_classifier, critic, k_critic=5, gamma=10, beta_ratio=0, **base_params, ): super(WDGRLTrainerVideo, self).__init__(dataset, feature_extractor, task_classifier, critic, k_critic, gamma, beta_ratio, **base_params) self.image_modality = image_modality self.rgb, self.flow = get_image_modality(self.image_modality) self.rgb_feat = self.feat["rgb"] self.flow_feat = self.feat["flow"]
def __init__( self, dataset, image_modality, feature_extractor, task_classifier, critic, use_entropy=False, use_random=False, random_dim=1024, **base_params, ): super(CDANTrainerVideo, self).__init__(dataset, feature_extractor, task_classifier, critic, use_entropy, use_random, random_dim, **base_params) self.image_modality = image_modality self.rgb, self.flow = get_image_modality(image_modality) self.rgb_feat = self.feat["rgb"] self.flow_feat = self.feat["flow"]
def get_video_feat_extractor(model_name, image_modality, attention, num_classes): """ Get the feature extractor w/o the pre-trained model and SELayers. The pre-trained models are saved in the path ``$XDG_CACHE_HOME/torch/hub/checkpoints/``. For Linux, default path is ``~/.cache/torch/hub/checkpoints/``. For Windows, default path is ``C:/Users/$USER_NAME/.cache/torch/hub/checkpoints/``. Provide four pre-trained models: "rgb_imagenet", "flow_imagenet", "rgb_charades", "flow_charades". Args: model_name (string): The name of the feature extractor. (Choices=["I3D", "R3D_18", "R2PLUS1D_18", "MC3_18"]) image_modality (string): Image type. (Choices=["rgb", "flow", "joint"]) attention (string): The attention type. (Choices=["SELayerC", "SELayerT", "SELayerCoC", "SELayerMC", "SELayerCT", "SELayerTC", "SELayerMAC"]) num_classes (int): The class number of specific dataset. (Default: No use) Returns: feature_network (dictionary): The network to extract features. class_feature_dim (int): The dimension of the feature network output for ClassNet. It is a convention when the input dimension and the network is fixed. domain_feature_dim (int): The dimension of the feature network output for DomainNet. """ rgb, flow = get_image_modality(image_modality) attention_list = ["SELayerC", "SELayerT", "SELayerCoC", "SELayerMC", "SELayerCT", "SELayerTC", "SELayerMAC"] model_list = ["I3D", "R3D_18", "MC3_18", "R2PLUS1D_18"] if attention in attention_list: att = True elif attention == "None": att = False else: raise ValueError("Wrong MODEL.ATTENTION. Current: {}".format(attention)) if model_name not in model_list: raise ValueError("Wrong MODEL.METHOD. Current:{}".format(model_name)) # Get I3D w/o SELayers for RGB, Flow or joint input if model_name == "I3D": rgb_pretrained_model = flow_pretrained_model = None if rgb: rgb_pretrained_model = "rgb_imagenet" # Options=["rgb_imagenet", "rgb_charades"] if flow: flow_pretrained_model = "flow_imagenet" # Options=["flow_imagenet", "flow_charades"] if rgb and flow: class_feature_dim = 2048 domain_feature_dim = class_feature_dim / 2 else: class_feature_dim = 1024 domain_feature_dim = class_feature_dim if not att: logging.info("{} without SELayer.".format(model_name)) feature_network = i3d_joint( rgb_pt=rgb_pretrained_model, flow_pt=flow_pretrained_model, num_classes=num_classes, pretrained=True ) else: logging.info("{} with {}.".format(model_name, attention)) feature_network = se_i3d_joint( rgb_pt=rgb_pretrained_model, flow_pt=flow_pretrained_model, attention=attention, num_classes=num_classes, pretrained=True, ) # Get R3D_18/R2PLUS1D_18/MC3_18 w/o SELayers for RGB, Flow or joint input elif model_name in ["R3D_18", "R2PLUS1D_18", "MC3_18"]: if rgb and flow: class_feature_dim = 1024 domain_feature_dim = class_feature_dim / 2 else: class_feature_dim = 512 domain_feature_dim = class_feature_dim if model_name == "R3D_18": if not att: logging.info("{} without SELayer.".format(model_name)) feature_network = r3d(rgb=rgb, flow=flow, pretrained=True) else: logging.info("{} with {}.".format(model_name, attention)) feature_network = se_r3d(rgb=rgb, flow=flow, pretrained=True, attention=attention) elif model_name == "R2PLUS1D_18": if not att: logging.info("{} without SELayer.".format(model_name)) feature_network = r2plus1d(rgb=rgb, flow=flow, pretrained=True) else: logging.info("{} with {}.".format(model_name, attention)) feature_network = se_r2plus1d(rgb=rgb, flow=flow, pretrained=True, attention=attention) elif model_name == "MC3_18": if not att: logging.info("{} without SELayer.".format(model_name)) feature_network = mc3(rgb=rgb, flow=flow, pretrained=True) else: logging.info("{} with {}.".format(model_name, attention)) feature_network = se_mc3(rgb=rgb, flow=flow, pretrained=True, attention=attention) return feature_network, int(class_feature_dim), int(domain_feature_dim)
def __init__( self, source_access_dict, target_access_dict, image_modality, seed, config_weight_type="natural", config_size_type=DatasetSizeType.Max, valid_split_ratio=0.1, source_sampling_config=None, target_sampling_config=None, n_fewshot=None, random_state=None, class_ids=None, ): """The class controlling how the source and target domains are iterated over when the input is joint. Inherited from MultiDomainDatasets. Args: source_access_dict (dictionary): dictionary of source RGB and flow dataset accessors target_access_dict (dictionary): dictionary of target RGB and flow dataset accessors image_modality (string): image type (RGB or Optical Flow) seed (int): seed value set manually. class_ids (list, optional): List of chosen subset of class ids. Defaults to None (=> All Classes). """ self._image_modality = image_modality self.rgb, self.flow = get_image_modality(self._image_modality) self._seed = seed if self.rgb: source_access = source_access_dict["rgb"] target_access = target_access_dict["rgb"] if self.flow: source_access = source_access_dict["flow"] target_access = target_access_dict["flow"] weight_type = WeightingType(config_weight_type) size_type = DatasetSizeType(config_size_type) if weight_type is WeightingType.PRESET0: self._source_sampling_config = FixedSeedSamplingConfig( class_weights=np.arange(source_access.n_classes(), 0, -1)) self._target_sampling_config = FixedSeedSamplingConfig( class_weights=np.random.randint( 1, 4, size=target_access.n_classes())) elif weight_type is WeightingType.BALANCED: self._source_sampling_config = FixedSeedSamplingConfig( balance=True) self._target_sampling_config = FixedSeedSamplingConfig( balance=True) elif weight_type not in WeightingType: raise ValueError(f"Unknown weighting method {weight_type}.") else: self._source_sampling_config = FixedSeedSamplingConfig( seed=self._seed) self._target_sampling_config = FixedSeedSamplingConfig( seed=self._seed) self._source_access_dict = source_access_dict self._target_access_dict = target_access_dict self._valid_split_ratio = valid_split_ratio self._rgb_source_by_split = {} self._flow_source_by_split = {} self._rgb_target_by_split = {} self._flow_target_by_split = {} self._size_type = size_type self._n_fewshot = n_fewshot self._random_state = check_random_state(random_state) self._source_by_split = {} self._labeled_target_by_split = None self._target_by_split = {} self.class_ids = class_ids
def test_get_image_modality(image_modality): rgb, flow = get_image_modality(image_modality) assert isinstance(rgb, bool) assert isinstance(flow, bool)