def __init__(
self,
dataset,
image_modality,
feature_extractor,
task_classifier,
critic,
method,
**base_params,
):
super(DANNTrainerVideo,
self).__init__(dataset, feature_extractor, task_classifier,
critic, method, **base_params)
self.image_modality = image_modality
self.rgb, self.flow = get_image_modality(self.image_modality)
self.rgb_feat = self.feat["rgb"]
self.flow_feat = self.feat["flow"]
def __init__(
self,
dataset,
image_modality,
feature_extractor,
task_classifier,
critic,
k_critic=5,
gamma=10,
beta_ratio=0,
**base_params,
):
super(WDGRLTrainerVideo,
self).__init__(dataset, feature_extractor, task_classifier,
critic, k_critic, gamma, beta_ratio,
**base_params)
self.image_modality = image_modality
self.rgb, self.flow = get_image_modality(self.image_modality)
self.rgb_feat = self.feat["rgb"]
self.flow_feat = self.feat["flow"]
def __init__(
self,
dataset,
image_modality,
feature_extractor,
task_classifier,
critic,
use_entropy=False,
use_random=False,
random_dim=1024,
**base_params,
):
super(CDANTrainerVideo,
self).__init__(dataset, feature_extractor, task_classifier,
critic, use_entropy, use_random, random_dim,
**base_params)
self.image_modality = image_modality
self.rgb, self.flow = get_image_modality(image_modality)
self.rgb_feat = self.feat["rgb"]
self.flow_feat = self.feat["flow"]
def get_video_feat_extractor(model_name, image_modality, attention, num_classes):
"""
Get the feature extractor w/o the pre-trained model and SELayers. The pre-trained models are saved in the path
``$XDG_CACHE_HOME/torch/hub/checkpoints/``. For Linux, default path is ``~/.cache/torch/hub/checkpoints/``.
For Windows, default path is ``C:/Users/$USER_NAME/.cache/torch/hub/checkpoints/``.
Provide four pre-trained models: "rgb_imagenet", "flow_imagenet", "rgb_charades", "flow_charades".
Args:
model_name (string): The name of the feature extractor. (Choices=["I3D", "R3D_18", "R2PLUS1D_18", "MC3_18"])
image_modality (string): Image type. (Choices=["rgb", "flow", "joint"])
attention (string): The attention type. (Choices=["SELayerC", "SELayerT", "SELayerCoC", "SELayerMC", "SELayerCT", "SELayerTC", "SELayerMAC"])
num_classes (int): The class number of specific dataset. (Default: No use)
Returns:
feature_network (dictionary): The network to extract features.
class_feature_dim (int): The dimension of the feature network output for ClassNet.
It is a convention when the input dimension and the network is fixed.
domain_feature_dim (int): The dimension of the feature network output for DomainNet.
"""
rgb, flow = get_image_modality(image_modality)
attention_list = ["SELayerC", "SELayerT", "SELayerCoC", "SELayerMC", "SELayerCT", "SELayerTC", "SELayerMAC"]
model_list = ["I3D", "R3D_18", "MC3_18", "R2PLUS1D_18"]
if attention in attention_list:
att = True
elif attention == "None":
att = False
else:
raise ValueError("Wrong MODEL.ATTENTION. Current: {}".format(attention))
if model_name not in model_list:
raise ValueError("Wrong MODEL.METHOD. Current:{}".format(model_name))
# Get I3D w/o SELayers for RGB, Flow or joint input
if model_name == "I3D":
rgb_pretrained_model = flow_pretrained_model = None
if rgb:
rgb_pretrained_model = "rgb_imagenet" # Options=["rgb_imagenet", "rgb_charades"]
if flow:
flow_pretrained_model = "flow_imagenet" # Options=["flow_imagenet", "flow_charades"]
if rgb and flow:
class_feature_dim = 2048
domain_feature_dim = class_feature_dim / 2
else:
class_feature_dim = 1024
domain_feature_dim = class_feature_dim
if not att:
logging.info("{} without SELayer.".format(model_name))
feature_network = i3d_joint(
rgb_pt=rgb_pretrained_model, flow_pt=flow_pretrained_model, num_classes=num_classes, pretrained=True
)
else:
logging.info("{} with {}.".format(model_name, attention))
feature_network = se_i3d_joint(
rgb_pt=rgb_pretrained_model,
flow_pt=flow_pretrained_model,
attention=attention,
num_classes=num_classes,
pretrained=True,
)
# Get R3D_18/R2PLUS1D_18/MC3_18 w/o SELayers for RGB, Flow or joint input
elif model_name in ["R3D_18", "R2PLUS1D_18", "MC3_18"]:
if rgb and flow:
class_feature_dim = 1024
domain_feature_dim = class_feature_dim / 2
else:
class_feature_dim = 512
domain_feature_dim = class_feature_dim
if model_name == "R3D_18":
if not att:
logging.info("{} without SELayer.".format(model_name))
feature_network = r3d(rgb=rgb, flow=flow, pretrained=True)
else:
logging.info("{} with {}.".format(model_name, attention))
feature_network = se_r3d(rgb=rgb, flow=flow, pretrained=True, attention=attention)
elif model_name == "R2PLUS1D_18":
if not att:
logging.info("{} without SELayer.".format(model_name))
feature_network = r2plus1d(rgb=rgb, flow=flow, pretrained=True)
else:
logging.info("{} with {}.".format(model_name, attention))
feature_network = se_r2plus1d(rgb=rgb, flow=flow, pretrained=True, attention=attention)
elif model_name == "MC3_18":
if not att:
logging.info("{} without SELayer.".format(model_name))
feature_network = mc3(rgb=rgb, flow=flow, pretrained=True)
else:
logging.info("{} with {}.".format(model_name, attention))
feature_network = se_mc3(rgb=rgb, flow=flow, pretrained=True, attention=attention)
return feature_network, int(class_feature_dim), int(domain_feature_dim)
def __init__(
self,
source_access_dict,
target_access_dict,
image_modality,
seed,
config_weight_type="natural",
config_size_type=DatasetSizeType.Max,
valid_split_ratio=0.1,
source_sampling_config=None,
target_sampling_config=None,
n_fewshot=None,
random_state=None,
class_ids=None,
):
"""The class controlling how the source and target domains are iterated over when the input is joint.
Inherited from MultiDomainDatasets.
Args:
source_access_dict (dictionary): dictionary of source RGB and flow dataset accessors
target_access_dict (dictionary): dictionary of target RGB and flow dataset accessors
image_modality (string): image type (RGB or Optical Flow)
seed (int): seed value set manually.
class_ids (list, optional): List of chosen subset of class ids. Defaults to None (=> All Classes).
"""
self._image_modality = image_modality
self.rgb, self.flow = get_image_modality(self._image_modality)
self._seed = seed
if self.rgb:
source_access = source_access_dict["rgb"]
target_access = target_access_dict["rgb"]
if self.flow:
source_access = source_access_dict["flow"]
target_access = target_access_dict["flow"]
weight_type = WeightingType(config_weight_type)
size_type = DatasetSizeType(config_size_type)
if weight_type is WeightingType.PRESET0:
self._source_sampling_config = FixedSeedSamplingConfig(
class_weights=np.arange(source_access.n_classes(), 0, -1))
self._target_sampling_config = FixedSeedSamplingConfig(
class_weights=np.random.randint(
1, 4, size=target_access.n_classes()))
elif weight_type is WeightingType.BALANCED:
self._source_sampling_config = FixedSeedSamplingConfig(
balance=True)
self._target_sampling_config = FixedSeedSamplingConfig(
balance=True)
elif weight_type not in WeightingType:
raise ValueError(f"Unknown weighting method {weight_type}.")
else:
self._source_sampling_config = FixedSeedSamplingConfig(
seed=self._seed)
self._target_sampling_config = FixedSeedSamplingConfig(
seed=self._seed)
self._source_access_dict = source_access_dict
self._target_access_dict = target_access_dict
self._valid_split_ratio = valid_split_ratio
self._rgb_source_by_split = {}
self._flow_source_by_split = {}
self._rgb_target_by_split = {}
self._flow_target_by_split = {}
self._size_type = size_type
self._n_fewshot = n_fewshot
self._random_state = check_random_state(random_state)
self._source_by_split = {}
self._labeled_target_by_split = None
self._target_by_split = {}
self.class_ids = class_ids
def test_get_image_modality(image_modality):
rgb, flow = get_image_modality(image_modality)
assert isinstance(rgb, bool)
assert isinstance(flow, bool)