def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
fusion_type = kwargs.get("fusion", "concatenate")
patch_feature_input_dim = kwargs.get("patch_feature_input_dim", 512)
patch_feature_hidden_dim = kwargs.get("patch_feature_hidden_dim", patch_feature_input_dim)
patch_feature_output_dim = kwargs.get("patch_feature_output_dim", patch_feature_input_dim)
if fusion_type == "concatenate":
num_channels = data_shape["skeleton"][-1] + patch_feature_output_dim
else:
num_channels = data_shape["skeleton"][-1]
self.patch_feature_dim_reducer = lambda x: x
if patch_feature_input_dim != patch_feature_output_dim:
self.patch_feature_dim_reducer = nn.Sequential(
nn.Linear(patch_feature_input_dim, patch_feature_hidden_dim),
nn.Linear(patch_feature_hidden_dim, patch_feature_output_dim)
)
agcn_input_shape = (data_shape["skeleton"][0], data_shape["skeleton"][1], graph.num_vertices,
num_channels)
self.agcn = agcn.Model(agcn_input_shape, num_classes, graph, num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
self.fusion = get_fusion(fusion_type, concatenate_dim=-1)
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
self.agcn = agcn.Model(data_shape["rgb"],
num_classes,
graph,
num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
shape = list(data_shape["skeleton"])
shape[-1] += data_shape["inertial"][-1] # add imu channels to shape
self.fusion = get_fusion("concatenate", concatenate_dim=-1)
self.agcn = agcn.Model(tuple(shape), num_classes, graph, num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
edges = kwargs["rgb_patch_groups_edges"]
edges = [tuple(map(int, edge.split(", "))) for edge in edges]
graph = Graph(edges)
self.agcn = agcn.Model(data_shape["rgb"],
num_classes,
graph,
num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
self.rgb_encoder = RgbCnnEncoder(rgb_num_vertices=graph.num_vertices,
**kwargs)
agcn_input_shape = (self.rgb_encoder.num_bodies, data_shape["rgb"][0],
self.rgb_encoder.num_vertices,
self.rgb_encoder.num_encoded_channels)
self.agcn = agcn.Model(agcn_input_shape,
num_classes,
graph,
num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
num_additional_nodes = kwargs.pop("num_additional_nodes", 3)
self.rgb_encoder = RgbR2P1DEncoder(num_encoded_channels=data_shape["skeleton"][-1],
num_additional_nodes=num_additional_nodes * data_shape["skeleton"][0],
**kwargs)
graph = graph.with_new_edges([(graph.num_vertices + i, graph.center_joint)
for i in range(num_additional_nodes)])
agcn_input_shape = list(data_shape["skeleton"])
agcn_input_shape[2] = graph.num_vertices
self.agcn = agcn.Model(tuple(agcn_input_shape), num_classes, graph, num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
fusion_type = kwargs.get("fusion", "concatenate")
self.rgb_encoder = RgbCnnEncoder(rgb_num_vertices=graph.num_vertices, **kwargs)
if fusion_type == "concatenate":
num_channels = data_shape["skeleton"][-1] + self.rgb_encoder.num_encoded_channels
else:
num_channels = data_shape["skeleton"][-1]
agcn_input_shape = (self.rgb_encoder.num_bodies, data_shape["rgb"][0], graph.num_vertices,
num_channels)
self.agcn = agcn.Model(agcn_input_shape, num_classes, graph, num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))
self.fusion = get_fusion(fusion_type, concatenate_dim=-1)
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
dropout = kwargs.get("dropout", 0.)
fusion_type = kwargs.get("fusion", "concatenate")
self.r2p1d = rgb_models.RgbR2p1DModel(data_shape["rgb"], num_classes, graph, without_fc=True, model_depth=18,
**kwargs)
self.agcn = agcn.Model(data_shape["skeleton"], num_classes, graph, num_layers=num_layers, without_fc=True,
dropout=dropout)
self.fusion = get_fusion(fusion_type, concatenate_dim=-1)
self.fc1 = nn.Linear(self.r2p1d.r2p1d.out_dim, self.agcn.out_channels)
if fusion_type == "concatenate":
out_dim = self.agcn.out_channels * 2
else:
out_dim = self.agcn.out_channels
self.fc2 = nn.Linear(out_dim, num_classes)
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
dropout = kwargs.get("dropout", 0.)
fusion_type = kwargs.get("fusion", "concatenate")
if kwargs.pop("skeleton_imu_spatial_fusion", False):
graph = get_skeleton_imu_fusion_graph(graph, **kwargs)
self.imu_gcn = imu_models.ImuGCN(data_shape, num_classes, inter_signal_back_connections=True,
include_additional_top_layer=True, without_fc=True, **kwargs)
self.agcn = agcn.Model(data_shape["skeleton"], num_classes, graph, num_layers=num_layers, without_fc=True,
dropout=dropout)
self.fusion = get_fusion(fusion_type, concatenate_dim=-1)
if fusion_type == "concatenate":
out_dim = self.agcn.out_channels * 2
else:
out_dim = self.agcn.out_channels
self.fc = nn.Linear(out_dim, num_classes)
def __init__(self, data_shape, num_classes: int, graph, **kwargs):
super().__init__()
num_layers = kwargs.get("num_layers", 10)
skeleton_imu_graph = get_skeleton_imu_fusion_graph(graph, **kwargs)
self.agcn = agcn.Model(data_shape["skeleton"], num_classes, skeleton_imu_graph, num_layers=num_layers,
without_fc=kwargs.get("without_fc", False))