def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: # check inputs -> if not spaces.is_determined(self._in_features): expected_input_dim = self.abstract_child["_in_features"].value else: expected_input_dim = spaces.get_determined_value(self._in_features) if input.size(-1) != expected_input_dim: raise ValueError( "Expect the input dim of {:} instead of {:}".format( expected_input_dim, input.size(-1))) # create the weight matrix if not spaces.is_determined(self._out_features): out_dim = self.abstract_child["_out_features"].value else: out_dim = spaces.get_determined_value(self._out_features) candidate_weight = self._super_weight[:out_dim, :expected_input_dim] # create the bias matrix if not spaces.is_determined(self._bias): if self.abstract_child["_bias"].value: candidate_bias = self._super_bias[:out_dim] else: candidate_bias = None else: if spaces.get_determined_value(self._bias): candidate_bias = self._super_bias[:out_dim] else: candidate_bias = None return F.linear(input, candidate_weight, candidate_bias)
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: # check inputs -> if not spaces.is_determined(self._in_features): expected_input_dim = self.abstract_child["_in_features"].value else: expected_input_dim = spaces.get_determined_value(self._in_features) if input.size(-1) != expected_input_dim: raise ValueError( "Expect the input dim of {:} instead of {:}".format( expected_input_dim, input.size(-1))) # create the weight and bias matrix for fc1 if not spaces.is_determined(self._hidden_multiplier): hmul = self.abstract_child[ "_hidden_multiplier"].value * expected_input_dim else: hmul = spaces.get_determined_value(self._hidden_multiplier) hidden_dim = int(expected_input_dim * hmul) _fc1_weight = self.fc1_super_weight[:hidden_dim, :expected_input_dim] _fc1_bias = self.fc1_super_bias[:hidden_dim] x = F.linear(input, _fc1_weight, _fc1_bias) x = self.act(x) x = self.drop(x) # create the weight and bias matrix for fc2 if not spaces.is_determined(self._out_features): out_dim = self.abstract_child["_out_features"].value else: out_dim = spaces.get_determined_value(self._out_features) _fc2_weight = self.fc2_super_weight[:out_dim, :hidden_dim] _fc2_bias = self.fc2_super_bias[:out_dim] x = F.linear(x, _fc2_weight, _fc2_bias) x = self.drop(x) return x
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: x = input.reshape(len(input), self._d_feat, -1) # [N, F*T] -> [N, F, T] x = x.permute(0, 2, 1) # [N, F, T] -> [N, T, F] if not spaces.is_determined(self._embed_dim): embed_dim = self.abstract_child["_embed_dim"].value else: embed_dim = spaces.get_determined_value(self._embed_dim) out = self.proj(x) * math.sqrt(embed_dim) return out
def forward_candidate( self, q_tensor, k_tensor, v_tensor, mask=None ) -> torch.Tensor: # check the num_heads: if not spaces.is_determined(self._num_heads): num_heads = self.abstract_child["_num_heads"].value else: num_heads = spaces.get_determined_value(self._num_heads) feats = self.forward_qkv(q_tensor, k_tensor, v_tensor, num_heads, mask) outs = self.proj(feats) outs = self.proj_drop(outs) return outs
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: # check the num_heads: if not spaces.is_determined(self._num_heads): num_heads = self.abstract_child["_num_heads"].value else: num_heads = spaces.get_determined_value(self._num_heads) feats = self.forward_qkv(input, num_heads) if self.proj is None: return feats else: outs = self.proj(feats) outs = self.proj_drop(outs) return outs
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: batch, seq, fdim = input.shape[:3] embeddings = self.pe[:, :seq] if not spaces.is_determined(self._d_model): expected_d_model = self.abstract_child["_d_model"].value else: expected_d_model = spaces.get_determined_value(self._d_model) assert fdim == expected_d_model, "{:} vs {:}".format( fdim, expected_d_model) embeddings = torch.nn.functional.interpolate(embeddings, size=(expected_d_model), mode="linear", align_corners=True) outs = self.dropout(input + embeddings) return outs
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: # check inputs -> if not spaces.is_determined(self._in_dim): expected_input_dim = self.abstract_child["_in_dim"].value else: expected_input_dim = spaces.get_determined_value(self._in_dim) if input.size(-1) != expected_input_dim: raise ValueError( "Expect the input dim of {:} instead of {:}".format( expected_input_dim, input.size(-1))) if self._elementwise_affine: weight = self.weight[:expected_input_dim] bias = self.bias[:expected_input_dim] else: weight, bias = None, None return F.layer_norm(input, (expected_input_dim, ), weight, bias, self.eps)
def forward_candidate(self, input: torch.Tensor) -> torch.Tensor: batch, flatten_size = input.shape feats = self.input_embed(input) # batch * 60 * 64 if not spaces.is_determined(self._embed_dim): embed_dim = self.abstract_child["_embed_dim"].value else: embed_dim = spaces.get_determined_value(self._embed_dim) cls_tokens = self.cls_token.expand(batch, -1, -1) cls_tokens = F.interpolate(cls_tokens, size=(embed_dim), mode="linear", align_corners=True) feats_w_ct = torch.cat((cls_tokens, feats), dim=1) feats_w_tp = self.pos_embed(feats_w_ct) xfeats = self.backbone(feats_w_tp) xfeats = xfeats[:, 0, :] # use the feature for the first token predicts = self.head(xfeats).squeeze(-1) return predicts