def __init__(self): super().__init__() self.virtual_dataset = _VirtualDataset() self.matmul1 = P.MatMul() self.matmul2 = P.MatMul() self.gelu = P.GeLU() self.bn1 = bn_with_initialize(2048)
def __init__(self, strategy1, strategy2, strategy3, strategy4, strategy5, strategy6): super().__init__() self.matmul1 = P.MatMul().shard(strategy1) self.matmul2 = P.MatMul().shard(strategy2) self.gelu = P.GeLU().shard(strategy3) self.tanh = P.Tanh().shard(strategy4) self.softmax = P.Softmax(axis=(0, 1)).shard(strategy5) self.logsoftmax = P.LogSoftmax().shard(strategy6)
def __init__(self, in_channels=786, out_channels=768, hidden_size=3072, hidden_dropout=0.1): super(FeedForward, self).__init__() self.c_fc = Conv1D(in_channels, hidden_size) self.c_proj = Conv1D(hidden_size, out_channels) # self.gelu = Gelu() self.layernorm = LayerNorm(in_channels=in_channels) self.residual_connect = ResidualConnection(dropout_prob=hidden_dropout) self.gelu_act = P.GeLU() self.dropout = nn.Dropout(1 - hidden_dropout) self.use_dropout = hidden_dropout > 0 self.reshape = P.Reshape()
def __init__(self): super(GeluNet, self).__init__() self.gelu = P.GeLU()
def __init__(self, strategy1, strategy2): super().__init__() self.matmul = P.MatMul().shard(strategy1) self.gelu = P.GeLU().shard(strategy2)
def __init__(self, strategy0, strategy1, strategy2, strategy3): super().__init__() self.virtual_dataset = _VirtualDataset().shard(strategy0) self.matmul1 = P.MatMul().shard(strategy1) self.matmul2 = P.MatMul().shard(strategy2) self.gelu = P.GeLU().shard(strategy3)
def __init__(self, strategy0, strategy1, strategy2): super().__init__() self.fc_nobias = P.MatMul(transpose_b=True).shard(strategy0) self.add = P.Add().shard(strategy1) self.gelu = P.GeLU().shard(strategy2)
def __init__(self): super(MEGeluLargeIn, self).__init__() self.matmul = P.MatMul() self.gelu = P.GeLU()
def __init__(self): super().__init__() self.matmul = P.MatMul(transpose_b=True) self.gelu = P.GeLU()
def __init__(self): super(GELU, self).__init__() self.gelu = P.GeLU()
def __init__(self): super(VirtualDatasetNet, self).__init__() self.virtual_dataset = _VirtualDataset() self.matmul1 = P.MatMul() self.matmul2 = P.MatMul() self.gelu = P.GeLU()