def __init__(self):
super().__init__()
self.virtual_dataset = _VirtualDataset()
self.matmul1 = P.MatMul()
self.matmul2 = P.MatMul()
self.gelu = P.GeLU()
self.bn1 = bn_with_initialize(2048)
def __init__(self, strategy1, strategy2, strategy3, strategy4, strategy5, strategy6):
super().__init__()
self.matmul1 = P.MatMul().shard(strategy1)
self.matmul2 = P.MatMul().shard(strategy2)
self.gelu = P.GeLU().shard(strategy3)
self.tanh = P.Tanh().shard(strategy4)
self.softmax = P.Softmax(axis=(0, 1)).shard(strategy5)
self.logsoftmax = P.LogSoftmax().shard(strategy6)
def __init__(self,
in_channels=786,
out_channels=768,
hidden_size=3072,
hidden_dropout=0.1):
super(FeedForward, self).__init__()
self.c_fc = Conv1D(in_channels, hidden_size)
self.c_proj = Conv1D(hidden_size, out_channels)
# self.gelu = Gelu()
self.layernorm = LayerNorm(in_channels=in_channels)
self.residual_connect = ResidualConnection(dropout_prob=hidden_dropout)
self.gelu_act = P.GeLU()
self.dropout = nn.Dropout(1 - hidden_dropout)
self.use_dropout = hidden_dropout > 0
self.reshape = P.Reshape()
def __init__(self):
super(GeluNet, self).__init__()
self.gelu = P.GeLU()
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().shard(strategy1)
self.gelu = P.GeLU().shard(strategy2)
def __init__(self, strategy0, strategy1, strategy2, strategy3):
super().__init__()
self.virtual_dataset = _VirtualDataset().shard(strategy0)
self.matmul1 = P.MatMul().shard(strategy1)
self.matmul2 = P.MatMul().shard(strategy2)
self.gelu = P.GeLU().shard(strategy3)
def __init__(self, strategy0, strategy1, strategy2):
super().__init__()
self.fc_nobias = P.MatMul(transpose_b=True).shard(strategy0)
self.add = P.Add().shard(strategy1)
self.gelu = P.GeLU().shard(strategy2)
def __init__(self):
super(MEGeluLargeIn, self).__init__()
self.matmul = P.MatMul()
self.gelu = P.GeLU()
def __init__(self):
super().__init__()
self.matmul = P.MatMul(transpose_b=True)
self.gelu = P.GeLU()
def __init__(self):
super(GELU, self).__init__()
self.gelu = P.GeLU()
def __init__(self):
super(VirtualDatasetNet, self).__init__()
self.virtual_dataset = _VirtualDataset()
self.matmul1 = P.MatMul()
self.matmul2 = P.MatMul()
self.gelu = P.GeLU()
