def test_silu_1(self):
'''
Unit test for SiLU activation function.
See :mod:`Echo.Activation.Torch.Silu`.
'''
fsilu = Silu()
input = torch.tensor((0.0, 0.0))
output = torch.tensor((0.0, 0.0))
# checking that silu(0) == 0
self.assertEqual((fsilu(input)).allclose(output), True)
def test_silu_2(self):
"""
Unit test for SiLU activation function.
See :mod:`Echo.Activation.Torch.Silu`.
"""
fsilu = Silu()
input = torch.tensor((1.0, 1.0))
output = torch.tensor((0.731058, 0.731058))
# checking that silu(1.0) == 0.7310
self.assertEqual((fsilu(input)).allclose(output), True)
def test_silu_3(self):
'''
Unit test for SiLU activation function.
See :mod:`Echo.Activation.Torch.Silu`.
'''
# checking an inplace implementation
fsilu = Silu(inplace=True)
input = torch.tensor((1.0, 1.0))
output = torch.tensor((0.731058, 0.731058))
# performing the inplace operation
fsilu(input)
# checking that value of an input is SiLU(input) == 0.7310 now
self.assertEqual((input).allclose(output), True)
def main():
"""
Demonstrate custom activation functions to classify Fashion MNIST
"""
# Parse command line arguments
parser = argparse.ArgumentParser(description="Argument parser")
# Add argument to choose one of the activation functions
parser.add_argument(
"--activation",
action="store",
default=WEIGHTED_TANH,
help="Activation function for demonstration.",
choices=[
WEIGHTED_TANH,
MISH,
SILU,
ARIA2,
ESWISH,
SWISH,
BMISH,
ELISH,
HELISH,
MILA,
SINERELU,
FTS,
SQNL,
ISRU,
ISRLU,
BENTID,
SC,
],
)
# Add argument to choose the way to initialize the model
parser.add_argument(
"--model_initialization",
action="store",
default="class",
help="Model initialization mode: use custom class or use Sequential.",
choices=["class", "sequential"],
)
# Add argument to choose the way to initialize the model
parser.add_argument(
"--inplace",
action="store_true",
default=False,
help="Use in-place of out-of-place version of activation function.",
)
# Parse command line arguments
results = parser.parse_args()
activation = results.activation
model_initialization = results.model_initialization
inplace = results.inplace
# Define a transform
transform = transforms.Compose([transforms.ToTensor()])
# Download and load the training data for Fashion MNIST
trainset = datasets.FashionMNIST("~/.pytorch/F_MNIST_data/",
download=True,
train=True,
transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=64,
shuffle=True)
# Download and load the test data for Fashion MNIST
testset = datasets.FashionMNIST("~/.pytorch/F_MNIST_data/",
download=True,
train=False,
transform=transform)
testloader = torch.utils.data.DataLoader(testset,
batch_size=64,
shuffle=True)
print("Create model with {activation} function.\n".format(
activation=activation))
# Initialize the model
if model_initialization == "class":
# Initialize the model using defined Classifier class
model = Classifier(activation=activation, inplace=inplace)
else:
# Setup the activation function
if activation == WEIGHTED_TANH:
activation_function = WeightedTanh(weight=1, inplace=inplace)
if activation == MISH:
activation_function = Mish()
if activation == SILU:
activation_function = Silu(inplace=inplace)
if activation == ARIA2:
activation_function = Aria2()
if activation == ESWISH:
activation_function = Eswish()
if activation == SWISH:
activation_function = Swish()
if activation == BMISH:
activation_function = BetaMish()
if activation == ELISH:
activation_function = Elish()
if activation == HELISH:
activation_function = HardElish()
if activation == MILA:
activation_function = Mila()
if activation == SINERELU:
activation_function = SineReLU()
if activation == FTS:
activation_function = FTS()
if activation == SQNL:
activation_function = SQNL()
if activation == ISRU:
activation_function = ISRU()
if activation == ISRLU:
activation_function = ISRLU()
if activation == BENTID:
activation_function = BentID()
if activation == SC:
activation_function = SoftClipping()
# Initialize the model using nn.Sequential
model = nn.Sequential(
OrderedDict([
("fc1", nn.Linear(784, 256)),
(
"activation",
activation_function,
), # use custom activation function
("fc2", nn.Linear(256, 128)),
("bn2", nn.BatchNorm1d(num_features=128)),
("relu2", nn.ReLU()),
("dropout", nn.Dropout(0.3)),
("fc3", nn.Linear(128, 64)),
("bn3", nn.BatchNorm1d(num_features=64)),
("relu3", nn.ReLU()),
("logits", nn.Linear(64, 10)),
("logsoftmax", nn.LogSoftmax(dim=1)),
]))
# Train the model
print(
"Training the model on Fashion MNIST dataset with {} activation function.\n"
.format(activation))
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=0.003)
epochs = 5
for e in range(epochs):
running_loss = 0
for images, labels in trainloader:
images = images.view(images.shape[0], -1)
log_ps = model(images)
loss = criterion(log_ps, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
running_loss += loss.item()
else:
print(f"Training loss: {running_loss}")