def torch(self,
cuda=None,
trainable=True,
train_numerator=True,
train_denominator=True):
"""
Returns a torch version of this activation function.
Arguments:
cuda (bool):
Use GPU CUDA version. If None, use cuda if available on \
the machine\n
Default ``None``
trainable (bool):
If the weights are trainable, i.e, if they are updated \
during backward pass\n
Default ``True``
Returns:
function: Rational torch function
"""
from rational.torch import Rational as Rational_torch
import torch.nn as nn
import torch
rtorch = Rational_torch(self.init_approximation, self.degrees, cuda,
self.version, trainable, train_numerator,
train_denominator)
rtorch.numerator = nn.Parameter(
torch.FloatTensor(self.numerator).to(rtorch.device),
requires_grad=trainable and train_numerator)
rtorch.denominator = nn.Parameter(
torch.FloatTensor(self.denominator).to(rtorch.device),
requires_grad=trainable and train_denominator)
return rtorch
def test_conversion_gpu_to_cpuC():
rational = Rational(version='C', cuda=True)
rational.cpu()
params = np.all([str(para.device) == 'cpu' for para in rational.parameters()])
cpu_f = "PYTORCH_C" in rational.activation_function.__qualname__
new_res = rational(inp).detach().numpy()
coherent_compute = np.all(np.isclose(new_res, expected_res, atol=5e-02))
assert params and cpu_f and coherent_compute
def test_conversion_cpu_to_gpuD():
rational = Rational(version='D', cuda=False, trainable=False)
rational.cuda()
params = np.all(['cuda' in str(para.device) for para in rational.parameters()])
cpu_f = "CUDA_D" in rational.activation_function.__qualname__
new_res = rational(cuda_inp).clone().detach().cpu().numpy()
coherent_compute = np.all(np.isclose(new_res, expected_res, atol=5e-02))
assert params and cpu_f and coherent_compute
def __init__(self,
input_shape,
output_shape,
recurrent=False,
cuda=False,
**kwargs):
super().__init__()
n_input = input_shape[0]
n_output = output_shape[0]
self._h1 = nn.Conv2d(n_input, 32, kernel_size=8, stride=4)
self._h2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self._h3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
self._h4 = nn.Linear(3136, self.n_features)
self._h5 = nn.Linear(self.n_features, n_output)
nn.init.xavier_uniform_(self._h1.weight,
gain=nn.init.calculate_gain('relu'))
nn.init.xavier_uniform_(self._h2.weight,
gain=nn.init.calculate_gain('relu'))
nn.init.xavier_uniform_(self._h3.weight,
gain=nn.init.calculate_gain('relu'))
nn.init.xavier_uniform_(self._h4.weight,
gain=nn.init.calculate_gain('relu'))
nn.init.xavier_uniform_(self._h5.weight,
gain=nn.init.calculate_gain('linear'))
if recurrent:
self.act_func1 = Rational(cuda=cuda)
self.act_func2 = self.act_func1
self.act_func3 = self.act_func1
self.act_func4 = self.act_func1
else:
self.act_func1 = Rational(cuda=cuda)
self.act_func2 = Rational(cuda=cuda)
self.act_func3 = Rational(cuda=cuda)
self.act_func4 = Rational(cuda=cuda)
if cuda:
self.cuda()
from rational.torch import Rational
rational_function = Rational("tanh") # Initialized closed to tanh
rational_function.show()
import numpy as np
rational_function.show(np.arange(-6, 12, 2))
rational_function.input_retrieve_mode()
# Retrieving input from now on.
import torch
means = torch.ones((50, 50)) * 2.
stds = torch.ones((50, 50)) * 3.
for _ in range(1500):
input = torch.normal(means, stds).to(rational_function.device)
rational_function(input)
# Training mode, no longer retrieving the input.
rational_function.show()
import torch
from torch.nn.functional import leaky_relu
from rational.torch import Rational
import numpy as np
t = torch.tensor([-2., -1, 0., 1., 2.])
expected_res = np.array(leaky_relu(t))
inp = torch.from_numpy(np.array(t)).reshape(-1)
cuda_inp = torch.tensor(np.array(t), dtype=torch.float, device="cuda").reshape(-1)
rationalA_lrelu_cpu = Rational(version='A', cuda=False)(inp).detach().numpy()
rationalB_lrelu_cpu = Rational(version='B', cuda=False)(inp).detach().numpy()
rationalC_lrelu_cpu = Rational(version='C', cuda=False)(inp).detach().numpy()
rationalD_lrelu_cpu = Rational(version='D', cuda=False, trainable=False)(inp).detach().numpy()
rationalA_lrelu_gpu = Rational(version='A', cuda=True)(cuda_inp).clone().detach().cpu().numpy()
rationalB_lrelu_gpu = Rational(version='B', cuda=True)(cuda_inp).clone().detach().cpu().numpy()
rationalC_lrelu_gpu = Rational(version='C', cuda=True)(cuda_inp).clone().detach().cpu().numpy()
rationalD_lrelu_gpu = Rational(version='D', cuda=True, trainable=False)(cuda_inp).clone().detach().cpu().numpy()
# Tests on cpu
def test_rationalA_cpu_lrelu():
assert np.all(np.isclose(rationalA_lrelu_cpu, expected_res, atol=5e-02))
def test_rationalB_cpu_lrelu():
assert np.all(np.isclose(rationalB_lrelu_cpu, expected_res, atol=5e-02))
from rational.torch import Rational
rational_function = Rational() # Initialized closed to Leaky ReLU
print(rational_function)
# Pade Activation Unit (version A) of degrees (5, 4) running on cuda:0
# or Pade Activation Unit (version A) of degrees (5, 4) running on cpu
rational_function.cpu()
rational_function.cuda()
print(rational_function.degrees)
# (5, 4)
print(rational_function.version)
# A
print(rational_function.training)
# True
import torch
import torch.nn as nn
class RationalNetwork(nn.Module):
n_features = 512
def __init__(self,
input_shape,
output_shape,
recurrent=False,
cuda=False,
**kwargs):
super().__init__()
find_weights(F.tanh)
# approximated function name: tanh
# approximated function name: tanh
# degree of the numerator P: 5
# degree of the denominator Q: 4
# lower bound: -3
# upper bound: 3
# Rational Version: B
# Found coeffient :
# P: [2.11729498e-09 9.99994250e-01 6.27633277e-07 1.07708645e-01
# 2.94655690e-08 8.71124374e-04]
# Q: [6.37690834e-07 4.41014181e-01 2.27476614e-07 1.45810399e-02]
# Do you want a plot of the result (y/n)y
# Do you want to store them in the json file ? (y/n)y
from rational.torch import Rational
rational_tanh_B = Rational("tanh", version="B")
print(rational_tanh_B.init_approximation)
# 'tanh'
print(rational_tanh_B.numerator.cpu().detach().numpy())
# [2.1172950e-09 9.9999428e-01 6.2763326e-07 1.0770865e-01 2.9465570e-08
# 8.7112439e-04]
print(rational_tanh_B.denominator.cpu().detach().numpy())
# [6.3769085e-07 4.4101417e-01 2.2747662e-07 1.4581040e-02]
