def __init__(self, approx_func="leaky_relu", degrees=(5, 4), cuda=None, version="A", trainable=True, train_numerator=True, train_denominator=True): super(Rational, self).__init__() if cuda is None: cuda = torch_cuda_available() if cuda is True: device = "cuda" elif cuda is False: device = "cpu" else: device = cuda w_numerator, w_denominator = get_parameters(version, degrees, approx_func) self.numerator = nn.Parameter(torch.FloatTensor(w_numerator).to(device), requires_grad=trainable and train_numerator) self.denominator = nn.Parameter(torch.FloatTensor(w_denominator).to(device), requires_grad=trainable and train_denominator) self.register_parameter("numerator", self.numerator) self.register_parameter("denominator", self.denominator) self.device = device self.degrees = degrees self.version = version self.training = trainable self.init_approximation = approx_func if "cuda" in str(device): if version == "A": rational_func = Rational_CUDA_A_F elif version == "B": rational_func = Rational_CUDA_B_F elif version == "C": rational_func = Rational_CUDA_C_F elif version == "D": rational_func = Rational_CUDA_D_F else: raise ValueError("version %s not implemented" % version) self.activation_function = rational_func.apply else: if version == "A": rational_func = Rational_PYTORCH_A_F elif version == "B": rational_func = Rational_PYTORCH_B_F elif version == "C": rational_func = Rational_PYTORCH_C_F elif version == "D": rational_func = Rational_PYTORCH_D_F else: raise ValueError("version %s not implemented" % version) self.activation_function = rational_func self._handle_retrieve_mode = None self.distribution = None self.best_fitted_function = None self.best_fitted_function_params = None
def is_torch_cuda_available(): """Wrapper for torch cuda availability check (torch.cuda.is_available) that takes an environment variable 'FORCE_CUDA' into account and returns also true iff FORCE_CUDA=1. This is necessary when building rational in a Dockerfile script since the docker build pass doesn't have access to cuda and thus torch.cuda.is_available always returns false, even when the docker image which is to be built in fact does have cuda. """ force_cuda = os.getenv("FORCE_CUDA", "0") == "1" return force_cuda or torch_cuda_available()
def __init__(self, w_numerator=init_w_numerator, w_denominator=init_w_denominator, center=center, cuda=None, version="A", trainable=True, train_center=True, train_numerator=True, train_denominator=True): super(PAU, self).__init__() if cuda is None: cuda = torch_cuda_available() self.center = nn.Parameter(torch.FloatTensor([center]), requires_grad=trainable and train_center) self.numerator = nn.Parameter(torch.FloatTensor(w_numerator), requires_grad=trainable and train_numerator) self.denominator = nn.Parameter(torch.FloatTensor(w_denominator), requires_grad=trainable and train_denominator) if cuda: if version == "A": pau_func = PAU_CUDA_A_F elif version == "B": pau_func = PAU_CUDA_B_F elif version == "C": pau_func = PAU_CUDA_C_F elif version == "D": pau_func = PAU_CUDA_D_F else: raise ValueError("version %s not implemented" % version) self.activation_function = pau_func.apply else: if version == "A": pau_func = PAU_PYTORCH_A_F elif version == "B": pau_func = PAU_PYTORCH_B_F elif version == "C": pau_func = PAU_PYTORCH_C_F elif version == "D": pau_func = PAU_PYTORCH_D_F else: raise ValueError("version %s not implemented" % version) self.activation_function = pau_func
\#include <curand.h> \#include <curand_kernel.h> \#include <curand_philox4x32_x.h> constexpr uint32_t THREADS_PER_BLOCK = 512; """ file_content = airspeed.Template(template + template_contents) content = file_content.merge(locals()) with open(fname, "w") as text_file: text_file.write(content) if torch_cuda_available(): version_names = [] template_contents = "" for template_fname in sorted(glob.glob("rational/_cuda/versions/*.cu")): version_names.append(Path(template_fname).stem) with open(template_fname) as infile: template_contents += infile.read() generate_cpp_module(fname='rational/_cuda/rational_cuda.cpp', versions=version_names) generate_cpp_kernels_module( fname='rational/_cuda/rational_cuda_kernels.cu', template_contents=template_contents) with open("README.md", "r") as fh: long_description = fh.read()
""" Padé Activation Units - Rational Activation Functions for pytorch ================================================================= This module allows you to create Rational Neural Networks using Padé Activation Units - Learnabe Rational activation functions. """ import torch.nn as nn from torch.cuda import is_available as torch_cuda_available from rational.utils.get_weights import get_parameters if torch_cuda_available(): try: from rational.torch.rational_cuda_functions import * except ImportError as ImpErr: print('\n\nError importing rational_cuda, is cuda not available?\n\n') print(ImpErr) exit(1) from rational.torch.rational_pytorch_functions import * class RecurrentRational(): """ Recurrent rational activation function - wrapper for Rational Arguments: approx_func (str): The name of the approximated function for initialisation. \ The different initialable functions are available in \ `rational.rationals_config.json`. \n