def regular_graph(degree, seed=1):
i2 = CompositeInitializer().with_seed(seed).uniform()
i = CompositeInitializer() \
.with_seed(seed) \
.uniform() \
.regular_graph(degree) \
.spectral_normalize() \
.scale(1.0)
w = WeightInitializer()
w.weight_hh_init = i
w.weight_ih_init = i2
return w
def init_parameters(self, initializer: WeightInitializer):
self.weight_ih = nn.Parameter(data=initializer.init_weight_ih(
weight=torch.Tensor(self.hidden_size, self.input_size), ),
requires_grad=self.requires_grad)
self.weight_hh = nn.Parameter(data=initializer.init_weight_hh(
weight=torch.Tensor(self.hidden_size, self.hidden_size), ),
requires_grad=self.requires_grad)
if self.bias:
self.bias_ih = nn.Parameter(data=initializer.init_bias_ih(
bias=torch.Tensor(self.hidden_size), ),
requires_grad=self.requires_grad)
self.bias_hh = nn.Parameter(data=initializer.init_bias_hh(
bias=torch.Tensor(self.hidden_size), ),
requires_grad=self.requires_grad)
def watts_strogatz_graph(neighbours, rewire_proba, seed=1):
input_weights = CompositeInitializer().with_seed(seed).uniform()
hidden_weight = CompositeInitializer() \
.with_seed(seed) \
.uniform() \
.watts_strogatz(neighbours=neighbours, rewire_proba=rewire_proba) \
.spectral_normalize() \
.scale(1.0)
return WeightInitializer(
weight_hh_init=hidden_weight,
weight_ih_init=input_weights
)
def __init__(self,
readout,
input_size: int = 1,
hidden_size: int = 500,
bias: bool = False,
initializer: WeightInitializer = WeightInitializer(),
num_layers=2,
activation: Activation = A.self_normalizing_default(),
washout: int = 30):
super().__init__(reservoir=MultiTimeSeriesHandler(
DeepESNCell(input_size, hidden_size, bias, initializer, num_layers,
activation)),
readout=readout,
washout=washout)
def __init__(self,
input_size: int,
hidden_size: int,
bias: bool,
initializer: WeightInitializer = WeightInitializer(),
num_chunks: int = 1,
requires_grad: bool = False,
init: bool = True):
super(ESNCellBase, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.requires_grad = requires_grad
self.bias = bias
self.weight_ih, self.weight_hh, self.bias_ih, self.bias_hh = None, None, None, None
if init:
self.init_parameters(initializer)
def __init__(self,
input_size: int,
hidden_size: int,
bias: bool = True,
initializer: WeightInitializer = WeightInitializer(),
activation: Activation = A.tanh(),
num_chunks: int = 1,
requires_grad: bool = False):
super(ESNCell, self).__init__(input_size,
hidden_size,
bias,
initializer=initializer,
num_chunks=num_chunks,
requires_grad=requires_grad)
self.requires_grad = requires_grad
self.activation = activation
self.hx = None
def __init__(self,
input_size: int = 1,
hidden_size: int = 250,
output_dim: int = 1,
bias: bool = False,
initializer: WeightInitializer = WeightInitializer(),
groups=4,
activation: Activation = A.self_normalizing_default(),
washout: int = 30,
regularization: float = 1.):
super().__init__(reservoir=MultiTimeSeriesHandler(
GroupOfESNCell(input_size, hidden_size, groups, activation, bias,
initializer)),
readout=SVDReadout(hidden_size * groups,
output_dim,
regularization=regularization),
washout=washout)
def __init__(self,
input_size: int = 1,
hidden_size: int = 500,
output_dim: int = 1,
bias: bool = False,
initializer: WeightInitializer = WeightInitializer(),
num_layers=2,
activation=A.self_normalizing_default(),
washout: int = 30,
regularization: float = 1.):
super().__init__(reservoir=MultiTimeSeriesHandler(
DeepESNCell(input_size, hidden_size, bias, initializer, num_layers,
activation)),
readout=SVDReadout(hidden_size * num_layers,
output_dim,
regularization=regularization),
washout=washout)
def regular_graph_initializer(seed, degree):
# initialize input weights with uniform distribution from -1 to 1 and specified seed to reproduce results
input_weight = CompositeInitializer().with_seed(seed).uniform()
# specified operations will be done one by one, so this "builder" can be seen as a list of transforms
# first set the seed and start with uniform distribution
# then treat the newly created dense matrix as adjacency matrix and transform it into regular graph with
# desired degree, then apply spectral normalization, so that spectral radius is 1.
# at the end scale the matrix by factor 0.9 and the initialization is done
reservoir_weight = CompositeInitializer() \
.with_seed(seed) \
.uniform() \
.regular_graph(degree) \
.spectral_normalize() \
.scale(0.9)
return WeightInitializer(weight_ih_init=input_weight,
weight_hh_init=reservoir_weight)
def __init__(self,
input_size: int,
hidden_size: int,
bias=False,
initializer: WeightInitializer = WeightInitializer(),
num_layers: int = 1,
activation: Activation = 'default'):
super().__init__()
if type(activation) != list:
activation = [activation] * num_layers
else:
activation = activation
self.layers = [
ESNCell(input_size, hidden_size, bias, initializer, activation[0])
]
if num_layers > 1:
self.layers += [
ESNCell(hidden_size, hidden_size, bias, initializer,
activation[i]) for i in range(1, num_layers)
]
self.gpu_enabled = False
def __init__(self,
input_size: int,
hidden_size: int,
groups,
activation=A.self_normalizing_default(),
bias: bool = False,
initializer: WeightInitializer = WeightInitializer()):
super(GroupOfESNCell, self).__init__()
num_groups = groups if type(groups) == int else len(groups)
if type(activation) != list:
activation = [activation] * num_groups
else:
activation = activation
if type(groups) != int:
self.groups = groups
else:
self.groups = [
ESNCell(input_size, hidden_size, bias, initializer,
activation[i]) for i in range(groups)
]
self.hidden_size = hidden_size
self.gpu_enabled = False
def __init__(self,
input_size: int = 1,
hidden_size: int = 250,
output_dim: int = 1,
bias: bool = False,
initializer: WeightInitializer = WeightInitializer(),
groups=2,
num_layers=(2, 2),
activation: Activation = A.self_normalizing_default(),
washout: int = 30,
regularization: float = 1.,
network_size=None):
hidden_size = hidden_size if network_size is None else network_size // sum(
num_layers)
super().__init__(reservoir=MultiTimeSeriesHandler(
GroupOfESNCell(input_size, hidden_size, [
DeepESNCell(input_size, hidden_size, bias, initializer, layers,
activation) for layers in num_layers
], activation, bias, initializer)),
readout=SVDReadout(hidden_size * groups,
output_dim,
regularization=regularization),
washout=washout)
sunspot = dl.loader_explicit(Sunspot, test_size=600)
nrmse = NRMSELoss()
if norm:
X, X_test, y, y_test, centr, spread = dl.norm_loader__(sunspot)
y_test = spread * y_test + centr
else:
X, X_test, y, y_test = sunspot()
i = CompositeInitializer()\
.with_seed(12)\
.uniform()\
.regular_graph(4)\
.spectral_normalize()\
.scale(0.9)
w = WeightInitializer()
w.weight_hh_init = i
esn = DeepESN(initializer= w, hidden_size=500, num_layers=2)
start = time.time()
# esn.to_cuda()
esn.fit(X, y)
if norm:
output = spread * esn(X_test) + centr
else:
output = esn(X_test)
print(time.time()-start)
n = nrmse(output, y_test).item()
print(n)
last = 50