def __init__(self,
rnn_type,
ntoken,
ninp,
nhid,
nlayers,
dropout=0.5,
tie_weights=False):
super(RNNModel, self).__init__()
self.rnn_type = rnn_type
self.nhid = nhid
self.nlayers = nlayers
self.initializer = Initializer() #初始化，这个utils的实现？
self.drop = nn.Dropout(dropout) #两个参数，dropout 随机失活概率；inplace:默认是False
self.encoder = nn.Embedding(
ntoken, ninp
) #ninp是每一个embedding向量的维数 ntoken是size of the dictionary of embeddings
self.initializer.init_embedding_(self.encoder) #初始化embedding
if rnn_type in ['LSTM', 'GRU']:
self.rnn = getattr(nn, rnn_type)(
ninp, nhid, nlayers, dropout=dropout,
batch_first=True) #getattr就是获得某属性的值，这里就是获取nn的网络类型值，如获得nn的LSTM
else:
try:
nonlinearity = {
'RNN_TANH': 'tanh',
'RNN_RELU': 'relu'
}[rnn_type] #获取字典的这个rnn_type值
except KeyError:
raise ValueError(
"""An invalid option for `--model` was supplied,
options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']"""
)
self.rnn = nn.RNN(ninp,
nhid,
nlayers,
nonlinearity=nonlinearity,
dropout=dropout,
batch_first=True) #激活函数的选择
self.decoder = nn.Linear(nhid, ntoken)
# Optionally tie weights as in:
# "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
# https://arxiv.org/abs/1608.05859
# and
# "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
# https://arxiv.org/abs/1611.01462
if tie_weights:
if nhid != ninp:
raise ValueError(
'When using the tied flag, nhid must be equal to emsize'
) #若是softmax和word embedding 权重绑定。则hidden层和emsize相等
self.decoder.weight = self.encoder.weight #若绑定就是embedding的encoding.weight 和 decoder.weight相等
else:
self.initializer.init_linear_(self.decoder, nhid,
dropout=dropout) #对线性层进行初始化
def stdp(self, wee, x, cutoff_weights):
""" Apply STDP rule : Regulates synaptic strength between the pre(Xj) and post(Xi) synaptic neurons"""
x = np.asarray(x)
xt_1 = x[:, 0]
xt = x[:, 1]
wee_t = wee.copy()
# STDP applies only on the neurons which are connected.
for i in range(len(wee_t[0])): # Each neuron i, Post-synaptic neuron
for j in range(len(wee_t[0:])): # Incoming connection from jth pre-synaptic neuron to ith neuron
if wee_t[j][i] != 0.: # Check connectivity
# Get the change in weight
delta_wee_t = self.eta_stdp * (xt[i] * xt_1[j] - xt_1[i] * xt[j])
# Update the weight between jth neuron to i ""Different from notation in article
wee_t[j][i] = wee[j][i] + delta_wee_t
""" Prune the smallest weights induced by plasticity mechanisms; Apply lower cutoff weight"""
wee_t = Initializer.prune_small_weights(wee_t, cutoff_weights[0])
"""Check and set all weights < upper cutoff weight """
wee_t = Initializer.set_max_cutoff_weight(wee_t, cutoff_weights[1])
return wee_t
def istdp(self, wei, x, y, cutoff_weights):
# Apply iSTDP rule : Regulates synaptic strength between the pre(Yj) and post(Xi) synaptic neurons
# Excitatory network activity
x = np.asarray(x) # Array sanity check
xt_1 = x[:, 0]
xt = x[:, 1]
# Inhibitory network activity
y = np.asarray(y)
yt_1 = y[:, 0]
yt = y[:, 1]
# iSTDP applies only on the neurons which are connected.
wei_t = wei.copy()
for i in range(len(wei_t[0])): # Each neuron i, Post-synaptic neuron: means for each column;
for j in range(len(wei_t[0:])): # Incoming connection from j, pre-synaptic neuron to ith neuron
if wei_t[j][i] != 0.: # Check connectivity
# Get the change in weight
delta_wei_t = - self.eta_inhib * yt_1[j] * (1 - xt[i] * (1 + 1 / self.mu_ip))
# Update the weight between jth neuron to i ""Different from notation in article
wei_t[j][i] = wei[j][i] + delta_wei_t
""" Prune the smallest weights induced by plasticity mechanisms; Apply lower cutoff weight"""
wei_t = Initializer.prune_small_weights(wei_t, cutoff_weights[0])
"""Check and set all weights < upper cutoff weight """
wei_t = Initializer.set_max_cutoff_weight(wei_t, cutoff_weights[1])
return wei_t
def initialize_plasticity():
"""## NOTE: DO NOT TRANSPOSE THE WEIGHT MATRIX WEI FOR SORN 2 MODEL"""
# Create and initialize sorn object and variables
sorn_init = Sorn()
WEE_init = sorn_init.initialize_weight_matrix(network_type='Sparse', synaptic_connection='EE',
self_connection='False',
lambd_w=20)
WEI_init = sorn_init.initialize_weight_matrix(network_type='Sparse', synaptic_connection='EI',
self_connection='False',
lambd_w=40)
WIE_init = sorn_init.initialize_weight_matrix(network_type='Dense', synaptic_connection='IE',
self_connection='False',
lambd_w=None)
Wee_init = Initializer.zero_sum_incoming_check(WEE_init)
# Wei_init = initializer.zero_sum_incoming_check(WEI_init.T)
Wei_init = Initializer.zero_sum_incoming_check(WEI_init)
Wie_init = Initializer.zero_sum_incoming_check(WIE_init)
c = np.count_nonzero(Wee_init)
v = np.count_nonzero(Wei_init)
b = np.count_nonzero(Wie_init)
print('Network Initialized')
print('Number of connections in Wee %s , Wei %s, Wie %s' %(c, v, b))
print('Shapes Wee %s Wei %s Wie %s' % (Wee_init.shape, Wei_init.shape, Wie_init.shape))
# Normalize the incoming weights
normalized_wee = Initializer.normalize_weight_matrix(Wee_init)
normalized_wei = Initializer.normalize_weight_matrix(Wei_init)
normalized_wie = Initializer.normalize_weight_matrix(Wie_init)
te_init, ti_init = sorn_init.initialize_threshold_matrix(Sorn.te_min, Sorn.te_max, Sorn.ti_min, Sorn.ti_max)
x_init, y_init = sorn_init.initialize_activity_vector(Sorn.ne, Sorn.ni)
# Initializing variables from sorn_initialize.py
wee = normalized_wee.copy()
wei = normalized_wei.copy()
wie = normalized_wie.copy()
te = te_init.copy()
ti = ti_init.copy()
x = x_init.copy()
y = y_init.copy()
return wee, wei, wie, te, ti, x, y
def structural_plasticity(wee):
""" Add new connection value to the smallest weight between excitatory units randomly"""
p_c = np.random.randint(0, 10, 1)
if p_c == 0: # p_c= 0.1
""" Do structural plasticity """
# Choose the smallest weights randomly from the weight matrix wee
indexes = Initializer.get_unconnected_indexes(wee)
# Choose any idx randomly
idx_rand = random.choice(indexes)
if idx_rand[0] == idx_rand[1]:
idx_rand = random.choice(indexes)
wee[idx_rand[0]][idx_rand[1]] = 0.001
return wee
def initialize_weight_matrix(network_type, synaptic_connection, self_connection, lambd_w):
"""
Args:
network_type(str) - Spare or Dense
synaptic_connection(str) - EE,EI,IE: Note that Spare connection is defined only for EE connections
self_connection(str) - True or False: i-->i ; Network is tested only using j-->i
lambd_w(int) - Average number of incoming and outgoing connections per neuron
Returns:
weight_matrix(array) - Array of connection strengths
"""
if (network_type == "Sparse") and (self_connection == "False"):
""" Generate weight matrix for E-E/ E-I connections with mean lamda incoming and
out-going connections per neuron """
weight_matrix = Initializer.generate_lambd_connections(synaptic_connection, Sorn.ne, Sorn.ni, lambd_w, lambd_std=1)
# Dense matrix for W_ie
elif (network_type == 'Dense') and (self_connection == 'False'):
# Gaussian distribution of weights
# weight_matrix = np.random.randn(Sorn.ne, Sorn.ni) + 2 # Small random values from gaussian distribution
# Centered around 1
# weight_matrix.reshape(Sorn.ne, Sorn.ni)
# weight_matrix *= 0.01 # Setting spectral radius
# Uniform distribution of weights
weight_matrix = np.random.uniform(0.0, 0.1, (Sorn.ne, Sorn.ni))
weight_matrix.reshape((Sorn.ne, Sorn.ni))
return weight_matrix
class RNNModel(nn.Module):
"""Container module with an encoder, a recurrent module, and a decoder."""
def __init__(self,
rnn_type,
ntoken,
ninp,
nhid,
nlayers,
dropout=0.5,
tie_weights=False):
super(RNNModel, self).__init__()
self.rnn_type = rnn_type
self.nhid = nhid
self.nlayers = nlayers
self.initializer = Initializer() #初始化，这个utils的实现？
self.drop = nn.Dropout(dropout) #两个参数，dropout 随机失活概率；inplace:默认是False
self.encoder = nn.Embedding(
ntoken, ninp
) #ninp是每一个embedding向量的维数 ntoken是size of the dictionary of embeddings
self.initializer.init_embedding_(self.encoder) #初始化embedding
if rnn_type in ['LSTM', 'GRU']:
self.rnn = getattr(nn, rnn_type)(
ninp, nhid, nlayers, dropout=dropout,
batch_first=True) #getattr就是获得某属性的值，这里就是获取nn的网络类型值，如获得nn的LSTM
else:
try:
nonlinearity = {
'RNN_TANH': 'tanh',
'RNN_RELU': 'relu'
}[rnn_type] #获取字典的这个rnn_type值
except KeyError:
raise ValueError(
"""An invalid option for `--model` was supplied,
options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']"""
)
self.rnn = nn.RNN(ninp,
nhid,
nlayers,
nonlinearity=nonlinearity,
dropout=dropout,
batch_first=True) #激活函数的选择
self.decoder = nn.Linear(nhid, ntoken)
# Optionally tie weights as in:
# "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
# https://arxiv.org/abs/1608.05859
# and
# "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
# https://arxiv.org/abs/1611.01462
if tie_weights:
if nhid != ninp:
raise ValueError(
'When using the tied flag, nhid must be equal to emsize'
) #若是softmax和word embedding 权重绑定。则hidden层和emsize相等
self.decoder.weight = self.encoder.weight #若绑定就是embedding的encoding.weight 和 decoder.weight相等
else:
self.initializer.init_linear_(self.decoder, nhid,
dropout=dropout) #对线性层进行初始化
# self.init_weights()
def init_weights(self):
initrange = 0.1
self.encoder.weight.data.uniform_(-initrange, initrange)
self.decoder.bias.data.zero_()
self.decoder.weight.data.uniform_(-initrange, initrange)
def forward(self, input, words_lengths):
''' Forward pass:
- word embedding
- 输入循环神经网络
- 一个线性层从hidden state转化为输出单词表
'''
emb = self.drop(self.encoder(
input)) # batch_size * seq_len * emb_dim #将输入向量嵌入后随机置0
emb = nn.utils.rnn.pack_padded_sequence(emb,
words_lengths,
batch_first=True)
output_packed, _ = self.rnn(emb) #通过网络得到输出
output, _ = nn.utils.rnn.pad_packed_sequence(
output_packed, batch_first=True
) # output: batch_size * seq_len * num_direct*hidden_dim, num_direct = 1
output = self.drop(output) #对输出进行随机置0
decoded = self.decoder(
output.contiguous().view(
output.size(0) * output.size(1), output.size(2))
) #decoder，output.contiguous()的一种可能解释是因为view要基于整块内存，而Tensor可能是零碎的，所以用contiguous变成连续
return decoded.view(
output.size(0), output.size(1),
decoded.size(1)) # batch_size * seq_len * vocab_size
def run_sorn(self, inp):
# Initialize/Get the weight, threshold matrices and activity vectors
matrix_collection = MatrixCollection(phase=self.phase, matrices=self.matrices)
# Collect the network activity at all time steps
X_all = [0] * self.time_steps
Y_all = [0] * self.time_steps
R_all = [0] * self.time_steps
frac_pos_active_conn = []
# To get the last activation status of Exc and Inh neurons
for i in tqdm.tqdm(range(self.time_steps)):
""" Generate white noise"""
white_noise_e = Initializer.white_gaussian_noise(mu=0., sigma=0.04, t=Sorn.ne)
white_noise_i = Initializer.white_gaussian_noise(mu=0., sigma=0.04, t=Sorn.ni)
# Generate inputs
# inp_ = np.expand_dims(Initializer.generate_normal_inp(10), 1)
network_state = NetworkState(inp) # Feed input and initialize network state
# Buffers to get the resulting x and y vectors at the current time step and update the master matrix
x_buffer, y_buffer = np.zeros((Sorn.ne, 2)), np.zeros((Sorn.ni, 2))
# TODO: Return te,ti values in next version
te_buffer, ti_buffer = np.zeros((Sorn.ne, 1)), np.zeros((Sorn.ni, 1))
# Get the matrices and rename them for ease of reading
Wee, Wei, Wie = matrix_collection.Wee, matrix_collection.Wei, matrix_collection.Wie
Te, Ti = matrix_collection.Te, matrix_collection.Ti
X, Y = matrix_collection.X, matrix_collection.Y
""" Fraction of active connections between E-E network"""
frac_pos_active_conn.append((Wee[i] > 0.0).sum())
""" Recurrent drive"""
r = network_state.recurrent_drive(Wee[i], Wei[i], Te[i], X[i], Y[i], white_noise_e)
"""Get excitatory states and inhibitory states given the weights and thresholds"""
# x(t+1), y(t+1)
excitatory_state_xt_buffer = network_state.excitatory_network_state(Wee[i], Wei[i], Te[i], X[i], Y[i],
white_noise_e)
inhibitory_state_yt_buffer = network_state.inhibitory_network_state(Wie[i], Ti[i], X[i], white_noise_i)
""" Update X and Y """
x_buffer[:, 0] = X[i][:, 1] # xt -->(becomes) xt_1
x_buffer[:, 1] = excitatory_state_xt_buffer.T # New_activation; x_buffer --> xt
y_buffer[:, 0] = Y[i][:, 1]
y_buffer[:, 1] = inhibitory_state_yt_buffer.T
"""Plasticity phase"""
plasticity = Plasticity()
# TODO
# Can be initialised outside loop--> Plasticity will receive dynamic args in future version
# STDP
Wee_t = plasticity.stdp(Wee[i], x_buffer, cutoff_weights=(0.0, 1.0))
# Intrinsic plasticity
Te_t = plasticity.ip(Te[i], x_buffer)
# Structural plasticity
Wee_t = plasticity.structural_plasticity(Wee_t)
# iSTDP
Wei_t = plasticity.istdp(Wei[i], x_buffer, y_buffer, cutoff_weights=(0.0, 1.0))
# Synaptic scaling Wee
Wee_t = Plasticity().ss(Wee_t)
# Synaptic scaling Wei
Wei_t = Plasticity().ss(Wei_t)
"""Assign the matrices to the matrix collections"""
matrix_collection.weight_matrix(Wee_t, Wei_t, Wie[i], i)
matrix_collection.threshold_matrix(Te_t, Ti[i], i)
matrix_collection.network_activity_t(x_buffer, y_buffer, i)
X_all[i] = x_buffer[:, 1]
Y_all[i] = y_buffer[:, 1]
R_all[i] = r
plastic_matrices = {'Wee': matrix_collection.Wee[-1],
'Wei': matrix_collection.Wei[-1],
'Wie': matrix_collection.Wie[-1],
'Te': matrix_collection.Te[-1], 'Ti': matrix_collection.Ti[-1],
'X': X[-1], 'Y': Y[-1]}
return plastic_matrices, X_all, Y_all, R_all, frac_pos_active_conn