def __init__(self, params):
super(ModelCell, self).__init__()
self.device = torch.device("cpu")
if torch.cuda.is_available():
self.device = torch.device("cuda")
# set params
self.params = params
# create memory
self.memory = ROM(params.memory_n, params.memory_m)
# create controller
self.controller = LSTMController(self.memory.M, params.controller_size,
params.controller_layers)
# create state
self.state = State(self.memory, self.controller)
# create variational model
self.vmodel = params.variationalmodel(params.sequence_width,
params.variational_hidden_size,
params.memory_m, params.memory_m)
# create FC layer for addressing using controller output
self.addressing_params_sizes = [self.memory.M, 1, 1, 3, 1]
self.fc1 = nn.Linear(params.controller_size,
sum(self.addressing_params_sizes))
self.to(self.device)
def __init__(self, params):
super(ModelCell, self).__init__()
self.device = torch.device("cpu")
if torch.cuda.is_available():
self.device = torch.device("cuda")
# set params
self.params = params
# create memory
self.memory = ROM(params.memory_n, params.memory_m)
# create controller
self.controller = LSTMController(
params.sequence_width + 1 + self.memory.M, params.controller_size,
params.controller_layers)
# create state
self.state = State(self.memory, self.controller)
# create FC layer for addressing using controller output
self.addressing_params_sizes = [self.memory.M, 1, 1, 3, 1]
self.fc1 = nn.Sequential(
nn.Linear(params.controller_size, sum(
self.addressing_params_sizes))) # no sigmoid needed here.
# create FC layer to make output from controller output and read value
self.fc2 = nn.Sequential(
nn.Linear(params.controller_size + self.memory.M,
params.sequence_width + 1), nn.Sigmoid())
self.to(self.device)
def __init__(self,
num_inputs,
num_outputs,
controller_size,
controller_layers,
num_heads,
N,
M,
K,
controller_type='MLP'):
"""Initialize an EncapsulatedNTM.
:param num_inputs: External number of inputs.
:param num_outputs: External number of outputs.
:param controller_size: The size of the internal representation. Note: require it to be the same as M
:param controller_layers: Controller number of layers.
:param num_heads: Number of heads.
:param N: Number of rows in the memory bank.
:param M: Number of cols/features in the memory bank.
:param K: Number of cols/features in the keys.
"""
super(EncapsulatedGCL, self).__init__()
# Save args
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.controller_size = controller_size
self.controller_type = controller_type
self.controller_layers = controller_layers
self.num_heads = num_heads
self.N = N
self.M = M
self.K = K
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
# Create the NTM components
self.memory = GCLMemory(N, M, K).to(self.device)
if controller_type == 'LSTM':
# controller = LSTMController(num_inputs + M*num_heads, controller_size, controller_layers)
controller = LSTMController(num_inputs, controller_size,
controller_layers)
elif controller_type == 'MLP':
# controller = MLPController(num_inputs + M*num_heads, controller_size)
controller = MLPController(num_inputs, controller_size,
controller_layers)
elif controller_type == 'simple':
controller = SimpleController(num_inputs, controller_size)
heads = nn.ModuleList([])
for i in range(num_heads):
heads += [
GCLReadHead(self.memory, controller_size),
GCLWriteHead(self.memory, controller_size)
]
self.gcl = GCL(num_inputs, num_outputs, controller, self.memory,
heads).to(self.device)
def __init__(self,
num_inputs,
num_outputs,
controller_size,
controller_layers,
num_heads,
N,
M,
controller_type='lstm'):
"""Initialize an EncapsulatedNTM.
:param num_inputs: External number of inputs.
:param num_outputs: External number of outputs.
:param controller_size: The size of the internal representation.
:param controller_layers: Controller number of layers.
:param num_heads: Number of heads.
:param N: Number of rows in the memory bank.
:param M: Number of cols/features in the memory bank.
"""
super(EncapsulatedNTM, self).__init__()
# Save args
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.controller_size = controller_size
self.controller_layers = controller_layers
self.num_heads = num_heads
self.N = N
self.M = M
# Create the NTM components
memory = NTMMemory(N, M)
if controller_type == 'lstm':
controller = LSTMController(num_inputs + M * num_heads,
controller_size, controller_layers)
else:
controller = MLPController(num_inputs + M * num_heads,
controller_size, controller_layers)
heads = nn.ModuleList([])
for i in range(num_heads):
heads += [
NTMReadHead(memory, controller_size),
NTMWriteHead(memory, controller_size)
]
self.ntm = NTM(num_inputs, num_outputs, controller, memory, heads)
self.memory = memory
def __init__(self, N, M, in_size, out_size, batch_size, lstm=False):
super(NTM, self).__init__()
if lstm:
self.controller = LSTMController(in_size, out_size, M, batch_size)
else:
self.controller = FeedForwardController(in_size, out_size, M, batch_size)
self.read_head = Head(batch_size, N, M)
self.write_head = Head(batch_size, N, M)
self.batch_size = batch_size
self.N = N
self.M = M
self.eps = 1e-8
self.memory = None
self.register_parameter('memory_bias',
nn.Parameter(torch.randn(1, N, M) / math.sqrt(N)))
def __init__(self,
num_inputs,
num_outputs,
controller_size,
controller_layers,
num_heads,
N,
M0,
M1,
K,
controller_type='MLP'):
"""Initialize an EncapsulatedNTM.
:param num_inputs: External number of inputs.
:param num_outputs: External number of outputs.
:param controller_size: The size of the internal representation.
:param controller_layers: Controller number of layers.
:param num_heads: Number of heads.
:param N: Number of rows in the memory bank.
:param M: Number of cols/features in the memory bank.
:param K: Number of cols/features in the keys.
"""
super(EncapsulatedGCLDual, self).__init__()
# Save args
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.controller_size = controller_size
self.controller_type = controller_type
self.controller_layers = controller_layers
self.num_heads = num_heads
self.N = N
self.M0 = M0
self.M1 = M1
self.K = K
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
# Create the NTM components
memory_hidden = GCLMemory(N, M0, K)
memory_output = GCLMemory(N, M1, K)
if controller_type == 'LSTM':
# controller = LSTMController(num_inputs + M*num_heads, controller_size, controller_layers)
controller = LSTMController(num_inputs, controller_size,
controller_layers)
elif controller_type == 'MLP':
# controller = MLPController(num_inputs + M*num_heads, controller_size)
controller = MLPController(num_inputs, controller_size)
elif controller_type == 'simple':
controller = SimpleController()
heads = nn.ModuleList([])
for i in range(num_heads): # each head has its specific memory
heads += [
GCLReadHead(memory_hidden, controller_size),
# GCLReadHead(memory_output, controller_size),
GCLWriteHead(memory_hidden, controller_size)
# GCLWriteHead(memory_output, controller_size)
]
heads.append(GCLReadHead(memory_output, controller_size))
heads.append(GCLWriteHead(memory_output, controller_size))
self.gcl_dual = GCLDual(num_inputs, num_outputs, controller,
memory_hidden, memory_output,
heads).to(self.device)
self.memory_hidden = memory_hidden.to(self.device)
self.memory_output = memory_output.to(self.device)
self.memory = [self.memory_hidden,
self.memory_output] # just for convenience
self.write_weights = []
self.memory_output_history = None