def __init__(self, visible_dim, hidden_dim, rnn_dimensions=(128, 128), **kwargs):
super(Rnnrbm, self).__init__(**kwargs)
self.rnn_dimensions = rnn_dimensions
self.visible_dim = visible_dim
self.hidden_dim = hidden_dim
# self.in_layer = Linear(input_dim=input_dim, output_dim=rnn_dimension * 4,
# weights_init=IsotropicGaussian(0.01),
# biases_init=Constant(0.0),
# use_bias=False,
# name="in_layer")
self.rbm = Rbm(visible_dim=visible_dim, hidden_dim=hidden_dim,
activation=Sigmoid(), weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.1),
name='rbm')
self.uv = Linear(input_dim=rnn_dimensions[-1], output_dim=visible_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True, name='uv')
self.uh = Linear(input_dim=rnn_dimensions[-1], output_dim=hidden_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True, name='uh')
self.rnn = Rnn([visible_dim] + list(rnn_dimensions), name='rnn')
self.children = [self.rbm, self.uv, self.uh, self.rnn] + self.rnn.children._items
def main(args):
print(args)
if args.seed != -1:
torch.manual_seed(args.seed)
np.random.seed(args.seed)
pattern_specs = OrderedDict(sorted(([int(y) for y in x.split("-")] for x in args.patterns.split("_")),
key=lambda t: t[0]))
n = args.num_train_instances
mlp_hidden_dim = args.mlp_hidden_dim
num_mlp_layers = args.num_mlp_layers
dev_vocab = vocab_from_text(args.vd)
print("Dev vocab size:", len(dev_vocab))
vocab, embeddings, word_dim = \
read_embeddings(args.embedding_file, dev_vocab)
num_padding_tokens = max(list(pattern_specs.keys())) - 1
dev_input, dev_text = read_docs(args.vd, vocab, num_padding_tokens=num_padding_tokens)
dev_labels = read_labels(args.vl)
dev_data = list(zip(dev_input, dev_labels))
if n is not None:
dev_data = dev_data[:n]
num_classes = len(set(dev_labels))
print("num_classes:", num_classes)
semiring = \
MaxPlusSemiring if args.maxplus else (
LogSpaceMaxTimesSemiring if args.maxtimes else ProbSemiring
)
if args.use_rnn:
rnn = Rnn(word_dim,
args.hidden_dim,
cell_type=LSTM,
gpu=args.gpu)
else:
rnn = None
model = SoftPatternClassifier(pattern_specs, mlp_hidden_dim, num_mlp_layers, num_classes, embeddings, vocab,
semiring, args.bias_scale_param, args.gpu, rnn=rnn, pre_computed_patterns=None)
if args.gpu:
print("Cuda!")
model.to_cuda(model)
state_dict = torch.load(args.input_model)
else:
state_dict = torch.load(args.input_model, map_location=lambda storage, loc: storage)
# Loading model
model.load_state_dict(state_dict)
interpret_documents(model, args.batch_size, dev_data, dev_text, args.ofile, args.max_doc_len)
return 0
class BeerTrackerWallGenotype(BeerTrackerGenotype):
num_input_nodes = 7
num_hidden_nodes = 8
num_output_nodes = 2
bits_per_weight = 8
rnn = Rnn(num_input_nodes, num_hidden_nodes, num_output_nodes)
GENOTYPE_SIZE = rnn.num_edges * bits_per_weight
class AveragingRnnClassifier(Module):
"""
A text classification model that runs a biLSTM (or biGRU) over a document,
averages the hidden states, then feeds that into an MLP.
"""
def __init__(self,
hidden_dim,
mlp_hidden_dim,
num_mlp_layers,
num_classes,
embeddings,
cell_type=LSTM,
gpu=False):
super(AveragingRnnClassifier, self).__init__()
self.embeddings = embeddings
self.rnn = \
Rnn(len(embeddings[0]),
hidden_dim,
cell_type=cell_type,
gpu=gpu)
self.mlp = \
MLP(self.rnn.num_directions * self.rnn.hidden_dim,
mlp_hidden_dim,
num_mlp_layers,
num_classes)
self.to_cuda = to_cuda(gpu)
print("# params:", sum(p.nelement() for p in self.parameters()))
def forward(self, batch, debug=0, dropout=None):
"""
Run a biLSTM over the batch of docs, average the hidden states, and
feed into an MLP.
"""
b = len(batch.docs)
outs = self.rnn.forward(batch, debug=debug, dropout=dropout)
padded, _ = pad_packed_sequence(
outs) # size: (max_doc_len, b, 2 * hidden_dim)
if dropout is not None:
padded = dropout(padded)
# average all the hidden states
outs_sum = torch.sum(padded, dim=0) # size: (b, 2 * hidden_dim)
outs_avg = torch.div(
outs_sum,
Variable(batch.doc_lens.float().view(b, 1)).expand(
b, self.rnn.num_directions *
self.rnn.hidden_dim)) # size: (b, 2 * hidden_dim)
return self.mlp.forward(outs_avg) # size: (b, num_classes)
def predict(self, batch, debug=0):
old_training = self.training
self.train(False)
output = self.forward(batch, debug=debug).data
_, am = torch.max(output, 1)
self.train(old_training)
return [int(x) for x in am]
def __init__(self,
visible_dim,
hidden_dim,
rnn_dimensions=(128, 128),
**kwargs):
super(Rnnrbm, self).__init__(**kwargs)
self.rnn_dimensions = rnn_dimensions
self.visible_dim = visible_dim
self.hidden_dim = hidden_dim
# self.in_layer = Linear(input_dim=input_dim, output_dim=rnn_dimension * 4,
# weights_init=IsotropicGaussian(0.01),
# biases_init=Constant(0.0),
# use_bias=False,
# name="in_layer")
self.rbm = Rbm(visible_dim=visible_dim,
hidden_dim=hidden_dim,
activation=Sigmoid(),
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.1),
name='rbm')
self.uv = Linear(input_dim=rnn_dimensions[-1],
output_dim=visible_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True,
name='uv')
self.uh = Linear(input_dim=rnn_dimensions[-1],
output_dim=hidden_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True,
name='uh')
self.rnn = Rnn([visible_dim] + list(rnn_dimensions), name='rnn')
self.children = [self.rbm, self.uv, self.uh, self.rnn
] + self.rnn.children._items
def run_language_model(dataset,
max_epochs,
hidden_size=100,
sequence_length=30,
learning_rate=1e-1,
sample_every=100):
vocab_size = len(dataset.sorted_chars)
RNN = Rnn(vocab_size, hidden_size, sequence_length,
learning_rate) # initialize the recurrent network
current_epoch = 0
batch = 0
#h0 = np.zeros((hidden_size, dataset.batch_size))
h0 = np.zeros((dataset.batch_size, hidden_size))
#TODO
seed = "HAN:\nIs that good or bad?\n\n" #"Lorem ipsum"#
n_sample = 300
average_loss = 0
while current_epoch < max_epochs:
e, x, y = dataset.next_minibatch()
if e:
current_epoch += 1
#h0 = np.zeros((hidden_size, dataset.batch_size))
h0 = np.zeros((dataset.batch_size, hidden_size))
# why do we reset the hidden state here?
# One-hot transform the x and y batches
x_oh, y_oh = dataset.one_hot(x, vocab_size), dataset.one_hot(
y, vocab_size)
# Run the recurrent network on the current batch
# Since we are using windows of a short length of characters,
# the step function should return the hidden state at the end
# of the unroll. You should then use that hidden state as the
# input for the next minibatch. In this way, we artificially
# preserve context between batches.
loss, h0 = RNN.step(h0, x_oh, y_oh)
average_loss += loss
if batch % sample_every == 0:
# run sampling (2.2)
print("epoch: %d \t batch: %d/%d \t" %
(current_epoch, batch % dataset.num_batches,
dataset.num_batches),
end="")
print("Average_loss : %f" % (average_loss /
(batch * dataset.batch_size)))
print(sample(RNN, seed, n_sample, dataset))
batch += 1
def __init__(self,
hidden_dim,
mlp_hidden_dim,
num_mlp_layers,
num_classes,
embeddings,
cell_type=LSTM,
gpu=False):
super(AveragingRnnClassifier, self).__init__()
self.embeddings = embeddings
self.rnn = \
Rnn(len(embeddings[0]),
hidden_dim,
cell_type=cell_type,
gpu=gpu)
self.mlp = \
MLP(self.rnn.num_directions * self.rnn.hidden_dim,
mlp_hidden_dim,
num_mlp_layers,
num_classes)
self.to_cuda = to_cuda(gpu)
print("# params:", sum(p.nelement() for p in self.parameters()))
def calculate_phenotype(self):
self.phenotype = Rnn(
BeerTrackerIndividual.genotype_class.num_input_nodes,
BeerTrackerIndividual.genotype_class.num_hidden_nodes,
BeerTrackerIndividual.genotype_class.num_output_nodes)
weights = []
calculate_weight = self.calculate_summed_weight
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['input_hidden']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_hidden']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['bias_hidden']):
weight = calculate_weight(i, 'internal_bias')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_output']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_gains']):
weight = calculate_weight(i, 'gain')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_time_constants']):
weight = calculate_weight(i, 'time_constant')
weights.append(weight)
self.phenotype.set_weights(weights)
from rnn import Rnn
from time import sleep
from math import *
from random import *
import numpy as np
#Parameters
populationSize = 100
breedingPoolSize = 30
hiddenLayerSize = 50
#Initiation
population = []
for x in range(0, populationSize):
print("Starting thread " + str(x + 1))
rnn = Rnn(hiddenLayerSize)
rnn.start()
population.append(rnn)
for i in range(1, 100):
#Run the population
print("Waiting for population to run")
sleep(6)
for rnn in population:
rnn.Stop()
error = []
for rnn in population:
state = rnn.GetState()
def main(args):
print(args)
pattern_specs = OrderedDict(
sorted(
([int(y) for y in x.split("-")] for x in args.patterns.split("_")),
key=lambda t: t[0]))
pre_computed_patterns = None
if args.pre_computed_patterns is not None:
pre_computed_patterns = read_patterns(args.pre_computed_patterns,
pattern_specs)
pattern_specs = OrderedDict(
sorted(pattern_specs.items(), key=lambda t: t[0]))
n = args.num_train_instances
mlp_hidden_dim = args.mlp_hidden_dim
num_mlp_layers = args.num_mlp_layers
if args.seed != -1:
torch.manual_seed(args.seed)
np.random.seed(args.seed)
dev_vocab = vocab_from_text(args.vd)
print("Dev vocab size:", len(dev_vocab))
train_vocab = vocab_from_text(args.td)
print("Train vocab size:", len(train_vocab))
dev_vocab |= train_vocab
vocab, embeddings, word_dim = \
read_embeddings(args.embedding_file, dev_vocab)
num_padding_tokens = max(list(pattern_specs.keys())) - 1
dev_input, _ = read_docs(args.vd,
vocab,
num_padding_tokens=num_padding_tokens)
dev_labels = read_labels(args.vl)
dev_data = list(zip(dev_input, dev_labels))
np.random.shuffle(dev_data)
num_iterations = args.num_iterations
train_input, _ = read_docs(args.td,
vocab,
num_padding_tokens=num_padding_tokens)
train_labels = read_labels(args.tl)
print("training instances:", len(train_input))
num_classes = len(set(train_labels))
# truncate data (to debug faster)
train_data = list(zip(train_input, train_labels))
np.random.shuffle(train_data)
print("num_classes:", num_classes)
if n is not None:
train_data = train_data[:n]
dev_data = dev_data[:n]
if args.use_rnn:
rnn = Rnn(word_dim, args.hidden_dim, cell_type=LSTM, gpu=args.gpu)
else:
rnn = None
semiring = \
MaxPlusSemiring if args.maxplus else (
LogSpaceMaxTimesSemiring if args.maxtimes else ProbSemiring
)
model = SoftPatternClassifier(pattern_specs, mlp_hidden_dim,
num_mlp_layers, num_classes, embeddings,
vocab, semiring, args.bias_scale_param,
args.gpu, rnn, pre_computed_patterns,
args.no_sl, args.shared_sl, args.no_eps,
args.eps_scale, args.self_loop_scale)
if args.gpu:
model.to_cuda(model)
model_file_prefix = 'model'
# Loading model
if args.input_model is not None:
state_dict = torch.load(args.input_model)
model.load_state_dict(state_dict)
model_file_prefix = 'model_retrained'
model_save_dir = args.model_save_dir
if model_save_dir is not None:
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
print("Training with", model_file_prefix)
train(train_data, dev_data, model, num_classes, model_save_dir,
num_iterations, model_file_prefix, args.learning_rate,
args.batch_size, args.scheduler, args.gpu, args.clip,
args.max_doc_len, args.debug, args.dropout, args.word_dropout,
args.patience)
return 0
default="standard"
)
args = arg_parser.parse_args()
with open('best_individual.json') as best_agent_weights:
weights = json.load(best_agent_weights)
if args.scenario == 'pull':
genotype_class = BeerTrackerPullGenotype
elif args.scenario == 'wall':
genotype_class = BeerTrackerWallGenotype
else:
genotype_class = BeerTrackerGenotype
nn = Rnn(
num_input_nodes=genotype_class.num_input_nodes,
num_hidden_nodes=genotype_class.num_hidden_nodes,
num_output_nodes=genotype_class.num_output_nodes
)
nn.set_weights(weights)
activations = [
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[1, 1, 0, 0, 0],
[0, 0, 1, 1, 0],
[1, 1, 0, 0, 0],
[0, 0, 0, 0, 0],
[1, 1, 0, 0, 0],
[1, 1, 1, 1, 0],
[1, 1, 1, 1, 1],
last_pitch = None
current_duration = 1
for note in notes:
midi_pitch = note + _LOWEST_NOTE
if last_pitch = midi_pitch:
current_duration += 1
elif last_pitch is not None:
midi_pitches.append(
{'pitch': last_pitch, 'duration': current_duration})
current_duration = 1
last_pitch = midi_pitch
midi_pitches.append(
{'pitch': last_pitch, 'duration': current_duration})
return midi_pitches
rnn = Rnn()
for progression in TRACKS:
notes, ticks_per_beat = read_file(progression.midi_filename)
input_data = _create_input_tensor(notes, chords, ticks_per_beat)
input_labels = [
[ _encode(nt[7]) for nt in note_tensors ] for note_tensors in input_data
]
del input_labels[0]
input_labels.append(0) # add on one fake prediction of a rest
rnn.train(
_split(input_data, SEQUENCE_LENGTH),
_split(input_labels, SEQUENCE_LENGTH)
)
notes = rnn.generate(TRACKS[0], 12) # pick a random start note about in the middle
def main(args):
print(args)
n = args.num_train_instances
mlp_hidden_dim = args.mlp_hidden_dim
num_mlp_layers = args.num_mlp_layers
dev_vocab = vocab_from_text(args.vd)
print("Dev vocab size:", len(dev_vocab))
vocab, embeddings, word_dim = \
read_embeddings(args.embedding_file, dev_vocab)
if args.seed != -1:
torch.manual_seed(args.seed)
np.random.seed(args.seed)
if args.dan or args.bilstm:
num_padding_tokens = 1
elif args.cnn:
num_padding_tokens = args.window_size - 1
else:
pattern_specs = OrderedDict(sorted(([int(y) for y in x.split("-")] for x in args.patterns.split("_")),
key=lambda t: t[0]))
num_padding_tokens = max(list(pattern_specs.keys())) - 1
dev_input, dev_text = read_docs(args.vd, vocab, num_padding_tokens=num_padding_tokens)
dev_labels = read_labels(args.vl)
dev_data = list(zip(dev_input, dev_labels))
if n is not None:
dev_data = dev_data[:n]
num_classes = len(set(dev_labels))
print("num_classes:", num_classes)
if args.dan:
model = DanClassifier(mlp_hidden_dim,
num_mlp_layers,
num_classes,
embeddings,
args.gpu)
elif args.bilstm:
cell_type = LSTM
model = AveragingRnnClassifier(args.hidden_dim,
mlp_hidden_dim,
num_mlp_layers,
num_classes,
embeddings,
cell_type=cell_type,
gpu=args.gpu)
elif args.cnn:
model = PooledCnnClassifier(args.window_size,
args.num_cnn_layers,
args.cnn_hidden_dim,
num_mlp_layers,
mlp_hidden_dim,
num_classes,
embeddings,
pooling=max_pool_seq,
gpu=args.gpu)
else:
semiring = \
MaxPlusSemiring if args.maxplus else (
LogSpaceMaxTimesSemiring if args.maxtimes else ProbSemiring
)
if args.use_rnn:
rnn = Rnn(word_dim,
args.hidden_dim,
cell_type=LSTM,
gpu=args.gpu)
else:
rnn = None
model = SoftPatternClassifier(pattern_specs, mlp_hidden_dim, num_mlp_layers, num_classes, embeddings, vocab,
semiring, args.bias_scale_param, args.gpu, rnn, None, args.no_sl, args.shared_sl,
args.no_eps, args.eps_scale, args.self_loop_scale)
if args.gpu:
state_dict = torch.load(args.input_model)
else:
state_dict = torch.load(args.input_model, map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
if args.gpu:
model.to_cuda(model)
test_acc = evaluate_accuracy(model, dev_data, args.batch_size, args.gpu)
print("Test accuracy: {:>8,.3f}%".format(100*test_acc))
return 0
class Rnnrbm(BaseRecurrent, Initializable):
@lazy(allocation=['visible_dim', 'hidden_dim'])
def __init__(self,
visible_dim,
hidden_dim,
rnn_dimensions=(128, 128),
**kwargs):
super(Rnnrbm, self).__init__(**kwargs)
self.rnn_dimensions = rnn_dimensions
self.visible_dim = visible_dim
self.hidden_dim = hidden_dim
# self.in_layer = Linear(input_dim=input_dim, output_dim=rnn_dimension * 4,
# weights_init=IsotropicGaussian(0.01),
# biases_init=Constant(0.0),
# use_bias=False,
# name="in_layer")
self.rbm = Rbm(visible_dim=visible_dim,
hidden_dim=hidden_dim,
activation=Sigmoid(),
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.1),
name='rbm')
self.uv = Linear(input_dim=rnn_dimensions[-1],
output_dim=visible_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True,
name='uv')
self.uh = Linear(input_dim=rnn_dimensions[-1],
output_dim=hidden_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True,
name='uh')
self.rnn = Rnn([visible_dim] + list(rnn_dimensions), name='rnn')
self.children = [self.rbm, self.uv, self.uh, self.rnn
] + self.rnn.children._items
def initialize(self, pretrained_rbm=None, pretrained_rnn=None, **kwargs):
super(Rnnrbm, self).initialize(**kwargs)
# if pretrained_rbm is not None:
# self.rbm.bv.set_value(pretrained_rbm.bv.get_value())
# self.uv.b.set_value(pretrained_rbm.bv.get_value())
# self.rbm.bh.set_value(pretrained_rbm.bh.get_value())
# self.uh.b.set_value(pretrained_rbm.bh.get_value())
# self.rbm.W.set_value(pretrained_rbm.W.get_value())
# if pretrained_rnn is not None:
# for param, trained_param in zip(itertools.chain(*[child.params for child in self.rnn.children]),
# itertools.chain(*[child.params for child in pretrained_rnn.children])):
# param.set_value(trained_param.get_value())
self.uv.b.name = 'buv'
self.uv.W.name = 'Wuv'
self.uh.W.name = 'Wuh'
self.uh.b.name = 'buh'
self.rbm.bh.name = 'buh'
self.rbm.bv.name = 'buv'
# self.rnn.input_transform.b.name = 'bvu'
# self.rnn.input_transform.W.name = 'Wvu'
# def _allocate(self):
# Wrbm = shared_floatx_nans((self.visible_dim, self.hidden_dim), name='Wrbm')
# add_role(Wrbm, WEIGHT)
# self.params.append(Wrbm)
# self.add_auxiliary_variable(Wrbm.norm(2), name='Wrbm_norm')
#
#
# def _initialize(self):
# for param in self.params:
# if has_roles(param, WEIGHT):
# self.weights_init.initialize(param, self.rng)
# elif has_roles(param, BIAS):
# self.biases_init.initialize(param, self.rng)
def get_dim(self, name):
dims = {
'visible': self.visible_dim,
'mask': self.visible_dim,
'gru_state': self.rnn_dimensions[0],
'lstm_state': self.rnn_dimensions[1],
'lstm_cells': self.rnn_dimensions[1]
}
return dims.get(name, None) or super(Rnnrbm, self).get_dim(name)
@recurrent(sequences=[],
contexts=['visible'],
states=['gru_state', 'lstm_state', 'lstm_cells'],
outputs=['visible', 'gru_state', 'lstm_state', 'lstm_cells'])
def generate(self,
visible=None,
gru_state=None,
lstm_state=None,
lstm_cells=None,
rbm_steps=25):
bv = self.uv.apply(lstm_state)
bh = self.uh.apply(lstm_state)
visible_start = rng.binomial(size=visible.shape,
n=1,
p=bv,
dtype=floatX)
_, visible = self.rbm.apply(visible=visible_start,
bv=bv,
bh=bh,
n_steps=rbm_steps,
batch_size=visible.shape[0])
visible = visible[-1]
gru_state, lstm_state, lstm_cells = self.rnn.rnn_apply(
inputs=visible,
gru_state=gru_state,
lstm_state=lstm_state,
lstm_cells=lstm_cells,
) # iterate=False)
# input_transform = self.vu.apply(visible)
# gru_state = self.gru_layer.apply(inputs=input_transform, states=gru_state,
# iterate=False)
# lstm_state, lstm_cells = self.lstm_layer.apply(inputs=gru_state, states=lstm_state, cells=lstm_state,
# iterate=False)
updates = ComputationGraph(lstm_state).updates
# output = 1.17 * self.out_layer.apply(hidden_state)
return [visible, gru_state, lstm_state, lstm_cells], updates
@recurrent(sequences=['visible', 'mask'],
contexts=[],
states=['gru_state', 'lstm_state', 'lstm_cells'],
outputs=['gru_state', 'lstm_state', 'lstm_cells', 'bv', 'bh'])
def training_biases(self,
visible,
mask=None,
gru_state=None,
lstm_state=None,
lstm_cells=None):
bv = self.uv.apply(lstm_state)
bh = self.uh.apply(lstm_state)
# inputs = rbm.apply(visible=visible, bv=bv, bh=bh, n_steps=25)
# input_transform = self.vu.apply(visible)
# gru_state = self.gru_layer.apply(inputs=input_transform, states=gru_state,
# iterate=False)
# lstm_state, lstm_cells = self.lstm_layer.apply(inputs=gru_state, states=lstm_state,
# cells=lstm_state,
# iterate=False)
gru_state, lstm_state, lstm_cells = self.rnn.rnn_apply(
visible,
mask=mask,
gru_state=gru_state,
lstm_state=lstm_state,
lstm_cells=lstm_cells,
) #iterate=False)
updates = ComputationGraph(lstm_state).updates
return [gru_state, lstm_state, lstm_cells, bv, bh], updates
def cost(self, examples, mask, k=10):
_, _, _, bv, bh = self.training_biases(visible=examples, mask=mask)
cost, v_samples = self.rbm.cost(visible=examples,
bv=bv,
bh=bh,
k=k,
batch_size=examples.shape[0])
return cost.astype(floatX), v_samples.astype(floatX)
# @application(inputs=['visible', 'k', 'batch_size', 'mask'], outputs=['vsample', 'cost'])
def rbm_pretrain_cost(self, visible, k, batch_size, mask=None):
return self.rbm.cost(visible=visible,
k=k,
batch_size=batch_size,
mask=mask)
# @application(inputs=['x', 'x_mask'], outputs=['output_'])
def rnn_pretrain_pred(self, x, x_mask):
return self.rnn.apply(inputs=x, mask=x_mask)
class Rnnrbm(BaseRecurrent, Initializable):
@lazy(allocation=['visible_dim', 'hidden_dim'])
def __init__(self, visible_dim, hidden_dim, rnn_dimensions=(128, 128), **kwargs):
super(Rnnrbm, self).__init__(**kwargs)
self.rnn_dimensions = rnn_dimensions
self.visible_dim = visible_dim
self.hidden_dim = hidden_dim
# self.in_layer = Linear(input_dim=input_dim, output_dim=rnn_dimension * 4,
# weights_init=IsotropicGaussian(0.01),
# biases_init=Constant(0.0),
# use_bias=False,
# name="in_layer")
self.rbm = Rbm(visible_dim=visible_dim, hidden_dim=hidden_dim,
activation=Sigmoid(), weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.1),
name='rbm')
self.uv = Linear(input_dim=rnn_dimensions[-1], output_dim=visible_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True, name='uv')
self.uh = Linear(input_dim=rnn_dimensions[-1], output_dim=hidden_dim,
weights_init=IsotropicGaussian(0.0001),
biases_init=Constant(0.001),
use_bias=True, name='uh')
self.rnn = Rnn([visible_dim] + list(rnn_dimensions), name='rnn')
self.children = [self.rbm, self.uv, self.uh, self.rnn] + self.rnn.children._items
def initialize(self, pretrained_rbm=None, pretrained_rnn=None, **kwargs):
super(Rnnrbm, self).initialize(**kwargs)
# if pretrained_rbm is not None:
# self.rbm.bv.set_value(pretrained_rbm.bv.get_value())
# self.uv.b.set_value(pretrained_rbm.bv.get_value())
# self.rbm.bh.set_value(pretrained_rbm.bh.get_value())
# self.uh.b.set_value(pretrained_rbm.bh.get_value())
# self.rbm.W.set_value(pretrained_rbm.W.get_value())
# if pretrained_rnn is not None:
# for param, trained_param in zip(itertools.chain(*[child.params for child in self.rnn.children]),
# itertools.chain(*[child.params for child in pretrained_rnn.children])):
# param.set_value(trained_param.get_value())
self.uv.b.name = 'buv'
self.uv.W.name = 'Wuv'
self.uh.W.name = 'Wuh'
self.uh.b.name = 'buh'
self.rbm.bh.name = 'buh'
self.rbm.bv.name = 'buv'
# self.rnn.input_transform.b.name = 'bvu'
# self.rnn.input_transform.W.name = 'Wvu'
# def _allocate(self):
# Wrbm = shared_floatx_nans((self.visible_dim, self.hidden_dim), name='Wrbm')
# add_role(Wrbm, WEIGHT)
# self.params.append(Wrbm)
# self.add_auxiliary_variable(Wrbm.norm(2), name='Wrbm_norm')
#
#
# def _initialize(self):
# for param in self.params:
# if has_roles(param, WEIGHT):
# self.weights_init.initialize(param, self.rng)
# elif has_roles(param, BIAS):
# self.biases_init.initialize(param, self.rng)
def get_dim(self, name):
dims = {'visible': self.visible_dim,
'mask': self.visible_dim,
'gru_state': self.rnn_dimensions[0],
'lstm_state': self.rnn_dimensions[1],
'lstm_cells': self.rnn_dimensions[1]
}
return dims.get(name, None) or super(Rnnrbm, self).get_dim(name)
@recurrent(sequences=[], contexts=['visible'], states=['gru_state', 'lstm_state', 'lstm_cells'],
outputs=['visible', 'gru_state', 'lstm_state', 'lstm_cells'])
def generate(self, visible=None, gru_state=None, lstm_state=None, lstm_cells=None, rbm_steps=25):
bv = self.uv.apply(lstm_state)
bh = self.uh.apply(lstm_state)
visible_start = rng.binomial(size=visible.shape, n=1, p=bv, dtype=floatX)
_, visible = self.rbm.apply(visible=visible_start, bv=bv, bh=bh, n_steps=rbm_steps,
batch_size=visible.shape[0])
visible = visible[-1]
gru_state, lstm_state, lstm_cells = self.rnn.rnn_apply(inputs=visible, gru_state=gru_state,
lstm_state=lstm_state,
lstm_cells=lstm_cells, ) # iterate=False)
# input_transform = self.vu.apply(visible)
# gru_state = self.gru_layer.apply(inputs=input_transform, states=gru_state,
# iterate=False)
# lstm_state, lstm_cells = self.lstm_layer.apply(inputs=gru_state, states=lstm_state, cells=lstm_state,
# iterate=False)
updates = ComputationGraph(lstm_state).updates
# output = 1.17 * self.out_layer.apply(hidden_state)
return [visible, gru_state, lstm_state, lstm_cells], updates
@recurrent(sequences=['visible', 'mask'], contexts=[],
states=['gru_state', 'lstm_state', 'lstm_cells'],
outputs=['gru_state', 'lstm_state', 'lstm_cells', 'bv', 'bh'])
def training_biases(self, visible, mask=None, gru_state=None, lstm_state=None, lstm_cells=None):
bv = self.uv.apply(lstm_state)
bh = self.uh.apply(lstm_state)
# inputs = rbm.apply(visible=visible, bv=bv, bh=bh, n_steps=25)
# input_transform = self.vu.apply(visible)
# gru_state = self.gru_layer.apply(inputs=input_transform, states=gru_state,
# iterate=False)
# lstm_state, lstm_cells = self.lstm_layer.apply(inputs=gru_state, states=lstm_state,
# cells=lstm_state,
# iterate=False)
gru_state, lstm_state, lstm_cells = self.rnn.rnn_apply(visible, mask=mask, gru_state=gru_state,
lstm_state=lstm_state,
lstm_cells=lstm_cells, ) #iterate=False)
updates = ComputationGraph(lstm_state).updates
return [gru_state, lstm_state, lstm_cells, bv, bh], updates
def cost(self, examples, mask, k=10):
_, _, _, bv, bh = self.training_biases(visible=examples, mask=mask)
cost, v_samples = self.rbm.cost(visible=examples, bv=bv, bh=bh, k=k,
batch_size=examples.shape[0])
return cost.astype(floatX), v_samples.astype(floatX)
# @application(inputs=['visible', 'k', 'batch_size', 'mask'], outputs=['vsample', 'cost'])
def rbm_pretrain_cost(self, visible, k, batch_size, mask=None):
return self.rbm.cost(visible=visible, k=k, batch_size=batch_size, mask=mask)
# @application(inputs=['x', 'x_mask'], outputs=['output_'])
def rnn_pretrain_pred(self, x, x_mask):
return self.rnn.apply(inputs=x, mask=x_mask)
required=False,
default="standard")
args = arg_parser.parse_args()
with open('best_individual.json') as best_agent_weights:
weights = json.load(best_agent_weights)
if args.scenario == 'pull':
genotype_class = ga.BeerTrackerPullGenotype
elif args.scenario == 'wall':
genotype_class = ga.BeerTrackerWallGenotype
else:
genotype_class = ga.BeerTrackerGenotype
nn = Rnn(num_input_nodes=genotype_class.num_input_nodes,
num_hidden_nodes=genotype_class.num_hidden_nodes,
num_output_nodes=genotype_class.num_output_nodes)
nn.set_weights(weights)
nodes = []
# input nodes
for i in range(nn.num_input_nodes):
x = float(i) / (nn.num_input_nodes - 1)
node = {
'id': 'i{}'.format(i),
'x': -0.2 + 1.4 * x,
'y': 1.5 * (x - 0.5)**2 - 0.5,
'size': 1,
'label': 'input{}'.format(i),
'type': 'input'
X = []
Y = []
Y2 = []
for l in f:
its = l.strip().split()
X.append(map(float, its[:-1]))
Y.append(mapping[its[-1][0]])
Y2.append(mapping[its[-1][1]])
data_x.append(np.array(X, dtype=np.float32))
data_y.append(np.array(Y, dtype=np.int32))
data_y2.append(np.array(Y2, dtype=np.int32))
print("done", sum(len(x) for x in refs))
sys.stdout.flush()
ntwk = Rnn(sys.argv[1])
print("net rdy")
s_arr = []
p_arr = []
subseq_size = 400
for s in range(len(data_x)):
s_arr += [s]
p_arr += [len(data_x[s]) - subseq_size]
sum_p = sum(p_arr)
for i in range(len(p_arr)):
p_arr[i] = 1. * p_arr[i] / sum_p
base_dir = str(datetime.datetime.now())
class BeerTrackerIndividual(Individual):
range_map = {
'weight': (-5.0, 5.0),
'internal_bias': (-10.0, 0.0),
'gain': (1.0, 5.0),
'time_constant': (1.0, 2.0)
}
genotype_class = None
def calculate_summed_weight(self, i, range_key):
j = i * BeerTrackerIndividual.genotype_class.bits_per_weight
weight = sum(self.genotype.
dna[j:j +
BeerTrackerIndividual.genotype_class.bits_per_weight])
weight = self.range_map[range_key][0] + \
(self.range_map[range_key][1] - self.range_map[range_key][0]) * \
float(weight) / BeerTrackerIndividual.genotype_class.bits_per_weight
return weight
def calculate_bitshifted_weight(self, i, range_key):
j = i * BeerTrackerIndividual.genotype_class.bits_per_weight
bits = self.genotype.dna[j:j + BeerTrackerIndividual.genotype_class.
bits_per_weight]
weight = 0
for bit in bits:
weight = (weight << 1) | bit
weight = self.range_map[range_key][0] + \
(self.range_map[range_key][1] - self.range_map[range_key][0]) * \
float(weight) / (2 ** BeerTrackerIndividual.genotype_class.bits_per_weight)
return weight
def calculate_phenotype(self):
self.phenotype = Rnn(
BeerTrackerIndividual.genotype_class.num_input_nodes,
BeerTrackerIndividual.genotype_class.num_hidden_nodes,
BeerTrackerIndividual.genotype_class.num_output_nodes)
weights = []
calculate_weight = self.calculate_summed_weight
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['input_hidden']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_hidden']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['bias_hidden']):
weight = calculate_weight(i, 'internal_bias')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_output']):
weight = calculate_weight(i, 'weight')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_gains']):
weight = calculate_weight(i, 'gain')
weights.append(weight)
for i in range(BeerTrackerIndividual.genotype_class.rnn.
edge_chunks['hidden_time_constants']):
weight = calculate_weight(i, 'time_constant')
weights.append(weight)
self.phenotype.set_weights(weights)
required=False,
default=0)
args = arg_parser.parse_args()
with open('best_individual.json') as best_agent_weights:
weights = json.load(best_agent_weights)
if args.scenario == 'pull':
genotype_class = BeerTrackerPullGenotype
elif args.scenario == 'wall':
genotype_class = BeerTrackerWallGenotype
else:
genotype_class = BeerTrackerGenotype
nn = Rnn(num_input_nodes=genotype_class.num_input_nodes,
num_hidden_nodes=genotype_class.num_hidden_nodes,
num_output_nodes=genotype_class.num_output_nodes)
nn.set_weights(weights)
g = gfx.Gfx()
g.fps = 8
for i in range(args.num_scenarios):
seed = i + ((997 * args.generation) if args.mode == 'dynamic' else 0)
print '---', 'seed', seed, '---'
bt = BeerTracker(nn=nn, seed=seed, scenario=args.scenario)
bt.gfx = g
bt.run()
print bt.world.agent.num_small_misses, 'small miss(es)'
print bt.world.agent.num_large_misses, 'large miss(es)'
print bt.world.agent.num_partial_captures, 'partial capture(s)'
data_path = config["data-path-cn"]
else:
data_path = config["data-path-en"]
config["rnn-head-word"] = 20000
config["rnn-model-name"] = "model-" + config["language"] + ".h5"
train_set = pickle.load(open(data_path + "train.pkl", "rb"))
test_set = pickle.load(open(data_path + "test.pkl", "rb"))
evaluate_set = pickle.load(open(data_path + "evaluate.pkl", "rb"))
print("Data loaded from", data_path)
algorithm = config.get("model", 'bayesian')
if algorithm == 'bayesian':
model = Bayesian(config)
elif algorithm == 'rnn':
model = Rnn(config)
print(
"============================ Training... ============================"
)
model.train(train_set, test_set)
print(
"============================ Predicting... ============================"
)
predict = model.predict(evaluate_set)
accuracy, f1, precision, recall = compare(
predict, list(map(lambda x: x[2], evaluate_set)))
print_result(evaluate_set, predict, accuracy, f1, precision, recall)
write_wrong_predict(evaluate_set, predict)
print("========================================================")
