def __init__(self, input_dim, output_dim, hidden_size, init_ranges,
**kwargs):
linear1 = LinearMaxout(input_dim=input_dim,
output_dim=hidden_size,
num_pieces=2,
name='linear1')
linear2 = LinearMaxout(input_dim=hidden_size,
output_dim=hidden_size,
num_pieces=2,
name='linear2')
linear3 = Linear(input_dim=hidden_size, output_dim=output_dim)
logistic = Logistic()
bricks = [
linear1,
BatchNormalization(input_dim=hidden_size, name='bn2'), linear2,
BatchNormalization(input_dim=hidden_size, name='bnl'), linear3,
logistic
]
for init_range, b in zip(init_ranges, (linear1, linear2, linear3)):
b.biases_init = initialization.Constant(0)
b.weights_init = initialization.Uniform(width=init_range)
kwargs.setdefault('use_bias', False)
super(ConcatenateClassifier, self).__init__([b.apply for b in bricks],
**kwargs)
def initialize2(brick, num_feature_maps):
fan_in = numpy.prod(brick.filter_size)
fan_out = numpy.prod(
brick.filter_size) * brick.num_filters / num_feature_maps
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
brick.weights_init = initialization.Uniform(width=W_bound)
brick.biases_init = initialization.Constant(0)
brick.initialize()
def initialize(to_init, width):
"""
Initialize weights according to Xavier Parameter Initialization
:param to_init the block to initialize
:param width width of uniform distribution
"""
to_init.weights_init = initialization.Uniform(width=width)
to_init.biases_init = initialization.Constant(0)
to_init.initialize()
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size,
init_ranges, **kwargs):
(visual_init_range, textual_init_range, gbu_init_range, linear_range_1,
linear_range_2, linear_range_3) = init_ranges
visual_mlp = Sequence([
BatchNormalization(input_dim=visual_dim).apply,
Linear(visual_dim,
hidden_size,
use_bias=False,
weights_init=initialization.Uniform(
width=visual_init_range)).apply,
],
name='visual_mlp')
textual_mlp = Sequence([
BatchNormalization(input_dim=textual_dim).apply,
Linear(textual_dim,
hidden_size,
use_bias=False,
weights_init=initialization.Uniform(
width=textual_init_range)).apply,
],
name='textual_mlp')
gbu = GatedBimodal(
hidden_size,
weights_init=initialization.Uniform(width=gbu_init_range))
logistic_mlp = MLPGenreClassifier(
hidden_size, output_dim, hidden_size,
[linear_range_1, linear_range_2, linear_range_3])
# logistic_mlp = Sequence([
# BatchNormalization(input_dim=hidden_size, name='bn1').apply,
# Linear(hidden_size, output_dim, name='linear_output', use_bias=False,
# weights_init=initialization.Uniform(width=linear_range_1)).apply,
# Logistic().apply
#], name='logistic_mlp')
children = [visual_mlp, textual_mlp, gbu, logistic_mlp]
kwargs.setdefault('use_bias', False)
kwargs.setdefault('children', children)
super(GatedClassifier, self).__init__(**kwargs)
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size,
init_ranges, **kwargs):
(visual_range, textual_range, linear_range_1, linear_range_2,
linear_range_3) = init_ranges
visual_layer = FeedforwardSequence([
BatchNormalization(input_dim=visual_dim).apply,
LinearMaxout(
input_dim=visual_dim,
output_dim=hidden_size,
weights_init=initialization.Uniform(width=visual_range),
use_bias=False,
biases_init=initialization.Constant(0),
num_pieces=2).apply
],
name='visual_layer')
textual_layer = FeedforwardSequence([
BatchNormalization(input_dim=textual_dim).apply,
LinearMaxout(
input_dim=textual_dim,
output_dim=hidden_size,
weights_init=initialization.Uniform(width=textual_range),
biases_init=initialization.Constant(0),
use_bias=False,
num_pieces=2).apply
],
name='textual_layer')
logistic_mlp = MLPGenreClassifier(
hidden_size, output_dim, hidden_size,
[linear_range_1, linear_range_2, linear_range_3])
# logistic_mlp = Sequence([
# BatchNormalization(input_dim=hidden_size, name='bn1').apply,
# Linear(hidden_size, output_dim, name='linear_output', use_bias=False,
# weights_init=initialization.Uniform(width=linear_range_1)).apply,
# Logistic().apply
#], name='logistic_mlp')
children = [visual_layer, textual_layer, logistic_mlp]
kwargs.setdefault('use_bias', False)
kwargs.setdefault('children', children)
super(LinearSumClassifier, self).__init__(**kwargs)
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size,
init_ranges, **kwargs):
(visual_range, textual_range, linear_range_1, linear_range_2,
linear_range_3) = init_ranges
manager_dim = visual_dim + textual_dim
visual_mlp = MLPGenreClassifier(
visual_dim,
output_dim,
hidden_size, [linear_range_1, linear_range_2, linear_range_3],
name='visual_mlp')
textual_mlp = MLPGenreClassifier(
textual_dim,
output_dim,
hidden_size, [linear_range_1, linear_range_2, linear_range_3],
name='textual_mlp')
# manager_mlp = MLPGenreClassifier(manager_dim, 2, hidden_size, [
# linear_range_1, linear_range_2, linear_range_3], output_act=Softmax,
# name='manager_mlp')
bn = BatchNormalization(input_dim=manager_dim, name='bn3')
manager_mlp = Sequence([
Linear(manager_dim,
2,
name='linear_output',
use_bias=False,
weights_init=initialization.Uniform(
width=linear_range_1)).apply,
],
name='manager_mlp')
fork = Fork(
input_dim=manager_dim,
output_dims=[2] * output_dim,
prototype=manager_mlp,
output_names=['linear_' + str(i) for i in range(output_dim)])
children = [visual_mlp, textual_mlp, fork, bn, NDimensionalSoftmax()]
kwargs.setdefault('use_bias', False)
kwargs.setdefault('children', children)
super(MoEClassifier, self).__init__(**kwargs)
def initialize(to_init, rndstd=0.01):
for bricks in to_init:
bricks.weights_init = initialization.Uniform(width=0.08)
bricks.biases_init = initialization.Constant(0)
bricks.initialize()
from blocks.algorithms import StepClipping, GradientDescent, CompositeRule, RMSProp
from dataset import T_H5PYDataset
source_path = 'dataset/normalized_syllables_rhythm_notes.json-seqlen-30.hdf5'
train_dataset = T_H5PYDataset(source_path, which_sets=('train', ))
hidden_layer_dim = 1000
x = tensor.lmatrix('syllables')
y = tensor.lmatrix('durations')
lookup_input = LookupTable(name='lookup_input',
length=train_dataset.syllables_vocab_size() + 1,
dim=hidden_layer_dim,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
lookup_input.initialize()
linear_input = Linear(name='linear_input',
input_dim=hidden_layer_dim,
output_dim=hidden_layer_dim,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
linear_input.initialize()
rnn = SimpleRecurrent(name='hidden',
dim=hidden_layer_dim,
activation=Tanh(),
weights_init=initialization.Uniform(width=0.01))
rnn.initialize()
def initialize_inout(brick, fan_in, fan_out,seed=1):
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
brick.weights_init = initialization.Uniform(width=2 * W_bound)
brick.biases_init = initialization.Constant(0)
brick.initialize()
def get_lookups(self, dim, vocab_list):
return [LookupTable(dim=dim, length=vocab, name='lookup' + str(index),
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0)) for index, vocab in enumerate(vocab_list)]
def __init__(self, input_sources_list, input_sources_vocab_size_list,
output_source, output_source_vocab_size,
lookup_dim=200, hidden_size=256, recurrent_stack_size=1):
self.InputSources = input_sources_list
self.InputSourcesVocab = input_sources_vocab_size_list
self.OutputSource = output_source
self.OutputSourceVocab = output_source_vocab_size
inputs = [tensor.lmatrix(source) for source in input_sources_list]
output = tensor.lmatrix(output_source)
lookups = self.get_lookups(lookup_dim, input_sources_vocab_size_list)
for lookup in lookups:
lookup.initialize()
merge = Merge([lookup.name for lookup in lookups], [lookup.dim for lookup in lookups], hidden_size,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
merge.initialize()
linear0 = Linear(input_dim=hidden_size, output_dim=hidden_size,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0), name='linear0')
linear0.initialize()
recurrent_blocks = []
for i in range(recurrent_stack_size):
recurrent_blocks.append(SimpleRecurrent(
dim=hidden_size, activation=Tanh(),
weights_init=initialization.Uniform(width=0.01),
use_bias=False))
for i, recurrent_block in enumerate(recurrent_blocks):
recurrent_block.name = 'recurrent'+str(i+1)
recurrent_block.initialize()
linear_out = Linear(input_dim=hidden_size, output_dim=output_source_vocab_size,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0), name='linear_out')
linear_out.initialize()
softmax = NDimensionalSoftmax(name='softmax')
lookup_outputs = [lookup.apply(input) for lookup, input in zip(lookups, inputs)]
m = merge.apply(*lookup_outputs)
r = linear0.apply(m)
for block in recurrent_blocks:
r = block.apply(r)
a = linear_out.apply(r)
self.Cost = softmax.categorical_cross_entropy(output, a, extra_ndim=1).mean()
self.Cost.name = 'cost'
y_hat = softmax.apply(a, extra_ndim=1)
y_hat.name = 'y_hat'
self.ComputationGraph = ComputationGraph(self.Cost)
self.Function = None
self.MainLoop = None
self.Model = Model(y_hat)
def __init__(self,
input1_size,
input2_size,
lookup1_dim=200,
lookup2_dim=200,
hidden_size=512):
self.hidden_size = hidden_size
self.input1_size = input1_size
self.input2_size = input2_size
self.lookup1_dim = lookup1_dim
self.lookup2_dim = lookup2_dim
x1 = tensor.lmatrix('durations')
x2 = tensor.lmatrix('syllables')
y = tensor.lmatrix('pitches')
lookup1 = LookupTable(dim=self.lookup1_dim,
length=self.input1_size,
name='lookup1',
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
lookup1.initialize()
lookup2 = LookupTable(dim=self.lookup2_dim,
length=self.input2_size,
name='lookup2',
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
lookup2.initialize()
merge = Merge(['lookup1', 'lookup2'],
[self.lookup1_dim, self.lookup2_dim],
self.hidden_size,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
merge.initialize()
recurrent_block = LSTM(
dim=self.hidden_size,
activation=Tanh(),
weights_init=initialization.Uniform(width=0.01)
) #RecurrentStack([LSTM(dim=self.hidden_size, activation=Tanh())] * 3)
recurrent_block.initialize()
linear = Linear(input_dim=self.hidden_size,
output_dim=self.input1_size,
weights_init=initialization.Uniform(width=0.01),
biases_init=Constant(0))
linear.initialize()
softmax = NDimensionalSoftmax()
l1 = lookup1.apply(x1)
l2 = lookup2.apply(x2)
m = merge.apply(l1, l2)
h = recurrent_block.apply(m)
a = linear.apply(h)
y_hat = softmax.apply(a, extra_ndim=1)
# ValueError: x must be 1-d or 2-d tensor of floats. Got TensorType(float64, 3D)
self.Cost = softmax.categorical_cross_entropy(y, a,
extra_ndim=1).mean()
self.ComputationGraph = ComputationGraph(self.Cost)
self.Model = Model(y_hat)
