def training_stream(training_frame_files):
split_streams = create_split_streams(training_frame_files)
split_streams = [data_io.buffered_random(s) for s in split_streams]
split_streams = [data_io.chop(s) for s in split_streams]
stream = data_io.random_select_stream(*split_streams)
stream = data_io.buffered_random(stream)
return stream
def training_stream(training_frame_files):
split_streams = create_split_streams(training_frame_files)
split_streams = [ data_io.buffered_random(s) for s in split_streams ]
split_streams = [ data_io.chop(s) for s in split_streams ]
stream = data_io.random_select_stream(*split_streams)
stream = data_io.buffered_random(stream)
return stream
def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream, key=lambda x: x[1].shape[0], buffer_items=128)
batched_stream = reader.batch_and_pad(stream, batch_size=16, mean=mean, std=std)
batched_stream = data_io.buffered_random(batched_stream, buffer_items=4)
return batched_stream
def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream,
key=lambda x: x[1].shape[0],
buffer_items=128)
batched_stream = reader.batch_and_pad(stream,
batch_size=16,
mean=mean,
std=std)
batched_stream = data_io.buffered_random(batched_stream,
buffer_items=4)
return batched_stream
P_learn = Parameters()
updates = updates.adam(parameters,gradients,learning_rate=0.00025,P=P_learn)
updates = normalise_weights(updates)
print "Compiling..."
train = theano.function(
inputs=[X,l],
outputs=batch_cost,
updates=updates,
)
test = theano.function(inputs=[X,l],outputs=batch_cost)
print "Calculating mean variance..."
rand_stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
mean, std, count = reader.get_normalise(rand_stream)
print "Dataset count:", count
def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream, key=lambda x: x[1].shape[0], buffer_items=128)
batched_stream = reader.batch_and_pad(stream, batch_size=16, mean=mean, std=std)
batched_stream = data_io.buffered_random(batched_stream, buffer_items=4)
return batched_stream
gradients,
learning_rate=0.00025,
P=P_learn)
updates = normalise_weights(updates)
print "Compiling..."
train = theano.function(
inputs=[X, l],
outputs=batch_cost,
updates=updates,
)
test = theano.function(inputs=[X, l], outputs=batch_cost)
print "Calculating mean variance..."
rand_stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
mean, std, count = reader.get_normalise(rand_stream)
print "Dataset count:", count
def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream,
key=lambda x: x[1].shape[0],
buffer_items=128)
batched_stream = reader.batch_and_pad(stream,
batch_size=16,