def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream, key=lambda x: x[1].shape[0], buffer_items=128)
batched_stream = reader.batch_and_pad(stream, batch_size=16, mean=mean, std=std)
batched_stream = data_io.buffered_random(batched_stream, buffer_items=4)
return batched_stream
def validate():
stream = data_io.stream_file('data/train.%02d.pklgz' % 0)
stream = data_io.buffered_sort(stream, key=lambda x: x[1].shape[0], buffer_items=128)
batched_stream = reader.batch_and_pad(stream, batch_size=32, mean=mean, std=std)
total_cost = 0
total_frames = 0
for data, lengths in batched_stream:
batch_avg_cost = test(data,lengths)
batch_frames = np.sum(lengths)
total_cost += batch_avg_cost * batch_frames
total_frames += batch_frames
return total_cost / total_frames
def stream():
stream = data_io.random_select_stream(*[
data_io.stream_file('data/train.%02d.pklgz' % i)
for i in xrange(1, 20)
])
stream = data_io.buffered_sort(stream,
key=lambda x: x[1].shape[0],
buffer_items=128)
batched_stream = reader.batch_and_pad(stream,
batch_size=16,
mean=mean,
std=std)
batched_stream = data_io.buffered_random(batched_stream,
buffer_items=4)
return batched_stream
def validate():
stream = data_io.stream_file('data/train.%02d.pklgz' % 0)
stream = data_io.buffered_sort(stream,
key=lambda x: x[1].shape[0],
buffer_items=128)
batched_stream = reader.batch_and_pad(stream,
batch_size=32,
mean=mean,
std=std)
total_cost = 0
total_frames = 0
for data, lengths in batched_stream:
batch_avg_cost = test(data, lengths)
batch_frames = np.sum(lengths)
total_cost += batch_avg_cost * batch_frames
total_frames += batch_frames
return total_cost / total_frames