def build_raw_interval_hdf5_dataset(youtube_id, hdf5_name, interval_size, window_size):
data_stream = YouTubeAudio(youtube_id).get_example_stream()
data_stream = Window(offset=interval_size,
source_window=interval_size*window_size,
target_window=interval_size*window_size,
overlapping=True,
data_stream=data_stream)
data_iterator = data_stream.get_epoch_iterator()
num_sequences = 0
for data in data_iterator:
num_sequences = num_sequences + 1
output_path = '{}.hdf5'.format(hdf5_name)
output_path = os.path.join(output_path)
print 'total num sequences : ', num_sequences
with h5py.File(output_path, mode='w') as h5file:
input_feature = h5file.create_dataset(name='input_feature' , shape=(num_sequences, window_size, interval_size), dtype='int16')
target_feature = h5file.create_dataset(name='target_feature', shape=(num_sequences, window_size, interval_size), dtype='int16')
data_iterator = data_stream.get_epoch_iterator()
# for each batch
for s_idx, sequence_data in enumerate(data_iterator):
# get data
source_data = sequence_data[0]
target_data = sequence_data[1]
# save data
input_feature[s_idx] = source_data.reshape(window_size, interval_size)
target_feature[s_idx] = target_data.reshape(window_size, interval_size)
# label each dataset axis
input_feature.dims[0].label = 'batch'
input_feature.dims[1].label = 'time'
input_feature.dims[2].label = 'feature'
target_feature.dims[0].label = 'batch'
target_feature.dims[1].label = 'time'
target_feature.dims[2].label = 'feature'
num_trains = int(num_sequences*0.8)
split_dict = {'train': {'input_feature' : ( 0, num_trains),
'target_feature': ( 0, num_trains)},
'valid': {'input_feature' : ( num_trains, num_sequences),
'target_feature': ( num_trains, num_sequences)},
}
h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
h5file.flush()
h5file.close()
return num_sequences
def get_stream(batch_size, source_window=4000, target_window=1000, num_examples=5000):
from fuel.datasets.youtube_audio import YouTubeAudio
data = YouTubeAudio('XqaJ2Ol5cC4')
train_stream = data.get_example_stream()
train_stream = ForceFloatX(train_stream)
window_stream = Window(0,source_window, target_window, overlapping=False, data_stream=train_stream)
source_stream = FilterSources(window_stream, sources=('features',))
feats_stream = Mapping(source_stream, mfcc)
targets_stream = FilterSources(window_stream, sources=('targets',))
targets_stream = Flatten(targets_stream)
stream = Merge((feats_stream,targets_stream), sources = ('features','targets'))
#Add a random Scheme?
it_scheme = ConstantScheme(batch_size,num_examples)
batched_stream = Batch(stream, it_scheme, strictness=1)
return batched_stream
def get_stream(batch_size,
source_window=4000,
target_window=1000,
num_examples=5000):
from fuel.datasets.youtube_audio import YouTubeAudio
data = YouTubeAudio('XqaJ2Ol5cC4')
train_stream = data.get_example_stream()
train_stream = ForceFloatX(train_stream)
window_stream = Window(0,
source_window,
target_window,
overlapping=False,
data_stream=train_stream)
source_stream = FilterSources(window_stream, sources=('features', ))
feats_stream = Mapping(source_stream, mfcc)
targets_stream = FilterSources(window_stream, sources=('targets', ))
targets_stream = Flatten(targets_stream)
stream = Merge((feats_stream, targets_stream),
sources=('features', 'targets'))
#Add a random Scheme?
it_scheme = ConstantScheme(batch_size, num_examples)
batched_stream = Batch(stream, it_scheme, strictness=1)
return batched_stream
from fuel.datasets.youtube_audio import YouTubeAudio
import matplotlib.pyplot as plt
import numpy as np
import scipy.io.wavfile
from random import randint
amp = 10000.
data = YouTubeAudio('XqaJ2Ol5cC4')
stream = data.get_example_stream()
it = stream.get_epoch_iterator()
track = next(it)
track = track[0].reshape(-1)
track = track/amp
sample_rate = scipy.io.wavfile.read("/home/alex/fuel_data/XqaJ2Ol5cC4.wav")[0]
total_len = track.shape[0]
#input shape is expected to be
#(batch_size, sequence_length, num_inputs)
secs = 0.75
num_inputs = int(sample_rate*secs)
seq_len = 20
example = seq_len*num_inputs
#Setting appropriate length to account for the size of an example
train_len = total_len*8/10/example*example
valid_len = (total_len-train_len)/2/example*example
test_len = (total_len-train_len)/2/example*example
unused = total_len - train_len - valid_len - test_len
def get_sound(name):
data = YouTubeAudio(name)
stream = data.get_example_stream()
it = stream.get_epoch_iterator()
return next(it)[0][:, 0]
def get_sound(name):
data = YouTubeAudio(name)
stream = data.get_example_stream()
it = stream.get_epoch_iterator()
return next(it)[0][:, 0]
