def test_window_stream():
sentences = [list(numpy.random.randint(10, size=sentence_length))
for sentence_length in [3, 5, 7]]
stream = DataStream(IterableDataset(sentences))
windows = Window(0, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source == target
assert i == 5 # Total of 6 windows
# Make sure that negative indices work
windows = Window(-2, 4, 4, False, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[-2:] == target[:2]
assert i == 1 # Should get 2 examples
# Even for overlapping negative indices should work
windows = Window(-2, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[:2] == target[-2:]
assert i == 1 # Should get 2 examples
# Test step_size param
sentences = [[i for i in range(10)]]
stream = DataStream(IterableDataset(sentences))
windows = Window(1, 4, 4, True, stream, step_size=3)
expected_sources = [[0, 1, 2, 3], [3, 4, 5, 6]]
expected_targets = [[1, 2, 3, 4], [4, 5, 6, 7]]
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source == expected_sources[i]
assert target == expected_targets[i]
assert i == 1 # Should get 2 examples
def test_window_stream():
sentences = [list(numpy.random.randint(10, size=sentence_length))
for sentence_length in [3, 5, 7]]
stream = DataStream(IterableDataset(sentences))
windows = Window(0, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source == target
assert i == 5 # Total of 6 windows
# Make sure that negative indices work
windows = Window(-2, 4, 4, False, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[-2:] == target[:2]
assert i == 1 # Should get 2 examples
# Even for overlapping negative indices should work
windows = Window(-2, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[:2] == target[-2:]
assert i == 1 # Should get 2 examples
def test_window_stream():
sentences = [
list(numpy.random.randint(10, size=sentence_length))
for sentence_length in [3, 5, 7]
]
stream = DataStream(IterableDataset(sentences))
windows = Window(0, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source == target
assert i == 5 # Total of 6 windows
# Make sure that negative indices work
windows = Window(-2, 4, 4, False, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[-2:] == target[:2]
assert i == 1 # Should get 2 examples
# Even for overlapping negative indices should work
windows = Window(-2, 4, 4, True, stream)
for i, (source, target) in enumerate(windows.get_epoch_iterator()):
assert source[:2] == target[-2:]
assert i == 1 # Should get 2 examples
print("retrieving data...")
data = YouTubeAudio(youTubeId)
stream = data.get_example_stream()
data_stream = Window(stride, sequenceSize, sequenceSize, True, stream)
# switch to configure training or audio generation
if mode == "train":
print("training begin...")
print("Input Size:", batchSize)
print("minibatches:", miniBatches)
print("stride:", stride)
print("hidden units:", hiddenUnits)
print("learning rate:", learningRate)
print("sequence size:", sequenceSize)
for batch_stream in data_stream.get_epoch_iterator():
# get samples
u, t = batch_stream
# Start somewhere (after 1 minute)
if idx >= idxBegin:
u = np.array(u, dtype=np.float64)
t = np.array(t, dtype=np.float64)
# reshape samples into minibatches
uBatch = np.reshape((u/0x8000), (miniBatches,batchSize)).swapaxes(0,1)
tBatch = np.reshape((t/0x8000), (miniBatches,batchSize)).swapaxes(0,1)
print(uBatch)
# train and find error
print("train...")
error = lstm.train(uBatch, tBatch, learningRate)
