def test_nowindow(input_seq, batch_size, seq_len):
"""
This test checks no-shuffle option, with non-overlapping windows
Goes over the dataset just once
"""
# Create the iterator
data_array = {'X': np.copy(input_seq),
'y': np.roll(input_seq, axis=0, shift=-1)}
iterations = None
it_array = SequentialArrayIterator(data_arrays=data_array, time_steps=seq_len,
batch_size=batch_size, tgt_key='y', shuffle=False,
total_iterations=iterations)
# Cut off the last samples of the input sequence that don't fit in a batch
no_samples = seq_len * batch_size * (len(input_seq) // seq_len // batch_size)
target_seq = input_seq[1:no_samples + 1]
input_seq = input_seq[:no_samples]
for idx, iter_val in enumerate(it_array):
# Start of the array needs to be time_steps * idx * batch_size
start_idx = seq_len * idx * batch_size
idcs = np.arange(start_idx, start_idx + seq_len * batch_size) % input_seq.shape[0]
# Reshape the input sequence into batches
reshape_dims = (batch_size, seq_len)
if len(input_seq.shape) > 1:
reshape_dims = (batch_size, seq_len, input_seq.shape[1])
# expected_x will have contigous non-overlapping samples
expected_x = input_seq[idcs].reshape(reshape_dims)
expected_y = target_seq[idcs].reshape(reshape_dims)
# We are not shuffling, consecutive samples are contiguous
# They will also be non-overlapping
assert np.array_equal(expected_x, iter_val['X'])
assert np.array_equal(expected_y, iter_val['y'])
def test_shuffle(input_seq, batch_size, seq_len):
"""
This test checks shuffle option, with non-overlapping windows
Goes over the dataset just once
"""
def sample2char(sample):
# Converts a numpy array into a single string
# sample is a numpy array
# We convert to string so that the data can be hashable to be used in a python set
str_array = ''.join(['%1.2f' % i for i in sample.flatten()])
return str_array
# create iterator
data_array = {
'X': np.copy(input_seq),
'y': np.roll(input_seq, axis=0, shift=-1)
}
it_array = SequentialArrayIterator(data_arrays=data_array,
time_steps=seq_len,
batch_size=batch_size,
tgt_key='y',
shuffle=True,
total_iterations=None)
# Cut off the last samples of the input sequence that don't fit in a batch
no_batches = len(input_seq) // seq_len // batch_size
no_samples = batch_size * no_batches
used_steps = seq_len * no_samples
input_seq = input_seq[:used_steps]
# Reshape the input sequence into batches
reshape_dims = (no_samples, seq_len)
if len(input_seq.shape) > 1:
reshape_dims = (no_samples, seq_len, input_seq.shape[1])
expected_x = input_seq.reshape(reshape_dims)
# Convert each sample into text and build a set
set_x = set([sample2char(sample) for sample in expected_x])
sample2loc = {
sample2char(sample): idx
for idx, sample in enumerate(expected_x)
}
count = 0
for idx, iter_val in enumerate(it_array):
# for every sample in this batch
for j, sample in enumerate(iter_val['X']):
txt_sample = sample2char(sample)
assert txt_sample in set_x # check sequence is valid
set_x.discard(txt_sample)
if (idx * batch_size + j) == sample2loc[txt_sample]:
count += 1
assert count < (len(expected_x) * .5) # check shuffle happened
assert len(set_x) == 0 # check every sequence appeared in iterator
def test_rolling_window(input_seq, batch_size, seq_len, strides):
# This test checks if the rolling window works
# We check if the first two samples in each batch are strided by strides
# Truncate input sequence such that last section that doesn't fit in a batch
# is thrown away
input_seq = input_seq[:seq_len * batch_size * (len(input_seq) // seq_len // batch_size)]
data_array = {'X': input_seq,
'y': np.roll(input_seq, axis=0, shift=-1)}
time_steps = seq_len
it_array = SequentialArrayIterator(data_arrays=data_array, time_steps=time_steps,
stride=strides, batch_size=batch_size, tgt_key='y',
shuffle=False)
for idx, iter_val in enumerate(it_array):
# Start of the array needs to be time_steps * idx
assert np.array_equal(iter_val['X'][0, strides:time_steps],
iter_val['X'][1, :time_steps - strides])
assert np.array_equal(iter_val['y'][0, strides:time_steps],
iter_val['y'][1, :time_steps - strides])
parser.add_argument('--use_lut',
action='store_true',
help='choose to use lut as first layer')
parser.set_defaults()
args = parser.parse_args()
# these hyperparameters are from the paper
args.batch_size = 50
time_steps = 150
hidden_size = 500
# download penn treebank
tree_bank_data = PTB(path=args.data_dir)
ptb_data = tree_bank_data.load_data()
train_set = SequentialArrayIterator(ptb_data['train'],
batch_size=args.batch_size,
time_steps=time_steps,
total_iterations=args.num_iterations)
valid_set = SequentialArrayIterator(ptb_data['valid'],
batch_size=args.batch_size,
time_steps=time_steps)
inputs = train_set.make_placeholders()
ax.Y.length = len(tree_bank_data.vocab)
def expand_onehot(x):
return ng.one_hot(x, axis=ax.Y)
# weight initialization
parser = NgraphArgparser(__doc__)
parser.set_defaults(batch_size=128, num_iterations=2000)
args = parser.parse_args()
# model parameters
time_steps = 5
hidden_size = 256
gradient_clip_value = 5
# download penn treebank
# set shift_target to be False, since it is going to predict the same sequence
tree_bank_data = PTB(path=args.data_dir, shift_target=False)
ptb_data = tree_bank_data.load_data()
train_set = SequentialArrayIterator(ptb_data['train'],
batch_size=args.batch_size,
time_steps=time_steps,
total_iterations=args.num_iterations,
reverse_target=True,
get_prev_target=True)
valid_set = SequentialArrayIterator(ptb_data['valid'],
batch_size=args.batch_size,
time_steps=time_steps,
total_iterations=10,
reverse_target=True,
get_prev_target=True)
inputs = train_set.make_placeholders()
ax.Y.length = len(tree_bank_data.vocab)
def generate_samples(inputs, encode, decode, num_time_steps):
"""
# Build an iterator that gets seq_len long chunks of characters
# Stride is by how many characters the window moves in each step
# if stride is set to seq_len, windows are non-overlapping
stride = seq_len // 8
shakes_data_train = {
'X': np.copy(shakes.train),
'y': np.roll(np.copy(shakes.train), shift=-1)
}
shakes_data_test = {
'X': np.copy(shakes.test),
'y': np.roll(np.copy(shakes.test), shift=-1)
}
shakes_train = SequentialArrayIterator(data_arrays=shakes_data_train,
total_iterations=num_iterations,
time_steps=seq_len,
batch_size=batch_size,
stride=stride,
include_iteration=True,
tgt_key='y',
shuffle=False)
shakes_test = SequentialArrayIterator(data_arrays=shakes_data_test,
time_steps=seq_len,
batch_size=batch_size,
stride=stride,
include_iteration=True,
tgt_key='y',
shuffle=False)
# Our input is of size (batch_size, seq_len)
# batch_axis must be named N
batch_axis = ng.make_axis(length=batch_size, name="N")
# time_axis must be named REC
time_axis = ng.make_axis(length=seq_len, name="REC")
parser.set_defaults(gen_be=False)
args = parser.parse_args()
# these hyperparameters are from the paper
args.batch_size = 50
time_steps = 5
hidden_size = 10
gradient_clip_value = 15
# download penn treebank
# set shift_target to be False, since it is going to predict the same sequence
tree_bank_data = PTB(path=args.data_dir, shift_target=False)
ptb_data = tree_bank_data.load_data()
train_set = SequentialArrayIterator(ptb_data['train'],
batch_size=args.batch_size,
time_steps=time_steps,
total_iterations=args.num_iterations,
reverse_target=True,
get_prev_target=True)
# weight initialization
init = UniformInit(low=-0.08, high=0.08)
# model initialization
one_hot_enc = Preprocess(functor=lambda x: ng.one_hot(x, axis=ax.Y))
enc = Recurrent(hidden_size,
init,
activation=Tanh(),
reset_cells=True,
return_sequence=False)
one_hot_dec = Preprocess(functor=lambda x: ng.one_hot(x, axis=ax.Y))
dec = Recurrent(hidden_size,
