def load_stream(batch_size=None,
source=None,
use_tanh=False,
discrete=False,
n_colors=16,
downsample_to=None):
if source is None:
raise ValueError('Source not provided.')
if batch_size is None:
raise ValueError('Batch size not provided.')
logger.info('Loading data from `{}`'.format(source))
train_data = H5PYDataset(source, which_sets=('train', ))
num_train = train_data.num_examples
logger.debug('Number of training examples: {}'.format(num_train))
scheme = ShuffledScheme(examples=num_train, batch_size=batch_size)
stream = DataStream(train_data, iteration_scheme=scheme)
viz_options = dict(use_tanh=use_tanh)
data = stream.get_epoch_iterator(as_dict=True).next()['features']
if discrete:
n_uniques = len(np.unique(data))
if n_uniques == 256 and np.max(data) == 255:
features = H5PYDataset(source,
which_sets=('train', ),
subset=slice(0, 1000),
sources=['features'])
handle = features.open()
arr, = features.get_data(handle, slice(0, 1000))
arr = arr.transpose(0, 2, 3, 1)
arr = arr.reshape((-1, arr.shape[2], arr.shape[3]))
img = Image.fromarray(arr).convert('P',
palette=Image.ADAPTIVE,
colors=n_colors)
viz_options['img'] = img
logger.info('Quantizing data to {} colors'.format(n_colors))
viz_options['quantized'] = True
if downsample_to is not None:
logger.info(
'Downsampling the data to {}'.format(downsample_to))
stream = Quantize(img=img,
n_colors=n_colors,
downsample_to=downsample_to,
data_stream=stream)
else:
logger.warning('Data already appears to be discretized. Skipping.')
else:
if np.max(data) != 255:
raise ValueError(
'Data needs to be RGB, this doesn\'t appear to be '
'the case')
stream = Rescale(stream, use_tanh=use_tanh)
data = stream.get_epoch_iterator(as_dict=True).next()['features']
shape = data.shape
shape = dict(dim_c=shape[1], dim_x=shape[2], dim_y=shape[3])
return stream, num_train, shape, viz_options
def test_sources_selection():
features = [5, 6, 7, 1]
targets = [1, 0, 1, 1]
stream = DataStream(IterableDataset(OrderedDict(
[('features', features), ('targets', targets)])))
assert list(stream.get_epoch_iterator()) == list(zip(features, targets))
stream = DataStream(IterableDataset(
{'features': features, 'targets': targets},
sources=('targets',)))
assert list(stream.get_epoch_iterator()) == list(zip(targets))
def get_seq_mnist_streams(hidden_dim, batch_size=100, drop_prob=0.5):
permutation = np.random.randint(0, 784, size=(784, ))
train_set, valid_set, test_set = load_data('mnist.pkl.gz')
train_x = train_set[0].reshape((50000 / batch_size, batch_size, 784))
train_x = np.swapaxes(train_x, 2, 1)
train_x = train_x[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
train_y = (np.zeros(train_set[0].shape) - 1)
# label for each time-step is -1 and for the last one is the real label
train_y[:, -1] = train_set[1]
train_y = train_y.reshape((50000 / batch_size, batch_size, 784))
train_y = np.swapaxes(train_y, 2, 1)
train_y = train_y[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
valid_x = valid_set[0].reshape((10000 / batch_size, batch_size, 784))
valid_x = np.swapaxes(valid_x, 2, 1)
valid_x = valid_x[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
valid_y = (np.zeros(valid_set[0].shape) - 1)
# label for each time-step is -1 and for the last one is the real label
valid_y[:, -1] = valid_set[1]
valid_y = valid_y.reshape((10000 / batch_size, batch_size, 784))
valid_y = np.swapaxes(valid_y, 2, 1)
valid_y = valid_y[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
train_x = train_x[:, permutation]
valid_x = valid_x[:, permutation]
train = IterableDataset({
'x': train_x.astype(floatX),
'y': train_y[:, -1, :, 0].astype('int32')
})
train_stream = DataStream(train)
train_stream = SampleDrops(train_stream, drop_prob, hidden_dim, False)
train_stream.sources = ('y', 'x', 'drops')
train_stream.get_epoch_iterator().next()
valid = IterableDataset({
'x': valid_x.astype(floatX),
'y': valid_y[:, -1, :, 0].astype('int32')
})
valid_stream = DataStream(valid)
valid_stream = SampleDrops(valid_stream, drop_prob, hidden_dim, True)
valid_stream.sources = ('y', 'x', 'drops')
return train_stream, valid_stream
def test_dataset():
data = [1, 2, 3]
# The default stream requests an example at a time
stream = DataStream(IterableDataset(data))
epoch = stream.get_epoch_iterator()
assert list(epoch) == list(zip(data))
# Check if iterating over multiple epochs works
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
# Check whether the returning as a dictionary of sources works
assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
def test_dataset():
data = [1, 2, 3]
# The default stream requests an example at a time
stream = DataStream(IterableDataset(data))
epoch = stream.get_epoch_iterator()
assert list(epoch) == list(zip(data))
# Check if iterating over multiple epochs works
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
# Check whether the returning as a dictionary of sources works
assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
def test_data_driven_epochs():
class TestDataset(IterableDataset):
sources = ('data',)
def __init__(self):
self.axis_labels = None
self.data = [[1, 2, 3, 4],
[5, 6, 7, 8]]
def open(self):
epoch_iter = iter(self.data)
data_iter = iter(next(epoch_iter))
return (epoch_iter, data_iter)
def next_epoch(self, state):
try:
data_iter = iter(next(state[0]))
return (state[0], data_iter)
except StopIteration:
return self.open()
def get_data(self, state, request):
data = []
for i in range(request):
data.append(next(state[1]))
return (data,)
epochs = []
epochs.append([([1],), ([2],), ([3],), ([4],)])
epochs.append([([5],), ([6],), ([7],), ([8],)])
stream = DataStream(TestDataset(), iteration_scheme=ConstantScheme(1))
assert list(stream.get_epoch_iterator()) == epochs[0]
assert list(stream.get_epoch_iterator()) == epochs[1]
assert list(stream.get_epoch_iterator()) == epochs[0]
stream.reset()
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == epochs[i]
# test scheme resetting between epochs
class TestScheme(BatchSizeScheme):
def get_request_iterator(self):
return iter([1, 2, 1, 3])
epochs = []
epochs.append([([1],), ([2, 3],), ([4],)])
epochs.append([([5],), ([6, 7],), ([8],)])
stream = DataStream(TestDataset(), iteration_scheme=TestScheme())
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == epochs[i]
def test_data_driven_epochs():
class TestDataset(IterableDataset):
sources = ('data',)
def __init__(self):
self.axis_labels = None
self.data = [[1, 2, 3, 4],
[5, 6, 7, 8]]
def open(self):
epoch_iter = iter(self.data)
data_iter = iter(next(epoch_iter))
return (epoch_iter, data_iter)
def next_epoch(self, state):
try:
data_iter = iter(next(state[0]))
return (state[0], data_iter)
except StopIteration:
return self.open()
def get_data(self, state, request):
data = []
for i in range(request):
data.append(next(state[1]))
return (data,)
epochs = []
epochs.append([([1],), ([2],), ([3],), ([4],)])
epochs.append([([5],), ([6],), ([7],), ([8],)])
stream = DataStream(TestDataset(), iteration_scheme=ConstantScheme(1))
assert list(stream.get_epoch_iterator()) == epochs[0]
assert list(stream.get_epoch_iterator()) == epochs[1]
assert list(stream.get_epoch_iterator()) == epochs[0]
stream.reset()
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == epochs[i]
# test scheme resetting between epochs
class TestScheme(BatchSizeScheme):
def get_request_iterator(self):
return iter([1, 2, 1, 3])
epochs = []
epochs.append([([1],), ([2, 3],), ([4],)])
epochs.append([([5],), ([6, 7],), ([8],)])
stream = DataStream(TestDataset(), iteration_scheme=TestScheme())
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == epochs[i]
def get_seq_mnist_streams(hidden_dim, batch_size=100, drop_prob=0.5):
permutation = np.random.randint(0, 784, size=(784,))
train_set, valid_set, test_set = load_data('mnist.pkl.gz')
train_x = train_set[0].reshape((50000 / batch_size, batch_size, 784))
train_x = np.swapaxes(train_x, 2, 1)
train_x = train_x[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
train_y = (np.zeros(train_set[0].shape) - 1)
# label for each time-step is -1 and for the last one is the real label
train_y[:, -1] = train_set[1]
train_y = train_y.reshape((50000 / batch_size, batch_size, 784))
train_y = np.swapaxes(train_y, 2, 1)
train_y = train_y[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
valid_x = valid_set[0].reshape((10000 / batch_size, batch_size, 784))
valid_x = np.swapaxes(valid_x, 2, 1)
valid_x = valid_x[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
valid_y = (np.zeros(valid_set[0].shape) - 1)
# label for each time-step is -1 and for the last one is the real label
valid_y[:, -1] = valid_set[1]
valid_y = valid_y.reshape((10000 / batch_size, batch_size, 784))
valid_y = np.swapaxes(valid_y, 2, 1)
valid_y = valid_y[:, :, :, np.newaxis]
# Now the dimension is num_batches x 784 x batch_size x 1
train_x = train_x[:, permutation]
valid_x = valid_x[:, permutation]
train = IterableDataset({'x': train_x.astype(floatX),
'y': train_y[:, -1, :, 0].astype('int32')})
train_stream = DataStream(train)
train_stream = SampleDrops(train_stream, drop_prob, hidden_dim, False)
train_stream.sources = ('y', 'x', 'drops')
train_stream.get_epoch_iterator().next()
valid = IterableDataset({'x': valid_x.astype(floatX),
'y': valid_y[:, -1, :, 0].astype('int32')})
valid_stream = DataStream(valid)
valid_stream = SampleDrops(valid_stream, drop_prob, hidden_dim, True)
valid_stream.sources = ('y', 'x', 'drops')
return train_stream, valid_stream
def test_in_memory():
skip_if_not_available(datasets=['mnist.hdf5'])
# Load MNIST and get two batches
mnist = MNIST('train', load_in_memory=True)
data_stream = DataStream(mnist,
iteration_scheme=SequentialScheme(
examples=mnist.num_examples, batch_size=256))
epoch = data_stream.get_epoch_iterator()
for i, (features, targets) in enumerate(epoch):
if i == 1:
break
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(256, 512))
mnist.close(handle)
assert numpy.all(features == known_features)
# Pickle the epoch and make sure that the data wasn't dumped
with tempfile.NamedTemporaryFile(delete=False) as f:
filename = f.name
cPickle.dump(epoch, f)
assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB
# Reload the epoch and make sure that the state was maintained
del epoch
with open(filename, 'rb') as f:
epoch = cPickle.load(f)
features, targets = next(epoch)
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(512, 768))
mnist.close(handle)
assert numpy.all(features == known_features)
def __init__(self,
dataset,
nb_class,
examples_per_class,
start=0,
**kwargs):
assert dataset.sources == ('features', 'targets')
super(SubSetStream, self).__init__(dataset, **kwargs)
total_examples = nb_class * examples_per_class
# build a list of indexes corresponding to the subset
print "Building subset indexes list..."
stream = DataStream(dataset=dataset,
iteration_scheme=SequentialScheme(
dataset.num_examples, 100))
epitr = stream.get_epoch_iterator()
statistics = np.zeros(nb_class)
self.statistics = np.zeros(nb_class)
indexes = []
for i in range(0, (dataset.num_examples // 100) * 100, 100):
if statistics.sum() == total_examples:
break
_, targets = next(epitr)
if i < start:
continue
for j in range(100):
if statistics[targets[j]].sum() < examples_per_class:
indexes += [i + j]
statistics[targets[j]] += 1
if statistics.sum() != total_examples:
raise RuntimeError("Transformer failed")
self.indexes = indexes
def test_in_memory():
skip_if_not_available(datasets=['mnist.hdf5'])
# Load MNIST and get two batches
mnist = MNIST(('train',), load_in_memory=True)
data_stream = DataStream(mnist, iteration_scheme=SequentialScheme(
examples=mnist.num_examples, batch_size=256))
epoch = data_stream.get_epoch_iterator()
for i, (features, targets) in enumerate(epoch):
if i == 1:
break
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(256, 512))
mnist.close(handle)
assert numpy.all(features == known_features)
# Pickle the epoch and make sure that the data wasn't dumped
with tempfile.NamedTemporaryFile(delete=False) as f:
filename = f.name
cPickle.dump(epoch, f)
assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB
# Reload the epoch and make sure that the state was maintained
del epoch
with open(filename, 'rb') as f:
epoch = cPickle.load(f)
features, targets = next(epoch)
handle = mnist.open()
known_features, _ = mnist.get_data(handle, slice(512, 768))
mnist.close(handle)
assert numpy.all(features == known_features)
def test_unpack_transformer():
data = range(10)
stream = DataStream(IterableDataset(data))
stream = Batch(stream, iteration_scheme=ConstantScheme(2))
stream = Unpack(stream)
epoch = stream.get_epoch_iterator()
for i, v in enumerate(epoch):
assert numpy.shape(v)[0] == 1
assert v[0] == i
def test_get_data_inmemory(self):
ds = ImagesFromFile('{}/*.jpeg'.format(self.ds_path),
load_in_memory=True)
stream = DataStream(ds, iteration_scheme=ShuffledScheme(
ds.num_examples, batch_size=10))
for imgs, file_paths in stream.get_epoch_iterator():
assert len(imgs) == 5
assert all([img.shape == (512, 512, 3) for img in imgs])
assert all([self.ds_path in fp for fp in file_paths])
assert len(set(file_paths)) == 5
def test_batch_iteration_scheme_with_lists(self):
"""Batch schemes should work with more than ndarrays."""
data = IndexableDataset(OrderedDict([('foo', list(range(50))),
('bar', list(range(1, 51)))]))
stream = DataStream(data,
iteration_scheme=ShuffledScheme(data.num_examples,
5))
returned = [sum(batches, []) for batches in
zip(*list(stream.get_epoch_iterator()))]
assert set(returned[0]) == set(range(50))
assert set(returned[1]) == set(range(1, 51))
def main():
X = T.matrix(name='input_ivecs',dtype='float32')
#load data
#train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',))
valid_set = H5PYDataset(VALID_FILE, which_sets=('valid',))
network = build_dnn(net_input=X)
dae_ivector = lasagne.layers.get_output(network, deterministic=True)
feature_extr = theano.function([X], dae_ivector)
h1=train_set.open()
h2=valid_set.open()
scheme = SequentialScheme(examples=train_set.num_examples, batch_size=512)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=512)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
for data in train_stream.get_epoch_iterator():
t_data, _,t_name = data
t_ivec = feature_extr(t_data)
for name,ivec in zip(t_name,t_ivec):
fname = os.path.join(SAVE_PATH,t_name)
scipy.io.savemat(fname,mdict={'ivec':ivec})
for data in valid_stream.get_epoch_iterator():
v_data, _,v_name = data
v_ivec = feature_extr(v_data)
for name,ivec in zip(v_name,v_ivec):
fname = os.path.join(SAVE_PATH,v_name)
scipy.io.savemat(fname,mdict={'ivec':ivec})
def load_voc2012_stream(batch_size, train_size=16500, validation_size=500,
test_size=100, shuffle=False):
fuel_root = fuel.config.data_path[0]
# データセットファイル保存場所
hdf5_filepath = os.path.join(
fuel_root, 'voc2012\hdf5_dataset\hdf5_dataset.hdf5')
valid_size = train_size + validation_size
test_size = valid_size + test_size
indices_train = range(0, train_size)
indices_valid = range(train_size, valid_size)
indices_test = range(valid_size, test_size)
h5py_file = h5py.File(hdf5_filepath)
dataset = H5PYDataset(h5py_file, ['train'])
scheme_class = ShuffledScheme if shuffle else SequentialScheme
scheme_train = scheme_class(indices_train, batch_size=batch_size)
scheme_valid = scheme_class(indices_valid, batch_size=batch_size)
scheme_test = scheme_class(indices_test, batch_size=batch_size)
stream_train = DataStream(dataset, iteration_scheme=scheme_train)
stream_valid = DataStream(dataset, iteration_scheme=scheme_valid)
stream_test = DataStream(dataset, iteration_scheme=scheme_test)
stream_train.get_epoch_iterator().next()
stream_valid.get_epoch_iterator().next()
stream_test.get_epoch_iterator().next()
return stream_train, stream_valid, stream_test
def test_text():
# Test word level and epochs.
with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
sentences1 = f.name
f.write("This is a sentence\n")
f.write("This another one")
with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
sentences2 = f.name
f.write("More sentences\n")
f.write("The last one")
dictionary = {'<UNK>': 0, '</S>': 1, 'this': 2, 'a': 3, 'one': 4}
text_data = TextFile(files=[sentences1, sentences2],
dictionary=dictionary,
bos_token=None,
preprocess=lower)
stream = DataStream(text_data)
epoch = stream.get_epoch_iterator()
assert len(list(epoch)) == 4
epoch = stream.get_epoch_iterator()
for sentence in zip(range(3), epoch):
pass
f = BytesIO()
cPickle.dump(epoch, f)
sentence = next(epoch)
f.seek(0)
epoch = cPickle.load(f)
assert next(epoch) == sentence
assert_raises(StopIteration, next, epoch)
# Test character level.
dictionary = dict([(chr(ord('a') + i), i)
for i in range(26)] + [(' ', 26)] + [('<S>', 27)] +
[('</S>', 28)] + [('<UNK>', 29)])
text_data = TextFile(files=[sentences1, sentences2],
dictionary=dictionary,
preprocess=lower,
level="character")
sentence = next(DataStream(text_data).get_epoch_iterator())[0]
assert sentence[:3] == [27, 19, 7]
assert sentence[-3:] == [2, 4, 28]
def test(model=None):
if model is not None:
#accuracies = []
testset = H5PYDataset('svhn_format_2.hdf5', which_sets=('test',), sources=('features', 'targets'))
teststream = DataStream(testset, iteration_scheme=SequentialScheme(examples=testset.num_examples, batch_size=500))
for data in teststream.get_epoch_iterator():
images, labels = data
images = np.swapaxes(images, axis1=1, axis2=3)
labels = keras.utils.to_categorical(labels)
loss, accuracy = model.test_on_batch(x=images, y=labels)
accuracies.append(accuracy)
trainstream.close()
return losses
def main():
X = T.matrix(name='input_ivecs', dtype='float32')
#load data
#train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',))
valid_set = H5PYDataset(VALID_FILE, which_sets=('valid', ))
network = build_dnn(net_input=X)
dae_ivector = lasagne.layers.get_output(network, deterministic=True)
feature_extr = theano.function([X], dae_ivector)
h1 = train_set.open()
h2 = valid_set.open()
scheme = SequentialScheme(examples=train_set.num_examples, batch_size=512)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=512)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
for data in train_stream.get_epoch_iterator():
t_data, _, t_name = data
t_ivec = feature_extr(t_data)
for name, ivec in zip(t_name, t_ivec):
fname = os.path.join(SAVE_PATH, t_name)
scipy.io.savemat(fname, mdict={'ivec': ivec})
for data in valid_stream.get_epoch_iterator():
v_data, _, v_name = data
v_ivec = feature_extr(v_data)
for name, ivec in zip(v_name, v_ivec):
fname = os.path.join(SAVE_PATH, v_name)
scipy.io.savemat(fname, mdict={'ivec': ivec})
def test_text():
# Test word level and epochs.
with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
sentences1 = f.name
f.write("This is a sentence\n")
f.write("This another one")
with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
sentences2 = f.name
f.write("More sentences\n")
f.write("The last one")
dictionary = {'<UNK>': 0, '</S>': 1, 'this': 2, 'a': 3, 'one': 4}
text_data = TextFile(files=[sentences1, sentences2],
dictionary=dictionary, bos_token=None,
preprocess=lower)
stream = DataStream(text_data)
epoch = stream.get_epoch_iterator()
assert len(list(epoch)) == 4
epoch = stream.get_epoch_iterator()
for sentence in zip(range(3), epoch):
pass
f = BytesIO()
cPickle.dump(epoch, f)
sentence = next(epoch)
f.seek(0)
epoch = cPickle.load(f)
assert next(epoch) == sentence
assert_raises(StopIteration, next, epoch)
# Test character level.
dictionary = dict([(chr(ord('a') + i), i) for i in range(26)] +
[(' ', 26)] + [('<S>', 27)] +
[('</S>', 28)] + [('<UNK>', 29)])
text_data = TextFile(files=[sentences1, sentences2],
dictionary=dictionary, preprocess=lower,
level="character")
sentence = next(DataStream(text_data).get_epoch_iterator())[0]
assert sentence[:3] == [27, 19, 7]
assert sentence[-3:] == [2, 4, 28]
def test_jsb():
jsb = MidiSequence('jsb')
print jsb.num_examples
dataset = DataStream(jsb,
iteration_scheme=SequentialScheme(
jsb.num_examples, 10))
for b in dataset.get_epoch_iterator():
print b[0].shape
# This is how to prepare this dataset to be used in Blocks
dataset = Padding(dataset)
def _transpose(data):
return tuple(np.rollaxis(array, 1, 0) for array in data)
dataset = Mapping(dataset, _transpose)
for b in dataset.get_epoch_iterator():
print len(b)
print b[1].shape
print b[0].shape
assert b[0].shape == (108, 9, 96)
def test_batch_iteration_scheme_with_lists(self):
"""Batch schemes should work with more than ndarrays."""
data = IndexableDataset(
OrderedDict([('foo', list(range(50))), ('bar', list(range(1,
51)))]))
stream = DataStream(data,
iteration_scheme=ShuffledScheme(
data.num_examples, 5))
returned = [
sum(batches, [])
for batches in zip(*list(stream.get_epoch_iterator()))
]
assert set(returned[0]) == set(range(50))
assert set(returned[1]) == set(range(1, 51))
def batch_iterator(self, dataset, batchsize, shuffle=False):
if isinstance(dataset, Dataset):
if shuffle:
train_scheme = ShuffledScheme(examples=dataset.num_examples, batch_size=batchsize)
else:
train_scheme = SequentialScheme(examples=dataset.num_examples, batch_size=batchsize)
stream = DataStream(dataset=dataset, iteration_scheme=train_scheme)
if self.fuel_stream_xform_fn is not None:
stream = self.fuel_stream_xform_fn(stream)
return stream.get_epoch_iterator()
elif _is_sequence_of_arrays(dataset):
return iterate_minibatches(dataset, batchsize, shuffle=shuffle)
else:
raise TypeError('dataset should be a fuel Dataset instance or a list of arrays')
def train(model=None):
if model is not None:
trainset = H5PYDataset('svhn_format_2.hdf5', which_sets=('train',), sources=('features', 'targets'))
trainstream = DataStream(trainset, iteration_scheme=SequentialScheme(examples=trainset.num_examples, batch_size=500))
for data in trainstream.get_epoch_iterator():
images, labels = data
#standardize the input images
m = images.mean(axis=(2,3), keepdims=True)
s = images.std(axis=(2,3), keepdims=True)
images = (images - m)/s
#change from "channel_first" to "channel_last"
images = np.transpose(images, (0,2,3,1))
labels = keras.utils.to_categorical(labels)
#print images.shape
model.train_on_batch(x=images, y=labels)
trainstream.close()
class FuelDataSet(object):
def __init__(self,dataset,batch_size=128,shuffle=False):
self.dataset = dataset
if shuffle:
self.datastream = DataStream(self.dataset,
iteration_scheme=ShuffledScheme(
examples=dataset.num_examples,
batch_size=batch_size))
else:
self.datastream = DataStream(self.dataset,
iteration_scheme=SequentialScheme(
examples=dataset.num_examples,
batch_size=batch_size))
def __iter__(self):
return self.datastream.get_epoch_iterator()
def infer(path, ae_encode):
'''
:param path: path of infer data
:param ae_encode: compiled theano function
:return: image saved path in string
'''
hf = h5py.File(path, 'r+')
split_dict = {
'test': {
'input': (0, 1),
'target': (0, 1)
},
}
hf.attrs['split'] = H5PYDataset.create_split_array(split_dict)
test_set = H5PYDataset(path, which_sets=('test', ))
batch_size = 1
test_scheme = SequentialScheme(examples=test_set.num_examples,
batch_size=batch_size)
test_stream = DataStream(test_set, iteration_scheme=test_scheme)
for te_train, te_target in test_stream.get_epoch_iterator():
break
te_out, te_ta = ae_encode(input_transform(te_train),
target_transform(te_target))
te_reshape = inverse(te_out)
te_target_reshape = inverse(te_ta)
new_size = (128 * 2, 160)
new_im = Image.new('RGB', new_size)
r = np.random.choice(1, 1, replace=False).reshape(1, 1)
for i in range(1):
for j in range(1):
index = r[i][j]
target_im = Image.fromarray(te_target_reshape[index])
train_im = Image.fromarray(te_train[index].astype(np.uint8))
im = Image.fromarray(te_reshape[index])
new_im.paste(train_im, (128 * (i * 2), 160 * j))
new_im.paste(im, (128 * (i * 2 + 1), 160 * j))
img_loc = "gen_images/%i.png" % int(time())
new_im.save(img_loc)
return img_loc
def test_data_stream():
from fuel.datasets import IndexableDataset
from fuel.streams import DataStream
from fuel.schemes import ShuffledScheme
seed = 1234
rng = numpy.random.RandomState(seed)
features = rng.randint(256, size=(8, 2, 2))
targets = rng.randint(4, size=(8, 1))
dataset = IndexableDataset(indexables=OrderedDict([('features', features),
('targets', targets)]),
axis_labels=OrderedDict([('features', ('batch', 'height', 'width')),
('targets', ('batch', 'index'))]))
scheme = ShuffledScheme(examples=dataset.num_examples, batch_size=10)
data_stream = DataStream(dataset=dataset, iteration_scheme=scheme)
for i, data in enumerate(data_stream.get_epoch_iterator()):
print('epoch '+str(i), data[0].shape, data[1].shape)
time.sleep(1)
def load_toy_stream(batch_size, ):
"""Parameters
----------
randomcirclesquaredataset : RandomCircleSqureDatasetクラスインスタンス.
batch_size : バッチサイズ"""
dataset = IterableDataset(RandomCircleSquareDataset(batch_size=batch_size))
stream_train = DataStream(dataset)
stream_valid = DataStream(dataset)
stream_test = DataStream(dataset)
stream_train.get_epoch_iterator().next()
stream_valid.get_epoch_iterator().next()
stream_test.get_epoch_iterator().next()
return stream_train, stream_valid, stream_test
def dataset_generator(dataset, batch_size=500):
while 1:
trainstream = DataStream(dataset, iteration_scheme=ShuffledScheme(examples=dataset.num_examples, batch_size=batch_size))
for data in trainstream.get_epoch_iterator():
images, labels = data
#resize images
#images = tf.image.resize_images(images, (48,48))
#images = images.eval()
#standardize the input images
m = images.mean(axis=(1,2,3), keepdims=True)
#s = images.std(axis=(1,2,3), keepdims=True)
images = (images - m)#/ (s + 1e-3)
#change from "channel_first" to "channel_last"
images = np.transpose(images, (0,2,3,1))
#images = keras.utils.normalize(images, axis=-1, order=2)
#convert to one_hot representation
#print labels
labels = keras.utils.to_categorical(labels, num_classes=11)
#print labels
yield(images, labels)
trainstream.close()
def test_in_memory():
skip_if_not_available(datasets=['mnist'])
# Load MNIST and get two batches
mnist = MNIST('train')
data_stream = DataStream(mnist, iteration_scheme=SequentialScheme(
num_examples=mnist.num_examples, batch_size=256))
epoch = data_stream.get_epoch_iterator()
for i, (features, targets) in enumerate(epoch):
if i == 1:
break
assert numpy.all(features == mnist.features[256:512])
# Pickle the epoch and make sure that the data wasn't dumped
with tempfile.NamedTemporaryFile(delete=False) as f:
filename = f.name
cPickle.dump(epoch, f)
assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB
# Reload the epoch and make sure that the state was maintained
del epoch
with open(filename, 'rb') as f:
epoch = cPickle.load(f)
features, targets = next(epoch)
assert numpy.all(features == mnist.features[512:768])
def main(config, tr_stream, dev_stream, use_bokeh=False):
print("~def main")
# Create Theano variables
logger.info('Creating theano variables')
source_sentence = tensor.lmatrix('source')
source_sentence_mask = tensor.matrix('source_mask')
target_sentence = tensor.lmatrix('target')
target_sentence_mask = tensor.matrix('target_mask')
sampling_input = tensor.lmatrix('input')
print("~sampling_input = tensor.lmatrix")
# Construct model
logger.info('Building RNN encoder-decoder')
encoder = BidirectionalEncoder(
config['src_vocab_size'], config['enc_embed'], config['enc_nhids'])
decoder = Decoder(
config['trg_vocab_size'], config['dec_embed'], config['dec_nhids'],
config['enc_nhids'] * 2)
cost = decoder.cost(
encoder.apply(source_sentence, source_sentence_mask),
source_sentence_mask, target_sentence, target_sentence_mask)
print("~source_sentence_mask, target_sentence, target_sentence_mask")
logger.info('Creating computational graph')
cg = ComputationGraph(cost)
print("~ComputationGraph")
# Initialize model
logger.info('Initializing model')
encoder.weights_init = decoder.weights_init = IsotropicGaussian(
config['weight_scale'])
encoder.biases_init = decoder.biases_init = Constant(0)
encoder.push_initialization_config()
decoder.push_initialization_config()
encoder.bidir.prototype.weights_init = Orthogonal()
decoder.transition.weights_init = Orthogonal()
encoder.initialize()
decoder.initialize()
print("~decoder.initialize()")
# apply dropout for regularization
if config['dropout'] < 1.0:
# dropout is applied to the output of maxout in ghog
logger.info('Applying dropout')
dropout_inputs = [x for x in cg.intermediary_variables
if x.name == 'maxout_apply_output']
cg = apply_dropout(cg, dropout_inputs, config['dropout'])
print("~cg = apply_dropout")
# Apply weight noise for regularization
if config['weight_noise_ff'] > 0.0:
logger.info('Applying weight noise to ff layers')
enc_params = Selector(encoder.lookup).get_params().values()
enc_params += Selector(encoder.fwd_fork).get_params().values()
enc_params += Selector(encoder.back_fork).get_params().values()
dec_params = Selector(
decoder.sequence_generator.readout).get_params().values()
dec_params += Selector(
decoder.sequence_generator.fork).get_params().values()
dec_params += Selector(decoder.state_init).get_params().values()
cg = apply_noise(cg, enc_params+dec_params, config['weight_noise_ff'])
print("~cg = apply_noise")
# Print shapes
shapes = [param.get_value().shape for param in cg.parameters]
logger.info("Parameter shapes: ")
for shape, count in Counter(shapes).most_common():
logger.info(' {:15}: {}'.format(shape, count))
logger.info("Total number of parameters: {}".format(len(shapes)))
print("~logger.info")
# Print parameter names
enc_dec_param_dict = merge(Selector(encoder).get_parameters(),
Selector(decoder).get_parameters())
logger.info("Parameter names: ")
for name, value in enc_dec_param_dict.items():
logger.info(' {:15}: {}'.format(value.get_value().shape, name))
logger.info("Total number of parameters: {}"
.format(len(enc_dec_param_dict)))
# Set up training model
logger.info("Building model")
training_model = Model(cost)
print("~training_model")
# Set extensions
logger.info("Initializing extensions")
extensions = [
FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'],
every_n_batches=config['save_freq'])
]
print("~every_n_batches=config")
# Set up beam search and sampling computation graphs if necessary
if config['hook_samples'] >= 1 or config['bleu_script'] is not None:
logger.info("Building sampling model")
sampling_representation = encoder.apply(
sampling_input, tensor.ones(sampling_input.shape))
generated = decoder.generate(sampling_input, sampling_representation)
search_model = Model(generated)
_, samples = VariableFilter(
bricks=[decoder.sequence_generator], name="outputs")(
ComputationGraph(generated[1])) # generated[1] is next_outputs
sample = Sampler(model=search_model, data_stream=tr_stream,
hook_samples=config['hook_samples'],
every_n_batches=config['sampling_freq'],
src_vocab_size=config['src_vocab_size'])
# Add sampling
if config['hook_samples'] >= 1:
logger.info("Building sampler")
extensions.append( sample )
# Add early stopping based on bleu
if config['bleu_script'] is not None:
logger.info("Building bleu validator")
extensions.append(
BleuValidator(sampling_input, samples=samples, config=config,
model=search_model, data_stream=dev_stream,
normalize=config['normalized_bleu'],
every_n_batches=config['bleu_val_freq']))
# Reload model if necessary
if config['reload']:
extensions.append(LoadNMT(config['saveto']))
# Plot cost in bokeh if necessary
if use_bokeh and BOKEH_AVAILABLE:
extensions.append(
Plot('Cs-En', channels=[['decoder_cost_cost']],
after_batch=True))
sampling_fn = search_model.get_theano_function()
print(" - - - - - - - - - - - - - - " )
sort_k_batches = 12
batch_size = 80
seq_len = 50
trg_ivocab = None
src_vocab_size = config['src_vocab_size']
trg_vocab_size = config['trg_vocab_size']
unk_id = config['unk_id']
src_vocab = config['src_vocab']
trg_vocab = config['trg_vocab']
src_vocab = ensure_special_tokens(
src_vocab if isinstance(src_vocab, dict)
else cPickle.load(open(src_vocab)),
bos_idx=0, eos_idx=src_vocab_size - 1, unk_idx=unk_id)
trg_vocab = ensure_special_tokens(
trg_vocab if isinstance(trg_vocab, dict) else
cPickle.load(open(trg_vocab)),
bos_idx=0, eos_idx=trg_vocab_size - 1, unk_idx=unk_id)
if not trg_ivocab:
trg_ivocab = {v: k for k, v in trg_vocab.items()}
src_data = config['src_data']
trg_data = config['trg_data']
src_dataset = TextFile([src_data], src_vocab, None)
trg_dataset = TextFile([trg_data], trg_vocab, None)
inputstringfile="inputstringfile.cs"
input_dataset = TextFile([inputstringfile], src_vocab, None)
stream = Merge([input_dataset.get_example_stream(),
trg_dataset.get_example_stream()],
('source', 'target'))
stream2 = Filter(stream,
predicate=_too_long(seq_len=seq_len))
stream3 = Mapping(stream2,
_oov_to_unk(src_vocab_size=src_vocab_size,
trg_vocab_size=trg_vocab_size,
unk_id=unk_id))
stream4 = Batch(stream3,
iteration_scheme=ConstantScheme(
batch_size*sort_k_batches))
stream5 = Mapping(stream4, SortMapping(_length))
stream6 = Unpack(stream5)
stream7 = Batch(
stream6, iteration_scheme=ConstantScheme(batch_size))
input_stream = DataStream(input_dataset)
print("dev_stream : ", type( dev_stream ) )
print("input_stream : ", type( input_stream ) )
epochone = input_stream.get_epoch_iterator()
vocab = input_stream.dataset.dictionary
unk_sym = input_stream.dataset.unk_token
eos_sym = input_stream.dataset.eos_token
for i, line in enumerate(epochone):
seq = oov_to_unk(
line[0], config['src_vocab_size'], unk_id)
input_ = numpy.tile(seq, ( 1 , 1))
print("seq : " , type( seq ) , seq )
print("input_ : ", type( input_ ) , input_ , inspect.getmembers( input_ ) )
_1, outputs, _2, _3, costs = ( sampling_fn( input_ ) )
outputs = outputs.flatten()
costs = costs.T
print(" outputs : " , outputs , type( outputs ) )
print("idx_to_word: ", idx_to_word(outputs , trg_ivocab))
print(" - - - - - - - - - - - - - - " )
data_stream = Batch(data_stream,
iteration_scheme=ConstantScheme(mini_batch_size))
#data_stream = Padding(data_stream, mask_sources=('tokens')) # Adds a mask fields to this stream field, type='floatX'
data_stream = Padding(
data_stream,
) # Adds a mask fields to all of this stream's fields, type='floatX'
data_stream = Mapping(
data_stream, _transpose
) # Flips stream so that sentences run down columns, batches along rows (strangely)
if False: # print sample for debugging Dataset / DataStream component
#t=0
max_len = 0
for i, data in enumerate(data_stream.get_epoch_iterator()):
#print(i)
#t=t + data[4].sum() + data[0].shape[1]
l = data[0].shape[0]
if l > max_len:
max_len = l
print(i, data[0].shape, max_len)
#print(data)
#break
exit(0)
"""
Comments in google-groups:blocks indicate that a reshaping has to be done,
so let's think about sizes of the arrays...
"""
x = tensor.matrix('tokens', dtype="int32")
########## start training ###########################
n_epochs = 15
if "n_epochs" in config:
n_epochs = int(config["n_epochs"])
bestF1 = 0
best_params = []
best_epoch = 0
epoch = 0
lrate = lrateOrig
while epoch < n_epochs:
time1 = time.time()
# train
time1Train = time.time()
for d in data_stream.get_epoch_iterator(as_dict=True):
x1_numpy = d['x1']
x2_numpy = d['x2']
x3_numpy = d['x3']
x4_numpy = d['x4']
e1_numpy = d['e1']
e2_numpy = d['e2']
y_numpy = d['y']
y1ET_numpy = d['y1ET']
y2ET_numpy = d['y2ET']
numSamples_numpy = numpy.ones_like(y1ET_numpy)
cost_ij = train(x1_numpy, x2_numpy, x3_numpy, x4_numpy, e1_numpy,
e2_numpy, y_numpy, y1ET_numpy, y2ET_numpy,
numSamples_numpy, lrate)
if numpy.isnan(cost_ij):
test_stream,
prefix="test")
#, FinishAfter(after_n_epochs=10)
, Printing()
#, Dump("out.pkl")
, ExperimentSaver("../VAOutCIFAR", ".")
, EpochProgress(dataset.num_examples // 128)
, Plot('cifar10',
channels=[['train_variational_cost', 'test_variational_cost', 'test_cost']])
])
main_loop.run()
_func_noisy = theano.function([m.X], m.noisy)
_func_produced = theano.function([m.X], m.produced)
batch = test_stream.get_epoch_iterator().next()[0]
out_noise = _func_noisy(batch)
out_produced = _func_produced(batch)
import cv2
for k in range(10):
print out_noise.shape
img = np.reshape(out_noise[k, :], (28, 28))
img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST)
cv2.imshow('img', img)
img = np.reshape(out_produced[k, :], (28, 28))
img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST)
cv2.imshow('masdf', img)
cv2.waitKey(0)
split_dict = {
'train': {
'images': (0, train_feature.shape[0]),
'targets': (0, train_target.shape[0])
}
}
f.attrs['split'] = H5PYDataset.create_split_array(split_dict)
f.flush()
f.close()
train_set = H5PYDataset('../../data/dataset.hdf5', which_sets=('train',))
#data_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
#state = train_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=4)
data_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
for data in data_stream.get_epoch_iterator():
print(data[0], data[1])
standardized_stream = ScaleAndShift(data_stream=data_stream,
scale=255,shift=0,
which_sources=('features',))
for data in standardized_stream.get_epoch_iterator():
print(data[0], data[1])
#train_set.close(state)
def main():
X = T.tensor3(name='input',dtype='float32')
MASK = T.matrix(name = 'mask', dtype='float32')
SPK_LABELS = T.ivector(name='spk_labels')
PHR_LABELS = T.ivector(name='phr_labels')
#load data
train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',))
valid_set = H5PYDataset(TRAIN_FILE, which_sets=('test',))
spk_network, phr_network = build_rnn(net_input=X, mask_input= MASK)
network_output_spk = lasagne.layers.get_output(spk_network)
network_output_phr = lasagne.layers.get_output(phr_network)
val_prediction_spk = lasagne.layers.get_output(spk_network, deterministic=True)
val_prediction_phr = lasagne.layers.get_output(phr_network, deterministic=True)
#needed for accuracy
val_acc_spk = T.mean(T.eq(T.argmax(val_prediction_spk, axis=1), SPK_LABELS), dtype=theano.config.floatX)
val_acc_phr = T.mean(T.eq(T.argmax(val_prediction_phr, axis=1), PHR_LABELS), dtype=theano.config.floatX)
#training accuracy
train_acc_spk = T.mean(T.eq(T.argmax(network_output_spk, axis=1), SPK_LABELS), dtype=theano.config.floatX)
train_acc_phr = T.mean(T.eq(T.argmax(network_output_phr, axis=1), PHR_LABELS), dtype=theano.config.floatX)
#cost function
spk_cost = lasagne.objectives.categorical_crossentropy(network_output_spk, SPK_LABELS)
phr_cost = lasagne.objectives.categorical_crossentropy(network_output_phr, PHR_LABELS)
total_cost = spk_cost + phr_cost
mean_cost = total_cost.mean()
#Validation cost
val_spk_cost = lasagne.objectives.categorical_crossentropy(val_prediction_spk, SPK_LABELS)
val_phr_cost = lasagne.objectives.categorical_crossentropy(val_prediction_phr, PHR_LABELS)
val_cost = val_spk_cost + val_phr_cost
val_mcost = val_cost.mean()
#Get parameters of both encoder and decoder
all_parameters = lasagne.layers.get_all_params([spk_network, phr_network], trainable=True)
print("Trainable Model Parameters")
print("-"*40)
for param in all_parameters:
print(param, param.get_value().shape)
print("-"*40)
#add grad clipping to avoid exploding gradients
all_grads = [T.clip(g,-3,3) for g in T.grad(mean_cost, all_parameters)]
#all_grads = lasagne.updates.total_norm_constraint(all_grads,3)
updates = lasagne.updates.adam(all_grads, all_parameters, learning_rate=LEARNING_RATE)
train_func = theano.function([X, MASK, SPK_LABELS, PHR_LABELS], [mean_cost, train_acc_spk, train_acc_phr], updates=updates)
val_func = theano.function([X, MASK, SPK_LABELS, PHR_LABELS], [val_mcost, val_acc_spk, val_acc_phr])
trainerr=[]
epoch=0 #set the epoch counter
min_val_loss = np.inf
max_val = np.inf*(-1)
patience=0
print("Starting training...")
# We iterate over epochs:
while 'true':
# In each epoch, we do a full pass over the training data:
train_err = 0
tr_acc_spk = 0
tr_acc_phr = 0
train_batches = 0
h1=train_set.open()
h2=valid_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=BATCH_SIZE)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=32)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
start_time = time.time()
for data in train_stream.get_epoch_iterator():
t_data, t_mask, t_plabs,_, t_labs,_ = data
terr, tacc_spk, tacc_phr = train_func(t_data, t_mask, t_labs, t_plabs)
train_err += terr
tr_acc_spk += tacc_spk
tr_acc_phr += tacc_phr
train_batches += 1
val_err = 0
val_acc_spk = 0
val_acc_phr = 0
val_batches = 0
for data in valid_stream.get_epoch_iterator():
v_data, v_mask, v_ptars, _, v_tars, _ = data
err, acc_spk, acc_phr = val_func(v_data, v_mask ,v_tars, v_ptars)
val_err += err
val_acc_spk += acc_spk
val_acc_phr += acc_phr
val_batches += 1
trainerr.append(train_err/train_batches)
epoch+=1
train_set.close(h1)
valid_set.close(h2)
#Display
if display:
print("Epoch {} of {} took {:.3f}s".format(
epoch, NUM_EPOCHS, time.time() - start_time))
print(" training loss:\t\t{:.6f}".format(train_err / train_batches))
print(" training accuracy (speaker):\t\t{:.2f} %".format(
tr_acc_spk / train_batches * 100))
print(" training accuracy (phrase):\t\t{:.2f} %".format(
tr_acc_phr / train_batches * 100))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
print(" validation accuracy (speaker):\t\t{:.2f} %".format(
val_acc_spk / val_batches * 100))
print(" validation accuracy (phrase):\t\t{:.2f} %".format(
val_acc_phr / val_batches * 100))
logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log')
flog1 = open(logfile,'ab')
flog1.write("Running %s on %s" % (SCRIPT,MSC))
flog1.write("Epoch {} of {} took {:.3f}s \n".format(
epoch, NUM_EPOCHS, time.time() - start_time))
flog1.write(" training loss:\t\t{:.6f}\n".format(train_err / train_batches))
flog1.write(" training accuracy (speaker):\t\t{:.2f} %\n".format(
tr_acc_spk / train_batches * 100))
flog1.write(" training accuracy (phrase):\t\t{:.2f} %\n".format(
tr_acc_phr / train_batches * 100))
flog1.write(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
flog1.write(" validation accuracy (speaker):\t\t{:.2f} %\n".format(
val_acc_spk / val_batches * 100))
flog1.write(" validation accuracy (phrase):\t\t{:.2f} %\n".format(
val_acc_phr / val_batches * 100))
flog1.write("\n")
flog1.close()
if epoch == NUM_EPOCHS:
break
valE = val_err/val_batches
if valE <= min_val_loss:
model_params1 = lasagne.layers.get_all_param_values([spk_network, phr_network])
model1_name = MODEL_NAME+'_minloss' + '.pkl'
vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name)
fsave = open(vpth1,'wb')
cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
patience=0
min_val_loss = valE
#Patience / Early stopping
else:
patience+=1
if patience==10:
break
#Save the final model
print('Saving Model ...')
model_params = lasagne.layers.get_all_param_values([spk_network, phr_network])
model1_name = MODEL_NAME+'_final' + '.pkl'
vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name)
fsave = open(vpth,'wb')
cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
train_data = H5PYDataset(DATASET_PATH,
which_sets=('train', ),
sources=('s_transition_obs', 'r_transition_obs',
'obs', 'actions'))
stream_train = DataStream(train_data,
iteration_scheme=SequentialScheme(
train_data.num_examples, batch_size))
valid_data = H5PYDataset(DATASET_PATH,
which_sets=('valid', ),
sources=('s_transition_obs', 'r_transition_obs',
'obs', 'actions'))
stream_valid = DataStream(valid_data,
iteration_scheme=SequentialScheme(
train_data.num_examples, batch_size))
iterator = stream_train.get_epoch_iterator(as_dict=True)
data = next(iterator)
length = data["actions"].shape[1]
match_env(env, env2)
video_recorder = VideoRecorder(env, 'sim+backlash.mp4', enabled=True)
video_recorder2 = VideoRecorder(env2, 'sim+.mp4', enabled=True)
for i, data in enumerate(stream_train.get_epoch_iterator(as_dict=True)):
for j in range(length):
action = data["actions"]
video_recorder.capture_frame()
video_recorder2.capture_frame()
new_obs, reward, done, info = env.step(action)
new_obs2, reward2, done2, info2 = env2.step(action)
def main():
X = T.tensor3(name='input',dtype='float32')
MASK = T.matrix(name = 'mask', dtype='float32')
LABELS = T.ivector(name='labels')
#load data
train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',))
valid_set = H5PYDataset(VALID_FILE, which_sets=('test',))
network = build_rnn(net_input=X, mask_input= MASK)
network_output = lasagne.layers.get_output(network)
val_prediction = lasagne.layers.get_output(network, deterministic=True)
#needed for accuracy
val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS), dtype=theano.config.floatX)
#training accuracy
train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS), dtype=theano.config.floatX)
#T.argmax(network_output, axis=1).eval({X: d1, Mask: m1})
#print network_output.eval({X: d1, Mask: m1})[1][97]
#cost function
total_cost = lasagne.objectives.categorical_crossentropy(network_output, LABELS)
#total_cost = -(labels*T.log(network_output) + (1-labels)*T.log(1-network_output))
mean_cost = total_cost.mean()
#accuracy function
val_cost = lasagne.objectives.categorical_crossentropy(val_prediction, LABELS)
val_mcost = val_cost.mean()
#Get parameters of both encoder and decoder
all_parameters = lasagne.layers.get_all_params([network], trainable=True)
print("Trainable Model Parameters")
print("-"*40)
for param in all_parameters:
print(param, param.get_value().shape)
print("-"*40)
#add grad clipping to avoid exploding gradients
all_grads = [T.clip(g,-3,3) for g in T.grad(mean_cost, all_parameters)]
all_grads = lasagne.updates.total_norm_constraint(all_grads,3)
updates = lasagne.updates.adam(all_grads, all_parameters, learning_rate=LEARNING_RATE)
train_func = theano.function([X, MASK, LABELS], [mean_cost, train_acc], updates=updates)
val_func = theano.function([X, MASK, LABELS], [val_mcost, val_acc])
trainerr=[]
epoch=0 #set the epoch counter
val_prev = np.inf
a_prev = -np.inf
print("Starting training...")
# We iterate over epochs:
while 'true':
# In each epoch, we do a full pass over the training data:
train_err = 0
tr_acc = 0
train_batches = 0
h1=train_set.open()
h2=valid_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=128)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=32)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
start_time = time.time()
for data in train_stream.get_epoch_iterator():
t_data, t_mask, _, t_labs = data
terr, tacc = train_func(t_data, t_mask, t_labs)
train_err += terr
tr_acc += tacc
train_batches += 1
val_err = 0
val_acc = 0
val_batches = 0
for data in valid_stream.get_epoch_iterator():
v_data, v_mask, _, v_tars = data
err, acc = val_func(v_data, v_mask ,v_tars)
val_err += err
val_acc += acc
val_batches += 1
trainerr.append(train_err/train_batches)
epoch+=1
train_set.close(h1)
valid_set.close(h2)
#Display
if display:
print("Epoch {} of {} took {:.3f}s ".format(
epoch, NUM_EPOCHS, time.time() - start_time))
print(" training loss:\t\t{:.6f} ".format(train_err / train_batches))
print(" training accuracy:\t\t{:.2f} % ".format(
tr_acc / train_batches * 100))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
print(" validation accuracy:\t\t{:.2f} % ".format(
val_acc / val_batches * 100))
logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log')
flog1 = open(logfile,'ab')
flog1.write("Epoch {} of {} took {:.3f}s\n ".format(
epoch, NUM_EPOCHS, time.time() - start_time))
flog1.write(" training loss:\t\t{:.6f} ".format(train_err / train_batches))
flog1.write(" training accuracy:\t\t{:.2f} %\n ".format(
tr_acc / train_batches * 100))
flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err / val_batches))
flog1.write(" validation accuracy:\t\t{:.2f} %\n ".format(
val_acc / val_batches * 100))
flog1.write("\n")
flog1.close()
if epoch == NUM_EPOCHS:
break
valE = val_err/val_batches
valA = val_acc / val_batches
#save the max accuracy model
max_val = a_prev
#save model with highest accuracy
if valA > max_val:
model_params1 = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME+'_acc' + '.pkl'
vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name)
fsave = open(vpth1,'wb')
cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
max_val = valA
#Patience
if valE > val_prev:
c+=1
#save the model incase
model_params1 = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME + '_ofit' + '.pkl'
vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name)
fsave = open(vpth1,'wb')
cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
val_prev=valE
else:
c=0
val_prev=valE
if c==5:
break
#Save the final model
print('Saving Model ...')
model_params = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME+'_final' + '.pkl'
vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name)
fsave = open(vpth,'wb')
cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
def main():
X = T.matrix(name='input_ivecs',dtype='float32')
X_mean = T.matrix(name = 'target_ivecs', dtype='float32')
#load data
train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',))
valid_set = H5PYDataset(VALID_FILE, which_sets=('valid',))
network = build_dnn(net_input=X)
network_output = lasagne.layers.get_output(network)
val_prediction = lasagne.layers.get_output(network, deterministic=True)
#needed for accuracy
#val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS), dtype=theano.config.floatX)
#training accuracy
#train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS), dtype=theano.config.floatX)
#T.argmax(network_output, axis=1).eval({X: d1, Mask: m1})
#print network_output.eval({X: d1, Mask: m1})[1][97]
#cost function
total_cost = lasagne.objectives.squared_error(network_output, X_mean)
mean_cost = total_cost.mean()
#accuracy function
val_cost = lasagne.objectives.squared_error(val_prediction, X_mean)
val_mcost = val_cost.mean()
#Get parameters of both encoder and decoder
all_parameters = lasagne.layers.get_all_params([network], trainable=True)
print("Trainable Model Parameters")
print("-"*40)
for param in all_parameters:
print(param, param.get_value().shape)
print("-"*40)
#add grad clipping to avoid exploding gradients
updates = lasagne.updates.adadelta(mean_cost, all_parameters)
train_func = theano.function([X, X_mean], mean_cost, updates=updates)
val_func = theano.function([X, X_mean], val_mcost)
trainerr=[]
epoch=0 #set the epoch counter
min_val_loss = np.inf
patience=0
print("Starting training...")
# We iterate over epochs:
while 'true':
# In each epoch, we do a full pass over the training data:
train_err = 0
train_batches = 0
h1=train_set.open()
h2=valid_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=BATCH_SIZE)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=64)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
start_time = time.time()
for data in train_stream.get_epoch_iterator():
t_data, t_targ,_ = data
terr = train_func(t_data, t_targ)
train_err += terr
train_batches += 1
val_err = 0
val_batches = 0
for data in valid_stream.get_epoch_iterator():
v_data, v_tars,_ = data
err = val_func(v_data, v_tars)
val_err += err
val_batches += 1
trainerr.append(train_err/train_batches)
epoch+=1
train_set.close(h1)
valid_set.close(h2)
#Display
if display:
print("Epoch {} of {} took {:.3f}s ".format(
epoch, NUM_EPOCHS, time.time() - start_time))
print(" training loss:\t\t{:.6f} ".format(train_err / train_batches))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log')
flog1 = open(logfile,'ab')
flog1.write('Running %s on %s' % (SCRIPT, MSC))
flog1.write("Epoch {} of {} took {:.3f}s\n ".format(
epoch, NUM_EPOCHS, time.time() - start_time))
flog1.write(" training loss:\t\t{:.6f} ".format(train_err / train_batches))
flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err / val_batches))
flog1.write("\n")
flog1.close()
if epoch == NUM_EPOCHS:
break
valE = val_err/val_batches
#Patience
if valE <= min_val_loss:
model_params1 = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME+'_minloss' + '.pkl'
vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name)
fsave = open(vpth1,'wb')
cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
min_val_loss = valE
patience=0
#Patience / Early stopping
else:
patience+=1
if patience==10:
break
#Save the final model
print('Saving Model ...')
model_params = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME+'_final' + '.pkl'
vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name)
fsave = open(vpth,'wb')
cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
class TestDataset(object):
def setUp(self):
self.data = [1, 2, 3]
self.stream = DataStream(IterableDataset(self.data))
def test_one_example_at_a_time(self):
assert_equal(
list(self.stream.get_epoch_iterator()), list(zip(self.data)))
def test_multiple_epochs(self):
for i, epoch in zip(range(2), self.stream.iterate_epochs()):
assert list(epoch) == list(zip(self.data))
def test_as_dict(self):
assert_equal(
next(self.stream.get_epoch_iterator(as_dict=True)), {"data": 1})
def test_value_error_on_no_provided_sources(self):
class FaultyDataset(Dataset):
def get_data(self, state=None, request=None):
pass
assert_raises(ValueError, FaultyDataset, self.data)
def test_value_error_on_nonexistent_sources(self):
def instantiate_dataset():
return IterableDataset(self.data, sources=('dummy',))
assert_raises(ValueError, instantiate_dataset)
def test_default_transformer(self):
class DoublingDataset(IterableDataset):
def apply_default_transformer(self, stream):
return Mapping(
stream, lambda sources: tuple(2 * s for s in sources))
dataset = DoublingDataset(self.data)
stream = dataset.apply_default_transformer(DataStream(dataset))
assert_equal(list(stream.get_epoch_iterator()), [(2,), (4,), (6,)])
def test_no_axis_labels(self):
assert IterableDataset(self.data).axis_labels is None
def test_axis_labels(self):
axis_labels = {'data': ('batch',)}
dataset = IterableDataset(self.data, axis_labels=axis_labels)
assert dataset.axis_labels == axis_labels
def test_attribute_error_on_no_example_iteration_scheme(self):
class FaultyDataset(Dataset):
provides_sources = ('data',)
def get_data(self, state=None, request=None):
pass
def get_example_iteration_scheme():
return FaultyDataset().example_iteration_scheme
assert_raises(AttributeError, get_example_iteration_scheme)
def test_example_iteration_scheme(self):
scheme = ConstantScheme(2)
class MinimalDataset(Dataset):
provides_sources = ('data',)
_example_iteration_scheme = scheme
def get_data(self, state=None, request=None):
pass
assert MinimalDataset().example_iteration_scheme is scheme
def test_filter_sources(self):
dataset = IterableDataset(
OrderedDict([('1', [1, 2]), ('2', [3, 4])]), sources=('1',))
assert_equal(dataset.filter_sources(([1, 2], [3, 4])), ([1, 2],))
out_dir_r = "{}/R/{}/".format(args.out, set_)
for out_dir in [out_dir_r, out_dir_s]:
if not os.path.exists(out_dir):
os.makedirs(out_dir)
reacher_data = H5PYDataset(name, which_sets=(set_,),
sources=('s_transition_img', 'r_transition_img', 'actions'))
stream = DataStream(
reacher_data,
iteration_scheme=ShuffledScheme(reacher_data.num_examples, batch)
)
iterator = stream.get_epoch_iterator(as_dict=True)
episode_idx = 0
for episode in tqdm(iterator):
# import ipdb; ipdb.set_trace()
s_trans_img = episode['s_transition_img'][0]
r_trans_img = episode['r_transition_img'][0]
tmp = episode['actions']
print(tmp.shape)
out_dir_s_episode = os.path.join(out_dir_s, "run-{}".format(episode_idx))
out_dir_r_episode = os.path.join(out_dir_r, "run-{}".format(episode_idx))
for out_dir in [out_dir_s_episode, out_dir_r_episode]:
if not os.path.exists(out_dir):
os.makedirs(out_dir)
tr_acc = 0
train_batches = 0
h1=train_set.open()
h2=valid_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=256)
scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=128)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
start_time = time.time()
for data in train_stream.get_epoch_iterator():
t_data, t_mask, _, t_labs = data
terr, tacc = train_func(t_data, t_mask, t_labs)
train_err += terr
tr_acc += tacc
train_batches += 1
val_err = 0
val_acc = 0
val_batches = 0
for data in valid_stream.get_epoch_iterator():
v_data, v_mask, _, v_tars = data
err, acc = val_func(v_data, v_mask ,v_tars)
val_err += err
val_acc += acc
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--t_dim',
type=int,
default=256,
help='Text feature dimension')
parser.add_argument('--batch_size',
type=int,
default=64,
help='Batch Size')
parser.add_argument('--image_size',
type=int,
default=64,
help='Image Size a, a x a')
parser.add_argument('--gf_dim',
type=int,
default=64,
help='Number of conv in the first layer gen.')
parser.add_argument('--df_dim',
type=int,
default=64,
help='Number of conv in the first layer discr.')
parser.add_argument(
'--gfc_dim',
type=int,
default=1024,
help='Dimension of gen untis for for fully connected layer 1024')
parser.add_argument('--caption_vector_length',
type=int,
default=2400,
help='Caption Vector Length')
parser.add_argument('--learning_rate',
type=float,
default=0.0002,
help='Learning Rate')
parser.add_argument('--beta1',
type=float,
default=0.5,
help='Momentum for Adam Update')
parser.add_argument('--epochs',
type=int,
default=600,
help='Max number of epochs')
parser.add_argument(
'--save_every',
type=int,
default=30,
help='Save Model/Samples every x iterations over batches')
parser.add_argument('--resume_model',
type=str,
default=None,
help='Pre-Trained Model Path, to resume from')
parser.add_argument('--data_dir',
type=str,
default="./",
help='Data Directory')
parser.add_argument(
'--model_path',
type=str,
default='./pdf_11/model/model_after_flowers_epoch_550.ckpt',
help='Trained Model Path')
parser.add_argument('--n_images',
type=int,
default=16,
help='Number of Images per Caption')
args = parser.parse_args()
model_options = {
't_dim': args.t_dim,
'batch_size': args.batch_size,
'image_size': args.image_size,
'gf_dim': args.gf_dim,
'df_dim': args.df_dim,
'gfc_dim': args.gfc_dim,
'caption_vector_length': args.caption_vector_length
}
gan = model_1.GAN(model_options)
_, _, _, _, _ = gan.build_model()
sess = tf.InteractiveSession()
saver = tf.train.Saver()
saver.restore(sess, args.model_path)
input_tensors, outputs = gan.build_generator()
nlen = 15
nt = 2400
n_updates = 0
n_epochs = 0
n_updates = 0
n_examples = 0
g_costs = []
d_costs = []
gl_costs = []
ntest = 1020
#path_last = "/home/xu/PycharmProjects/tensorflowlstmgan/pdf_1/test2/gen"
dataPath = './flower_15.h5'
test_set = H5PYDataset(dataPath, which_sets=('test', ))
test_scheme = SequentialScheme(examples=test_set.num_examples,
batch_size=args.batch_size)
test_stream = DataStream(test_set, iteration_scheme=test_scheme)
it_test = test_stream.get_epoch_iterator()
for epoch in range(16):
try:
annotationtr, image_featuretr, skip_vectortr, word_vectortr = it_test.next(
)
image_featuretr = image_featuretr / np.float32(255)
except:
it_test = test_stream.get_epoch_iterator()
annotationtr, image_featuretr, skip_vectortr, word_vectortr = it_test.next(
)
image_featuretr = image_featuretr / np.float32(255)
image_featuretr = np.reshape(image_featuretr,
[image_featuretr.shape[0], -1])
image_featuretr = np.reshape(image_featuretr,
(image_featuretr.shape[0], 64, 64, 3))
# image_featuretr = np.transpose(image_featuretr, [0, 3, 1, 2])
image_featuretr_new = np.empty((args.batch_size, nlen, 64, 64, 3))
for m in range(args.batch_size):
for mm in range(nlen):
image_featuretr_new[m][mm] = image_featuretr[m]
image_featuretr_new = np.asarray(image_featuretr_new, dtype='float32')
imb0 = image_featuretr
imb = image_featuretr # batchsize*length*3*64*64
y_sum = np.sum(word_vectortr, axis=2)
y_len = np.empty((args.batch_size, ), dtype='int32')
for j in range(args.batch_size):
y_len[j] = np.count_nonzero(y_sum[j]) - 1
mask = np.empty((args.batch_size, 15, 400), dtype='float32')
for i in range(args.batch_size):
for j in range(15):
if j < (y_len[i] + 1):
mask[i][j] = 1.0
else:
mask[i][j] = 0.0
con = skip_vectortr[:, 0:2400] # batchsize*2400
con = np.reshape(con, (args.batch_size, nt)) # batchsize*2400
zmb = word_vectortr # batchsize*length*400
# GEN UPDATE TWICE, to make sure d_loss does not go to 0
[gen_image] = sess.run([outputs['generator']],
feed_dict={
input_tensors['t_z']: zmb,
input_tensors['mask']: mask
})
n_updates += 1
n_examples += len(imb)
n_epochs += 1
# if n_epochs > 50:
# progress = float(epoch) / num_epochs
# eta.set_value(lasagne.utils.floatX(initial_eta * 2 * (1 - progress)))
sys.stdout.flush()
annotationtr1 = annotationtr
reconst_img2 = imb0
for u in range(64):
file = open("./pdf_11/test/txt/%s.txt" % str(n_epochs), "a")
xxxx = annotationtr1[u][0]
file.write(str(u + 1) + ' ' + xxxx + '\n')
file.close()
scipy.misc.imsave("./pdf_11/test/real/%i_%i.png" % (epoch, u),
(reconst_img2[u] * 255.0).astype(int))
# generative image
reconst_img = gen_image # length*batchsize*(64*64*3)
print reconst_img.shape
for m in range(64):
gen_img = reconst_img[m][14]
scipy.misc.imsave("./pdf_11/test/gen/%i_%i.png" % (epoch, m),
(gen_img * 255.0).astype(int))
print "%i finished" % (epoch)
class TestDataset(object):
def setUp(self):
self.data = [1, 2, 3]
self.stream = DataStream(IterableDataset(self.data))
def test_one_example_at_a_time(self):
assert_equal(list(self.stream.get_epoch_iterator()),
list(zip(self.data)))
def test_multiple_epochs(self):
for i, epoch in zip(range(2), self.stream.iterate_epochs()):
assert list(epoch) == list(zip(self.data))
def test_as_dict(self):
assert_equal(next(self.stream.get_epoch_iterator(as_dict=True)),
{"data": 1})
def test_value_error_on_no_provided_sources(self):
class FaultyDataset(Dataset):
def get_data(self, state=None, request=None):
pass
assert_raises(ValueError, FaultyDataset, self.data)
def test_value_error_on_nonexistent_sources(self):
def instantiate_dataset():
return IterableDataset(self.data, sources=('dummy', ))
assert_raises(ValueError, instantiate_dataset)
def test_default_transformer(self):
class DoublingDataset(IterableDataset):
def apply_default_transformer(self, stream):
return Mapping(stream,
lambda sources: tuple(2 * s for s in sources))
dataset = DoublingDataset(self.data)
stream = dataset.apply_default_transformer(DataStream(dataset))
assert_equal(list(stream.get_epoch_iterator()), [(2, ), (4, ), (6, )])
def test_no_axis_labels(self):
assert IterableDataset(self.data).axis_labels is None
def test_axis_labels(self):
axis_labels = {'data': ('batch', )}
dataset = IterableDataset(self.data, axis_labels=axis_labels)
assert dataset.axis_labels == axis_labels
def test_attribute_error_on_no_example_iteration_scheme(self):
class FaultyDataset(Dataset):
provides_sources = ('data', )
def get_data(self, state=None, request=None):
pass
def get_example_iteration_scheme():
return FaultyDataset().example_iteration_scheme
assert_raises(AttributeError, get_example_iteration_scheme)
def test_example_iteration_scheme(self):
scheme = ConstantScheme(2)
class MinimalDataset(Dataset):
provides_sources = ('data', )
_example_iteration_scheme = scheme
def get_data(self, state=None, request=None):
pass
assert MinimalDataset().example_iteration_scheme is scheme
def test_filter_sources(self):
dataset = IterableDataset(OrderedDict([('1', [1, 2]), ('2', [3, 4])]),
sources=('1', ))
assert_equal(dataset.filter_sources(([1, 2], [3, 4])), ([1, 2], ))
def main(name, model, epochs, batch_size, learning_rate, bokeh, layers, gamma,
rectifier, predict, dropout, qlinear, sparse):
runname = "vae%s-L%s%s%s%s-l%s-g%s-b%d" % (name, layers,
'r' if rectifier else '',
'd' if dropout else '',
'l' if qlinear else '',
shnum(learning_rate), shnum(gamma), batch_size//100)
if rectifier:
activation = Rectifier()
full_weights_init = Orthogonal()
else:
activation = Tanh()
full_weights_init = Orthogonal()
if sparse:
runname += '-s%d'%sparse
weights_init = Sparse(num_init=sparse, weights_init=full_weights_init)
else:
weights_init = full_weights_init
layers = map(int,layers.split(','))
encoder_layers = layers[:-1]
encoder_mlp = MLP([activation] * (len(encoder_layers)-1),
encoder_layers,
name="MLP_enc", biases_init=Constant(0.), weights_init=weights_init)
enc_dim = encoder_layers[-1]
z_dim = layers[-1]
if qlinear:
sampler = Qlinear(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init)
else:
sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init)
decoder_layers = layers[:] ## includes z_dim as first layer
decoder_layers.reverse()
decoder_mlp = MLP([activation] * (len(decoder_layers)-2) + [Sigmoid()],
decoder_layers,
name="MLP_dec", biases_init=Constant(0.), weights_init=weights_init)
vae = VAEModel(encoder_mlp, sampler, decoder_mlp)
vae.initialize()
x = tensor.matrix('features')
if predict:
mean_z, enc = vae.mean_z(x)
# cg = ComputationGraph([mean_z, enc])
newmodel = Model([mean_z,enc])
else:
x_recons, kl_terms = vae.reconstruct(x)
recons_term = BinaryCrossEntropy().apply(x, x_recons)
recons_term.name = "recons_term"
cost = recons_term + kl_terms.mean()
cg = ComputationGraph([cost])
if gamma > 0:
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
cost += gamma * blocks.theano_expressions.l2_norm(weights)
cost.name = "nll_bound"
newmodel = Model(cost)
if dropout:
weights = [v for k,v in newmodel.get_params().iteritems()
if k.find('MLP')>=0 and k.endswith('.W') and not k.endswith('MLP_enc/linear_0.W')]
cg = apply_dropout(cg,weights,0.5)
target_cost = cg.outputs[0]
else:
target_cost = cost
if name == 'mnist':
if predict:
train_ds = MNIST("train")
else:
train_ds = MNIST("train", sources=['features'])
test_ds = MNIST("test")
else:
datasource_dir = os.path.join(fuel.config.data_path, name)
datasource_fname = os.path.join(datasource_dir , name+'.hdf5')
if predict:
train_ds = H5PYDataset(datasource_fname, which_set='train')
else:
train_ds = H5PYDataset(datasource_fname, which_set='train', sources=['features'])
test_ds = H5PYDataset(datasource_fname, which_set='test')
train_s = DataStream(train_ds,
iteration_scheme=SequentialScheme(
train_ds.num_examples, batch_size))
test_s = DataStream(test_ds,
iteration_scheme=SequentialScheme(
test_ds.num_examples, batch_size))
if predict:
from itertools import chain
fprop = newmodel.get_theano_function()
allpdata = None
alledata = None
f = train_s.sources.index('features')
assert f == test_s.sources.index('features')
sources = test_s.sources
alllabels = dict((s,[]) for s in sources if s != 'features')
for data in chain(train_s.get_epoch_iterator(), test_s.get_epoch_iterator()):
for s,d in zip(sources,data):
if s != 'features':
alllabels[s].extend(list(d))
pdata, edata = fprop(data[f])
if allpdata is None:
allpdata = pdata
else:
allpdata = np.vstack((allpdata, pdata))
if alledata is None:
alledata = edata
else:
alledata = np.vstack((alledata, edata))
print 'Saving',allpdata.shape,'intermidiate layer, for all training and test examples, to',name+'_z.npy'
np.save(name+'_z', allpdata)
print 'Saving',alledata.shape,'last encoder layer to',name+'_e.npy'
np.save(name+'_e', alledata)
print 'Saving additional labels/targets:',','.join(alllabels.keys()),
print ' of size',','.join(map(lambda x: str(len(x)),alllabels.values())),
print 'to',name+'_labels.pkl'
with open(name+'_labels.pkl','wb') as fp:
pickle.dump(alllabels, fp, -1)
else:
algorithm = GradientDescent(
cost=target_cost, params=cg.parameters,
step_rule=Adam(learning_rate) # Scale(learning_rate=learning_rate)
)
extensions = []
if model:
extensions.append(LoadFromDump(model))
extensions += [Timing(),
FinishAfter(after_n_epochs=epochs),
DataStreamMonitoring(
[recons_term, cost],
test_s,
prefix="test"),
TrainingDataMonitoring(
[cost,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Dump(runname, every_n_epochs=10),
Printing()]
if bokeh:
extensions.append(Plot(
'Auto',
channels=[
['test_recons_term','test_nll_bound','train_nll_bound'
],
['train_total_gradient_norm']]))
main_loop = MainLoop(
algorithm,
train_s,
model=newmodel,
extensions=extensions)
main_loop.run()
m = VAModel()
# load parameters
model = Model(m.variational_cost)
print "loading params"
params = load_parameter_values(sys.argv[1])
model.set_param_values(params)
test_dataset = MNIST('test', sources=['features'])
test_scheme = ShuffledScheme(test_dataset.num_examples, 128)
test_stream = DataStream(test_dataset, iteration_scheme=test_scheme)
_func_noisy = theano.function([m.X], m.noisy)
_func_produced = theano.function([m.X], m.produced)
batch = test_stream.get_epoch_iterator().next()[0]
out_noise = _func_noisy(batch)
out_produced = _func_produced(batch)
import cv2
for k in range(10):
print out_noise.shape
img = np.reshape(out_noise[k, :], (28, 28))
img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST)
cv2.imshow('img', img)
img = np.reshape(out_produced[k, :], (28, 28))
img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST)
cv2.imshow('masdf', img)
cv2.waitKey(0)
import numpy as np
DATASET_PATH_REL = "/data/lisa/data/sim2real/"
DATASET_PATH = DATASET_PATH_REL + "mujoco_data_pusher3dof_big_backl.h5"
# DATASET_PATH = "/Tmp/alitaiga/mujoco_data_pusher3dof_big_backl.h5"
batch_size = 1
train_data = H5PYDataset(
DATASET_PATH, which_sets=('train',), sources=('s_transition_obs','r_transition_obs', 'obs', 'actions')
)
stream_train = DataStream(train_data, iteration_scheme=SequentialScheme(train_data.num_examples, train_data.num_examples))
valid_data = H5PYDataset(
DATASET_PATH, which_sets=('valid',), sources=('s_transition_obs','r_transition_obs', 'obs', 'actions')
)
stream_valid = DataStream(valid_data, iteration_scheme=SequentialScheme(train_data.num_examples, batch_size))
iterator = stream_train.get_epoch_iterator(as_dict=True)
data = next(iterator)
import ipdb; ipdb.set_trace()
## Max, min mean std obs:
[2.56284189, 2.3500247, 2.39507723, 7.40329409, 9.20471668, 15.37792397]
[-0.12052701, -0.17479207, -0.73818409, -1.25026512, -3.95599413, -4.73272848]
[2.25090551, 1.94997263, 1.6495719, 0.43379614, 0.3314755, 0.43763939]
[0.5295766 , 0.51998389, 0.57609886, 1.35480666, 1.40806067, 2.43865967]
# max_obs = np.zeros((6,))
# max_sim = np.zeros((6,))
# max_real = np.zeros((6,))
# max_act = np.zeros((3,))
train_input_mean = 1470614.1
train_input_std = 3256577.0
train_input_mean_reshape = (train_input_mean*numpy.ones(test_input.shape[1])).astype(numpy.float32)
train_input_std_reshape = (train_input_std*numpy.ones(test_input.shape[1])).astype(numpy.float32)
test_input = (test_input - train_input_mean_reshape[None,:,None])/train_input_std_reshape[None,:,None]
test_product = test[:,2,0]
ref = {int(k):i for i, k in enumerate(open('listid.txt'))}
test_product = numpy.array([ref[test_product[i]] for i in range(len(test_product))])
test_id = test[:,0]
ds_test = IndexableDataset({'id':test_id,'input':test_input,'product':test_product})
scheme_test = SequentialScheme(batch_size=10000,examples= ds_test.num_examples)
stream_test = DataStream(ds_test,iteration_scheme = scheme_test)
pred_test = ComputationGraph([m.pred]).get_theano_function()
with open('results_deep_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s.csv'
%(config.name,config.couches,config.hidden_dim,
config.activation_function_name,config.batch_size,config.w_noise_std,
config.i_dropout, config.algo,config.learning_rate_value,
config.momentum_value,config.decay_rate_value,config.StepClipping_value)
, 'wb') as csvfile:
print "Writing results on test set..."
csv_func = csv.writer(csvfile, delimiter=',')
csv_func.writerow(['ID','TARGET'])
for d in stream_test.get_epoch_iterator(as_dict=True):
print d['id'][0]
output, = pred_test(**{x: d[x] for x in ['input','product']})
for i in range(output.shape[0]):
csv_func.writerow([int(test_id[i]),float(output[i])])
def main(mode, save_to, num_epochs, load_params=None,
feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, stride=None, repeat_times=None,
batch_size=None, num_batches=None, algo=None,
test_set=None, valid_examples=None,
dropout=None, max_norm=None, weight_decay=None,
batch_norm=None):
if feature_maps is None:
feature_maps = [20, 50, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5, 5]
if pool_sizes is None:
pool_sizes = [2, 2, 2]
if repeat_times is None:
repeat_times = [1, 1, 1]
if batch_size is None:
batch_size = 500
if valid_examples is None:
valid_examples = 2500
if stride is None:
stride = 1
if test_set is None:
test_set = 'test'
if algo is None:
algo = 'rmsprop'
if batch_norm is None:
batch_norm = False
image_size = (128, 128)
output_size = 2
if (len(feature_maps) != len(conv_sizes) or
len(feature_maps) != len(pool_sizes) or
len(feature_maps) != len(repeat_times)):
raise ValueError("OMG, inconsistent arguments")
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 3, image_size,
stride=stride,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
repeat_times=repeat_times,
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
batch_norm=batch_norm,
weights_init=Glorot(),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
single_x = tensor.tensor3('image_features')
x = tensor.tensor4('image_features')
single_y = tensor.lvector('targets')
y = tensor.lmatrix('targets')
# Training
with batch_normalization(convnet):
probs = convnet.apply(x)
cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
.copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
cg = ComputationGraph([cost, error_rate])
extra_updates = []
if batch_norm: # batch norm:
logger.debug("Apply batch norm")
pop_updates = get_batch_normalization_updates(cg)
# p stands for population mean
# m stands for minibatch
alpha = 0.005
extra_updates = [(p, m * alpha + p * (1 - alpha))
for p, m in pop_updates]
population_statistics = [p for p, m in extra_updates]
if dropout:
relu_outputs = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg)
cg = apply_dropout(cg, relu_outputs, dropout)
cost, error_rate = cg.outputs
if weight_decay:
logger.debug("Apply weight decay {}".format(weight_decay))
cost += weight_decay * l2_norm(cg.parameters)
cost.name = 'cost'
# Validation
valid_probs = convnet.apply_5windows(single_x)
valid_cost = (CategoricalCrossEntropy().apply(single_y, valid_probs)
.copy(name='cost'))
valid_error_rate = (MisclassificationRate().apply(
single_y, valid_probs).copy(name='error_rate'))
model = Model([cost, error_rate])
if load_params:
logger.info("Loaded params from {}".format(load_params))
with open(load_params, 'r') as src:
model.set_parameter_values(load_parameters(src))
# Training stream with random cropping
train = DogsVsCats(("train",), subset=slice(None, 25000 - valid_examples, None))
train_str = DataStream(
train, iteration_scheme=ShuffledScheme(train.num_examples, batch_size))
train_str = add_transformers(train_str, random_crop=True)
# Validation stream without cropping
valid = DogsVsCats(("train",), subset=slice(25000 - valid_examples, None, None))
valid_str = DataStream(
valid, iteration_scheme=SequentialExampleScheme(valid.num_examples))
valid_str = add_transformers(valid_str)
if mode == 'train':
directory, _ = os.path.split(sys.argv[0])
env = dict(os.environ)
env['THEANO_FLAGS'] = 'floatX=float32'
port = numpy.random.randint(1025, 10000)
server = subprocess.Popen(
[directory + '/server.py',
str(25000 - valid_examples), str(batch_size), str(port)],
env=env, stderr=subprocess.STDOUT)
train_str = ServerDataStream(
('image_features', 'targets'), produces_examples=False,
port=port)
save_to_base, save_to_extension = os.path.splitext(save_to)
# Train with simple SGD
if algo == 'rmsprop':
step_rule = RMSProp(decay_rate=0.999, learning_rate=0.0003)
elif algo == 'adam':
step_rule = Adam()
else:
assert False
if max_norm:
conv_params = VariableFilter(bricks=[Convolutional], roles=[WEIGHT])(cg)
linear_params = VariableFilter(bricks=[Linear], roles=[WEIGHT])(cg)
step_rule = CompositeRule(
[step_rule,
Restrict(VariableClipping(max_norm, axis=0), linear_params),
Restrict(VariableClipping(max_norm, axis=(1, 2, 3)), conv_params)])
algorithm = GradientDescent(
cost=cost, parameters=model.parameters,
step_rule=step_rule)
algorithm.add_updates(extra_updates)
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(every_n_batches=100),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
DataStreamMonitoring(
[valid_cost, valid_error_rate],
valid_str,
prefix="valid"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
TrackTheBest("valid_error_rate"),
Checkpoint(save_to, save_separately=['log'],
parameters=cg.parameters +
(population_statistics if batch_norm else []),
before_training=True, after_epoch=True)
.add_condition(
['after_epoch'],
OnLogRecord("valid_error_rate_best_so_far"),
(save_to_base + '_best' + save_to_extension,)),
Printing(every_n_batches=100)]
model = Model(cost)
main_loop = MainLoop(
algorithm,
train_str,
model=model,
extensions=extensions)
try:
main_loop.run()
finally:
server.terminate()
elif mode == 'test':
classify = theano.function([single_x], valid_probs.argmax())
test = DogsVsCats((test_set,))
test_str = DataStream(
test, iteration_scheme=SequentialExampleScheme(test.num_examples))
test_str = add_transformers(test_str)
correct = 0
with open(save_to, 'w') as dst:
print("id", "label", sep=',', file=dst)
for index, example in enumerate(test_str.get_epoch_iterator()):
image = example[0]
prediction = classify(image)
print(index + 1, classify(image), sep=',', file=dst)
if len(example) > 1 and prediction == example[1]:
correct += 1
print(correct / float(test.num_examples))
else:
assert False
def main(save, load, sample, path, **kwargs):
input_dim = 784
hidden_dim = 2
batch_size = 100
features = tensor.matrix('features')
vae = VariationalAutoEncoder(input_dim, hidden_dim,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.))
vae.initialize()
mu, logsigma, x_hat = vae.apply(features)
cost = vae.cost(features)
cost.name = 'cost'
regularization_cost = vae.regularization_cost(mu, logsigma).mean()
regularization_cost.name = 'regularization_cost'
reconstruction_cost = vae.reconstruction_cost(features, x_hat).mean()
reconstruction_cost.name = 'reconstruction_cost'
cg = ComputationGraph([cost, reconstruction_cost, regularization_cost])
model = Model(cost)
algorithm = GradientDescent(step_rule=RMSProp(1e-4), params=cg.parameters,
cost=cost)
extensions = []
if load:
extensions.append(LoadFromDump(path))
if save:
extensions.append(Dump(path, after_epoch=True))
extensions.append(FinishAfter(after_n_epochs=6001))
train_dataset = MNIST('train', binary=False, sources=('features',))
train_stream = DataStream(train_dataset,
iteration_scheme=ShuffledScheme(
examples=train_dataset.num_examples,
batch_size=batch_size))
train_monitor = TrainingDataMonitoring(
[cost, regularization_cost, reconstruction_cost],
prefix='train', after_epoch=True)
test_dataset = MNIST('test', binary=True, sources=('features',))
test_stream = DataStream(test_dataset,
iteration_scheme=ShuffledScheme(
examples=test_dataset.num_examples,
batch_size=batch_size))
test_monitor = DataStreamMonitoring([cost], test_stream, prefix='test')
extensions.extend([train_monitor, test_monitor])
extensions.extend([Timing(), Printing()])
main_loop = MainLoop(model=model, algorithm=algorithm,
data_stream=train_stream,
extensions=extensions)
if not sample:
main_loop.run()
else:
parameters = load_parameter_values(path + '/params.npz')
model.set_param_values(parameters)
num_samples = 10
samples = vae.sample(num_samples)
samples = function([], samples)()
z = tensor.matrix('z')
decode_z = function([z], vae.decoder.apply(z))
from matplotlib import pyplot as plt
sample = numpy.zeros((28, 0))
size = 40
z_val = numpy.zeros((size ** 2, 2))
for i in xrange(size):
for j in xrange(size):
z_val[i * size + j, :] = numpy.array(
[i / float(0.3 * size) - .5 / .3,
j / float(0.3 * size) - .5 / .3])
samples = decode_z(z_val)
samples = samples.reshape((size, size, 28, 28))
samples = numpy.concatenate(samples, axis=1)
samples = numpy.concatenate(samples, axis=1)
plt.imshow(samples, cmap=plt.get_cmap('Greys'))
plt.show()
f = function([features], x_hat)
for data in train_stream.get_epoch_iterator():
data_hat = f(data[0])
for image, image_hat in zip(data[0], data_hat):
im = numpy.concatenate([image_hat.reshape((28, 28)),
image.reshape((28, 28))])
plt.imshow(im, cmap=plt.get_cmap('Greys'))
plt.show()
_filename = 'cars196/cars196.hdf5'
def __init__(self, which_sets, **kwargs):
try:
#path = find_in_data_path(self._filename)
path = "datasets/data/cars196/cars196.hdf5"
except IOError as e:
msg = str(e) + (""".
You need to download the dataset and convert it to hdf5 before.""")
raise IOError(msg)
super(Cars196Dataset, self).__init__(file_or_path=path,
which_sets=which_sets,
**kwargs)
def load_as_ndarray(which_sets=['train', 'test']):
datasets = []
for split in which_sets:
data = Cars196Dataset([split], load_in_memory=True).data_sources
datasets.append(data)
return datasets
if __name__ == '__main__':
dataset = Cars196Dataset(['train'])
st = DataStream(dataset,
iteration_scheme=SequentialScheme(dataset.num_examples, 1))
it = st.get_epoch_iterator()
it.next()
scheme_train = ShuffledScheme(examples=train_set.num_examples, batch_size=conf.batch_size)
scheme_val = SequentialScheme(examples=val_set.num_examples, batch_size=conf.batch_size)
scheme_test = SequentialScheme(examples=test_set.num_examples, batch_size=conf.batch_size)
stream_train = DataStream(dataset=train_set, iteration_scheme=scheme_train)
stream_val = DataStream(dataset=val_set, iteration_scheme=scheme_val)
stream_test = DataStream(dataset=test_set, iteration_scheme=scheme_test)
stream_train = Mapping(data_stream = stream_train, mapping=ShuffleDim)
stream_val = Mapping(data_stream = stream_val, mapping=ShuffleDim)
stream_test = Mapping(data_stream = stream_test, mapping=ShuffleDim)
stream_train = AddInputNoise(data_stream = stream_train, std=conf.std_input_noise)
epoch_iterator=stream_train.get_epoch_iterator(as_dict = True)
test_batch=next(epoch_iterator)
num_features=test_batch[conf.input_theano].shape[-1]
if conf.transition=='GRU':
transition=GatedRecurrent
elif conf.transition=='LSTM':
transition=LSTM
elif conf.transition=='RNN':
transition=SimpleRecurrent
else:
raise ValueError, conf.transition
def main():
X = T.tensor3(name='input', dtype='float32')
MASK = T.matrix(name='mask', dtype='float32')
LABELS = T.ivector(name='labels')
#load data
train_set = H5PYDataset(TRAIN_FILE, which_sets=('train', ))
valid_set = H5PYDataset(TRAIN_FILE, which_sets=('test', ))
network = build_rnn(net_input=X, mask_input=MASK)
network_output = lasagne.layers.get_output(network)
val_prediction = lasagne.layers.get_output(network, deterministic=True)
#needed for accuracy
val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS),
dtype=theano.config.floatX)
#training accuracy
train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS),
dtype=theano.config.floatX)
#T.argmax(network_output, axis=1).eval({X: d1, Mask: m1})
#print network_output.eval({X: d1, Mask: m1})[1][97]
#cost function
total_cost = lasagne.objectives.categorical_crossentropy(
network_output, LABELS)
#total_cost = -(labels*T.log(network_output) + (1-labels*T.log(1-network_output))
masked_cost = total_cost * MASK.flatten()
mean_cost = total_cost.mean()
#accuracy function
val_cost = lasagne.objectives.categorical_crossentropy(
val_prediction, LABELS)
val_cost = val_cost * MASK.flatten()
val_mcost = val_cost.mean()
#Get parameters of both encoder and decoder
all_parameters = lasagne.layers.get_all_params([network], trainable=True)
print("Trainable Model Parameters")
print("-" * 40)
for param in all_parameters:
print(param, param.get_value().shape)
print("-" * 40)
#add grad clipping to avoid exploding gradients
all_grads = [T.clip(g, -3, 3) for g in T.grad(mean_cost, all_parameters)]
all_grads = lasagne.updates.total_norm_constraint(all_grads, 3)
updates = lasagne.updates.adam(all_grads,
all_parameters,
learning_rate=LEARNING_RATE)
train_func = theano.function([X, MASK, LABELS], [mean_cost, train_acc],
updates=updates)
val_func = theano.function([X, MASK, LABELS], [val_mcost, val_acc])
trainerr = []
epoch = 0 #set the epoch counter
min_val_loss = np.inf
patience = 0
print("Starting training...")
# We iterate over epochs:
while 'true':
# In each epoch, we do a full pass over the training data:
train_err = 0
tr_acc = 0
train_batches = 0
h1 = train_set.open()
h2 = valid_set.open()
scheme = ShuffledScheme(examples=train_set.num_examples,
batch_size=BATCH_SIZE)
scheme1 = SequentialScheme(examples=valid_set.num_examples,
batch_size=32)
train_stream = DataStream(dataset=train_set, iteration_scheme=scheme)
valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1)
start_time = time.time()
for data in train_stream.get_epoch_iterator():
t_data, t_mask, _, _, _, t_labs = data
t_labs = t_labs.flatten()
terr, tacc = train_func(t_data, t_mask, t_labs)
train_err += terr
tr_acc += tacc
train_batches += 1
val_err = 0
val_acc = 0
val_batches = 0
for data in valid_stream.get_epoch_iterator():
v_data, v_mask, _, _, _, v_tars = data
v_tars = v_tars.flatten()
err, acc = val_func(v_data, v_mask, v_tars)
val_err += err
val_acc += acc
val_batches += 1
trainerr.append(train_err / train_batches)
epoch += 1
train_set.close(h1)
valid_set.close(h2)
#Display
if display:
print("Epoch {} of {} took {:.3f}s ".format(
epoch, NUM_EPOCHS,
time.time() - start_time))
print(" training loss:\t\t{:.6f} ".format(train_err /
train_batches))
print(" training accuracy:\t\t{:.2f} % ".format(
tr_acc / train_batches * 100))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
print(" validation accuracy:\t\t{:.2f} % ".format(
val_acc / val_batches * 100))
logfile = os.path.join(SAVE_PATH, MODEL_NAME, 'logs',
MODEL_NAME + '.log')
flog1 = open(logfile, 'ab')
flog1.write('Running %s on %s\n' % (SCRIPT, MSC))
flog1.write("Epoch {} of {} took {:.3f}s\n ".format(
epoch, NUM_EPOCHS,
time.time() - start_time))
flog1.write(" training loss:\t\t{:.6f} ".format(train_err /
train_batches))
flog1.write(" training accuracy:\t\t{:.2f} %\n ".format(
tr_acc / train_batches * 100))
flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err /
val_batches))
flog1.write(" validation accuracy:\t\t{:.2f} %\n ".format(
val_acc / val_batches * 100))
flog1.write("\n")
flog1.close()
if epoch == NUM_EPOCHS:
break
valE = val_err / val_batches
#Patience
if valE <= min_val_loss:
model_params1 = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME + '_minloss' + '.pkl'
vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights', model1_name)
fsave = open(vpth1, 'wb')
cPickle.dump(model_params1,
fsave,
protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
min_val_loss = valE
#reset the patience
patience = 0
#Patience / Early stopping
else:
patience += 1
if patience == 10:
break
#Save the final model
print('Saving Model ...')
model_params = lasagne.layers.get_all_param_values(network)
model1_name = MODEL_NAME + '_final' + '.pkl'
vpth = os.path.join(SAVE_PATH, MODEL_NAME, 'weights', model1_name)
fsave = open(vpth, 'wb')
cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL)
fsave.close()
dataset = CoNLLTextFile(data_paths, dictionary=word2code, unknown_token='<UNK>')
data_stream = DataStream(dataset)
data_stream = Filter(data_stream, _filter_long)
#data_stream = Mapping(data_stream, reverse_words, add_sources=("targets",))
data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(mini_batch_size))
#data_stream = Padding(data_stream, mask_sources=('tokens')) # Adds a mask fields to this stream field, type='floatX'
data_stream = Padding(data_stream, ) # Adds a mask fields to all of this stream's fields, type='floatX'
data_stream = Mapping(data_stream, _transpose) # Flips stream so that sentences run down columns, batches along rows (strangely)
if False: # print sample for debugging Dataset / DataStream component
#t=0
max_len=0
for i, data in enumerate(data_stream.get_epoch_iterator()):
#print(i)
#t=t + data[4].sum() + data[0].shape[1]
l=data[0].shape[0]
if l>max_len:
max_len = l
print(i, data[0].shape, max_len)
#print(data)
#break
exit(0)
"""
Comments in google-groups:blocks indicate that a reshaping has to be done,
so let's think about sizes of the arrays...
"""
