def load_stream(batch_size=None, source=None, use_tanh=False, discrete=False, n_colors=16, downsample_to=None): if source is None: raise ValueError('Source not provided.') if batch_size is None: raise ValueError('Batch size not provided.') logger.info('Loading data from `{}`'.format(source)) train_data = H5PYDataset(source, which_sets=('train', )) num_train = train_data.num_examples logger.debug('Number of training examples: {}'.format(num_train)) scheme = ShuffledScheme(examples=num_train, batch_size=batch_size) stream = DataStream(train_data, iteration_scheme=scheme) viz_options = dict(use_tanh=use_tanh) data = stream.get_epoch_iterator(as_dict=True).next()['features'] if discrete: n_uniques = len(np.unique(data)) if n_uniques == 256 and np.max(data) == 255: features = H5PYDataset(source, which_sets=('train', ), subset=slice(0, 1000), sources=['features']) handle = features.open() arr, = features.get_data(handle, slice(0, 1000)) arr = arr.transpose(0, 2, 3, 1) arr = arr.reshape((-1, arr.shape[2], arr.shape[3])) img = Image.fromarray(arr).convert('P', palette=Image.ADAPTIVE, colors=n_colors) viz_options['img'] = img logger.info('Quantizing data to {} colors'.format(n_colors)) viz_options['quantized'] = True if downsample_to is not None: logger.info( 'Downsampling the data to {}'.format(downsample_to)) stream = Quantize(img=img, n_colors=n_colors, downsample_to=downsample_to, data_stream=stream) else: logger.warning('Data already appears to be discretized. Skipping.') else: if np.max(data) != 255: raise ValueError( 'Data needs to be RGB, this doesn\'t appear to be ' 'the case') stream = Rescale(stream, use_tanh=use_tanh) data = stream.get_epoch_iterator(as_dict=True).next()['features'] shape = data.shape shape = dict(dim_c=shape[1], dim_x=shape[2], dim_y=shape[3]) return stream, num_train, shape, viz_options
def test_sources_selection(): features = [5, 6, 7, 1] targets = [1, 0, 1, 1] stream = DataStream(IterableDataset(OrderedDict( [('features', features), ('targets', targets)]))) assert list(stream.get_epoch_iterator()) == list(zip(features, targets)) stream = DataStream(IterableDataset( {'features': features, 'targets': targets}, sources=('targets',))) assert list(stream.get_epoch_iterator()) == list(zip(targets))
def get_seq_mnist_streams(hidden_dim, batch_size=100, drop_prob=0.5): permutation = np.random.randint(0, 784, size=(784, )) train_set, valid_set, test_set = load_data('mnist.pkl.gz') train_x = train_set[0].reshape((50000 / batch_size, batch_size, 784)) train_x = np.swapaxes(train_x, 2, 1) train_x = train_x[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 train_y = (np.zeros(train_set[0].shape) - 1) # label for each time-step is -1 and for the last one is the real label train_y[:, -1] = train_set[1] train_y = train_y.reshape((50000 / batch_size, batch_size, 784)) train_y = np.swapaxes(train_y, 2, 1) train_y = train_y[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 valid_x = valid_set[0].reshape((10000 / batch_size, batch_size, 784)) valid_x = np.swapaxes(valid_x, 2, 1) valid_x = valid_x[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 valid_y = (np.zeros(valid_set[0].shape) - 1) # label for each time-step is -1 and for the last one is the real label valid_y[:, -1] = valid_set[1] valid_y = valid_y.reshape((10000 / batch_size, batch_size, 784)) valid_y = np.swapaxes(valid_y, 2, 1) valid_y = valid_y[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 train_x = train_x[:, permutation] valid_x = valid_x[:, permutation] train = IterableDataset({ 'x': train_x.astype(floatX), 'y': train_y[:, -1, :, 0].astype('int32') }) train_stream = DataStream(train) train_stream = SampleDrops(train_stream, drop_prob, hidden_dim, False) train_stream.sources = ('y', 'x', 'drops') train_stream.get_epoch_iterator().next() valid = IterableDataset({ 'x': valid_x.astype(floatX), 'y': valid_y[:, -1, :, 0].astype('int32') }) valid_stream = DataStream(valid) valid_stream = SampleDrops(valid_stream, drop_prob, hidden_dim, True) valid_stream.sources = ('y', 'x', 'drops') return train_stream, valid_stream
def test_dataset(): data = [1, 2, 3] # The default stream requests an example at a time stream = DataStream(IterableDataset(data)) epoch = stream.get_epoch_iterator() assert list(epoch) == list(zip(data)) # Check if iterating over multiple epochs works for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == list(zip(data)) # Check whether the returning as a dictionary of sources works assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
def test_dataset(): data = [1, 2, 3] # The default stream requests an example at a time stream = DataStream(IterableDataset(data)) epoch = stream.get_epoch_iterator() assert list(epoch) == list(zip(data)) # Check if iterating over multiple epochs works for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == list(zip(data)) # Check whether the returning as a dictionary of sources works assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
def test_data_driven_epochs(): class TestDataset(IterableDataset): sources = ('data',) def __init__(self): self.axis_labels = None self.data = [[1, 2, 3, 4], [5, 6, 7, 8]] def open(self): epoch_iter = iter(self.data) data_iter = iter(next(epoch_iter)) return (epoch_iter, data_iter) def next_epoch(self, state): try: data_iter = iter(next(state[0])) return (state[0], data_iter) except StopIteration: return self.open() def get_data(self, state, request): data = [] for i in range(request): data.append(next(state[1])) return (data,) epochs = [] epochs.append([([1],), ([2],), ([3],), ([4],)]) epochs.append([([5],), ([6],), ([7],), ([8],)]) stream = DataStream(TestDataset(), iteration_scheme=ConstantScheme(1)) assert list(stream.get_epoch_iterator()) == epochs[0] assert list(stream.get_epoch_iterator()) == epochs[1] assert list(stream.get_epoch_iterator()) == epochs[0] stream.reset() for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == epochs[i] # test scheme resetting between epochs class TestScheme(BatchSizeScheme): def get_request_iterator(self): return iter([1, 2, 1, 3]) epochs = [] epochs.append([([1],), ([2, 3],), ([4],)]) epochs.append([([5],), ([6, 7],), ([8],)]) stream = DataStream(TestDataset(), iteration_scheme=TestScheme()) for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == epochs[i]
def test_data_driven_epochs(): class TestDataset(IterableDataset): sources = ('data',) def __init__(self): self.axis_labels = None self.data = [[1, 2, 3, 4], [5, 6, 7, 8]] def open(self): epoch_iter = iter(self.data) data_iter = iter(next(epoch_iter)) return (epoch_iter, data_iter) def next_epoch(self, state): try: data_iter = iter(next(state[0])) return (state[0], data_iter) except StopIteration: return self.open() def get_data(self, state, request): data = [] for i in range(request): data.append(next(state[1])) return (data,) epochs = [] epochs.append([([1],), ([2],), ([3],), ([4],)]) epochs.append([([5],), ([6],), ([7],), ([8],)]) stream = DataStream(TestDataset(), iteration_scheme=ConstantScheme(1)) assert list(stream.get_epoch_iterator()) == epochs[0] assert list(stream.get_epoch_iterator()) == epochs[1] assert list(stream.get_epoch_iterator()) == epochs[0] stream.reset() for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == epochs[i] # test scheme resetting between epochs class TestScheme(BatchSizeScheme): def get_request_iterator(self): return iter([1, 2, 1, 3]) epochs = [] epochs.append([([1],), ([2, 3],), ([4],)]) epochs.append([([5],), ([6, 7],), ([8],)]) stream = DataStream(TestDataset(), iteration_scheme=TestScheme()) for i, epoch in zip(range(2), stream.iterate_epochs()): assert list(epoch) == epochs[i]
def get_seq_mnist_streams(hidden_dim, batch_size=100, drop_prob=0.5): permutation = np.random.randint(0, 784, size=(784,)) train_set, valid_set, test_set = load_data('mnist.pkl.gz') train_x = train_set[0].reshape((50000 / batch_size, batch_size, 784)) train_x = np.swapaxes(train_x, 2, 1) train_x = train_x[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 train_y = (np.zeros(train_set[0].shape) - 1) # label for each time-step is -1 and for the last one is the real label train_y[:, -1] = train_set[1] train_y = train_y.reshape((50000 / batch_size, batch_size, 784)) train_y = np.swapaxes(train_y, 2, 1) train_y = train_y[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 valid_x = valid_set[0].reshape((10000 / batch_size, batch_size, 784)) valid_x = np.swapaxes(valid_x, 2, 1) valid_x = valid_x[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 valid_y = (np.zeros(valid_set[0].shape) - 1) # label for each time-step is -1 and for the last one is the real label valid_y[:, -1] = valid_set[1] valid_y = valid_y.reshape((10000 / batch_size, batch_size, 784)) valid_y = np.swapaxes(valid_y, 2, 1) valid_y = valid_y[:, :, :, np.newaxis] # Now the dimension is num_batches x 784 x batch_size x 1 train_x = train_x[:, permutation] valid_x = valid_x[:, permutation] train = IterableDataset({'x': train_x.astype(floatX), 'y': train_y[:, -1, :, 0].astype('int32')}) train_stream = DataStream(train) train_stream = SampleDrops(train_stream, drop_prob, hidden_dim, False) train_stream.sources = ('y', 'x', 'drops') train_stream.get_epoch_iterator().next() valid = IterableDataset({'x': valid_x.astype(floatX), 'y': valid_y[:, -1, :, 0].astype('int32')}) valid_stream = DataStream(valid) valid_stream = SampleDrops(valid_stream, drop_prob, hidden_dim, True) valid_stream.sources = ('y', 'x', 'drops') return train_stream, valid_stream
def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST('train', load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def __init__(self, dataset, nb_class, examples_per_class, start=0, **kwargs): assert dataset.sources == ('features', 'targets') super(SubSetStream, self).__init__(dataset, **kwargs) total_examples = nb_class * examples_per_class # build a list of indexes corresponding to the subset print "Building subset indexes list..." stream = DataStream(dataset=dataset, iteration_scheme=SequentialScheme( dataset.num_examples, 100)) epitr = stream.get_epoch_iterator() statistics = np.zeros(nb_class) self.statistics = np.zeros(nb_class) indexes = [] for i in range(0, (dataset.num_examples // 100) * 100, 100): if statistics.sum() == total_examples: break _, targets = next(epitr) if i < start: continue for j in range(100): if statistics[targets[j]].sum() < examples_per_class: indexes += [i + j] statistics[targets[j]] += 1 if statistics.sum() != total_examples: raise RuntimeError("Transformer failed") self.indexes = indexes
def test_in_memory(): skip_if_not_available(datasets=['mnist.hdf5']) # Load MNIST and get two batches mnist = MNIST(('train',), load_in_memory=True) data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(256, 512)) mnist.close(handle) assert numpy.all(features == known_features) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) handle = mnist.open() known_features, _ = mnist.get_data(handle, slice(512, 768)) mnist.close(handle) assert numpy.all(features == known_features)
def test_unpack_transformer(): data = range(10) stream = DataStream(IterableDataset(data)) stream = Batch(stream, iteration_scheme=ConstantScheme(2)) stream = Unpack(stream) epoch = stream.get_epoch_iterator() for i, v in enumerate(epoch): assert numpy.shape(v)[0] == 1 assert v[0] == i
def test_get_data_inmemory(self): ds = ImagesFromFile('{}/*.jpeg'.format(self.ds_path), load_in_memory=True) stream = DataStream(ds, iteration_scheme=ShuffledScheme( ds.num_examples, batch_size=10)) for imgs, file_paths in stream.get_epoch_iterator(): assert len(imgs) == 5 assert all([img.shape == (512, 512, 3) for img in imgs]) assert all([self.ds_path in fp for fp in file_paths]) assert len(set(file_paths)) == 5
def test_batch_iteration_scheme_with_lists(self): """Batch schemes should work with more than ndarrays.""" data = IndexableDataset(OrderedDict([('foo', list(range(50))), ('bar', list(range(1, 51)))])) stream = DataStream(data, iteration_scheme=ShuffledScheme(data.num_examples, 5)) returned = [sum(batches, []) for batches in zip(*list(stream.get_epoch_iterator()))] assert set(returned[0]) == set(range(50)) assert set(returned[1]) == set(range(1, 51))
def main(): X = T.matrix(name='input_ivecs',dtype='float32') #load data #train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',)) valid_set = H5PYDataset(VALID_FILE, which_sets=('valid',)) network = build_dnn(net_input=X) dae_ivector = lasagne.layers.get_output(network, deterministic=True) feature_extr = theano.function([X], dae_ivector) h1=train_set.open() h2=valid_set.open() scheme = SequentialScheme(examples=train_set.num_examples, batch_size=512) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=512) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) for data in train_stream.get_epoch_iterator(): t_data, _,t_name = data t_ivec = feature_extr(t_data) for name,ivec in zip(t_name,t_ivec): fname = os.path.join(SAVE_PATH,t_name) scipy.io.savemat(fname,mdict={'ivec':ivec}) for data in valid_stream.get_epoch_iterator(): v_data, _,v_name = data v_ivec = feature_extr(v_data) for name,ivec in zip(v_name,v_ivec): fname = os.path.join(SAVE_PATH,v_name) scipy.io.savemat(fname,mdict={'ivec':ivec})
def load_voc2012_stream(batch_size, train_size=16500, validation_size=500, test_size=100, shuffle=False): fuel_root = fuel.config.data_path[0] # データセットファイル保存場所 hdf5_filepath = os.path.join( fuel_root, 'voc2012\hdf5_dataset\hdf5_dataset.hdf5') valid_size = train_size + validation_size test_size = valid_size + test_size indices_train = range(0, train_size) indices_valid = range(train_size, valid_size) indices_test = range(valid_size, test_size) h5py_file = h5py.File(hdf5_filepath) dataset = H5PYDataset(h5py_file, ['train']) scheme_class = ShuffledScheme if shuffle else SequentialScheme scheme_train = scheme_class(indices_train, batch_size=batch_size) scheme_valid = scheme_class(indices_valid, batch_size=batch_size) scheme_test = scheme_class(indices_test, batch_size=batch_size) stream_train = DataStream(dataset, iteration_scheme=scheme_train) stream_valid = DataStream(dataset, iteration_scheme=scheme_valid) stream_test = DataStream(dataset, iteration_scheme=scheme_test) stream_train.get_epoch_iterator().next() stream_valid.get_epoch_iterator().next() stream_test.get_epoch_iterator().next() return stream_train, stream_valid, stream_test
def test_text(): # Test word level and epochs. with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences1 = f.name f.write("This is a sentence\n") f.write("This another one") with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences2 = f.name f.write("More sentences\n") f.write("The last one") dictionary = {'<UNK>': 0, '</S>': 1, 'this': 2, 'a': 3, 'one': 4} text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, bos_token=None, preprocess=lower) stream = DataStream(text_data) epoch = stream.get_epoch_iterator() assert len(list(epoch)) == 4 epoch = stream.get_epoch_iterator() for sentence in zip(range(3), epoch): pass f = BytesIO() cPickle.dump(epoch, f) sentence = next(epoch) f.seek(0) epoch = cPickle.load(f) assert next(epoch) == sentence assert_raises(StopIteration, next, epoch) # Test character level. dictionary = dict([(chr(ord('a') + i), i) for i in range(26)] + [(' ', 26)] + [('<S>', 27)] + [('</S>', 28)] + [('<UNK>', 29)]) text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, preprocess=lower, level="character") sentence = next(DataStream(text_data).get_epoch_iterator())[0] assert sentence[:3] == [27, 19, 7] assert sentence[-3:] == [2, 4, 28]
def test(model=None): if model is not None: #accuracies = [] testset = H5PYDataset('svhn_format_2.hdf5', which_sets=('test',), sources=('features', 'targets')) teststream = DataStream(testset, iteration_scheme=SequentialScheme(examples=testset.num_examples, batch_size=500)) for data in teststream.get_epoch_iterator(): images, labels = data images = np.swapaxes(images, axis1=1, axis2=3) labels = keras.utils.to_categorical(labels) loss, accuracy = model.test_on_batch(x=images, y=labels) accuracies.append(accuracy) trainstream.close() return losses
def main(): X = T.matrix(name='input_ivecs', dtype='float32') #load data #train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',)) valid_set = H5PYDataset(VALID_FILE, which_sets=('valid', )) network = build_dnn(net_input=X) dae_ivector = lasagne.layers.get_output(network, deterministic=True) feature_extr = theano.function([X], dae_ivector) h1 = train_set.open() h2 = valid_set.open() scheme = SequentialScheme(examples=train_set.num_examples, batch_size=512) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=512) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) for data in train_stream.get_epoch_iterator(): t_data, _, t_name = data t_ivec = feature_extr(t_data) for name, ivec in zip(t_name, t_ivec): fname = os.path.join(SAVE_PATH, t_name) scipy.io.savemat(fname, mdict={'ivec': ivec}) for data in valid_stream.get_epoch_iterator(): v_data, _, v_name = data v_ivec = feature_extr(v_data) for name, ivec in zip(v_name, v_ivec): fname = os.path.join(SAVE_PATH, v_name) scipy.io.savemat(fname, mdict={'ivec': ivec})
def test_text(): # Test word level and epochs. with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences1 = f.name f.write("This is a sentence\n") f.write("This another one") with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences2 = f.name f.write("More sentences\n") f.write("The last one") dictionary = {'<UNK>': 0, '</S>': 1, 'this': 2, 'a': 3, 'one': 4} text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, bos_token=None, preprocess=lower) stream = DataStream(text_data) epoch = stream.get_epoch_iterator() assert len(list(epoch)) == 4 epoch = stream.get_epoch_iterator() for sentence in zip(range(3), epoch): pass f = BytesIO() cPickle.dump(epoch, f) sentence = next(epoch) f.seek(0) epoch = cPickle.load(f) assert next(epoch) == sentence assert_raises(StopIteration, next, epoch) # Test character level. dictionary = dict([(chr(ord('a') + i), i) for i in range(26)] + [(' ', 26)] + [('<S>', 27)] + [('</S>', 28)] + [('<UNK>', 29)]) text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, preprocess=lower, level="character") sentence = next(DataStream(text_data).get_epoch_iterator())[0] assert sentence[:3] == [27, 19, 7] assert sentence[-3:] == [2, 4, 28]
def test_jsb(): jsb = MidiSequence('jsb') print jsb.num_examples dataset = DataStream(jsb, iteration_scheme=SequentialScheme( jsb.num_examples, 10)) for b in dataset.get_epoch_iterator(): print b[0].shape # This is how to prepare this dataset to be used in Blocks dataset = Padding(dataset) def _transpose(data): return tuple(np.rollaxis(array, 1, 0) for array in data) dataset = Mapping(dataset, _transpose) for b in dataset.get_epoch_iterator(): print len(b) print b[1].shape print b[0].shape assert b[0].shape == (108, 9, 96)
def test_batch_iteration_scheme_with_lists(self): """Batch schemes should work with more than ndarrays.""" data = IndexableDataset( OrderedDict([('foo', list(range(50))), ('bar', list(range(1, 51)))])) stream = DataStream(data, iteration_scheme=ShuffledScheme( data.num_examples, 5)) returned = [ sum(batches, []) for batches in zip(*list(stream.get_epoch_iterator())) ] assert set(returned[0]) == set(range(50)) assert set(returned[1]) == set(range(1, 51))
def batch_iterator(self, dataset, batchsize, shuffle=False): if isinstance(dataset, Dataset): if shuffle: train_scheme = ShuffledScheme(examples=dataset.num_examples, batch_size=batchsize) else: train_scheme = SequentialScheme(examples=dataset.num_examples, batch_size=batchsize) stream = DataStream(dataset=dataset, iteration_scheme=train_scheme) if self.fuel_stream_xform_fn is not None: stream = self.fuel_stream_xform_fn(stream) return stream.get_epoch_iterator() elif _is_sequence_of_arrays(dataset): return iterate_minibatches(dataset, batchsize, shuffle=shuffle) else: raise TypeError('dataset should be a fuel Dataset instance or a list of arrays')
def train(model=None): if model is not None: trainset = H5PYDataset('svhn_format_2.hdf5', which_sets=('train',), sources=('features', 'targets')) trainstream = DataStream(trainset, iteration_scheme=SequentialScheme(examples=trainset.num_examples, batch_size=500)) for data in trainstream.get_epoch_iterator(): images, labels = data #standardize the input images m = images.mean(axis=(2,3), keepdims=True) s = images.std(axis=(2,3), keepdims=True) images = (images - m)/s #change from "channel_first" to "channel_last" images = np.transpose(images, (0,2,3,1)) labels = keras.utils.to_categorical(labels) #print images.shape model.train_on_batch(x=images, y=labels) trainstream.close()
class FuelDataSet(object): def __init__(self,dataset,batch_size=128,shuffle=False): self.dataset = dataset if shuffle: self.datastream = DataStream(self.dataset, iteration_scheme=ShuffledScheme( examples=dataset.num_examples, batch_size=batch_size)) else: self.datastream = DataStream(self.dataset, iteration_scheme=SequentialScheme( examples=dataset.num_examples, batch_size=batch_size)) def __iter__(self): return self.datastream.get_epoch_iterator()
def infer(path, ae_encode): ''' :param path: path of infer data :param ae_encode: compiled theano function :return: image saved path in string ''' hf = h5py.File(path, 'r+') split_dict = { 'test': { 'input': (0, 1), 'target': (0, 1) }, } hf.attrs['split'] = H5PYDataset.create_split_array(split_dict) test_set = H5PYDataset(path, which_sets=('test', )) batch_size = 1 test_scheme = SequentialScheme(examples=test_set.num_examples, batch_size=batch_size) test_stream = DataStream(test_set, iteration_scheme=test_scheme) for te_train, te_target in test_stream.get_epoch_iterator(): break te_out, te_ta = ae_encode(input_transform(te_train), target_transform(te_target)) te_reshape = inverse(te_out) te_target_reshape = inverse(te_ta) new_size = (128 * 2, 160) new_im = Image.new('RGB', new_size) r = np.random.choice(1, 1, replace=False).reshape(1, 1) for i in range(1): for j in range(1): index = r[i][j] target_im = Image.fromarray(te_target_reshape[index]) train_im = Image.fromarray(te_train[index].astype(np.uint8)) im = Image.fromarray(te_reshape[index]) new_im.paste(train_im, (128 * (i * 2), 160 * j)) new_im.paste(im, (128 * (i * 2 + 1), 160 * j)) img_loc = "gen_images/%i.png" % int(time()) new_im.save(img_loc) return img_loc
def test_data_stream(): from fuel.datasets import IndexableDataset from fuel.streams import DataStream from fuel.schemes import ShuffledScheme seed = 1234 rng = numpy.random.RandomState(seed) features = rng.randint(256, size=(8, 2, 2)) targets = rng.randint(4, size=(8, 1)) dataset = IndexableDataset(indexables=OrderedDict([('features', features), ('targets', targets)]), axis_labels=OrderedDict([('features', ('batch', 'height', 'width')), ('targets', ('batch', 'index'))])) scheme = ShuffledScheme(examples=dataset.num_examples, batch_size=10) data_stream = DataStream(dataset=dataset, iteration_scheme=scheme) for i, data in enumerate(data_stream.get_epoch_iterator()): print('epoch '+str(i), data[0].shape, data[1].shape) time.sleep(1)
def load_toy_stream(batch_size, ): """Parameters ---------- randomcirclesquaredataset : RandomCircleSqureDatasetクラスインスタンス. batch_size : バッチサイズ""" dataset = IterableDataset(RandomCircleSquareDataset(batch_size=batch_size)) stream_train = DataStream(dataset) stream_valid = DataStream(dataset) stream_test = DataStream(dataset) stream_train.get_epoch_iterator().next() stream_valid.get_epoch_iterator().next() stream_test.get_epoch_iterator().next() return stream_train, stream_valid, stream_test
def dataset_generator(dataset, batch_size=500): while 1: trainstream = DataStream(dataset, iteration_scheme=ShuffledScheme(examples=dataset.num_examples, batch_size=batch_size)) for data in trainstream.get_epoch_iterator(): images, labels = data #resize images #images = tf.image.resize_images(images, (48,48)) #images = images.eval() #standardize the input images m = images.mean(axis=(1,2,3), keepdims=True) #s = images.std(axis=(1,2,3), keepdims=True) images = (images - m)#/ (s + 1e-3) #change from "channel_first" to "channel_last" images = np.transpose(images, (0,2,3,1)) #images = keras.utils.normalize(images, axis=-1, order=2) #convert to one_hot representation #print labels labels = keras.utils.to_categorical(labels, num_classes=11) #print labels yield(images, labels) trainstream.close()
def test_in_memory(): skip_if_not_available(datasets=['mnist']) # Load MNIST and get two batches mnist = MNIST('train') data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( num_examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break assert numpy.all(features == mnist.features[256:512]) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name cPickle.dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) assert numpy.all(features == mnist.features[512:768])
def main(config, tr_stream, dev_stream, use_bokeh=False): print("~def main") # Create Theano variables logger.info('Creating theano variables') source_sentence = tensor.lmatrix('source') source_sentence_mask = tensor.matrix('source_mask') target_sentence = tensor.lmatrix('target') target_sentence_mask = tensor.matrix('target_mask') sampling_input = tensor.lmatrix('input') print("~sampling_input = tensor.lmatrix") # Construct model logger.info('Building RNN encoder-decoder') encoder = BidirectionalEncoder( config['src_vocab_size'], config['enc_embed'], config['enc_nhids']) decoder = Decoder( config['trg_vocab_size'], config['dec_embed'], config['dec_nhids'], config['enc_nhids'] * 2) cost = decoder.cost( encoder.apply(source_sentence, source_sentence_mask), source_sentence_mask, target_sentence, target_sentence_mask) print("~source_sentence_mask, target_sentence, target_sentence_mask") logger.info('Creating computational graph') cg = ComputationGraph(cost) print("~ComputationGraph") # Initialize model logger.info('Initializing model') encoder.weights_init = decoder.weights_init = IsotropicGaussian( config['weight_scale']) encoder.biases_init = decoder.biases_init = Constant(0) encoder.push_initialization_config() decoder.push_initialization_config() encoder.bidir.prototype.weights_init = Orthogonal() decoder.transition.weights_init = Orthogonal() encoder.initialize() decoder.initialize() print("~decoder.initialize()") # apply dropout for regularization if config['dropout'] < 1.0: # dropout is applied to the output of maxout in ghog logger.info('Applying dropout') dropout_inputs = [x for x in cg.intermediary_variables if x.name == 'maxout_apply_output'] cg = apply_dropout(cg, dropout_inputs, config['dropout']) print("~cg = apply_dropout") # Apply weight noise for regularization if config['weight_noise_ff'] > 0.0: logger.info('Applying weight noise to ff layers') enc_params = Selector(encoder.lookup).get_params().values() enc_params += Selector(encoder.fwd_fork).get_params().values() enc_params += Selector(encoder.back_fork).get_params().values() dec_params = Selector( decoder.sequence_generator.readout).get_params().values() dec_params += Selector( decoder.sequence_generator.fork).get_params().values() dec_params += Selector(decoder.state_init).get_params().values() cg = apply_noise(cg, enc_params+dec_params, config['weight_noise_ff']) print("~cg = apply_noise") # Print shapes shapes = [param.get_value().shape for param in cg.parameters] logger.info("Parameter shapes: ") for shape, count in Counter(shapes).most_common(): logger.info(' {:15}: {}'.format(shape, count)) logger.info("Total number of parameters: {}".format(len(shapes))) print("~logger.info") # Print parameter names enc_dec_param_dict = merge(Selector(encoder).get_parameters(), Selector(decoder).get_parameters()) logger.info("Parameter names: ") for name, value in enc_dec_param_dict.items(): logger.info(' {:15}: {}'.format(value.get_value().shape, name)) logger.info("Total number of parameters: {}" .format(len(enc_dec_param_dict))) # Set up training model logger.info("Building model") training_model = Model(cost) print("~training_model") # Set extensions logger.info("Initializing extensions") extensions = [ FinishAfter(after_n_batches=config['finish_after']), TrainingDataMonitoring([cost], after_batch=True), Printing(after_batch=True), CheckpointNMT(config['saveto'], every_n_batches=config['save_freq']) ] print("~every_n_batches=config") # Set up beam search and sampling computation graphs if necessary if config['hook_samples'] >= 1 or config['bleu_script'] is not None: logger.info("Building sampling model") sampling_representation = encoder.apply( sampling_input, tensor.ones(sampling_input.shape)) generated = decoder.generate(sampling_input, sampling_representation) search_model = Model(generated) _, samples = VariableFilter( bricks=[decoder.sequence_generator], name="outputs")( ComputationGraph(generated[1])) # generated[1] is next_outputs sample = Sampler(model=search_model, data_stream=tr_stream, hook_samples=config['hook_samples'], every_n_batches=config['sampling_freq'], src_vocab_size=config['src_vocab_size']) # Add sampling if config['hook_samples'] >= 1: logger.info("Building sampler") extensions.append( sample ) # Add early stopping based on bleu if config['bleu_script'] is not None: logger.info("Building bleu validator") extensions.append( BleuValidator(sampling_input, samples=samples, config=config, model=search_model, data_stream=dev_stream, normalize=config['normalized_bleu'], every_n_batches=config['bleu_val_freq'])) # Reload model if necessary if config['reload']: extensions.append(LoadNMT(config['saveto'])) # Plot cost in bokeh if necessary if use_bokeh and BOKEH_AVAILABLE: extensions.append( Plot('Cs-En', channels=[['decoder_cost_cost']], after_batch=True)) sampling_fn = search_model.get_theano_function() print(" - - - - - - - - - - - - - - " ) sort_k_batches = 12 batch_size = 80 seq_len = 50 trg_ivocab = None src_vocab_size = config['src_vocab_size'] trg_vocab_size = config['trg_vocab_size'] unk_id = config['unk_id'] src_vocab = config['src_vocab'] trg_vocab = config['trg_vocab'] src_vocab = ensure_special_tokens( src_vocab if isinstance(src_vocab, dict) else cPickle.load(open(src_vocab)), bos_idx=0, eos_idx=src_vocab_size - 1, unk_idx=unk_id) trg_vocab = ensure_special_tokens( trg_vocab if isinstance(trg_vocab, dict) else cPickle.load(open(trg_vocab)), bos_idx=0, eos_idx=trg_vocab_size - 1, unk_idx=unk_id) if not trg_ivocab: trg_ivocab = {v: k for k, v in trg_vocab.items()} src_data = config['src_data'] trg_data = config['trg_data'] src_dataset = TextFile([src_data], src_vocab, None) trg_dataset = TextFile([trg_data], trg_vocab, None) inputstringfile="inputstringfile.cs" input_dataset = TextFile([inputstringfile], src_vocab, None) stream = Merge([input_dataset.get_example_stream(), trg_dataset.get_example_stream()], ('source', 'target')) stream2 = Filter(stream, predicate=_too_long(seq_len=seq_len)) stream3 = Mapping(stream2, _oov_to_unk(src_vocab_size=src_vocab_size, trg_vocab_size=trg_vocab_size, unk_id=unk_id)) stream4 = Batch(stream3, iteration_scheme=ConstantScheme( batch_size*sort_k_batches)) stream5 = Mapping(stream4, SortMapping(_length)) stream6 = Unpack(stream5) stream7 = Batch( stream6, iteration_scheme=ConstantScheme(batch_size)) input_stream = DataStream(input_dataset) print("dev_stream : ", type( dev_stream ) ) print("input_stream : ", type( input_stream ) ) epochone = input_stream.get_epoch_iterator() vocab = input_stream.dataset.dictionary unk_sym = input_stream.dataset.unk_token eos_sym = input_stream.dataset.eos_token for i, line in enumerate(epochone): seq = oov_to_unk( line[0], config['src_vocab_size'], unk_id) input_ = numpy.tile(seq, ( 1 , 1)) print("seq : " , type( seq ) , seq ) print("input_ : ", type( input_ ) , input_ , inspect.getmembers( input_ ) ) _1, outputs, _2, _3, costs = ( sampling_fn( input_ ) ) outputs = outputs.flatten() costs = costs.T print(" outputs : " , outputs , type( outputs ) ) print("idx_to_word: ", idx_to_word(outputs , trg_ivocab)) print(" - - - - - - - - - - - - - - " )
data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(mini_batch_size)) #data_stream = Padding(data_stream, mask_sources=('tokens')) # Adds a mask fields to this stream field, type='floatX' data_stream = Padding( data_stream, ) # Adds a mask fields to all of this stream's fields, type='floatX' data_stream = Mapping( data_stream, _transpose ) # Flips stream so that sentences run down columns, batches along rows (strangely) if False: # print sample for debugging Dataset / DataStream component #t=0 max_len = 0 for i, data in enumerate(data_stream.get_epoch_iterator()): #print(i) #t=t + data[4].sum() + data[0].shape[1] l = data[0].shape[0] if l > max_len: max_len = l print(i, data[0].shape, max_len) #print(data) #break exit(0) """ Comments in google-groups:blocks indicate that a reshaping has to be done, so let's think about sizes of the arrays... """ x = tensor.matrix('tokens', dtype="int32")
########## start training ########################### n_epochs = 15 if "n_epochs" in config: n_epochs = int(config["n_epochs"]) bestF1 = 0 best_params = [] best_epoch = 0 epoch = 0 lrate = lrateOrig while epoch < n_epochs: time1 = time.time() # train time1Train = time.time() for d in data_stream.get_epoch_iterator(as_dict=True): x1_numpy = d['x1'] x2_numpy = d['x2'] x3_numpy = d['x3'] x4_numpy = d['x4'] e1_numpy = d['e1'] e2_numpy = d['e2'] y_numpy = d['y'] y1ET_numpy = d['y1ET'] y2ET_numpy = d['y2ET'] numSamples_numpy = numpy.ones_like(y1ET_numpy) cost_ij = train(x1_numpy, x2_numpy, x3_numpy, x4_numpy, e1_numpy, e2_numpy, y_numpy, y1ET_numpy, y2ET_numpy, numSamples_numpy, lrate) if numpy.isnan(cost_ij):
test_stream, prefix="test") #, FinishAfter(after_n_epochs=10) , Printing() #, Dump("out.pkl") , ExperimentSaver("../VAOutCIFAR", ".") , EpochProgress(dataset.num_examples // 128) , Plot('cifar10', channels=[['train_variational_cost', 'test_variational_cost', 'test_cost']]) ]) main_loop.run() _func_noisy = theano.function([m.X], m.noisy) _func_produced = theano.function([m.X], m.produced) batch = test_stream.get_epoch_iterator().next()[0] out_noise = _func_noisy(batch) out_produced = _func_produced(batch) import cv2 for k in range(10): print out_noise.shape img = np.reshape(out_noise[k, :], (28, 28)) img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST) cv2.imshow('img', img) img = np.reshape(out_produced[k, :], (28, 28)) img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST) cv2.imshow('masdf', img) cv2.waitKey(0)
split_dict = { 'train': { 'images': (0, train_feature.shape[0]), 'targets': (0, train_target.shape[0]) } } f.attrs['split'] = H5PYDataset.create_split_array(split_dict) f.flush() f.close() train_set = H5PYDataset('../../data/dataset.hdf5', which_sets=('train',)) #data_stream = DataStream(dataset=train_set, iteration_scheme=scheme) #state = train_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=4) data_stream = DataStream(dataset=train_set, iteration_scheme=scheme) for data in data_stream.get_epoch_iterator(): print(data[0], data[1]) standardized_stream = ScaleAndShift(data_stream=data_stream, scale=255,shift=0, which_sources=('features',)) for data in standardized_stream.get_epoch_iterator(): print(data[0], data[1]) #train_set.close(state)
def main(): X = T.tensor3(name='input',dtype='float32') MASK = T.matrix(name = 'mask', dtype='float32') SPK_LABELS = T.ivector(name='spk_labels') PHR_LABELS = T.ivector(name='phr_labels') #load data train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',)) valid_set = H5PYDataset(TRAIN_FILE, which_sets=('test',)) spk_network, phr_network = build_rnn(net_input=X, mask_input= MASK) network_output_spk = lasagne.layers.get_output(spk_network) network_output_phr = lasagne.layers.get_output(phr_network) val_prediction_spk = lasagne.layers.get_output(spk_network, deterministic=True) val_prediction_phr = lasagne.layers.get_output(phr_network, deterministic=True) #needed for accuracy val_acc_spk = T.mean(T.eq(T.argmax(val_prediction_spk, axis=1), SPK_LABELS), dtype=theano.config.floatX) val_acc_phr = T.mean(T.eq(T.argmax(val_prediction_phr, axis=1), PHR_LABELS), dtype=theano.config.floatX) #training accuracy train_acc_spk = T.mean(T.eq(T.argmax(network_output_spk, axis=1), SPK_LABELS), dtype=theano.config.floatX) train_acc_phr = T.mean(T.eq(T.argmax(network_output_phr, axis=1), PHR_LABELS), dtype=theano.config.floatX) #cost function spk_cost = lasagne.objectives.categorical_crossentropy(network_output_spk, SPK_LABELS) phr_cost = lasagne.objectives.categorical_crossentropy(network_output_phr, PHR_LABELS) total_cost = spk_cost + phr_cost mean_cost = total_cost.mean() #Validation cost val_spk_cost = lasagne.objectives.categorical_crossentropy(val_prediction_spk, SPK_LABELS) val_phr_cost = lasagne.objectives.categorical_crossentropy(val_prediction_phr, PHR_LABELS) val_cost = val_spk_cost + val_phr_cost val_mcost = val_cost.mean() #Get parameters of both encoder and decoder all_parameters = lasagne.layers.get_all_params([spk_network, phr_network], trainable=True) print("Trainable Model Parameters") print("-"*40) for param in all_parameters: print(param, param.get_value().shape) print("-"*40) #add grad clipping to avoid exploding gradients all_grads = [T.clip(g,-3,3) for g in T.grad(mean_cost, all_parameters)] #all_grads = lasagne.updates.total_norm_constraint(all_grads,3) updates = lasagne.updates.adam(all_grads, all_parameters, learning_rate=LEARNING_RATE) train_func = theano.function([X, MASK, SPK_LABELS, PHR_LABELS], [mean_cost, train_acc_spk, train_acc_phr], updates=updates) val_func = theano.function([X, MASK, SPK_LABELS, PHR_LABELS], [val_mcost, val_acc_spk, val_acc_phr]) trainerr=[] epoch=0 #set the epoch counter min_val_loss = np.inf max_val = np.inf*(-1) patience=0 print("Starting training...") # We iterate over epochs: while 'true': # In each epoch, we do a full pass over the training data: train_err = 0 tr_acc_spk = 0 tr_acc_phr = 0 train_batches = 0 h1=train_set.open() h2=valid_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=BATCH_SIZE) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=32) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) start_time = time.time() for data in train_stream.get_epoch_iterator(): t_data, t_mask, t_plabs,_, t_labs,_ = data terr, tacc_spk, tacc_phr = train_func(t_data, t_mask, t_labs, t_plabs) train_err += terr tr_acc_spk += tacc_spk tr_acc_phr += tacc_phr train_batches += 1 val_err = 0 val_acc_spk = 0 val_acc_phr = 0 val_batches = 0 for data in valid_stream.get_epoch_iterator(): v_data, v_mask, v_ptars, _, v_tars, _ = data err, acc_spk, acc_phr = val_func(v_data, v_mask ,v_tars, v_ptars) val_err += err val_acc_spk += acc_spk val_acc_phr += acc_phr val_batches += 1 trainerr.append(train_err/train_batches) epoch+=1 train_set.close(h1) valid_set.close(h2) #Display if display: print("Epoch {} of {} took {:.3f}s".format( epoch, NUM_EPOCHS, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / train_batches)) print(" training accuracy (speaker):\t\t{:.2f} %".format( tr_acc_spk / train_batches * 100)) print(" training accuracy (phrase):\t\t{:.2f} %".format( tr_acc_phr / train_batches * 100)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy (speaker):\t\t{:.2f} %".format( val_acc_spk / val_batches * 100)) print(" validation accuracy (phrase):\t\t{:.2f} %".format( val_acc_phr / val_batches * 100)) logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log') flog1 = open(logfile,'ab') flog1.write("Running %s on %s" % (SCRIPT,MSC)) flog1.write("Epoch {} of {} took {:.3f}s \n".format( epoch, NUM_EPOCHS, time.time() - start_time)) flog1.write(" training loss:\t\t{:.6f}\n".format(train_err / train_batches)) flog1.write(" training accuracy (speaker):\t\t{:.2f} %\n".format( tr_acc_spk / train_batches * 100)) flog1.write(" training accuracy (phrase):\t\t{:.2f} %\n".format( tr_acc_phr / train_batches * 100)) flog1.write(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) flog1.write(" validation accuracy (speaker):\t\t{:.2f} %\n".format( val_acc_spk / val_batches * 100)) flog1.write(" validation accuracy (phrase):\t\t{:.2f} %\n".format( val_acc_phr / val_batches * 100)) flog1.write("\n") flog1.close() if epoch == NUM_EPOCHS: break valE = val_err/val_batches if valE <= min_val_loss: model_params1 = lasagne.layers.get_all_param_values([spk_network, phr_network]) model1_name = MODEL_NAME+'_minloss' + '.pkl' vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name) fsave = open(vpth1,'wb') cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close() patience=0 min_val_loss = valE #Patience / Early stopping else: patience+=1 if patience==10: break #Save the final model print('Saving Model ...') model_params = lasagne.layers.get_all_param_values([spk_network, phr_network]) model1_name = MODEL_NAME+'_final' + '.pkl' vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name) fsave = open(vpth,'wb') cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close()
train_data = H5PYDataset(DATASET_PATH, which_sets=('train', ), sources=('s_transition_obs', 'r_transition_obs', 'obs', 'actions')) stream_train = DataStream(train_data, iteration_scheme=SequentialScheme( train_data.num_examples, batch_size)) valid_data = H5PYDataset(DATASET_PATH, which_sets=('valid', ), sources=('s_transition_obs', 'r_transition_obs', 'obs', 'actions')) stream_valid = DataStream(valid_data, iteration_scheme=SequentialScheme( train_data.num_examples, batch_size)) iterator = stream_train.get_epoch_iterator(as_dict=True) data = next(iterator) length = data["actions"].shape[1] match_env(env, env2) video_recorder = VideoRecorder(env, 'sim+backlash.mp4', enabled=True) video_recorder2 = VideoRecorder(env2, 'sim+.mp4', enabled=True) for i, data in enumerate(stream_train.get_epoch_iterator(as_dict=True)): for j in range(length): action = data["actions"] video_recorder.capture_frame() video_recorder2.capture_frame() new_obs, reward, done, info = env.step(action) new_obs2, reward2, done2, info2 = env2.step(action)
def main(): X = T.tensor3(name='input',dtype='float32') MASK = T.matrix(name = 'mask', dtype='float32') LABELS = T.ivector(name='labels') #load data train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',)) valid_set = H5PYDataset(VALID_FILE, which_sets=('test',)) network = build_rnn(net_input=X, mask_input= MASK) network_output = lasagne.layers.get_output(network) val_prediction = lasagne.layers.get_output(network, deterministic=True) #needed for accuracy val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS), dtype=theano.config.floatX) #training accuracy train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS), dtype=theano.config.floatX) #T.argmax(network_output, axis=1).eval({X: d1, Mask: m1}) #print network_output.eval({X: d1, Mask: m1})[1][97] #cost function total_cost = lasagne.objectives.categorical_crossentropy(network_output, LABELS) #total_cost = -(labels*T.log(network_output) + (1-labels)*T.log(1-network_output)) mean_cost = total_cost.mean() #accuracy function val_cost = lasagne.objectives.categorical_crossentropy(val_prediction, LABELS) val_mcost = val_cost.mean() #Get parameters of both encoder and decoder all_parameters = lasagne.layers.get_all_params([network], trainable=True) print("Trainable Model Parameters") print("-"*40) for param in all_parameters: print(param, param.get_value().shape) print("-"*40) #add grad clipping to avoid exploding gradients all_grads = [T.clip(g,-3,3) for g in T.grad(mean_cost, all_parameters)] all_grads = lasagne.updates.total_norm_constraint(all_grads,3) updates = lasagne.updates.adam(all_grads, all_parameters, learning_rate=LEARNING_RATE) train_func = theano.function([X, MASK, LABELS], [mean_cost, train_acc], updates=updates) val_func = theano.function([X, MASK, LABELS], [val_mcost, val_acc]) trainerr=[] epoch=0 #set the epoch counter val_prev = np.inf a_prev = -np.inf print("Starting training...") # We iterate over epochs: while 'true': # In each epoch, we do a full pass over the training data: train_err = 0 tr_acc = 0 train_batches = 0 h1=train_set.open() h2=valid_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=128) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=32) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) start_time = time.time() for data in train_stream.get_epoch_iterator(): t_data, t_mask, _, t_labs = data terr, tacc = train_func(t_data, t_mask, t_labs) train_err += terr tr_acc += tacc train_batches += 1 val_err = 0 val_acc = 0 val_batches = 0 for data in valid_stream.get_epoch_iterator(): v_data, v_mask, _, v_tars = data err, acc = val_func(v_data, v_mask ,v_tars) val_err += err val_acc += acc val_batches += 1 trainerr.append(train_err/train_batches) epoch+=1 train_set.close(h1) valid_set.close(h2) #Display if display: print("Epoch {} of {} took {:.3f}s ".format( epoch, NUM_EPOCHS, time.time() - start_time)) print(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) print(" training accuracy:\t\t{:.2f} % ".format( tr_acc / train_batches * 100)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy:\t\t{:.2f} % ".format( val_acc / val_batches * 100)) logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log') flog1 = open(logfile,'ab') flog1.write("Epoch {} of {} took {:.3f}s\n ".format( epoch, NUM_EPOCHS, time.time() - start_time)) flog1.write(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) flog1.write(" training accuracy:\t\t{:.2f} %\n ".format( tr_acc / train_batches * 100)) flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err / val_batches)) flog1.write(" validation accuracy:\t\t{:.2f} %\n ".format( val_acc / val_batches * 100)) flog1.write("\n") flog1.close() if epoch == NUM_EPOCHS: break valE = val_err/val_batches valA = val_acc / val_batches #save the max accuracy model max_val = a_prev #save model with highest accuracy if valA > max_val: model_params1 = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME+'_acc' + '.pkl' vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name) fsave = open(vpth1,'wb') cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close() max_val = valA #Patience if valE > val_prev: c+=1 #save the model incase model_params1 = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME + '_ofit' + '.pkl' vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name) fsave = open(vpth1,'wb') cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close() val_prev=valE else: c=0 val_prev=valE if c==5: break #Save the final model print('Saving Model ...') model_params = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME+'_final' + '.pkl' vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name) fsave = open(vpth,'wb') cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close()
def main(): X = T.matrix(name='input_ivecs',dtype='float32') X_mean = T.matrix(name = 'target_ivecs', dtype='float32') #load data train_set = H5PYDataset(TRAIN_FILE, which_sets=('train',)) valid_set = H5PYDataset(VALID_FILE, which_sets=('valid',)) network = build_dnn(net_input=X) network_output = lasagne.layers.get_output(network) val_prediction = lasagne.layers.get_output(network, deterministic=True) #needed for accuracy #val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS), dtype=theano.config.floatX) #training accuracy #train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS), dtype=theano.config.floatX) #T.argmax(network_output, axis=1).eval({X: d1, Mask: m1}) #print network_output.eval({X: d1, Mask: m1})[1][97] #cost function total_cost = lasagne.objectives.squared_error(network_output, X_mean) mean_cost = total_cost.mean() #accuracy function val_cost = lasagne.objectives.squared_error(val_prediction, X_mean) val_mcost = val_cost.mean() #Get parameters of both encoder and decoder all_parameters = lasagne.layers.get_all_params([network], trainable=True) print("Trainable Model Parameters") print("-"*40) for param in all_parameters: print(param, param.get_value().shape) print("-"*40) #add grad clipping to avoid exploding gradients updates = lasagne.updates.adadelta(mean_cost, all_parameters) train_func = theano.function([X, X_mean], mean_cost, updates=updates) val_func = theano.function([X, X_mean], val_mcost) trainerr=[] epoch=0 #set the epoch counter min_val_loss = np.inf patience=0 print("Starting training...") # We iterate over epochs: while 'true': # In each epoch, we do a full pass over the training data: train_err = 0 train_batches = 0 h1=train_set.open() h2=valid_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=BATCH_SIZE) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=64) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) start_time = time.time() for data in train_stream.get_epoch_iterator(): t_data, t_targ,_ = data terr = train_func(t_data, t_targ) train_err += terr train_batches += 1 val_err = 0 val_batches = 0 for data in valid_stream.get_epoch_iterator(): v_data, v_tars,_ = data err = val_func(v_data, v_tars) val_err += err val_batches += 1 trainerr.append(train_err/train_batches) epoch+=1 train_set.close(h1) valid_set.close(h2) #Display if display: print("Epoch {} of {} took {:.3f}s ".format( epoch, NUM_EPOCHS, time.time() - start_time)) print(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) logfile = os.path.join(SAVE_PATH,MODEL_NAME,'logs',MODEL_NAME+'.log') flog1 = open(logfile,'ab') flog1.write('Running %s on %s' % (SCRIPT, MSC)) flog1.write("Epoch {} of {} took {:.3f}s\n ".format( epoch, NUM_EPOCHS, time.time() - start_time)) flog1.write(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err / val_batches)) flog1.write("\n") flog1.close() if epoch == NUM_EPOCHS: break valE = val_err/val_batches #Patience if valE <= min_val_loss: model_params1 = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME+'_minloss' + '.pkl' vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights',model1_name) fsave = open(vpth1,'wb') cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close() min_val_loss = valE patience=0 #Patience / Early stopping else: patience+=1 if patience==10: break #Save the final model print('Saving Model ...') model_params = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME+'_final' + '.pkl' vpth = os.path.join(SAVE_PATH, MODEL_NAME,'weights',model1_name) fsave = open(vpth,'wb') cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close()
class TestDataset(object): def setUp(self): self.data = [1, 2, 3] self.stream = DataStream(IterableDataset(self.data)) def test_one_example_at_a_time(self): assert_equal( list(self.stream.get_epoch_iterator()), list(zip(self.data))) def test_multiple_epochs(self): for i, epoch in zip(range(2), self.stream.iterate_epochs()): assert list(epoch) == list(zip(self.data)) def test_as_dict(self): assert_equal( next(self.stream.get_epoch_iterator(as_dict=True)), {"data": 1}) def test_value_error_on_no_provided_sources(self): class FaultyDataset(Dataset): def get_data(self, state=None, request=None): pass assert_raises(ValueError, FaultyDataset, self.data) def test_value_error_on_nonexistent_sources(self): def instantiate_dataset(): return IterableDataset(self.data, sources=('dummy',)) assert_raises(ValueError, instantiate_dataset) def test_default_transformer(self): class DoublingDataset(IterableDataset): def apply_default_transformer(self, stream): return Mapping( stream, lambda sources: tuple(2 * s for s in sources)) dataset = DoublingDataset(self.data) stream = dataset.apply_default_transformer(DataStream(dataset)) assert_equal(list(stream.get_epoch_iterator()), [(2,), (4,), (6,)]) def test_no_axis_labels(self): assert IterableDataset(self.data).axis_labels is None def test_axis_labels(self): axis_labels = {'data': ('batch',)} dataset = IterableDataset(self.data, axis_labels=axis_labels) assert dataset.axis_labels == axis_labels def test_attribute_error_on_no_example_iteration_scheme(self): class FaultyDataset(Dataset): provides_sources = ('data',) def get_data(self, state=None, request=None): pass def get_example_iteration_scheme(): return FaultyDataset().example_iteration_scheme assert_raises(AttributeError, get_example_iteration_scheme) def test_example_iteration_scheme(self): scheme = ConstantScheme(2) class MinimalDataset(Dataset): provides_sources = ('data',) _example_iteration_scheme = scheme def get_data(self, state=None, request=None): pass assert MinimalDataset().example_iteration_scheme is scheme def test_filter_sources(self): dataset = IterableDataset( OrderedDict([('1', [1, 2]), ('2', [3, 4])]), sources=('1',)) assert_equal(dataset.filter_sources(([1, 2], [3, 4])), ([1, 2],))
out_dir_r = "{}/R/{}/".format(args.out, set_) for out_dir in [out_dir_r, out_dir_s]: if not os.path.exists(out_dir): os.makedirs(out_dir) reacher_data = H5PYDataset(name, which_sets=(set_,), sources=('s_transition_img', 'r_transition_img', 'actions')) stream = DataStream( reacher_data, iteration_scheme=ShuffledScheme(reacher_data.num_examples, batch) ) iterator = stream.get_epoch_iterator(as_dict=True) episode_idx = 0 for episode in tqdm(iterator): # import ipdb; ipdb.set_trace() s_trans_img = episode['s_transition_img'][0] r_trans_img = episode['r_transition_img'][0] tmp = episode['actions'] print(tmp.shape) out_dir_s_episode = os.path.join(out_dir_s, "run-{}".format(episode_idx)) out_dir_r_episode = os.path.join(out_dir_r, "run-{}".format(episode_idx)) for out_dir in [out_dir_s_episode, out_dir_r_episode]: if not os.path.exists(out_dir): os.makedirs(out_dir)
tr_acc = 0 train_batches = 0 h1=train_set.open() h2=valid_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=256) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=128) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) start_time = time.time() for data in train_stream.get_epoch_iterator(): t_data, t_mask, _, t_labs = data terr, tacc = train_func(t_data, t_mask, t_labs) train_err += terr tr_acc += tacc train_batches += 1 val_err = 0 val_acc = 0 val_batches = 0 for data in valid_stream.get_epoch_iterator(): v_data, v_mask, _, v_tars = data err, acc = val_func(v_data, v_mask ,v_tars) val_err += err val_acc += acc
def main(): parser = argparse.ArgumentParser() parser.add_argument('--t_dim', type=int, default=256, help='Text feature dimension') parser.add_argument('--batch_size', type=int, default=64, help='Batch Size') parser.add_argument('--image_size', type=int, default=64, help='Image Size a, a x a') parser.add_argument('--gf_dim', type=int, default=64, help='Number of conv in the first layer gen.') parser.add_argument('--df_dim', type=int, default=64, help='Number of conv in the first layer discr.') parser.add_argument( '--gfc_dim', type=int, default=1024, help='Dimension of gen untis for for fully connected layer 1024') parser.add_argument('--caption_vector_length', type=int, default=2400, help='Caption Vector Length') parser.add_argument('--learning_rate', type=float, default=0.0002, help='Learning Rate') parser.add_argument('--beta1', type=float, default=0.5, help='Momentum for Adam Update') parser.add_argument('--epochs', type=int, default=600, help='Max number of epochs') parser.add_argument( '--save_every', type=int, default=30, help='Save Model/Samples every x iterations over batches') parser.add_argument('--resume_model', type=str, default=None, help='Pre-Trained Model Path, to resume from') parser.add_argument('--data_dir', type=str, default="./", help='Data Directory') parser.add_argument( '--model_path', type=str, default='./pdf_11/model/model_after_flowers_epoch_550.ckpt', help='Trained Model Path') parser.add_argument('--n_images', type=int, default=16, help='Number of Images per Caption') args = parser.parse_args() model_options = { 't_dim': args.t_dim, 'batch_size': args.batch_size, 'image_size': args.image_size, 'gf_dim': args.gf_dim, 'df_dim': args.df_dim, 'gfc_dim': args.gfc_dim, 'caption_vector_length': args.caption_vector_length } gan = model_1.GAN(model_options) _, _, _, _, _ = gan.build_model() sess = tf.InteractiveSession() saver = tf.train.Saver() saver.restore(sess, args.model_path) input_tensors, outputs = gan.build_generator() nlen = 15 nt = 2400 n_updates = 0 n_epochs = 0 n_updates = 0 n_examples = 0 g_costs = [] d_costs = [] gl_costs = [] ntest = 1020 #path_last = "/home/xu/PycharmProjects/tensorflowlstmgan/pdf_1/test2/gen" dataPath = './flower_15.h5' test_set = H5PYDataset(dataPath, which_sets=('test', )) test_scheme = SequentialScheme(examples=test_set.num_examples, batch_size=args.batch_size) test_stream = DataStream(test_set, iteration_scheme=test_scheme) it_test = test_stream.get_epoch_iterator() for epoch in range(16): try: annotationtr, image_featuretr, skip_vectortr, word_vectortr = it_test.next( ) image_featuretr = image_featuretr / np.float32(255) except: it_test = test_stream.get_epoch_iterator() annotationtr, image_featuretr, skip_vectortr, word_vectortr = it_test.next( ) image_featuretr = image_featuretr / np.float32(255) image_featuretr = np.reshape(image_featuretr, [image_featuretr.shape[0], -1]) image_featuretr = np.reshape(image_featuretr, (image_featuretr.shape[0], 64, 64, 3)) # image_featuretr = np.transpose(image_featuretr, [0, 3, 1, 2]) image_featuretr_new = np.empty((args.batch_size, nlen, 64, 64, 3)) for m in range(args.batch_size): for mm in range(nlen): image_featuretr_new[m][mm] = image_featuretr[m] image_featuretr_new = np.asarray(image_featuretr_new, dtype='float32') imb0 = image_featuretr imb = image_featuretr # batchsize*length*3*64*64 y_sum = np.sum(word_vectortr, axis=2) y_len = np.empty((args.batch_size, ), dtype='int32') for j in range(args.batch_size): y_len[j] = np.count_nonzero(y_sum[j]) - 1 mask = np.empty((args.batch_size, 15, 400), dtype='float32') for i in range(args.batch_size): for j in range(15): if j < (y_len[i] + 1): mask[i][j] = 1.0 else: mask[i][j] = 0.0 con = skip_vectortr[:, 0:2400] # batchsize*2400 con = np.reshape(con, (args.batch_size, nt)) # batchsize*2400 zmb = word_vectortr # batchsize*length*400 # GEN UPDATE TWICE, to make sure d_loss does not go to 0 [gen_image] = sess.run([outputs['generator']], feed_dict={ input_tensors['t_z']: zmb, input_tensors['mask']: mask }) n_updates += 1 n_examples += len(imb) n_epochs += 1 # if n_epochs > 50: # progress = float(epoch) / num_epochs # eta.set_value(lasagne.utils.floatX(initial_eta * 2 * (1 - progress))) sys.stdout.flush() annotationtr1 = annotationtr reconst_img2 = imb0 for u in range(64): file = open("./pdf_11/test/txt/%s.txt" % str(n_epochs), "a") xxxx = annotationtr1[u][0] file.write(str(u + 1) + ' ' + xxxx + '\n') file.close() scipy.misc.imsave("./pdf_11/test/real/%i_%i.png" % (epoch, u), (reconst_img2[u] * 255.0).astype(int)) # generative image reconst_img = gen_image # length*batchsize*(64*64*3) print reconst_img.shape for m in range(64): gen_img = reconst_img[m][14] scipy.misc.imsave("./pdf_11/test/gen/%i_%i.png" % (epoch, m), (gen_img * 255.0).astype(int)) print "%i finished" % (epoch)
class TestDataset(object): def setUp(self): self.data = [1, 2, 3] self.stream = DataStream(IterableDataset(self.data)) def test_one_example_at_a_time(self): assert_equal(list(self.stream.get_epoch_iterator()), list(zip(self.data))) def test_multiple_epochs(self): for i, epoch in zip(range(2), self.stream.iterate_epochs()): assert list(epoch) == list(zip(self.data)) def test_as_dict(self): assert_equal(next(self.stream.get_epoch_iterator(as_dict=True)), {"data": 1}) def test_value_error_on_no_provided_sources(self): class FaultyDataset(Dataset): def get_data(self, state=None, request=None): pass assert_raises(ValueError, FaultyDataset, self.data) def test_value_error_on_nonexistent_sources(self): def instantiate_dataset(): return IterableDataset(self.data, sources=('dummy', )) assert_raises(ValueError, instantiate_dataset) def test_default_transformer(self): class DoublingDataset(IterableDataset): def apply_default_transformer(self, stream): return Mapping(stream, lambda sources: tuple(2 * s for s in sources)) dataset = DoublingDataset(self.data) stream = dataset.apply_default_transformer(DataStream(dataset)) assert_equal(list(stream.get_epoch_iterator()), [(2, ), (4, ), (6, )]) def test_no_axis_labels(self): assert IterableDataset(self.data).axis_labels is None def test_axis_labels(self): axis_labels = {'data': ('batch', )} dataset = IterableDataset(self.data, axis_labels=axis_labels) assert dataset.axis_labels == axis_labels def test_attribute_error_on_no_example_iteration_scheme(self): class FaultyDataset(Dataset): provides_sources = ('data', ) def get_data(self, state=None, request=None): pass def get_example_iteration_scheme(): return FaultyDataset().example_iteration_scheme assert_raises(AttributeError, get_example_iteration_scheme) def test_example_iteration_scheme(self): scheme = ConstantScheme(2) class MinimalDataset(Dataset): provides_sources = ('data', ) _example_iteration_scheme = scheme def get_data(self, state=None, request=None): pass assert MinimalDataset().example_iteration_scheme is scheme def test_filter_sources(self): dataset = IterableDataset(OrderedDict([('1', [1, 2]), ('2', [3, 4])]), sources=('1', )) assert_equal(dataset.filter_sources(([1, 2], [3, 4])), ([1, 2], ))
def main(name, model, epochs, batch_size, learning_rate, bokeh, layers, gamma, rectifier, predict, dropout, qlinear, sparse): runname = "vae%s-L%s%s%s%s-l%s-g%s-b%d" % (name, layers, 'r' if rectifier else '', 'd' if dropout else '', 'l' if qlinear else '', shnum(learning_rate), shnum(gamma), batch_size//100) if rectifier: activation = Rectifier() full_weights_init = Orthogonal() else: activation = Tanh() full_weights_init = Orthogonal() if sparse: runname += '-s%d'%sparse weights_init = Sparse(num_init=sparse, weights_init=full_weights_init) else: weights_init = full_weights_init layers = map(int,layers.split(',')) encoder_layers = layers[:-1] encoder_mlp = MLP([activation] * (len(encoder_layers)-1), encoder_layers, name="MLP_enc", biases_init=Constant(0.), weights_init=weights_init) enc_dim = encoder_layers[-1] z_dim = layers[-1] if qlinear: sampler = Qlinear(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init) else: sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init) decoder_layers = layers[:] ## includes z_dim as first layer decoder_layers.reverse() decoder_mlp = MLP([activation] * (len(decoder_layers)-2) + [Sigmoid()], decoder_layers, name="MLP_dec", biases_init=Constant(0.), weights_init=weights_init) vae = VAEModel(encoder_mlp, sampler, decoder_mlp) vae.initialize() x = tensor.matrix('features') if predict: mean_z, enc = vae.mean_z(x) # cg = ComputationGraph([mean_z, enc]) newmodel = Model([mean_z,enc]) else: x_recons, kl_terms = vae.reconstruct(x) recons_term = BinaryCrossEntropy().apply(x, x_recons) recons_term.name = "recons_term" cost = recons_term + kl_terms.mean() cg = ComputationGraph([cost]) if gamma > 0: weights = VariableFilter(roles=[WEIGHT])(cg.variables) cost += gamma * blocks.theano_expressions.l2_norm(weights) cost.name = "nll_bound" newmodel = Model(cost) if dropout: weights = [v for k,v in newmodel.get_params().iteritems() if k.find('MLP')>=0 and k.endswith('.W') and not k.endswith('MLP_enc/linear_0.W')] cg = apply_dropout(cg,weights,0.5) target_cost = cg.outputs[0] else: target_cost = cost if name == 'mnist': if predict: train_ds = MNIST("train") else: train_ds = MNIST("train", sources=['features']) test_ds = MNIST("test") else: datasource_dir = os.path.join(fuel.config.data_path, name) datasource_fname = os.path.join(datasource_dir , name+'.hdf5') if predict: train_ds = H5PYDataset(datasource_fname, which_set='train') else: train_ds = H5PYDataset(datasource_fname, which_set='train', sources=['features']) test_ds = H5PYDataset(datasource_fname, which_set='test') train_s = DataStream(train_ds, iteration_scheme=SequentialScheme( train_ds.num_examples, batch_size)) test_s = DataStream(test_ds, iteration_scheme=SequentialScheme( test_ds.num_examples, batch_size)) if predict: from itertools import chain fprop = newmodel.get_theano_function() allpdata = None alledata = None f = train_s.sources.index('features') assert f == test_s.sources.index('features') sources = test_s.sources alllabels = dict((s,[]) for s in sources if s != 'features') for data in chain(train_s.get_epoch_iterator(), test_s.get_epoch_iterator()): for s,d in zip(sources,data): if s != 'features': alllabels[s].extend(list(d)) pdata, edata = fprop(data[f]) if allpdata is None: allpdata = pdata else: allpdata = np.vstack((allpdata, pdata)) if alledata is None: alledata = edata else: alledata = np.vstack((alledata, edata)) print 'Saving',allpdata.shape,'intermidiate layer, for all training and test examples, to',name+'_z.npy' np.save(name+'_z', allpdata) print 'Saving',alledata.shape,'last encoder layer to',name+'_e.npy' np.save(name+'_e', alledata) print 'Saving additional labels/targets:',','.join(alllabels.keys()), print ' of size',','.join(map(lambda x: str(len(x)),alllabels.values())), print 'to',name+'_labels.pkl' with open(name+'_labels.pkl','wb') as fp: pickle.dump(alllabels, fp, -1) else: algorithm = GradientDescent( cost=target_cost, params=cg.parameters, step_rule=Adam(learning_rate) # Scale(learning_rate=learning_rate) ) extensions = [] if model: extensions.append(LoadFromDump(model)) extensions += [Timing(), FinishAfter(after_n_epochs=epochs), DataStreamMonitoring( [recons_term, cost], test_s, prefix="test"), TrainingDataMonitoring( [cost, aggregation.mean(algorithm.total_gradient_norm)], prefix="train", after_epoch=True), Dump(runname, every_n_epochs=10), Printing()] if bokeh: extensions.append(Plot( 'Auto', channels=[ ['test_recons_term','test_nll_bound','train_nll_bound' ], ['train_total_gradient_norm']])) main_loop = MainLoop( algorithm, train_s, model=newmodel, extensions=extensions) main_loop.run()
m = VAModel() # load parameters model = Model(m.variational_cost) print "loading params" params = load_parameter_values(sys.argv[1]) model.set_param_values(params) test_dataset = MNIST('test', sources=['features']) test_scheme = ShuffledScheme(test_dataset.num_examples, 128) test_stream = DataStream(test_dataset, iteration_scheme=test_scheme) _func_noisy = theano.function([m.X], m.noisy) _func_produced = theano.function([m.X], m.produced) batch = test_stream.get_epoch_iterator().next()[0] out_noise = _func_noisy(batch) out_produced = _func_produced(batch) import cv2 for k in range(10): print out_noise.shape img = np.reshape(out_noise[k, :], (28, 28)) img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST) cv2.imshow('img', img) img = np.reshape(out_produced[k, :], (28, 28)) img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_NEAREST) cv2.imshow('masdf', img) cv2.waitKey(0)
import numpy as np DATASET_PATH_REL = "/data/lisa/data/sim2real/" DATASET_PATH = DATASET_PATH_REL + "mujoco_data_pusher3dof_big_backl.h5" # DATASET_PATH = "/Tmp/alitaiga/mujoco_data_pusher3dof_big_backl.h5" batch_size = 1 train_data = H5PYDataset( DATASET_PATH, which_sets=('train',), sources=('s_transition_obs','r_transition_obs', 'obs', 'actions') ) stream_train = DataStream(train_data, iteration_scheme=SequentialScheme(train_data.num_examples, train_data.num_examples)) valid_data = H5PYDataset( DATASET_PATH, which_sets=('valid',), sources=('s_transition_obs','r_transition_obs', 'obs', 'actions') ) stream_valid = DataStream(valid_data, iteration_scheme=SequentialScheme(train_data.num_examples, batch_size)) iterator = stream_train.get_epoch_iterator(as_dict=True) data = next(iterator) import ipdb; ipdb.set_trace() ## Max, min mean std obs: [2.56284189, 2.3500247, 2.39507723, 7.40329409, 9.20471668, 15.37792397] [-0.12052701, -0.17479207, -0.73818409, -1.25026512, -3.95599413, -4.73272848] [2.25090551, 1.94997263, 1.6495719, 0.43379614, 0.3314755, 0.43763939] [0.5295766 , 0.51998389, 0.57609886, 1.35480666, 1.40806067, 2.43865967] # max_obs = np.zeros((6,)) # max_sim = np.zeros((6,)) # max_real = np.zeros((6,)) # max_act = np.zeros((3,))
train_input_mean = 1470614.1 train_input_std = 3256577.0 train_input_mean_reshape = (train_input_mean*numpy.ones(test_input.shape[1])).astype(numpy.float32) train_input_std_reshape = (train_input_std*numpy.ones(test_input.shape[1])).astype(numpy.float32) test_input = (test_input - train_input_mean_reshape[None,:,None])/train_input_std_reshape[None,:,None] test_product = test[:,2,0] ref = {int(k):i for i, k in enumerate(open('listid.txt'))} test_product = numpy.array([ref[test_product[i]] for i in range(len(test_product))]) test_id = test[:,0] ds_test = IndexableDataset({'id':test_id,'input':test_input,'product':test_product}) scheme_test = SequentialScheme(batch_size=10000,examples= ds_test.num_examples) stream_test = DataStream(ds_test,iteration_scheme = scheme_test) pred_test = ComputationGraph([m.pred]).get_theano_function() with open('results_deep_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s_%s.csv' %(config.name,config.couches,config.hidden_dim, config.activation_function_name,config.batch_size,config.w_noise_std, config.i_dropout, config.algo,config.learning_rate_value, config.momentum_value,config.decay_rate_value,config.StepClipping_value) , 'wb') as csvfile: print "Writing results on test set..." csv_func = csv.writer(csvfile, delimiter=',') csv_func.writerow(['ID','TARGET']) for d in stream_test.get_epoch_iterator(as_dict=True): print d['id'][0] output, = pred_test(**{x: d[x] for x in ['input','product']}) for i in range(output.shape[0]): csv_func.writerow([int(test_id[i]),float(output[i])])
def main(mode, save_to, num_epochs, load_params=None, feature_maps=None, mlp_hiddens=None, conv_sizes=None, pool_sizes=None, stride=None, repeat_times=None, batch_size=None, num_batches=None, algo=None, test_set=None, valid_examples=None, dropout=None, max_norm=None, weight_decay=None, batch_norm=None): if feature_maps is None: feature_maps = [20, 50, 50] if mlp_hiddens is None: mlp_hiddens = [500] if conv_sizes is None: conv_sizes = [5, 5, 5] if pool_sizes is None: pool_sizes = [2, 2, 2] if repeat_times is None: repeat_times = [1, 1, 1] if batch_size is None: batch_size = 500 if valid_examples is None: valid_examples = 2500 if stride is None: stride = 1 if test_set is None: test_set = 'test' if algo is None: algo = 'rmsprop' if batch_norm is None: batch_norm = False image_size = (128, 128) output_size = 2 if (len(feature_maps) != len(conv_sizes) or len(feature_maps) != len(pool_sizes) or len(feature_maps) != len(repeat_times)): raise ValueError("OMG, inconsistent arguments") # Use ReLUs everywhere and softmax for the final prediction conv_activations = [Rectifier() for _ in feature_maps] mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()] convnet = LeNet(conv_activations, 3, image_size, stride=stride, filter_sizes=zip(conv_sizes, conv_sizes), feature_maps=feature_maps, pooling_sizes=zip(pool_sizes, pool_sizes), repeat_times=repeat_times, top_mlp_activations=mlp_activations, top_mlp_dims=mlp_hiddens + [output_size], border_mode='full', batch_norm=batch_norm, weights_init=Glorot(), biases_init=Constant(0)) # We push initialization config to set different initialization schemes # for convolutional layers. convnet.initialize() logging.info("Input dim: {} {} {}".format( *convnet.children[0].get_dim('input_'))) for i, layer in enumerate(convnet.layers): if isinstance(layer, Activation): logging.info("Layer {} ({})".format( i, layer.__class__.__name__)) else: logging.info("Layer {} ({}) dim: {} {} {}".format( i, layer.__class__.__name__, *layer.get_dim('output'))) single_x = tensor.tensor3('image_features') x = tensor.tensor4('image_features') single_y = tensor.lvector('targets') y = tensor.lmatrix('targets') # Training with batch_normalization(convnet): probs = convnet.apply(x) cost = (CategoricalCrossEntropy().apply(y.flatten(), probs) .copy(name='cost')) error_rate = (MisclassificationRate().apply(y.flatten(), probs) .copy(name='error_rate')) cg = ComputationGraph([cost, error_rate]) extra_updates = [] if batch_norm: # batch norm: logger.debug("Apply batch norm") pop_updates = get_batch_normalization_updates(cg) # p stands for population mean # m stands for minibatch alpha = 0.005 extra_updates = [(p, m * alpha + p * (1 - alpha)) for p, m in pop_updates] population_statistics = [p for p, m in extra_updates] if dropout: relu_outputs = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg) cg = apply_dropout(cg, relu_outputs, dropout) cost, error_rate = cg.outputs if weight_decay: logger.debug("Apply weight decay {}".format(weight_decay)) cost += weight_decay * l2_norm(cg.parameters) cost.name = 'cost' # Validation valid_probs = convnet.apply_5windows(single_x) valid_cost = (CategoricalCrossEntropy().apply(single_y, valid_probs) .copy(name='cost')) valid_error_rate = (MisclassificationRate().apply( single_y, valid_probs).copy(name='error_rate')) model = Model([cost, error_rate]) if load_params: logger.info("Loaded params from {}".format(load_params)) with open(load_params, 'r') as src: model.set_parameter_values(load_parameters(src)) # Training stream with random cropping train = DogsVsCats(("train",), subset=slice(None, 25000 - valid_examples, None)) train_str = DataStream( train, iteration_scheme=ShuffledScheme(train.num_examples, batch_size)) train_str = add_transformers(train_str, random_crop=True) # Validation stream without cropping valid = DogsVsCats(("train",), subset=slice(25000 - valid_examples, None, None)) valid_str = DataStream( valid, iteration_scheme=SequentialExampleScheme(valid.num_examples)) valid_str = add_transformers(valid_str) if mode == 'train': directory, _ = os.path.split(sys.argv[0]) env = dict(os.environ) env['THEANO_FLAGS'] = 'floatX=float32' port = numpy.random.randint(1025, 10000) server = subprocess.Popen( [directory + '/server.py', str(25000 - valid_examples), str(batch_size), str(port)], env=env, stderr=subprocess.STDOUT) train_str = ServerDataStream( ('image_features', 'targets'), produces_examples=False, port=port) save_to_base, save_to_extension = os.path.splitext(save_to) # Train with simple SGD if algo == 'rmsprop': step_rule = RMSProp(decay_rate=0.999, learning_rate=0.0003) elif algo == 'adam': step_rule = Adam() else: assert False if max_norm: conv_params = VariableFilter(bricks=[Convolutional], roles=[WEIGHT])(cg) linear_params = VariableFilter(bricks=[Linear], roles=[WEIGHT])(cg) step_rule = CompositeRule( [step_rule, Restrict(VariableClipping(max_norm, axis=0), linear_params), Restrict(VariableClipping(max_norm, axis=(1, 2, 3)), conv_params)]) algorithm = GradientDescent( cost=cost, parameters=model.parameters, step_rule=step_rule) algorithm.add_updates(extra_updates) # `Timing` extension reports time for reading data, aggregating a batch # and monitoring; # `ProgressBar` displays a nice progress bar during training. extensions = [Timing(every_n_batches=100), FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches), DataStreamMonitoring( [valid_cost, valid_error_rate], valid_str, prefix="valid"), TrainingDataMonitoring( [cost, error_rate, aggregation.mean(algorithm.total_gradient_norm)], prefix="train", after_epoch=True), TrackTheBest("valid_error_rate"), Checkpoint(save_to, save_separately=['log'], parameters=cg.parameters + (population_statistics if batch_norm else []), before_training=True, after_epoch=True) .add_condition( ['after_epoch'], OnLogRecord("valid_error_rate_best_so_far"), (save_to_base + '_best' + save_to_extension,)), Printing(every_n_batches=100)] model = Model(cost) main_loop = MainLoop( algorithm, train_str, model=model, extensions=extensions) try: main_loop.run() finally: server.terminate() elif mode == 'test': classify = theano.function([single_x], valid_probs.argmax()) test = DogsVsCats((test_set,)) test_str = DataStream( test, iteration_scheme=SequentialExampleScheme(test.num_examples)) test_str = add_transformers(test_str) correct = 0 with open(save_to, 'w') as dst: print("id", "label", sep=',', file=dst) for index, example in enumerate(test_str.get_epoch_iterator()): image = example[0] prediction = classify(image) print(index + 1, classify(image), sep=',', file=dst) if len(example) > 1 and prediction == example[1]: correct += 1 print(correct / float(test.num_examples)) else: assert False
def main(save, load, sample, path, **kwargs): input_dim = 784 hidden_dim = 2 batch_size = 100 features = tensor.matrix('features') vae = VariationalAutoEncoder(input_dim, hidden_dim, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0.)) vae.initialize() mu, logsigma, x_hat = vae.apply(features) cost = vae.cost(features) cost.name = 'cost' regularization_cost = vae.regularization_cost(mu, logsigma).mean() regularization_cost.name = 'regularization_cost' reconstruction_cost = vae.reconstruction_cost(features, x_hat).mean() reconstruction_cost.name = 'reconstruction_cost' cg = ComputationGraph([cost, reconstruction_cost, regularization_cost]) model = Model(cost) algorithm = GradientDescent(step_rule=RMSProp(1e-4), params=cg.parameters, cost=cost) extensions = [] if load: extensions.append(LoadFromDump(path)) if save: extensions.append(Dump(path, after_epoch=True)) extensions.append(FinishAfter(after_n_epochs=6001)) train_dataset = MNIST('train', binary=False, sources=('features',)) train_stream = DataStream(train_dataset, iteration_scheme=ShuffledScheme( examples=train_dataset.num_examples, batch_size=batch_size)) train_monitor = TrainingDataMonitoring( [cost, regularization_cost, reconstruction_cost], prefix='train', after_epoch=True) test_dataset = MNIST('test', binary=True, sources=('features',)) test_stream = DataStream(test_dataset, iteration_scheme=ShuffledScheme( examples=test_dataset.num_examples, batch_size=batch_size)) test_monitor = DataStreamMonitoring([cost], test_stream, prefix='test') extensions.extend([train_monitor, test_monitor]) extensions.extend([Timing(), Printing()]) main_loop = MainLoop(model=model, algorithm=algorithm, data_stream=train_stream, extensions=extensions) if not sample: main_loop.run() else: parameters = load_parameter_values(path + '/params.npz') model.set_param_values(parameters) num_samples = 10 samples = vae.sample(num_samples) samples = function([], samples)() z = tensor.matrix('z') decode_z = function([z], vae.decoder.apply(z)) from matplotlib import pyplot as plt sample = numpy.zeros((28, 0)) size = 40 z_val = numpy.zeros((size ** 2, 2)) for i in xrange(size): for j in xrange(size): z_val[i * size + j, :] = numpy.array( [i / float(0.3 * size) - .5 / .3, j / float(0.3 * size) - .5 / .3]) samples = decode_z(z_val) samples = samples.reshape((size, size, 28, 28)) samples = numpy.concatenate(samples, axis=1) samples = numpy.concatenate(samples, axis=1) plt.imshow(samples, cmap=plt.get_cmap('Greys')) plt.show() f = function([features], x_hat) for data in train_stream.get_epoch_iterator(): data_hat = f(data[0]) for image, image_hat in zip(data[0], data_hat): im = numpy.concatenate([image_hat.reshape((28, 28)), image.reshape((28, 28))]) plt.imshow(im, cmap=plt.get_cmap('Greys')) plt.show()
_filename = 'cars196/cars196.hdf5' def __init__(self, which_sets, **kwargs): try: #path = find_in_data_path(self._filename) path = "datasets/data/cars196/cars196.hdf5" except IOError as e: msg = str(e) + (""". You need to download the dataset and convert it to hdf5 before.""") raise IOError(msg) super(Cars196Dataset, self).__init__(file_or_path=path, which_sets=which_sets, **kwargs) def load_as_ndarray(which_sets=['train', 'test']): datasets = [] for split in which_sets: data = Cars196Dataset([split], load_in_memory=True).data_sources datasets.append(data) return datasets if __name__ == '__main__': dataset = Cars196Dataset(['train']) st = DataStream(dataset, iteration_scheme=SequentialScheme(dataset.num_examples, 1)) it = st.get_epoch_iterator() it.next()
scheme_train = ShuffledScheme(examples=train_set.num_examples, batch_size=conf.batch_size) scheme_val = SequentialScheme(examples=val_set.num_examples, batch_size=conf.batch_size) scheme_test = SequentialScheme(examples=test_set.num_examples, batch_size=conf.batch_size) stream_train = DataStream(dataset=train_set, iteration_scheme=scheme_train) stream_val = DataStream(dataset=val_set, iteration_scheme=scheme_val) stream_test = DataStream(dataset=test_set, iteration_scheme=scheme_test) stream_train = Mapping(data_stream = stream_train, mapping=ShuffleDim) stream_val = Mapping(data_stream = stream_val, mapping=ShuffleDim) stream_test = Mapping(data_stream = stream_test, mapping=ShuffleDim) stream_train = AddInputNoise(data_stream = stream_train, std=conf.std_input_noise) epoch_iterator=stream_train.get_epoch_iterator(as_dict = True) test_batch=next(epoch_iterator) num_features=test_batch[conf.input_theano].shape[-1] if conf.transition=='GRU': transition=GatedRecurrent elif conf.transition=='LSTM': transition=LSTM elif conf.transition=='RNN': transition=SimpleRecurrent else: raise ValueError, conf.transition
def main(): X = T.tensor3(name='input', dtype='float32') MASK = T.matrix(name='mask', dtype='float32') LABELS = T.ivector(name='labels') #load data train_set = H5PYDataset(TRAIN_FILE, which_sets=('train', )) valid_set = H5PYDataset(TRAIN_FILE, which_sets=('test', )) network = build_rnn(net_input=X, mask_input=MASK) network_output = lasagne.layers.get_output(network) val_prediction = lasagne.layers.get_output(network, deterministic=True) #needed for accuracy val_acc = T.mean(T.eq(T.argmax(val_prediction, axis=1), LABELS), dtype=theano.config.floatX) #training accuracy train_acc = T.mean(T.eq(T.argmax(network_output, axis=1), LABELS), dtype=theano.config.floatX) #T.argmax(network_output, axis=1).eval({X: d1, Mask: m1}) #print network_output.eval({X: d1, Mask: m1})[1][97] #cost function total_cost = lasagne.objectives.categorical_crossentropy( network_output, LABELS) #total_cost = -(labels*T.log(network_output) + (1-labels*T.log(1-network_output)) masked_cost = total_cost * MASK.flatten() mean_cost = total_cost.mean() #accuracy function val_cost = lasagne.objectives.categorical_crossentropy( val_prediction, LABELS) val_cost = val_cost * MASK.flatten() val_mcost = val_cost.mean() #Get parameters of both encoder and decoder all_parameters = lasagne.layers.get_all_params([network], trainable=True) print("Trainable Model Parameters") print("-" * 40) for param in all_parameters: print(param, param.get_value().shape) print("-" * 40) #add grad clipping to avoid exploding gradients all_grads = [T.clip(g, -3, 3) for g in T.grad(mean_cost, all_parameters)] all_grads = lasagne.updates.total_norm_constraint(all_grads, 3) updates = lasagne.updates.adam(all_grads, all_parameters, learning_rate=LEARNING_RATE) train_func = theano.function([X, MASK, LABELS], [mean_cost, train_acc], updates=updates) val_func = theano.function([X, MASK, LABELS], [val_mcost, val_acc]) trainerr = [] epoch = 0 #set the epoch counter min_val_loss = np.inf patience = 0 print("Starting training...") # We iterate over epochs: while 'true': # In each epoch, we do a full pass over the training data: train_err = 0 tr_acc = 0 train_batches = 0 h1 = train_set.open() h2 = valid_set.open() scheme = ShuffledScheme(examples=train_set.num_examples, batch_size=BATCH_SIZE) scheme1 = SequentialScheme(examples=valid_set.num_examples, batch_size=32) train_stream = DataStream(dataset=train_set, iteration_scheme=scheme) valid_stream = DataStream(dataset=valid_set, iteration_scheme=scheme1) start_time = time.time() for data in train_stream.get_epoch_iterator(): t_data, t_mask, _, _, _, t_labs = data t_labs = t_labs.flatten() terr, tacc = train_func(t_data, t_mask, t_labs) train_err += terr tr_acc += tacc train_batches += 1 val_err = 0 val_acc = 0 val_batches = 0 for data in valid_stream.get_epoch_iterator(): v_data, v_mask, _, _, _, v_tars = data v_tars = v_tars.flatten() err, acc = val_func(v_data, v_mask, v_tars) val_err += err val_acc += acc val_batches += 1 trainerr.append(train_err / train_batches) epoch += 1 train_set.close(h1) valid_set.close(h2) #Display if display: print("Epoch {} of {} took {:.3f}s ".format( epoch, NUM_EPOCHS, time.time() - start_time)) print(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) print(" training accuracy:\t\t{:.2f} % ".format( tr_acc / train_batches * 100)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy:\t\t{:.2f} % ".format( val_acc / val_batches * 100)) logfile = os.path.join(SAVE_PATH, MODEL_NAME, 'logs', MODEL_NAME + '.log') flog1 = open(logfile, 'ab') flog1.write('Running %s on %s\n' % (SCRIPT, MSC)) flog1.write("Epoch {} of {} took {:.3f}s\n ".format( epoch, NUM_EPOCHS, time.time() - start_time)) flog1.write(" training loss:\t\t{:.6f} ".format(train_err / train_batches)) flog1.write(" training accuracy:\t\t{:.2f} %\n ".format( tr_acc / train_batches * 100)) flog1.write(" validation loss:\t\t{:.6f}\n".format(val_err / val_batches)) flog1.write(" validation accuracy:\t\t{:.2f} %\n ".format( val_acc / val_batches * 100)) flog1.write("\n") flog1.close() if epoch == NUM_EPOCHS: break valE = val_err / val_batches #Patience if valE <= min_val_loss: model_params1 = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME + '_minloss' + '.pkl' vpth1 = os.path.join(SAVE_PATH, MODEL_NAME, 'weights', model1_name) fsave = open(vpth1, 'wb') cPickle.dump(model_params1, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close() min_val_loss = valE #reset the patience patience = 0 #Patience / Early stopping else: patience += 1 if patience == 10: break #Save the final model print('Saving Model ...') model_params = lasagne.layers.get_all_param_values(network) model1_name = MODEL_NAME + '_final' + '.pkl' vpth = os.path.join(SAVE_PATH, MODEL_NAME, 'weights', model1_name) fsave = open(vpth, 'wb') cPickle.dump(model_params, fsave, protocol=cPickle.HIGHEST_PROTOCOL) fsave.close()
dataset = CoNLLTextFile(data_paths, dictionary=word2code, unknown_token='<UNK>') data_stream = DataStream(dataset) data_stream = Filter(data_stream, _filter_long) #data_stream = Mapping(data_stream, reverse_words, add_sources=("targets",)) data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(mini_batch_size)) #data_stream = Padding(data_stream, mask_sources=('tokens')) # Adds a mask fields to this stream field, type='floatX' data_stream = Padding(data_stream, ) # Adds a mask fields to all of this stream's fields, type='floatX' data_stream = Mapping(data_stream, _transpose) # Flips stream so that sentences run down columns, batches along rows (strangely) if False: # print sample for debugging Dataset / DataStream component #t=0 max_len=0 for i, data in enumerate(data_stream.get_epoch_iterator()): #print(i) #t=t + data[4].sum() + data[0].shape[1] l=data[0].shape[0] if l>max_len: max_len = l print(i, data[0].shape, max_len) #print(data) #break exit(0) """ Comments in google-groups:blocks indicate that a reshaping has to be done, so let's think about sizes of the arrays... """