def launch_data_server(dataset, port, config):
"""
"""
n_items = dataset.num_examples
batch_sz = config.hyper_parameters.batch_size
it_schm = ShuffledScheme(n_items, batch_sz)
data_stream = DataStream(dataset=dataset, iteration_scheme=it_schm)
try:
start_server(data_stream, port=port, hwm=config.data_server.hwm)
except KeyboardInterrupt as ke:
print(ke)
finally:
data_stream.close()
def test_h5py_dataset_datastream_pickles():
try:
h5file = h5py.File(name='tmp.hdf5', mode="w")
features = h5file.create_dataset('features', (10, 5), dtype='float32')
features[...] = numpy.arange(50, dtype='float32').reshape((10, 5))
split_dict = {'train': {'features': (0, 10)}}
h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
h5file.flush()
h5file.close()
dataset = H5PYDataset(path='tmp.hdf5', which_set='train')
stream = DataStream(dataset)
pickle.loads(pickle.dumps(stream))
finally:
stream.close()
if os.path.exists('tmp.hdf5'):
os.remove('tmp.hdf5')
def train(model=None):
if model is not None:
trainset = H5PYDataset('svhn_format_2.hdf5', which_sets=('train',), sources=('features', 'targets'))
trainstream = DataStream(trainset, iteration_scheme=SequentialScheme(examples=trainset.num_examples, batch_size=500))
for data in trainstream.get_epoch_iterator():
images, labels = data
#standardize the input images
m = images.mean(axis=(2,3), keepdims=True)
s = images.std(axis=(2,3), keepdims=True)
images = (images - m)/s
#change from "channel_first" to "channel_last"
images = np.transpose(images, (0,2,3,1))
labels = keras.utils.to_categorical(labels)
#print images.shape
model.train_on_batch(x=images, y=labels)
trainstream.close()
def test_h5py_dataset_datastream_pickles():
try:
h5file = h5py.File(name='tmp.hdf5', mode="w")
features = h5file.create_dataset('features', (10, 5), dtype='float32')
features[...] = numpy.arange(50, dtype='float32').reshape((10, 5))
split_dict = {'train': {'features': (0, 10)}}
h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
h5file.flush()
h5file.close()
dataset = H5PYDataset(path='tmp.hdf5', which_set='train')
stream = DataStream(dataset)
pickle.loads(pickle.dumps(stream))
finally:
stream.close()
if os.path.exists('tmp.hdf5'):
os.remove('tmp.hdf5')
def dataset_generator(dataset, batch_size=500):
while 1:
trainstream = DataStream(dataset, iteration_scheme=ShuffledScheme(examples=dataset.num_examples, batch_size=batch_size))
for data in trainstream.get_epoch_iterator():
images, labels = data
#resize images
#images = tf.image.resize_images(images, (48,48))
#images = images.eval()
#standardize the input images
m = images.mean(axis=(1,2,3), keepdims=True)
#s = images.std(axis=(1,2,3), keepdims=True)
images = (images - m)#/ (s + 1e-3)
#change from "channel_first" to "channel_last"
images = np.transpose(images, (0,2,3,1))
#images = keras.utils.normalize(images, axis=-1, order=2)
#convert to one_hot representation
#print labels
labels = keras.utils.to_categorical(labels, num_classes=11)
#print labels
yield(images, labels)
trainstream.close()
def test_data_stream_pickling(self):
stream = DataStream(H5PYDataset(self.h5file, which_sets=('train', )),
iteration_scheme=SequentialScheme(100, 10))
cPickle.loads(cPickle.dumps(stream))
stream.close()
def test_data_stream_pickling(self):
stream = DataStream(H5PYDataset(self.h5file, which_sets=('train',)),
iteration_scheme=SequentialScheme(100, 10))
cPickle.loads(cPickle.dumps(stream))
stream.close()
def test_data_stream_pickling(self):
stream = DataStream(H5PYDataset(self.h5file, which_set='train'))
cPickle.loads(cPickle.dumps(stream))
stream.close()