def get_data_specs(self):
axes = ('b', 0, 1, 'c')
return (CompositeSpace([
Conv2DSpace(shape=(96, 96), num_channels=3, axes=axes),
MatrixSpace(196, 96)
]), ('features', 'targets'))
def _update_topo_space(self):
"""Update self.topo_space from self.shape and self.axes"""
rows, cols, channels = self.shape
self.topo_space = Conv2DSpace(shape=(rows, cols),
num_channels=channels,
axes=self.axes)
def set_input_space(self, space):
""" Note: this resets parameters! """
# set up detector space and initialize transformer
setup_detector_layer_b01tc(layer=self,
input_space=space,
rng=self.mlp.rng,
irange=self.irange)
rng = self.mlp.rng
detector_shape = self.detector_space.shape
#def handle_pool_shape(idx):
# if self.pool_shape[idx] < 1:
# raise ValueError("bad pool shape: " + str(self.pool_shape))
# if self.pool_shape[idx] > detector_shape[idx]:
# if self.fix_pool_shape:
# assert detector_shape[idx] > 0
# self.pool_shape[idx] = detector_shape[idx]
# else:
# raise ValueError("Pool shape exceeds detector layer shape on axis %d" % idx)
#map(handle_pool_shape, [0, 1, 2])
### Check some precondition
assert self.pool_shape[0] == self.pool_shape[1]
assert self.pool_stride[0] == self.pool_stride[1]
assert all(
isinstance(elem, py_integer_types) for elem in self.pool_stride)
for i in xrange(0, 2):
assert self.pool_stride[i] <= self.pool_shape[i]
assert all(
isinstance(elem, py_integer_types) for elem in self.pool_stride)
dummy_shape = [self.input_space.shape[0], self.input_space.shape[1]]
# added to find out output space shape after temporal and spatial pooling "max_pool_c01b"
dummy_output_shape = [
int(np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))) + 1
for i_sh, k_sh, k_st in zip(dummy_shape, self.kernel_shape,
self.kernel_stride)
]
dummy_output_shape = [dummy_output_shape[0], dummy_output_shape[1]]
#print dummy_output_shape
dummy_detector_space = Conv2DSpace(shape=dummy_output_shape,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
# picked only 16 channels and 1 image in order to do a fast dummy maxpooling (16 because Alex's code needs at least 16 channels)
dummy_detector = sharedX(
dummy_detector_space.get_origin_batch(2)[0:16, :, :, :])
dummy_p = max_pool_c01b(c01b=dummy_detector,
pool_shape=self.pool_shape,
pool_stride=self.pool_stride)
dummy_p = dummy_p.eval()
# set space after temporal pooling with overlap
if self.pool_temporal_stride[1] > self.pool_temporal_shape[1]:
if self.fix_pool_stride:
warnings.warn("Fixing the pool stride")
ps = self.pool_temporal_shape[1]
assert isinstance(ps, py_integer_types)
self.pool_stride = [1, ps]
else:
raise ValueError("Stride too big.")
# (0*1,'t')
dummy_temp_image = [(dummy_p.shape[1] * dummy_p.shape[2]),
self.detector_space.shape[2]]
#overlapped temporal max pooling image_shape
self.temp_pool_input_shape = dummy_temp_image
dummy_temp_space = Conv2DSpace(shape=dummy_temp_image,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
temp_input = sharedX(
dummy_temp_space.get_origin_batch(2)[0:16, :, :, :])
dummy_temp_p = temporal_max_pool_c01b(
c01b=temp_input,
pool_shape=self.pool_temporal_shape,
pool_stride=self.pool_temporal_stride,
image_shape=dummy_temp_image)
dummy_temp_p = dummy_temp_p.eval()
self.output_space = Conv3DSpace(
shape=[dummy_p.shape[1], dummy_p.shape[2], dummy_temp_p.shape[2]],
num_channels=self.num_channels,
axes=('b', 0, 1, 't', 'c'))
# Print spaces
print "Input shape: ", self.input_space.shape
print "Detector space: ", self.detector_space.shape
print "Output space: ", self.output_space.shape
if retrain:
print "training on", X_train.X.shape, 'testing on', X_test.X.shape
trainer = sgd.SGD(learning_rate=learn_rate, batch_size=batch_size,
learning_rule=learning_rule.Momentum(momentum_start),
cost=Dropout(
input_include_probs={'l1':1., 'l2':1., 'l3':1., 'l4':1., 'l5':1., 'l6':1.},
input_scales={'l1':1., 'l2':1., 'l3':1., 'l4':1., 'l5':1., 'l6':1.}
),
termination_criterion=EpochCounter(max_epochs=max_epochs),
monitoring_dataset={'train':X_train, 'valid':X_test},
)
input_space = Conv2DSpace(shape=(central_window_shape, central_window_shape),
axes = axes,
num_channels = 1)
ann = mlp.MLP(layers, input_space=input_space)
velocity = learning_rule.MomentumAdjustor(final_momentum=momentum_end,
start=1,
saturate=momentum_saturate)
watcher = best_params.MonitorBasedSaveBest(channel_name='valid_y_nll',
save_path=save_path)
decay = sgd.LinearDecayOverEpoch(start=1, saturate=decay_saturate, decay_factor=decay_factor)
ra = RealtimeAugment(window_shape=[img_dim, img_dim], randomize=[X_train, X_test],
scale_diff=scale_diff, translation=translation, center_shape=center_shape, center=[X_train, X_test],
def setup_deconv_detector_layer_c01b(layer,
input_space,
rng,
irange="not specified"):
"""
layer. This function sets up only the detector layer.
Does the following:
* raises a RuntimeError if cuda is not available
* sets layer.input_space to input_space
* sets up addition of dummy channels for compatibility with cuda-convnet:
- layer.dummy_channels: # of dummy channels that need to be added
(You might want to check this and raise an Exception if it's not 0)
- layer.dummy_space: The Conv2DSpace representing the input with dummy
channels added
* sets layer.detector_space to the space for the detector layer
* sets layer.transformer to be a Conv2D instance
* sets layer.b to the right value
Parameters
----------
layer : object
Any python object that allows the modifications described below and
has the following attributes:
* pad : int describing amount of zero padding to add
* kernel_shape : 2-element tuple or list describing spatial shape of
kernel
* fix_kernel_shape : bool, if true, will shrink the kernel shape to
make it feasible, as needed (useful for hyperparameter searchers)
* detector_channels : The number of channels in the detector layer
* init_bias : numeric constant added to a tensor of zeros to
initialize the bias
* tied_b : If true, biases are shared across all spatial locations
input_space : WRITEME
A Conv2DSpace to be used as input to the layer
rng : WRITEME
A numpy RandomState or equivalent
"""
if irange != "not specified":
raise AssertionError(
"There was a bug in setup_detector_layer_c01b."
"It uses layer.irange instead of the irange parameter to the "
"function. The irange parameter is now disabled by this "
"AssertionError, so that this error message can alert you that "
"the bug affected your code and explain why the interface is "
"changing. The irange parameter to the function and this "
"error message may be removed after April 21, 2014.")
# Use "self" to refer to layer from now on, so we can pretend we're
# just running in the set_input_space method of the layer
self = layer
# Make sure cuda is available
check_cuda(str(type(self)))
# Validate input
if not isinstance(input_space, Conv2DSpace):
raise TypeError("The input to a convolutional layer should be a "
"Conv2DSpace, but layer " + self.layer_name + " got " +
str(type(self.input_space)))
if not hasattr(self, 'detector_channels'):
raise ValueError("layer argument must have a 'detector_channels' "
"attribute specifying how many channels to put in "
"the convolution kernel stack.")
# Store the input space
self.input_space = input_space
# Make sure number of channels is supported by cuda-convnet
# (multiple of 4 or <= 3)
# If not supported, pad the input with dummy channels
ch = self.detector_channels
rem = ch % 4
if ch > 3 and rem != 0:
raise NotImplementedError("Need to do dummy channels on the output")
# self.dummy_channels = 4 - rem
#else:
# self.dummy_channels = 0
#self.dummy_space = Conv2DSpace(
# shape=input_space.shape,
# channels=input_space.num_channels + self.dummy_channels,
# axes=('c', 0, 1, 'b')
#)
if hasattr(self, 'output_stride'):
kernel_stride = self.output_stride
else:
assert False # not sure if I got the name right, remove this assert if I did
kernel_stride = [1, 1]
#o_sh = int(np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))) + 1
#o_sh -1 = np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))
#inv_ceil(o_sh -1) = (i_sh + 2. * self.pad - k_sh) / float(k_st)
#float(k_st) inv_cel(o_sh -1) = (i_sh + 2 * self.pad -k_sh)
# i_sh = k_st inv_ceil(o_sh-1) - 2 * self.pad + k_sh
output_shape = \
[k_st * (i_sh - 1) - 2 * self.pad_out + k_sh
for i_sh, k_sh, k_st in zip(self.input_space.shape,
self.kernel_shape, kernel_stride)]
if self.input_space.num_channels < 16:
raise ValueError("Cuda-convnet requires the input to lmul_T to have "
"at least 16 channels.")
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
if hasattr(self, 'partial_sum'):
partial_sum = self.partial_sum
else:
partial_sum = 1
if hasattr(self, 'sparse_init') and self.sparse_init is not None:
self.transformer = \
checked_call(make_sparse_random_conv2D,
OrderedDict([('num_nonzero', self.sparse_init),
('input_space', self.detector_space),
('output_space', self.input_space),
('kernel_shape', self.kernel_shape),
('pad', self.pad),
('partial_sum', partial_sum),
('kernel_stride', kernel_stride),
('rng', rng)]))
else:
self.transformer = make_random_conv2D(
irange=self.irange,
input_axes=self.detector_space.axes,
output_axes=self.input_space.axes,
input_channels=self.detector_space.num_channels,
output_channels=self.input_space.num_channels,
kernel_shape=self.kernel_shape,
pad=self.pad_out,
partial_sum=partial_sum,
kernel_stride=kernel_stride,
rng=rng,
input_shape=self.detector_space.shape)
W, = self.transformer.get_params()
W.name = self.layer_name + '_W'
if self.tied_b:
self.b = sharedX(
np.zeros(self.detector_space.num_channels) + self.init_bias)
else:
self.b = sharedX(self.detector_space.get_origin() + self.init_bias)
self.b.name = self.layer_name + '_b'
logger.info('Input shape: {0}'.format(self.input_space.shape))
print layer.layer_name + ' detector space: {0}'.format(
self.detector_space.shape)
from pylearn2.train_extensions import best_params, window_flip
from pylearn2.utils import serial
trn = kaggle_cifar10('train',
one_hot=True,
datapath='/home/kkastner/kaggle_data/kaggle-cifar10',
max_count=40000,
axes=('c', 0, 1, 'b'))
tst = cifar10.CIFAR10('test',
toronto_prepro=False,
one_hot=True,
axes=('c', 0, 1, 'b'))
in_space = Conv2DSpace(shape=(32, 32),
num_channels=3,
axes=('c', 0, 1, 'b'))
l1 = maxout.MaxoutConvC01B(layer_name='l1',
pad=4,
tied_b=1,
W_lr_scale=.05,
b_lr_scale=.05,
num_channels=96,
num_pieces=2,
kernel_shape=(8, 8),
pool_shape=(4, 4),
pool_stride=(2, 2),
irange=.005,
max_kernel_norm=.9,
partial_sum=33)
if isinstance(space, VectorSpace):
# For some reason format_as from VectorSpace is not working right
samples = model.generator.mlp.fprop(Z).eval()
is_color = samples.shape[-1] % 3 == 0 and samples.shape[-1] != 48 * 48
samples = dataset.get_topological_view(samples)
else:
total_dimension = space.get_total_dimension()
import numpy as np
num_colors = 1
if total_dimension % 3 == 0:
num_colors = 3
w = int(np.sqrt(total_dimension / num_colors))
from pylearn2.space import Conv2DSpace
desired_space = Conv2DSpace(shape=[w, w], num_channels=num_colors, axes=('b',0,1,'c'))
samples = space.format_as(batch=model.generator.mlp.fprop(Z),
space=desired_space).eval()
is_color = samples.shape[-1] == 3
from pylearn2.utils import image
for i in xrange(endpoints * steps_per_point):
img = samples[i, :, :, :]
img /= np.abs(img).max()
number = str(i)
while len(number) < len(str(endpoints * steps_per_point)):
number = '0' + number
path = 'video/' + number + '.png'
image.save(path, img)
def get_maxout(dim_input):
config = {
'batch_size':
bsize,
'input_space':
Conv2DSpace(shape=dim_input[:2],
num_channels=dim_input[2],
axes=['c', 0, 1, 'b']),
'layers': [
MaxoutConvC01B(layer_name='h0',
num_channels=96,
num_pieces=2,
irange=.005,
tied_b=1,
max_kernel_norm=.9,
kernel_shape=[8, 8],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=.05,
b_lr_scale=.05),
MaxoutConvC01B(layer_name='h1',
num_channels=128,
num_pieces=2,
irange=.005,
tied_b=1,
max_kernel_norm=0.9,
kernel_shape=[7, 7],
pad=3,
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=.05,
b_lr_scale=.05),
MaxoutConvC01B(layer_name='h2',
num_channels=160,
num_pieces=3,
irange=.005,
tied_b=1,
max_kernel_norm=0.9,
kernel_shape=[6, 6],
pad=2,
pool_shape=[2, 2],
pool_stride=[2, 2],
W_lr_scale=.05,
b_lr_scale=.05),
MaxoutConvC01B(layer_name='h3',
num_channels=192,
num_pieces=4,
irange=.005,
tied_b=1,
max_kernel_norm=0.9,
kernel_shape=[5, 5],
pad=1,
pool_shape=[2, 2],
pool_stride=[2, 2],
W_lr_scale=.05,
b_lr_scale=.05),
Maxout(layer_name='h4',
irange=.005,
num_units=500,
num_pieces=5,
max_col_norm=1.9),
Softmax(layer_name='y',
n_classes=nclass,
irange=.005,
max_col_norm=1.9)
]
}
return MLP(**config)
# End of test_dtype_setter() function
shape = np.array([2, 3, 4], dtype='int')
assert len(shape) == 3 # This test depends on this being true
dtypes = ('floatX', None) + all_scalar_dtypes
#
# spaces with the same number of elements
#
vector_spaces = tuple(
VectorSpace(dim=shape.prod(), dtype=dt, sparse=s) for dt in dtypes
for s in (True, False))
conv2d_spaces = tuple(
Conv2DSpace(shape=shape[:2], dtype=dt, num_channels=shape[2])
for dt in dtypes)
# no need to make CompositeSpaces with components spanning all possible
# dtypes. Just try 2 dtype combos. No need to try different sparsities
# either. That will be tested by the non-composite space conversions.
n_dtypes = 2
old_nchannels = shape[2]
shape[2] = old_nchannels / 2
assert shape[2] * 2 == old_nchannels, \
("test code is broken: # of channels should start as an even "
"number, not %d." % old_nchannels)
def make_composite_space(dtype0, dtype1, use_conv2d):
if use_conv2d:
second_space = Conv2DSpace(shape=shape[:2],
def __init__(
self,
db, # data source
name='', # optional name
selectors=dict(),
partitioner=None,
meta_sources=[], # optional sources other than 'features' and 'targets' from metadata
channel_filter=NoChannelFilter(
), # optional channel filter, default: keep all
channel_names=None, # optional channel names (for metadata)
label_attribute='label', # metadata attribute to be used as label
label_map=None, # optional conversion of labels
use_targets=True, # use targets if provides, otherwise labels are used
remove_dc_offset=False, # optional subtraction of channel mean, usually done already earlier
resample=None, # optional down-sampling
normalize=True, # normalize to max=1
# optional sub-sequences selection
start_sample=0,
stop_sample=None, # optional for selection of sub-sequences
zero_padding=True, # if True (default) trials that are too short will be padded with
# otherwise they will rejected.
# optional signal filter to by applied before splitting the signal
signal_filter=None,
trial_processors=[], # optional processing of the trials
target_processor=None, # optional processing of the targets, e.g. zero-padding
transformers=[], # optional transformations of the dataset
layout='tf', # (0,1)-axes layout tf=time x features or ft=features x time
debug=False,
):
'''
Constructor
'''
# save params
self.params = locals().copy()
del self.params['self']
# print self.params
self.name = name
self.debug = debug
metadb = DatasetMetaDB(db.metadata, selectors.keys())
if partitioner is not None:
pass # FIXME
selected_trial_ids = metadb.select(selectors)
log.info('selectors: {}'.format(selectors))
log.info('selected trials: {}'.format(selected_trial_ids))
if normalize:
log.info(
'Data will be normalized to max amplitude 1 per channel (normalize=True).'
)
trials = list()
labels = list()
targets = list()
meta = list()
if stop_sample == 'auto-min':
stop_sample = np.min(
[db.data[trial_i].shape[-1] for trial_i in selected_trial_ids])
log.info('Using minimum trial length. stop_sample={}'.format(
stop_sample))
elif stop_sample == 'auto-max':
stop_sample = np.max(
[db.data[trial_i].shape[-1] for trial_i in selected_trial_ids])
log.info('Using maximum trial length. stop_sample={}'.format(
stop_sample))
for trial_i in selected_trial_ids:
trial_meta = db.metadata[trial_i]
if use_targets:
if targets is None:
target = None
else:
target = db.targets[trial_i]
assert not np.isnan(np.sum(target))
if target_processor is not None:
target = target_processor.process(target, trial_meta)
assert not np.isnan(np.sum(target))
else:
# get and process label
label = db.metadata[trial_i][label_attribute]
if label_map is not None:
label = label_map[label]
processed_trial = []
trial = db.data[trial_i]
if np.isnan(np.sum(trial)):
print trial_i, trial
assert not np.isnan(np.sum(trial))
rejected = False # flag for trial rejection
trial = np.atleast_2d(trial)
# process 1 channel at a time
for channel in xrange(trial.shape[0]):
# filter channels
if not channel_filter.keep_channel(channel):
continue
samples = trial[channel, :]
# subtract channel mean
if remove_dc_offset:
samples -= samples.mean()
# down-sample if requested
if resample is not None and resample[0] != resample[1]:
samples = librosa.resample(samples,
resample[0],
resample[1],
res_type='sinc_best')
# apply optional signal filter after down-sampling -> requires lower order
if signal_filter is not None:
samples = signal_filter.process(samples)
# get sub-sequence in resampled space
# log.info('using samples {}..{} of {}'.format(start_sample,stop_sample, samples.shape))
if stop_sample is not None and stop_sample > len(samples):
if zero_padding:
tmp = np.zeros(stop_sample)
tmp[:len(samples)] = samples
samples = tmp
else:
rejected = True
break # stop processing this trial
s = samples[start_sample:stop_sample]
# TODO optional channel processing
# normalize to max amplitude 1
if normalize:
s = librosa.util.normalize(s)
# add 2nd data dimension
s = s.reshape(s.shape[0], 1)
# print s.shape
s = np.asfarray(s, dtype=theano.config.floatX)
processed_trial.append(s)
### end of channel iteration ###
if rejected:
continue # next trial
processed_trial = np.asfarray([processed_trial],
dtype=theano.config.floatX)
# processed_trial = processed_trial.reshape((1, processed_trial.shape))
processed_trial = np.rollaxis(processed_trial, 1, 4)
# optional (external) trial processing, e.g. windowing
# trials will be in b01c format with tf layout for 01-axes
for trial_processor in trial_processors:
processed_trial = trial_processor.process(
processed_trial, trial_meta)
trials.append(processed_trial)
for k in range(len(processed_trial)):
meta.append(trial_meta)
if use_targets:
targets.append(target)
else:
labels.append(label)
### end of datafile iteration ###
# turn into numpy arrays
self.trials = np.vstack(trials)
assert not np.isnan(np.sum(self.trials))
# prepare targets / labels
if use_targets:
self.targets = np.vstack(targets)
assert not np.isnan(np.sum(self.targets))
else:
labels = np.hstack(labels)
if label_map is None:
one_hot_formatter = OneHotFormatter(max(labels) + 1)
else:
one_hot_formatter = OneHotFormatter(
max(label_map.values()) + 1)
one_hot_y = one_hot_formatter.format(labels)
self.targets = one_hot_y
self.metadata = meta
if layout == 'ft': # swap axes to (batch, feature, time, channels)
self.trials = self.trials.swapaxes(1, 2)
# transform after finalizing the data structure
for transformer in transformers:
self.trials, self.targets = transformer.process(
self.trials, self.targets)
self.trials = np.asarray(self.trials, dtype=theano.config.floatX)
log.debug('final dataset shape: {} (b,0,1,c)'.format(
self.trials.shape))
# super(EEGEpochsDataset, self).__init__(topo_view=self.trials, y=self.targets, axes=['b', 0, 1, 'c'])
self.X = self.trials.reshape(self.trials.shape[0],
np.prod(self.trials.shape[1:]))
self.y = self.targets
log.info('generated dataset "{}" with shape X={}={} y={} targets={} '.
format(self.name, self.X.shape, self.trials.shape,
self.y.shape, self.targets.shape))
# determine data specs
features_space = Conv2DSpace(
shape=[self.trials.shape[1], self.trials.shape[2]],
num_channels=self.trials.shape[3])
features_source = 'features'
targets_space = VectorSpace(dim=self.targets.shape[-1])
targets_source = 'targets'
space_components = [features_space, targets_space]
source_components = [features_source, targets_source]
# additional support for meta information
self.meta_maps = dict()
for meta_source in meta_sources:
self.meta_maps[meta_source] = sorted(
list(set([m[meta_source] for m in self.metadata])))
space_components.extend([VectorSpace(dim=1)])
source_components.extend([meta_source])
log.info('Generated meta-source "{}" with value map: {}'.format(
meta_source, self.meta_maps[meta_source]))
space = CompositeSpace(space_components)
source = tuple(source_components)
self.data_specs = (space, source)
log.debug('data specs: {}'.format(self.data_specs))
def set_input_space(self, space):
""" Note: this resets parameters! """
self.input_space = space
rng = self.mlp.rng
if self.border_mode == 'valid':
output_shape = [(self.input_space.shape[0] - self.kernel_shape[0]) / self.kernel_stride[0] + 1,
(self.input_space.shape[1] - self.kernel_shape[1]) / self.kernel_stride[1] + 1]
elif self.border_mode == 'full':
output_shape = [(self.input_space.shape[0] + self.kernel_shape[0]) / self.kernel_stride[0] - 1,
(self.input_space.shape[1] + self.kernel_shape[1]) / self.kernel_stride_stride[1] - 1]
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels = self.output_channels,
axes = ('b', 'c', 0, 1))
if not (self.can_alter_transformer and hasattr(self, "transformer")
and self.transformer is not None):
if self.irange is not None:
self.transformer = conv2d.make_random_conv2D(
irange = self.irange,
input_space = self.input_space,
output_space = self.detector_space,
kernel_shape = self.kernel_shape,
batch_size = self.mlp.batch_size,
subsample = self.kernel_stride,
border_mode = self.border_mode,
rng = rng)
else:
filters_shape = self.transformer._filters.get_value().shape
if (self.input_space.filters_shape[-2:-1] != self.kernel_shape or
self.input_space.filters_shape != filters_shape):
raise ValueError("The filters and input space don't have compatible input space.")
if self.input_space.num_channels != filters_shape[1]:
raise ValueError("The filters and input space don't have compatible number of channels.")
W, = self.transformer.get_params()
W.name = 'W'
if not (self.can_alter_biases and hasattr(self, "b")
and self.b is not None):
self.b = sharedX(self.detector_space.get_origin() + self.init_bias)
self.b.name = 'b'
print 'Input shape: ', self.input_space.shape
print 'Detector space: ', self.detector_space.shape
dummy_batch_size = self.mlp.batch_size
if dummy_batch_size is None:
dummy_batch_size = 2
dummy_detector = sharedX(self.detector_space.get_origin_batch(dummy_batch_size))
if self.pool_type is not None:
assert self.pool_type in ['max', 'mean']
if self.pool_type == 'max':
dummy_p = max_pool(bc01=dummy_detector, pool_shape=self.pool_shape,
pool_stride=self.pool_stride,
image_shape=self.detector_space.shape)
elif self.pool_type == 'mean':
dummy_p = mean_pool(bc01=dummy_detector, pool_shape=self.pool_shape,
pool_stride=self.pool_stride,
image_shape=self.detector_space.shape)
dummy_p = dummy_p.eval()
self.output_space = Conv2DSpace(shape=[dummy_p.shape[2], dummy_p.shape[3]],
num_channels = self.output_channels, axes = ('b', 'c', 0, 1) )
else:
dummy_detector = dummy_detector.eval()
self.output_space = Conv2DSpace(shape=[dummy_detector.shape[2], dummy_detector.shape[3]],
num_channels = self.output_channels, axes = ('b', 'c', 0, 1) )
print 'Output space: ', self.output_space.shape
def set_input_space(self, space):
""" Note: this resets parameters! """
self.input_space = space
rng = self.mlp.rng
if self.border_mode == 'valid':
output_shape = [
self.input_space.shape[0] - self.kernel_shape[0] + 1,
self.input_space.shape[1] - self.kernel_shape[1] + 1
]
elif self.border_mode == 'full':
output_shape = [
self.input_space.shape[0] + self.kernel_shape[0] - 1,
self.input_space.shape[1] + self.kernel_shape[1] - 1
]
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels=self.output_channels,
axes=('b', 'c', 0, 1))
if self.irange is not None:
assert self.sparse_init is None
self.transformer = conv2d.make_random_conv2D(
irange=self.irange,
input_space=self.input_space,
output_space=self.detector_space,
kernel_shape=self.kernel_shape,
batch_size=self.mlp.batch_size,
subsample=(1, 1),
border_mode=self.border_mode,
rng=rng)
elif self.sparse_init is not None:
self.transformer = conv2d.make_sparse_random_conv2D(
num_nonzero=self.sparse_init,
input_space=self.input_space,
output_space=self.detector_space,
kernel_shape=self.kernel_shape,
batch_size=self.mlp.batch_size,
subsample=(1, 1),
border_mode=self.border_mode,
rng=rng)
W, = self.transformer.get_params()
W.name = 'W'
self.b = sharedX(self.detector_space.get_origin() + self.init_bias)
self.b.name = 'b'
print 'Input shape: ', self.input_space.shape
print 'Detector space: ', self.detector_space.shape
if self.mlp.batch_size is None:
raise ValueError(
"Tried to use a convolutional layer with an MLP that has "
"no batch size specified. You must specify the batch size of the "
"model because theano requires the batch size to be known at "
"graph construction time for convolution.")
assert self.pool_type in ['max', 'mean']
dummy_detector = sharedX(
self.detector_space.get_origin_batch(self.mlp.batch_size))
if self.pool_type == 'max':
dummy_p = max_pool(bc01=dummy_detector,
pool_shape=self.pool_shape,
pool_stride=self.pool_stride,
image_shape=self.detector_space.shape)
elif self.pool_type == 'mean':
dummy_p = mean_pool(bc01=dummy_detector,
pool_shape=self.pool_shape,
pool_stride=self.pool_stride,
image_shape=self.detector_space.shape)
dummy_p = dummy_p.eval()
self.output_space = Conv2DSpace(
shape=[dummy_p.shape[2], dummy_p.shape[3]],
num_channels=self.output_channels,
axes=('b', 'c', 0, 1))
print 'Output space: ', self.output_space.shape
def check_case(conv_nonlinearity, mlp_nonlinearity, cost_implemented=True):
"""Check that ConvNonLinearity and MLPNonlinearity are consistent.
This is done by building an MLP with a ConvElemwise layer with the
supplied non-linearity, an MLP with a dense layer, and checking that
the outputs (and costs if applicable) are consistent.
Parameters
----------
conv_nonlinearity: instance of `ConvNonlinearity`
The non-linearity to provide to a `ConvElemwise` layer.
mlp_nonlinearity: subclass of `mlp.Linear`
The fully-connected MLP layer (including non-linearity).
check_implemented: bool
If `True`, check that both costs give consistent results.
If `False`, check that both costs raise `NotImplementedError`.
"""
# Create fake data
np.random.seed(12345)
r = 31
s = 21
shape = [r, s]
nvis = r * s
output_channels = 13
batch_size = 103
x = np.random.rand(batch_size, r, s, 1)
y = np.random.randint(2, size=[batch_size, output_channels, 1, 1])
x = x.astype(config.floatX)
y = y.astype(config.floatX)
x_mlp = x.flatten().reshape(batch_size, nvis)
y_mlp = y.flatten().reshape(batch_size, output_channels)
# Initialize convnet with random weights.
conv_model = MLP(input_space=Conv2DSpace(shape=shape,
axes=['b', 0, 1, 'c'],
num_channels=1),
layers=[
ConvElemwise(layer_name='conv',
nonlinearity=conv_nonlinearity,
output_channels=output_channels,
kernel_shape=shape,
pool_shape=[1, 1],
pool_stride=shape,
irange=1.0)
],
batch_size=batch_size)
X = conv_model.get_input_space().make_theano_batch()
Y = conv_model.get_target_space().make_theano_batch()
Y_hat = conv_model.fprop(X)
g = theano.function([X], Y_hat)
# Construct an equivalent MLP which gives the same output
# after flattening both.
mlp_model = MLP(layers=[
mlp_nonlinearity(dim=output_channels, layer_name='mlp', irange=1.0)
],
batch_size=batch_size,
nvis=nvis)
W, b = conv_model.get_param_values()
W_mlp = np.zeros(shape=(output_channels, nvis), dtype=config.floatX)
for k in range(output_channels):
W_mlp[k] = W[k, 0].flatten()[::-1]
W_mlp = W_mlp.T
b_mlp = b.flatten()
mlp_model.set_param_values([W_mlp, b_mlp])
X1 = mlp_model.get_input_space().make_theano_batch()
Y1 = mlp_model.get_target_space().make_theano_batch()
Y1_hat = mlp_model.fprop(X1)
f = theano.function([X1], Y1_hat)
# Check that the two models give the same output
assert_allclose(f(x_mlp).flatten(), g(x).flatten(), rtol=1e-5, atol=5e-5)
if cost_implemented:
# Check that the two models have the same costs
mlp_cost = theano.function([X1, Y1], mlp_model.cost(Y1, Y1_hat))
conv_cost = theano.function([X, Y], conv_model.cost(Y, Y_hat))
assert_allclose(conv_cost(x, y), mlp_cost(x_mlp, y_mlp))
else:
# Check that both costs are not implemented
assert_raises(NotImplementedError, conv_model.cost, Y, Y_hat)
assert_raises(NotImplementedError, mlp_model.cost, Y1, Y1_hat)
'bs': 128,
'dw': 1,
'dh': 1,
}]
for i in range(4):
run = runs[i]
# params for run:
ni, no, kw, kh, bs, iw, ih, dw, dh = run['ni'], run['no'], run['kw'], run[
'kh'], run['bs'], run['iw'], run['ih'], run['dw'], run['dh']
print ''
print 'CONFIG: input =', ni, 'x', iw, 'x', ih, '* ker =', ni, 'x', no, 'x', kw, 'x', kh, '( bs =', bs, ', stride =', dw, ')'
conv = MLP(batch_size=bs,
input_space=Conv2DSpace((ih, iw),
num_channels=ni,
axes=('b', 'c', 0, 1)),
layers=[
ConvElemwise(no, (kw, kh),
'ConvTest',
ConvNonlinearity(),
irange=0.1)
])
inputBatch = np.random.randn(bs, ni, ih, iw)
sharedX = theano.shared(inputBatch.astype('float32'))
sharedY = theano.shared(
np.random.randn(bs, no, (ih - kh) / dh + 1,
(iw - kw) / dw + 1).astype('float32'))
X = theano.tensor.tensor4()
def __init__(self, rows, cols, channels):
dim = rows * cols * channels
self.input_space = Conv2DSpace((rows, cols), channels)
self.dim = dim
rng = np.random.RandomState([2012,9,25])
self.P = sharedX( rng.uniform(-1.,1.,(dim,)))
def test_np_format_as_composite_composite():
"""
Test using CompositeSpace.np_format_as() to convert between
composite spaces that have the same tree structure, but different
leaf spaces.
"""
def make_composite_space(image_space):
"""
Returns a compsite space with a particular tree structure.
"""
return CompositeSpace((CompositeSpace(
(image_space, ) * 2), VectorSpace(dim=1)))
shape = np.array([8, 11])
channels = 3
datum_size = channels * shape.prod()
composite_topo = make_composite_space(
Conv2DSpace(shape=shape, num_channels=channels))
composite_flat = make_composite_space(VectorSpace(dim=datum_size))
def make_vector_data(batch_size, space):
"""
Returns a batch of synthetic data appropriate to the provided space.
Supports VectorSpaces, and CompositeSpaces of VectorSpaces. synthetic
data.
"""
if isinstance(space, CompositeSpace):
return tuple(
make_vector_data(batch_size, subspace)
for subspace in space.components)
else:
assert isinstance(space, VectorSpace)
result = np.random.rand(batch_size, space.dim)
if space.dtype is not None:
return np.asarray(result, dtype=space.dtype)
else:
return result
batch_size = 5
flat_data = make_vector_data(batch_size, composite_flat)
composite_flat.np_validate(flat_data)
topo_data = composite_flat.np_format_as(flat_data, composite_topo)
composite_topo.np_validate(topo_data)
new_flat_data = composite_topo.np_format_as(topo_data, composite_flat)
def get_shape(batch):
"""
Returns the (nested) shape(s) of a (nested) batch.
"""
if isinstance(batch, np.ndarray):
return batch.shape
else:
return tuple(get_shape(b) for b in batch)
def batch_equals(batch_0, batch_1):
"""
Returns true if all corresponding elements of two batches are
equal. Supports composite data (i.e. nested tuples of data).
"""
assert type(batch_0) == type(batch_1)
if isinstance(batch_0, tuple):
if len(batch_0) != len(batch_1):
return False
return np.all(
tuple(
batch_equals(b0, b1) for b0, b1 in zip(batch_0, batch_1)))
else:
assert isinstance(batch_0, np.ndarray)
return np.all(batch_0 == batch_1)
assert batch_equals(new_flat_data, flat_data)
def load_dataset(which_set, dataset_types):
# we need to have at least 2 types otherwise this func is useless
assert len(dataset_types) > 1
print "loading.. ", which_set
if which_set == 'test':
start_set = 0
stop_set = 10000
elif which_set == 'valid':
which_set = 'train'
start_set = 40000
stop_set = 50000
else:
#train
start_set = 0
stop_set = 40000
n_classes = 10
data = []
for prepro in dataset_types:
if prepro == 'gcn':
print "LOADING GCN..."
input_data = CIFAR10(which_set=which_set,
start=start_set,
stop=stop_set,
gcn=55.,
axes=['b', 0, 1, 'c'])
# gcn_data = input_data.get_topological_view()
data.append(input_data.get_topological_view())
if prepro == 'toronto':
print "LOADING TOR..."
input_data = CIFAR10(which_set=which_set,
start=start_set,
stop=stop_set,
axes=['b', 0, 1, 'c'],
toronto_prepro=1)
# tor_data = input_data.get_topological_view()
data.append(input_data.get_topological_view())
if prepro == 'zca':
print "LOADING ZCA..."
data_dir = string_utils.preprocess('${PYLEARN2_DATA_PATH}/cifar10')
input_data = ZCA_Dataset(
preprocessed_dataset=serial.load(data_dir +
"/pylearn2_gcn_whitened/" +
which_set + ".pkl"),
preprocessor=serial.load(
data_dir + "/pylearn2_gcn_whitened/preprocessor.pkl"),
start=start_set,
stop=stop_set,
axes=['b', 0, 1, 'c'])
# zca_data = input_data.get_topological_view()
data.append(input_data.get_topological_view())
target_data = OneHotFormatter(n_classes).format(input_data.y,
mode="concatenate")
data.append(target_data)
data_source = []
for i in range(len(dataset_types)):
data_source.append('features' + str(i))
data_source.append('targets')
################################## DEFINE SPACES ##################################
spaces = []
# add input spaces as b01c
for i in range(0, len(dataset_types)):
spaces.append(
Conv2DSpace(shape=(32, 32), num_channels=3, axes=('b', 0, 1, 'c')))
# add output space
spaces.append(VectorSpace(n_classes))
set = VectorSpacesDataset(tuple(data),
(CompositeSpace(spaces), tuple(data_source)))
return set
def set_input_space(self, space):
self.input_space = space
if not isinstance(space, Conv2DSpace):
raise BadInputSpaceError("ConvRectifiedLinear.set_input_space "
"expected a Conv2DSpace, got " +
str(space) + " of type " +
str(type(space)))
rng = self.mlp.rng
if self.border_mode == 'valid':
output_shape = [
(self.input_space.shape[0] - self.kernel_shape[0]) /
self.kernel_stride[0] + 1,
(self.input_space.shape[1] - self.kernel_shape[1]) /
self.kernel_stride[1] + 1
]
elif self.border_mode == 'full':
output_shape = [
(self.input_space.shape[0] + self.kernel_shape[0]) /
self.kernel_stride[0] - 1,
(self.input_space.shape[1] + self.kernel_shape[1]) /
self.kernel_stride[1] - 1
]
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels=self.output_channels,
axes=('b', 'c', 0, 1))
if self.irange is not None:
assert self.sparse_init is None
self.transformer = conv2d.make_random_conv2D(
irange=self.irange,
input_space=self.input_space,
output_space=self.detector_space,
kernel_shape=self.kernel_shape,
batch_size=self.mlp.batch_size,
subsample=self.kernel_stride,
border_mode=self.border_mode,
rng=rng)
elif self.sparse_init is not None:
self.transformer = conv2d.make_sparse_random_conv2D(
num_nonzero=self.sparse_init,
input_space=self.input_space,
output_space=self.detector_space,
kernel_shape=self.kernel_shape,
batch_size=self.mlp.batch_size,
subsample=self.kernel_stride,
border_mode=self.border_mode,
rng=rng)
W, = self.transformer.get_params()
W.name = 'W'
self.b = sharedX(
np.zeros(((self.num_pieces * self.output_channels), )) +
self.init_bias)
self.b.name = 'b'
print 'Input shape: ', self.input_space.shape
print 'Detector space: ', self.detector_space.shape
assert self.pool_type in ['max', 'mean']
dummy_batch_size = self.mlp.batch_size
if dummy_batch_size is None:
dummy_batch_size = 2
dummy_detector = sharedX(
self.detector_space.get_origin_batch(dummy_batch_size))
#dummy_p = dummy_p.eval()
# self.output_space = Conv2DSpace(shape=[1, 1],
# num_channels=self.output_channels,
# axes=('b', 'c', 0, 1))
self.output_space = self.detector_space
W = rng.uniform(-self.irange, self.irange,
(426, (self.num_pieces * self.output_channels)))
W = sharedX(W)
W.name = self.layer_name + "_w"
self.transformer = MatrixMul(W)
print 'Output space: ', self.output_space.shape
pool_stride=[2, 2],
layer_name="h1",
irange=0.1,
border_mode="full")
h2 = ConvRectifiedLinear(output_channels=32,
kernel_shape=[5, 5],
pool_shape=[2, 2],
pool_stride=[2, 2],
layer_name="h2",
irange=0.1,
border_mode="full")
y = Softmax(n_classes=2, layer_name="y", irange=0.1)
inputSpace = Conv2DSpace(shape=[cropSize, cropSize], num_channels=3)
model = MLP(layers=[h0, h1, h2, y],
batch_size=batchSize,
input_space=inputSpace)
algorithm = SGD(learning_rate=0.01,
cost=MethodCost("cost_from_X"),
batch_size=batchSize,
monitoring_batch_size=batchSize,
monitoring_dataset={
'train': train,
'valid': valid
},
monitor_iteration_mode="even_batchwise_shuffled_sequential",
termination_criterion=EpochCounter(max_epochs=200),
def set_input_space(self, space):
desired = Conv2DSpace([32, 32], 3, axes=('c', 0, 1, 'b'))
if not (space == desired):
print space
assert False
self.output_space = space
from pylearn2.training_algorithms.sgd import SGD
from pylearn2.termination_criteria import EpochCounter
from DBL_layer import DBL_ConvLayers
from DBL_model import DBL_model
import os
import cPickle
import numpy as np
from faceModule import faceDetector
import cv2.cv as cv
import time
from DBL_util import DataPylearn2
from pylearn2.space import Conv2DSpace
ishape = Conv2DSpace(shape=[48, 48], num_channels=1)
def loadModel(pklname):
ishape = Conv2DSpace(shape=[48, 48], num_channels=1)
nclass = 7
# create layers
#nk = [30, 40]
nk = [32, 20, 10]
ks = [[8, 8], [5, 5], [3, 3]]
ir = [0.05, 0.05, 0.05]
ps = [[4, 4], [4, 4], [2, 2]]
pd = [[2, 2], [2, 2], [2, 2]]
kn = [0.9, 0.9, 0.9]
def main():
#creating layers
#2 convolutional rectified layers, border mode valid
batch_size = 48
lr = 1.0 #0.1/4
finMomentum = 0.9
maxout_units = 2000
num_pcs = 4
lay1_reg = lay2_reg = maxout_reg = None
#save_path = './models/no_maxout/titan_lr_0.1_btch_64_momFinal_0.9_maxout_2000_4.joblib'
#best_path = '/models/no_maxout/titan_bart10_gpu2_best.joblib'
#save_path = './models/'+params.host+'_'+params.device+'_'+sys.argv[1]+'.joblib'
#best_path = './models/'+params.host+'_'+params.device+'_'+sys.argv[1]+'best.joblib'
save_path = '/Tmp/zumerjer/bart10_meancost_adadelta_ema.joblib'
best_path = '/Tmp/zumerjer/bart10_meancost_adadelta_ema_best.joblib'
#numBatches = 400000/batch_size
'''
print 'Applying preprocessing'
ddmTrain = EmotiwKeypoints(start=0, stop =40000)
ddmValid = EmotiwKeypoints(start=40000, stop = 44000)
ddmTest = EmotiwKeypoints(start=44000)
stndrdz = preprocessing.Standardize()
stndrdz.applyLazily(ddmTrain, can_fit=True, name = 'train')
stndrdz.applyLazily(ddmValid, can_fit=False, name = 'val')
stndrdz.applyLazily(ddmTest, can_fit=False, name = 'test')
GCN = preprocessing.GlobalContrastNormalization(batch_size = 1000)
GCN.apply(ddmTrain, can_fit =True, name = 'train')
GCN.apply(ddmValid, can_fit =False, name = 'val')
GCN.apply(ddmTest, can_fit = False, name = 'test')
return
'''
ddmTrain = ComboDatasetPyTable('/Tmp/zumerjer/all_', which_set='train')
ddmValid = ComboDatasetPyTable('/Tmp/zumerjer/all_', which_set='valid')
ddmSmallTrain = ComboDatasetPyTable('/Tmp/zumerjer/all_', which_set='small_train')
layer1 = ConvRectifiedLinear(layer_name = 'convRect1',
output_channels = 64,
irange = .05,
kernel_shape = [5, 5],
pool_shape = [4, 4],
pool_stride = [2, 2],
W_lr_scale = 0.1,
max_kernel_norm = lay1_reg)
layer2 = ConvRectifiedLinear(layer_name = 'convRect2',
output_channels = 128,
irange = .05,
kernel_shape = [5, 5],
pool_shape = [3, 3],
pool_stride = [2, 2],
W_lr_scale = 0.1,
max_kernel_norm = lay2_reg)
# Rectified linear units
#layer3 = RectifiedLinear(dim = 3000,
# sparse_init = 15,
# layer_name = 'RectLin3')
#Maxout layer
maxout = Maxout(layer_name= 'maxout',
irange= .005,
num_units= maxout_units,
num_pieces= num_pcs,
W_lr_scale = 0.1,
max_col_norm= maxout_reg)
#multisoftmax
n_groups = 196
n_classes = 96
layer_name = 'multisoftmax'
layerMS = MultiSoftmax(n_groups=n_groups,irange = 0.05, n_classes=n_classes, layer_name= layer_name)
#setting up MLP
MLPerc = MLP(batch_size = batch_size,
input_space = Conv2DSpace(shape = [96, 96],
num_channels = 3, axes=('b', 0, 1, 'c')),
layers = [ layer1, layer2, maxout, layerMS])
#mlp_cost
missing_target_value = -1
mlp_cost = MLPCost(cost_type='default',
missing_target_value=missing_target_value )
#mlp_cost.setup_dropout(input_include_probs= { 'convRect1' : 0.8 }, input_scales= { 'convRect1': 1. })
#dropout_cost = Dropout(input_include_probs= { 'convRect1' : .8 },
# input_scales= { 'convRect1': 1. })
#algorithm
monitoring_dataset = {'validation':ddmValid, 'mini-train':ddmSmallTrain}
term_crit = MonitorBased(prop_decrease = 1e-7, N = 100, channel_name = 'validation_objective')
kp_ada = KeypointADADELTA(decay_factor = 0.95,
#init_momentum = 0.5,
monitoring_dataset = monitoring_dataset, batch_size = batch_size,
termination_criterion = term_crit,
cost = mlp_cost)
#train extension
#train_ext = ExponentialDecayOverEpoch(decay_factor = 0.998, min_lr_scale = 0.001)
#train_ext = LinearDecayOverEpoch(start= 1,saturate= 250,decay_factor= .01)
#train_ext = ADADELTA(0.95)
#train object
train = Train(dataset = ddmTrain,
save_path= save_path,
save_freq=10,
model = MLPerc,
algorithm= kp_ada,
extensions = [#train_ext,
MonitorBasedSaveBest(channel_name='validation_objective',
save_path= best_path)#,
# MomentumAdjustor(start = 1,#
# saturate = 25,
# final_momentum = finMomentum)
] )
train.main_loop()
train.save()
def setup_detector_layer_c01b(layer, input_space, rng, irange):
"""
Takes steps to set up an object for use as being some kind of convolutional layer.
This function sets up only the detector layer.
Parameters
----------
layer: Any python object that allows the modifications described below and has
the following attributes:
pad: int describing amount of zero padding to add
kernel_shape: 2-element tuple or list describing spatial shape of kernel
fix_kernel_shape: bool, if true, will shrink the kernel shape to make it
feasible, as needed (useful for hyperparameter searchers)
detector_channels: The number of channels in the detector layer
init_bias: A numeric constant added to a tensor of zeros to initialize the
bias
tied_b: If true, biases are shared across all spatial locations
input_space: A Conv2DSpace to be used as input to the layer
rng: a numpy RandomState or equivalent
irange: float. kernel elements are initialized randomly from U(-irange, irange)
Does the following:
raises a RuntimeError if cuda is not available
sets layer.input_space to input_space
sets up addition of dummy channels for compatibility with cuda-convnet:
layer.dummy_channels: # of dummy channels that need to be added
(You might want to check this and raise an Exception if it's not 0)
layer.dummy_space: The Conv2DSpace representing the input with dummy channels
added
sets layer.detector_space to the space for the detector layer
sets layer.transformer to be a Conv2D instance
sets layer.b to the right value
"""
# Use "self" to refer to layer from now on, so we can pretend we're just running
# in the set_input_space method of the layer
self = layer
# Make sure cuda is available
check_cuda(str(type(self)))
# Validate input
if not isinstance(input_space, Conv2DSpace):
raise TypeError(
"The input to a convolutional layer should be a Conv2DSpace, "
" but layer " + self.layer_name + " got " +
str(type(self.input_space)))
if not hasattr(self, 'detector_channels'):
raise ValueError(
'layer argument must have a "detector_channels" attribute specifying how many channels to put in the convolution kernel stack.'
)
# Store the input space
self.input_space = input_space
# Make sure number of channels is supported by cuda-convnet
# (multiple of 4 or <= 3)
# If not supported, pad the input with dummy channels
ch = self.input_space.num_channels
rem = ch % 4
if ch > 3 and rem != 0:
self.dummy_channels = 4 - rem
else:
self.dummy_channels = 0
self.dummy_space = Conv2DSpace(shape=input_space.shape,
channels=input_space.num_channels +
self.dummy_channels,
axes=('c', 0, 1, 'b'))
if hasattr(self, 'kernel_stride'):
kernel_stride = self.kernel_stride
else:
kernel_stride = [1, 1]
output_shape = [
int(np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))) + 1
for i_sh, k_sh, k_st in zip(self.input_space.shape, self.kernel_shape,
kernel_stride)
]
def handle_kernel_shape(idx):
if self.kernel_shape[idx] < 1:
raise ValueError(
"kernel must have strictly positive size on all axes but has shape: "
+ str(self.kernel_shape))
if output_shape[idx] <= 0:
if self.fix_kernel_shape:
self.kernel_shape[
idx] = self.input_space.shape[idx] + 2 * self.pad
assert self.kernel_shape[idx] != 0
output_shape[idx] = 1
warnings.warn(
"Had to change the kernel shape to make network feasible")
else:
raise ValueError(
"kernel too big for input (even with zero padding)")
map(handle_kernel_shape, [0, 1])
if self.detector_channels < 16:
raise ValueError(
"Cuda-convnet requires the detector layer to have at least 16 channels."
)
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
if hasattr(self, 'partial_sum'):
partial_sum = self.partial_sum
else:
partial_sum = 1
self.transformer = make_random_conv2D(
irange=self.irange,
input_axes=self.input_space.axes,
output_axes=self.detector_space.axes,
input_channels=self.dummy_space.num_channels,
output_channels=self.detector_space.num_channels,
kernel_shape=self.kernel_shape,
pad=self.pad,
partial_sum=partial_sum,
kernel_stride=kernel_stride,
rng=rng)
W, = self.transformer.get_params()
W.name = 'W'
if self.tied_b:
self.b = sharedX(
np.zeros((self.detector_space.num_channels)) + self.init_bias)
else:
self.b = sharedX(self.detector_space.get_origin() + self.init_bias)
self.b.name = 'b'
print 'Input shape: ', self.input_space.shape
print 'Detector space: ', self.detector_space.shape
def DBL_model_test1(basepath, cutoff=[-1, -1], pklname='', newdata=None):
# data
ishape = Conv2DSpace(shape=[48, 48], num_channels=1)
preproc = [0, 0]
nclass = 7
DBL = DBL_model(basepath, nclass, np.append(ishape.shape, 1), preproc,
cutoff)
# create layers
nk = [30]
#nk = [40,30,20]
ks = [[8, 8], [5, 5], [3, 3]]
ir = [0.05, 0.05, 0.05]
ps = [[4, 4], [4, 4], [2, 2]]
pd = [[2, 2], [2, 2], [2, 2]]
kn = [0.9, 0.9, 0.9]
layers = DBL_ConvLayers(nk, ks, ir, ps, pd, kn)
layer_soft = Softmax(
layer_name='y',
#max_col_norm = 1.9365,
n_classes=nclass,
init_bias_target_marginals=DBL.ds_train,
#istdev = .05
irange=.0)
layers.append(layer_soft)
# create DBL_model
model = MLP(layers, input_space=ishape)
if pklname != '' and os.path.isfile(pklname):
# load and rebuild model
layer_params = cPickle.load(open(pklname + '.cpu'))
layer_id = 0
for layer in model.layers:
if layer_id < len(layers) - 1:
layer.set_weights(layer_params[layer_id][0])
layer.set_biases(layer_params[layer_id][1])
else:
layer.set_weights(layer_params[layer_id][1])
layer.set_biases(layer_params[layer_id][0])
layer_id = layer_id + 1
DBL.model = model
DBL.test_raw(newdata)
else:
algo_term = EpochCounter(500) # number of epoch iteration
algo = SGD(learning_rate=0.001,
batch_size=500,
init_momentum=.5,
monitoring_dataset=DBL.ds_valid,
termination_criterion=algo_term)
DBL.run_model(model, algo)
# save the model
if pklname != '':
layer_params = []
for layer in layers:
param = layer.get_params()
print param
print param[0].get_value()
layer_params.append(
[param[0].get_value(), param[1].get_value()])
#cPickle.dump(DBL,open(pklname, 'wb'))
#cPickle.dump(layer_params, open(pklname + '.cpu', 'wb'))
cPickle.dump(layer_params, open(pklname + '.cpu', 'wb'))
print DBL.result_valid[1], DBL.result_test[1]
return DBL.result_valid[1], DBL.result_test[1]
def setup_detector_layer_c01b(layer, input_space, rng, irange="not specified"):
"""
.. todo::
WRITEME properly
Takes steps to set up an object for use as being some kind of convolutional
layer. This function sets up only the detector layer.
Does the following:
* raises a RuntimeError if cuda is not available
* sets layer.input_space to input_space
* sets up addition of dummy channels for compatibility with cuda-convnet:
- layer.dummy_channels: # of dummy channels that need to be added
(You might want to check this and raise an Exception if it's not 0)
- layer.dummy_space: The Conv2DSpace representing the input with dummy
channels added
* sets layer.detector_space to the space for the detector layer
* sets layer.transformer to be a Conv2D instance
* sets layer.b to the right value
Parameters
----------
layer : object
Any python object that allows the modifications described below and \
has the following attributes: \
* pad: int describing amount of zero padding to add \
* kernel_shape: 2-element tuple or list describing spatial shape of \
kernel \
* fix_kernel_shape: bool, if true, will shrink the kernel shape to \
make it feasible, as needed (useful for hyperparameter searchers) \
* detector_channels: The number of channels in the detector layer \
* init_bias: numeric constant added to a tensor of zeros to \
initialize the bias \
* tied_b: If true, biases are shared across all spatial locations
input_space : WRITEME
A Conv2DSpace to be used as input to the layer
rng : WRITEME
A numpy RandomState or equivalent
"""
if irange != "not specified":
raise AssertionError(
"There was a bug in setup_detector_layer_c01b."
"It uses layer.irange instead of the irange parameter to the "
"function. The irange parameter is now disabled by this "
"AssertionError, so that this error message can alert you that "
"the bug affected your code and explain why the interface is "
"changing. The irange parameter to the function and this "
"error message may be removed after April 21, 2014.")
# Use "self" to refer to layer from now on, so we can pretend we're just running
# in the set_input_space method of the layer
self = layer
# Make sure cuda is available
check_cuda(str(type(self)))
# Validate input
if not isinstance(input_space, Conv2DSpace):
raise TypeError(
"The input to a convolutional layer should be a Conv2DSpace, "
" but layer " + self.layer_name + " got " +
str(type(self.input_space)))
if not hasattr(self, 'detector_channels'):
raise ValueError(
'layer argument must have a "detector_channels" attribute specifying how many channels to put in the convolution kernel stack.'
)
# Store the input space
self.input_space = input_space
# Make sure number of channels is supported by cuda-convnet
# (multiple of 4 or <= 3)
# If not supported, pad the input with dummy channels
ch = self.input_space.num_channels
rem = ch % 4
if ch > 3 and rem != 0:
self.dummy_channels = 4 - rem
else:
self.dummy_channels = 0
self.dummy_space = Conv2DSpace(shape=input_space.shape,
channels=input_space.num_channels +
self.dummy_channels,
axes=('c', 0, 1, 'b'))
if hasattr(self, 'kernel_stride'):
kernel_stride = self.kernel_stride
else:
kernel_stride = [1, 1]
output_shape = [
int(np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))) + 1
for i_sh, k_sh, k_st in zip(self.input_space.shape, self.kernel_shape,
kernel_stride)
]
def handle_kernel_shape(idx):
if self.kernel_shape[idx] < 1:
raise ValueError(
"kernel must have strictly positive size on all axes but has shape: "
+ str(self.kernel_shape))
if output_shape[idx] <= 0:
if self.fix_kernel_shape:
self.kernel_shape[
idx] = self.input_space.shape[idx] + 2 * self.pad
assert self.kernel_shape[idx] != 0
output_shape[idx] = 1
warnings.warn(
"Had to change the kernel shape to make network feasible")
else:
raise ValueError(
"kernel too big for input (even with zero padding)")
map(handle_kernel_shape, [0, 1])
if self.detector_channels < 16:
raise ValueError(
"Cuda-convnet requires the detector layer to have at least 16 channels."
)
self.detector_space = Conv2DSpace(shape=output_shape,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
if hasattr(self, 'partial_sum'):
partial_sum = self.partial_sum
else:
partial_sum = 1
if hasattr(self, 'sparse_init') and self.sparse_init is not None:
self.transformer = checked_call(
make_sparse_random_conv2D,
OrderedDict([('num_nonzero', self.sparse_init),
('input_space', self.input_space),
('output_space', self.detector_space),
('kernel_shape', self.kernel_shape),
('pad', self.pad), ('partial_sum', partial_sum),
('kernel_stride', kernel_stride), ('rng', rng)]))
else:
self.transformer = make_random_conv2D(
irange=self.irange,
input_axes=self.input_space.axes,
output_axes=self.detector_space.axes,
input_channels=self.dummy_space.num_channels,
output_channels=self.detector_space.num_channels,
kernel_shape=self.kernel_shape,
pad=self.pad,
partial_sum=partial_sum,
kernel_stride=kernel_stride,
rng=rng)
W, = self.transformer.get_params()
W.name = self.layer_name + '_W'
if self.tied_b:
self.b = sharedX(
np.zeros((self.detector_space.num_channels)) + self.init_bias)
else:
self.b = sharedX(self.detector_space.get_origin() + self.init_bias)
self.b.name = self.layer_name + '_b'
print 'Input shape: ', self.input_space.shape
print 'Detector space: ', self.detector_space.shape
def test_works():
load = True
if load == False:
ddmTrain = FacialKeypoint(which_set='train', start=0, stop=6000)
ddmValid = FacialKeypoint(which_set='train', start=6000, stop=7049)
# valid can_fit = false
pipeline = preprocessing.Pipeline()
stndrdz = preprocessing.Standardize()
stndrdz.apply(ddmTrain, can_fit=True)
#doubt, how about can_fit = False?
stndrdz.apply(ddmValid, can_fit=False)
GCN = preprocessing.GlobalContrastNormalization()
GCN.apply(ddmTrain, can_fit=True)
GCN.apply(ddmValid, can_fit=False)
pcklFile = open('kpd.pkl', 'wb')
obj = (ddmTrain, ddmValid)
pickle.dump(obj, pcklFile)
pcklFile.close()
return
else:
pcklFile = open('kpd.pkl', 'rb')
(ddmTrain, ddmValid) = pickle.load(pcklFile)
pcklFile.close()
#creating layers
#2 convolutional rectified layers, border mode valid
layer1 = ConvRectifiedLinear(layer_name='convRect1',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[3, 3],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
layer2 = ConvRectifiedLinear(layer_name='convRect2',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[3, 3],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
# Rectified linear units
layer3 = RectifiedLinear(dim=3000, sparse_init=15, layer_name='RectLin3')
#multisoftmax
n_groups = 30
n_classes = 98
irange = 0
layer_name = 'multisoftmax'
layerMS = MultiSoftmax(n_groups=n_groups,
irange=0.05,
n_classes=n_classes,
layer_name=layer_name)
#setting up MLP
MLPerc = MLP(batch_size=8,
input_space=Conv2DSpace(shape=[96, 96], num_channels=1),
layers=[layer1, layer2, layer3, layerMS])
#mlp_cost
missing_target_value = -1
mlp_cost = MLPCost(cost_type='default',
missing_target_value=missing_target_value)
#algorithm
# learning rate, momentum, batch size, monitoring dataset, cost, termination criteria
term_crit = MonitorBased(prop_decrease=0.00001,
N=30,
channel_name='validation_objective')
kpSGD = KeypointSGD(learning_rate=0.001,
init_momentum=0.5,
monitoring_dataset={
'validation': ddmValid,
'training': ddmTrain
},
batch_size=8,
batches_per_iter=750,
termination_criterion=term_crit,
train_iteration_mode='random_uniform',
cost=mlp_cost)
#train extension
train_ext = ExponentialDecayOverEpoch(decay_factor=0.998,
min_lr_scale=0.01)
#train object
train = Train(dataset=ddmTrain,
save_path='kpd_model2.pkl',
save_freq=1,
model=MLPerc,
algorithm=kpSGD,
extensions=[
train_ext,
MonitorBasedSaveBest(channel_name='validation_objective',
save_path='kpd_best.pkl'),
MomentumAdjustor(start=1, saturate=20, final_momentum=.9)
])
train.main_loop()
train.save()
def train(d):
print 'Creating dataset'
# load mnist here
# X = d.train_X
# y = d.train_Y
# test_X = d.test_X
# test_Y = d.test_Y
# nb_classes = len(np.unique(y))
# train_y = convert_one_hot(y)
# train_set = DenseDesignMatrix(X=X, y=y)
train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y))
valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y))
test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y))
print 'Setting up'
batch_size = 1000
conv = mlp.ConvRectifiedLinear(
layer_name='c0',
output_channels=20,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout = MaxoutConvC01B(layer_name='m0',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout2 = MaxoutConvC01B(layer_name='m1',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
sigmoid = mlp.Sigmoid(
layer_name='Sigmoid',
dim=500,
sparse_init=15,
)
smax = mlp.Softmax(layer_name='y', n_classes=10, irange=0.)
in_space = Conv2DSpace(shape=[28, 28],
num_channels=1,
axes=['c', 0, 1, 'b'])
net = mlp.MLP(
layers=[mout, mout2, smax],
input_space=in_space,
# nvis=784,
)
trainer = bgd.BGD(batch_size=batch_size,
line_search_mode='exhaustive',
conjugate=1,
updates_per_batch=10,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer = sgd.SGD(learning_rate=0.15,
cost=dropout.Dropout(),
batch_size=batch_size,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer.setup(net, train)
epoch = 0
while True:
print 'Training...', epoch
trainer.train(dataset=train)
net.monitor()
epoch += 1
def agent_init(self,task_spec_string):
"""
This function is called once at the beginning of an experiment.
Arguments: task_spec_string - A string defining the task. This string
is decoded using
TaskSpecVRLGLUE3.TaskSpecParser
"""
self.start_time = time.time()
self.image = None
self.show_ale = False
self.total_reward = 0
self.mini_batch_size = 32
self.num_mini_batches = 1
self.frame_count = 0
self.frames_trained = 0
self.qvalue_sum = 0
self.qvalue_count = 0
self.predicted_reward
learning_rate = .00001
self.testing_policy = False
self.epoch_counter = 0
self.epochs_until_test = 5
self.policy_test_file_name = "results.csv"
load_file = False
load_file_name = "cnnparams.pkl"
self.save_file_name = "cnnparams.pkl"
self.counter = 0
self.cur_action = 0
#starting value for epsilon-greedy
self.epsilon = 1
TaskSpec = TaskSpecVRLGLUE3.TaskSpecParser(task_spec_string)
if TaskSpec.valid:
assert ((len(TaskSpec.getIntObservations())== 0) != \
(len(TaskSpec.getDoubleObservations()) == 0 )), \
"expecting continous or discrete observations. Not both."
assert len(TaskSpec.getDoubleActions())==0, \
"expecting no continuous actions"
assert not TaskSpec.isSpecial(TaskSpec.getIntActions()[0][0]), \
" expecting min action to be a number not a special value"
assert not TaskSpec.isSpecial(TaskSpec.getIntActions()[0][1]), \
" expecting max action to be a number not a special value"
self.num_actions=TaskSpec.getIntActions()[0][1]+1
self.num_actions = 3
self.int_states = len(TaskSpec.getIntObservations()) > 0
# Create neural network and initialize trainer and dataset
if load_file:
thefile = open(load_file_name, "r")
self.cnn = cPickle.load(thefile)
else:
self.first_conv_layer = maxout.MaxoutConvC01B(16, 1, (8, 8), (1, 1),
(1, 1), "first conv layer", irange=.1,
kernel_stride=(4, 4), min_zero=True)
self.second_conv_layer = maxout.MaxoutConvC01B(32, 1, (4, 4),
(1, 1), (1, 1), "second conv layer", irange=.1,
kernel_stride=(2, 2), min_zero=True)
self.rect_layer = mlp.RectifiedLinear(dim=256,
layer_name="rectified layer", irange=.1)
self.output_layer = mlp.Linear(self.num_actions, "output layer",
irange=.1)
layers = [self.first_conv_layer, self.second_conv_layer,
self.rect_layer, self.output_layer]
self.cnn = mlp.MLP(layers, input_space = Conv2DSpace((80, 80),
num_channels=4, axes=('c', 0, 1, 'b')),
batch_size=self.mini_batch_size)
self.data = nqd.NeuralRewardPredictorDataset(self.cnn, mini_batch_size = self.mini_batch_size,
num_mini_batches = self.num_mini_batches,
learning_rate=learning_rate)
#Create appropriate RL-Glue objects for storing these.
self.last_action=Action()
self.last_observation=Observation()
thefile = open(self.policy_test_file_name, "w")
thefile.write("Reward, Predicted reward, Frames trained\n")
thefile.close()
count = 0
for i in range(len(data)):
current_phone = data[i]
current_phone_len = len(current_phone)
for j in range(current_phone_len - record_len):
frame_end = j + self.frame_length
target_end = frame_end + self.target_width
X[count, :] = current_phone[j:frame_end]
y[count, :] = current_phone[frame_end:target_end]
count += 1
return (X, y)
if __name__ == "__main__":
valid_timit = TIMIT("valid",
frame_length=1,
frames_per_example=100,
audio_only=False)
data_specs = (Conv2DSpace(shape=[100, 1],
num_channels=1,
axes=('b', 0, 1, 'c')), 'features')
it = valid_timit.iterator(mode='sequential',
data_specs=data_specs,
num_batches=1,
batch_size=10)
for rval in it:
import pdb
pdb.set_trace()
print[val.shape for val in rval]
def set_input_space(self, space):
""" Note: this resets parameters! """
# setup_detector_layer_bct01(layer=self,
# input_space=space,
# rng=self.mlp.rng,
# irange=self.irange)
# Use theano conv3d instead
setup_detector_layer_b01tc(layer=self,
input_space=space,
rng=self.mlp.rng,
irange=self.irange)
rng = self.mlp.rng
detector_shape = self.detector_space.shape
def handle_pool_shape(idx):
if self.pool_shape[idx] < 1:
raise ValueError("bad pool shape: " + str(self.pool_shape))
if self.pool_shape[idx] > detector_shape[idx]:
if self.fix_pool_shape:
assert detector_shape[idx] > 0
self.pool_shape[idx] = detector_shape[idx]
else:
raise ValueError(
"Pool shape exceeds detector layer shape on axis %d" %
idx)
map(handle_pool_shape, [0, 1, 2])
### Check some precondition
assert self.pool_shape[0] == self.pool_shape[1]
assert self.pool_stride[0] == self.pool_stride[1]
assert all(
isinstance(elem, py_integer_types) for elem in self.pool_stride)
for i in xrange(0, 2):
assert self.pool_stride[i] <= self.pool_shape[i]
assert all(
isinstance(elem, py_integer_types) for elem in self.pool_stride)
# Find space shape after convolution
dummy_output_shape = [
int(np.ceil((i_sh + 2. * self.pad - k_sh) / float(k_st))) + 1
for i_sh, k_sh, k_st in zip(self.input_space.shape,
self.kernel_shape, self.kernel_stride)
]
dummy_output_sequence_length = dummy_output_shape[2]
### Find the space shape after spatial pooling
dummy_output_shape = [dummy_output_shape[0], dummy_output_shape[1]]
dummy_detector_space = Conv2DSpace(shape=dummy_output_shape,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
dummy_detector = sharedX(
dummy_detector_space.get_origin_batch(2)[0:16, :, :, :])
dummy_p = max_pool_c01b(c01b=dummy_detector,
pool_shape=self.pool_shape,
pool_stride=self.pool_stride)
dummy_p = dummy_p.eval()
### Find the space shape after temporal pooling
# (0*1,'t')
dummy_temp_image = [
dummy_p.shape[1] * dummy_p.shape[2], dummy_output_sequence_length
]
self.temp_pool_input_shape = dummy_temp_image
dummy_temp_space = Conv2DSpace(shape=dummy_temp_image,
num_channels=self.detector_channels,
axes=('c', 0, 1, 'b'))
temp_input = sharedX(
dummy_temp_space.get_origin_batch(2)[0:16, :, :, :])
dummy_temp_p = temporal_max_pool_c01b(
c01b=temp_input,
pool_shape=[1, self.pool_shape[2]],
pool_stride=[1, self.pool_stride[2]],
image_shape=dummy_temp_image)
dummy_temp_p = dummy_temp_p.eval()
self.output_space = Conv3DSpace(
shape=[dummy_p.shape[1], dummy_p.shape[2], dummy_temp_p.shape[2]],
num_channels=self.num_channels,
axes=('b', 0, 1, 't', 'c'))
# Print spaces
print "Input shape: ", self.input_space.shape
print "Detector space: ", self.detector_space.shape
print "Output space: ", self.output_space.shape