def jpype_compile(source_files, force_recompile = False):
if isinstance(source_files, str):
source_files = source_files.split(':')
if isinstance(source_files, list):
source_files = tuple(source_files)
if source_files in _compiled_srcs and not force_recompile:
return
for src in source_files:
assert os.path.exists(src), 'File "%s" does not exist!' % (src, )
if src.endswith('.java'):
return_code = subprocess.call(['javac', src])
elif src.endswith('.scala'):
curdir = os.getcwd() # scalac is a bit different when it comes to placing class files so we need to compensate.
os.chdir('..')
return_code = subprocess.call(['scalac', src])
print '%s COMPILED!!!' % (src, )
os.chdir(curdir)
else:
bad_value(source_files, "Can only handle java and scala files.")
assert return_code == 0, 'Error while compiling "%s". See message above.' % (src, )
_compiled_srcs.add(source_files)
def jpype_compile(source_files, force_recompile=False):
if isinstance(source_files, str):
source_files = source_files.split(':')
if isinstance(source_files, list):
source_files = tuple(source_files)
if source_files in _compiled_srcs and not force_recompile:
return
for src in source_files:
assert os.path.exists(src), 'File "%s" does not exist!' % (src, )
if src.endswith('.java'):
return_code = subprocess.call(['javac', src])
elif src.endswith('.scala'):
curdir = os.getcwd(
) # scalac is a bit different when it comes to placing class files so we need to compensate.
os.chdir('..')
return_code = subprocess.call(['scalac', src])
print '%s COMPILED!!!' % (src, )
os.chdir(curdir)
else:
bad_value(source_files, "Can only handle java and scala files.")
assert return_code == 0, 'Error while compiling "%s". See message above.' % (
src, )
_compiled_srcs.add(source_files)
def demo_rbm_mnist(
vis_activation = 'bernoulli',
hid_activation = 'bernoulli',
n_hidden = 500,
plot = True,
eta = 0.01,
optimizer = 'sgd',
w_init_mag = 0.001,
minibatch_size = 9,
persistent = False,
n_epochs = 100,
plot_interval = 100,
):
"""
In this demo we train an RBM on the MNIST input data (labels are ignored). We plot the state of a markov chanin
that is being simulaniously sampled from the RBM, and the parameters of the RBM.
What you see:
A plot will appear with 6 subplots. The subplots are as follows:
hidden-neg-chain: The activity of the hidden layer for each of the persistent CD chains for draewing negative samples.
visible-neg-chain: The probabilities of the visible activations corresponding to the state of hidden-neg-chain.
w: A subset of the weight vectors, reshaped to the shape of the input.
b: The bias of the hidden units.
b_rev: The bias of the visible units.
visible-sample: The probabilities of the visible samples drawin from an independent free-sampling chain (outside the
training function).
As learning progresses, visible-neg-chain and visible-sample should increasingly resemble the data.
"""
with EnableOmniscence():
if is_test_mode():
n_epochs = 0.01
data = get_mnist_dataset(flat = True).training_set.input
rbm = simple_rbm(
visible_layer = StochasticNonlinearity(vis_activation),
bridge=FullyConnectedBridge(w = w_init_mag*np.random.randn(28*28, n_hidden).astype(theano.config.floatX), b=0, b_rev = 0),
hidden_layer = StochasticNonlinearity(hid_activation)
)
optimizer = \
SimpleGradientDescent(eta = eta) if optimizer == 'sgd' else \
AdaMax(alpha=eta) if optimizer == 'adamax' else \
bad_value(optimizer)
train_function = rbm.get_training_fcn(n_gibbs = 1, persistent = persistent, optimizer = optimizer).compile()
def plot_fcn():
lv = train_function.locals()
dbplot({
'visible-pos-chain': lv['wake_visible'].reshape((-1, 28, 28)),
'visible-neg-chain': lv['sleep_visible'].reshape((-1, 28, 28)),
})
for i, visible_data in enumerate(minibatch_iterate(data, minibatch_size=minibatch_size, n_epochs=n_epochs)):
train_function(visible_data)
if plot and i % plot_interval == 0:
plot_fcn()
def __init__(self, input_size, hidden_sizes, output_size, distribution = 'gaussian', hidden_activation = 'sig', w_init = lambda n_in, n_out: 0.01*np.random.randn(n_in, n_out)):
"""
:param input_size: The dimensionality of the input
:param hidden_sizes: A list indicating the sizes of each hidden layer.
:param output_size: The dimensionality of the output
:param distribution: The form of the output distribution (currently 'gaussian' or 'bernoulli')
:param hidden_activation: A string indicating the type of each hidden layer.
{'sig', 'tanh', 'rect-lin', 'lin', 'softmax'}
:param w_init: A function which, given input dims, output dims, returns an initial weight matrix
"""
all_layer_sizes = [input_size]+hidden_sizes
all_layer_activations = [hidden_activation] * len(hidden_sizes)
processing_chain = sum([[
FullyConnectedTransform(w = w_init(pre_size, post_size)),
get_named_activation_function(activation_fcn)
] for (pre_size, post_size), activation_fcn in zip(zip(all_layer_sizes[:-1], all_layer_sizes[1:]), all_layer_activations)
], [])
distribution_function = \
Branch(
FullyConnectedTransform(w = w_init(all_layer_sizes[-1], output_size)),
FullyConnectedTransform(w_init(all_layer_sizes[-1], output_size))) \
if distribution == 'gaussian' else \
Chain(FullyConnectedTransform(w = w_init(all_layer_sizes[-1], output_size)), get_named_activation_function('sig')) \
if distribution=='bernoulli' else \
bad_value(distribution)
self.distribution = distribution
self.chain = Chain(*processing_chain+[distribution_function])
def get_plot_from_data(data, mode, **plot_preference_kwargs):
return \
get_live_plot_from_data(data, **plot_preference_kwargs) if mode == 'live' else \
get_static_plot_from_data(data, **plot_preference_kwargs) if mode == 'static' else \
ImagePlot(**plot_preference_kwargs) if mode == 'image' else \
bad_value(mode, 'Unknown plot modee: %s' % (mode, ))
def __init__(self, specied_path, default_smooth = False):
"""
:param specied_path: Can be defined in two ways:
(1) As a list of source/dest signals (with an optional third argument defining whether the pass should be "smooth". e.g.
[('vis', 'hid'), ('hid', 'ass'), ('ass', 'lab', True)]
[('vis', 'hid'), (('hid', 'lab'), 'ass)), ('ass', ('hid', 'lab')), ('hid', 'vis', True)]
(2) As a list of signals to compute in order:
['vis', 'hid', 'ass', 'lab']
:param default_smooth: For steps where smooth is not specified, define whether the pass should be smooth.
"""
assert isinstance(specied_path, list)
assert isinstance(default_smooth, bool)
spec_type = \
1 if all(isinstance(el, tuple) and len(el) in (2, 3) for el in specied_path) else \
2 if all(isinstance(el, (int, str)) for el in specied_path) else \
bad_value(specied_path, 'Could not interpret the path %s - see docstring of InferencePath for required format.')
if spec_type == 1:
path = []
for el in specied_path:
srcs = _tuplefy_singles(el[0])
for dest in _tuplefy_singles(el[1]):
is_smooth = el[2] if len(el)==3 else default_smooth
assert isinstance(is_smooth, bool), 'The third element of the tuple in your specified path must be ' \
'a boolean indicating whether to do a smooth pass. It was %s' % (is_smooth, )
path.append((srcs, dest, is_smooth))
else:
path = [((src, ), dest, default_smooth) for src, dest in zip(specied_path[:-1], specied_path[1:])]
self._path = path
def __init__(self, x_dim, z_dim, encoder_hidden_sizes = [100], decoder_hidden_sizes = [100],
hidden_activation = 'tanh', w_init = lambda n_in, n_out: 0.1*np.random.randn(n_in, n_out),
x_distribution = 'gaussian', z_distribution = 'gaussian'
):
self._n_observed_dim = x_dim
self._n_latent_dim = z_dim
self.p_net = DistributionMLP(input_size = z_dim, hidden_sizes = encoder_hidden_sizes,
output_size=x_dim, hidden_activation=hidden_activation, w_init=w_init, distribution = x_distribution)
self.q_net = DistributionMLP(input_size = x_dim, hidden_sizes = decoder_hidden_sizes,
output_size=z_dim, hidden_activation=hidden_activation, w_init=w_init, distribution=z_distribution)
self._prior = \
StandardUniformDistribution(z_dim) if z_distribution == 'bernoulli' else \
StandardNormalDistribution(z_dim) if z_distribution == 'gaussian' else \
bad_value(z_distribution)
def minibatch_index_generator(n_samples, minibatch_size, n_epochs = 1, final_treatment = 'stop', slice_when_possible = True):
"""
Generates the indices for minibatch-iteration.
:param n_samples: Number of samples in the data you want to iterate through
:param minibatch_size: Number of samples in the minibatch
:param n_epochs: Number of epochs to iterate for
:param final_treatment: How to terminate. Options are:
'stop': Stop when you can no longer get a complete minibatch
'truncate': Produce a runt-minibatch at the end.
:param slice_when_possible: Return slices, instead of indices, as long as the indexing does not wrap around. This
can be more efficient, since it avoids array copying, but you have to be careful not to modify your source array.
:yield: IIndices that you can use to slice arrays for minibatch iteration.
"""
true_minibatch_size = n_samples if minibatch_size == 'full' else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
remaining_samples = int(n_epochs * n_samples)
base_indices = np.arange(minibatch_size)
standard_indices = (lambda: slice(i, i+minibatch_size)) if slice_when_possible else (lambda: base_indices+i)
i = 0
while True:
next_i = i + true_minibatch_size
if remaining_samples < minibatch_size: # Final minibatch case
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
yield np.arange(i, i+remaining_samples) % n_samples
break
else:
raise Exception('Unknown final treatment: %s' % final_treatment)
elif next_i < n_samples: # Standard case
segment = standard_indices()
else: # Wraparound case
segment = np.arange(i, next_i) % n_samples
next_i = next_i % n_samples
yield segment
i = next_i
remaining_samples -= minibatch_size
def iterator(data_collection):
"""
:param data_collection: A DataCollection object
:yield: A 2-tuple of (input_data, label_data)
"""
assert isinstance(data_collection, DataCollection)
i = 0
n_samples = data_collection.n_samples
total_samples = epochs * n_samples
true_minibatch_size = n_samples if minibatch_size == 'full' else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
if single_channel:
input_data = data_collection.input
target_data = data_collection.target
else:
input_data = data_collection.inputs
target_data = data_collection.targets
while i < total_samples:
next_i = i + true_minibatch_size
segment = np.arange(i, next_i) % n_samples
if next_i > total_samples:
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
next_i = total_samples
else:
raise Exception('Unknown final treatment: %s' % final_treatment)
if single_channel:
input_minibatch = input_data[segment]
target_minibatch = target_data[segment]
else:
input_minibatch, = [d[segment] for d in input_data]
target_minibatch, = [d[segment] for d in target_data]
yield next_i, input_minibatch, target_minibatch
i = next_i
def get_execution_path(self, inference_path):
"""
Given a path defined in terms of variables int the factor graph, return a full inference that defines the order
in which to compute factors and sample variables.
:param varible_path: A list of 2-tuples, identifying the source, destination layers for the update.
:return: path: An OrderedDict<(tuple<*int>, int): function> indicating the path to take.
"""
if not isinstance(inference_path, InferencePath):
inference_path = InferencePath(inference_path)
path = OrderedDict()
for src_vars, dest_var, is_smooth in inference_path:
# Add the factor nodes pointing to the given variable.
for src_var in src_vars:
path[(src_var, ), factor_name(src_var, dest_var)] = \
self._factors[src_var, dest_var] if (src_var, dest_var) in self._factors else \
self._factors[dest_var, src_var].reverse if (dest_var, src_var) in self._factors else \
bad_value((src_var, dest_var), 'Factor %s does not exist in the graph: %s' % ((src_var, dest_var), self._factors.keys()))
# Add the variable node
path[tuple(factor_name(src_var, dest_var) for src_var in src_vars), dest_var] = self._variables[dest_var].smooth if is_smooth else self._variables[dest_var]
return ExecutionPath(path)
def __init__(self, w, b = 0, normalize_minibatch = False, scale = False, use_bias = True):
"""
:param w: Initial weight value. Can be:
- A numpy array, in which case a shared variable is instantiated from this data.
- A symbolic variable that is either a shared variabe or descended from a shared variable.
This is used when there are shared parameters.
:param b: Can be:
- A numpy vector representing the initial bias on the hidden layer, where len(b) = w.shape[1]
- A scaler, which just initializes the full vector to this value
:param normalize_minibatch: Set to True to normalize over the minibatch. This has been shown to cause better optimization
:param scale: Set to True to include an scale term (per output). Generally this only makes sense if
normalize_minibatch is True.
:param use_bias: Use a bias term? Generally, the answer is "True", a bias term helps.
"""
self.w = create_shared_variable(w, name = 'w')
self.b = create_shared_variable(b, shape = w.shape[1] if w.ndim==2 else (w.shape[0], w.shape[2]) if w.ndim==3 else bad_value(w.shape), name = 'b')
self.log_scale = create_shared_variable(0 if scale else None, shape = w.shape[1], name = 'log_scale') if scale else None
self.normalizer = \
batch_normalize if normalize_minibatch is True else \
None if normalize_minibatch is False else \
normalize_minibatch
self._use_bias = use_bias
def struct_to_layer(struct):
layer_type = struct[1][0]
layer_name = str(struct[0][0])
assert isinstance(layer_type, basestring)
if layer_type == 'conv':
w_orig = struct[2][0, 0] # (n_rows, n_cols, n_in_maps, n_out_maps)
# (n_out_maps, n_in_maps, n_rows, n_cols) (Theano conventions)
w = w_orig.T.swapaxes(2, 3)
b = struct[2][0, 1][:, 0]
padding = 0 if layer_name.startswith('fc') else 1 if layer_name.startswith('conv') else bad_value(layer_name)
layer = ConvLayer(w, b, force_shared_parameters=force_shared_parameters, border_mode=padding, filter_flip=False) # Note: Should filter_flip be true...? Need to check
elif layer_type in ('relu', 'softmax'):
layer = Nonlinearity(layer_type)
elif layer_type == 'pool':
layer = Pooler(region=tuple(struct[3][0].astype(int)), stride=tuple(
struct[4][0].astype(int)), mode=pooling_mode)
else:
raise Exception(
"Don't know about this '%s' layer type." % layer_type)
return layer_name, layer
def dataset_to_testing_sets(dataset, test_on = 'training+test'):
return \
{'Training': (dataset.training_set.input, dataset.training_set.target), 'Test': (dataset.test_set.input, dataset.test_set.target)} if test_on == 'training+test' else \
{'Test': (dataset.test_set.input, dataset.test_set.target)} if test_on == 'test' else \
{'Training': (dataset.training_set.input, dataset.training_set.target)} if test_on == 'training' else \
bad_value(test_on)
def compare_predictors(dataset, online_predictors={}, offline_predictors={}, minibatch_size = 'full',
evaluation_function = 'mse', test_epochs = sqrtspace(0, 1, 10), report_test_scores = True,
test_on = 'training+test', test_batch_size = None, accumulators = None):
"""
Compare a set of predictors by running them on a dataset, and return the learning curves for each predictor.
:param dataset: A DataSet object
:param online_predictors: A dict<str:IPredictor> of online predictors. An online predictor is
sequentially fed minibatches of data and updates its parameters with each minibatch.
:param offline_predictors: A dict<str:object> of offline predictors. Offline predictors obey sklearn's
Estimator/Predictor interfaces - ie they methods
estimator = object.fit(data, targets) and
prediction = object.predict(data)
:param minibatch_size: Size of the minibatches to use for online predictors. Can be:
An int, in which case it represents the minibatch size for all classifiers.
A dict<str: int>, in which case you can set the minibatch size per-classifier.
In place of the int, you can put 'all' if you want to train on the whole dataset in each iteration.
:param test_epochs: Test points to use for online predictors. Can be:
A list of integers - in which case the classifier is after seeing this many samples.
A list of floats - in which case the classifier is tested after seeing this many epochs.
'always' - In which case a test is performed after every training step
The final test point determines the end of training.
:param evaluation_function: Function used to evaluate output of predictors
:param report_test_scores: Boolean indicating whether you'd like to report results online.
:return: An OrderedDict<LearningCurveData>
"""
all_keys = online_predictors.keys()+offline_predictors.keys()
assert len(all_keys) > 0, 'You have to give at least one predictor. Is that too much to ask?'
assert len(all_keys) == len(np.unique(all_keys)), "You have multiple predictors using the same names. Change that."
type_constructor_dict = OrderedDict(
[(k, ('offline', offline_predictors[k])) for k in sorted(offline_predictors.keys())] +
[(k, ('online', online_predictors[k])) for k in sorted(online_predictors.keys())]
)
if not isinstance(minibatch_size, dict):
minibatch_size = {predictor_name: minibatch_size for predictor_name in online_predictors.keys()}
else:
assert online_predictors.viewkeys() == minibatch_size.viewkeys()
if not isinstance(accumulators, dict):
accumulators = {predictor_name: accumulators for predictor_name in online_predictors.keys()}
else:
assert online_predictors.viewkeys() == accumulators.viewkeys()
test_epochs = np.array(test_epochs)
# test_epochs_float = test_epochs.dtype == float
# if test_epochs_float:
# test_epochs = (test_epochs * dataset.training_set.n_samples).astype(int)
if isinstance(evaluation_function, str):
evaluation_function = get_evaluation_function(evaluation_function)
records = OrderedDict()
# Run the offline predictors
for predictor_name, (predictor_type, predictor) in type_constructor_dict.iteritems():
print '%s\nRunning predictor %s\n%s' % ('='*20, predictor_name, '-'*20)
records[predictor_name] = \
assess_offline_predictor(
predictor=predictor,
dataset = dataset,
evaluation_function = evaluation_function,
report_test_scores = report_test_scores,
test_on = test_on,
test_batch_size = test_batch_size
) if predictor_type == 'offline' else \
assess_online_predictor(
predictor=predictor,
dataset = dataset,
evaluation_function = evaluation_function,
test_epochs = test_epochs,
accumulator = accumulators[predictor_name],
minibatch_size = minibatch_size[predictor_name],
report_test_scores = report_test_scores,
test_on = test_on,
test_batch_size = test_batch_size,
) if predictor_type == 'online' else \
bad_value(predictor_type)
print 'Done!'
return records