def get_normalized_vgg_net(up_to_layer=None, force_shared_parameters=True):
"""
Load the normalized version of VGG19 discussed here: https://bethgelab.org/deepneuralart/
"""
norm_vgg19_file = get_file(
relative_name='data/norm-vgg-19.pkl',
url = 'https://s3.amazonaws.com/lasagne/recipes/pretrained/imagenet/vgg19_normalized.pkl',
)
with open(norm_vgg19_file) as f:
vgg_struct = pickle.load(f)
layer_names = ['conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',
'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4', 'pool3', 'conv4_1',
'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3', 'relu4_3', 'conv4_4', 'relu4_4', 'pool4', 'conv5_1', 'relu5_1',
'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4', 'pool5', 'fc6', 'relu6', 'fc7', 'relu7',
'fc8', 'prob']
if isinstance(up_to_layer, list):
up_to_layer = up_to_layer[np.argmax([layer_names.index(layer_name) for layer_name in up_to_layer])]
assert up_to_layer is not None and layer_names.index(up_to_layer) < layer_names.index('fc6'), "This can only be used to load the convolutional portion of vggnet. Set "
net_spec = OrderedDict()
param_iterator = (p for p in vgg_struct['param values'])
for layer_name in layer_names:
if layer_name.startswith('conv'):
w = param_iterator.next()
b = param_iterator.next()
assert w.ndim==4 and b.ndim==1
layer = ConvolverSpec(w=w, b=b, mode = 'same')
elif layer_name.startswith('relu'):
layer = NonlinearitySpec('relu')
elif layer_name.startswith('pool'):
layer = PoolerSpec(region=2, stride=2, mode='max')
elif layer_name.startswith('fc'):
w = param_iterator.next()
b = param_iterator.next()
# Here we'll express the "full" layers as convolutional.
if layer_name == 'fc6':
w = w.T.reshape(4096, 512, 7, 7)
elif layer_name == 'fc7':
w = w.T.reshape(4096, 4096, 1, 1)
elif layer_name == 'fc8':
w = w.T.reshape(1000, 4096, 1, 1)
else:
bad_value(layer_name)
layer = ConvolverSpec(w=w, b=b, mode = 'valid')
elif layer_name == 'prob':
layer = NonlinearitySpec('softmax')
else:
raise Exception("Don't know how to handle layer: '%s'" % (layer_name, ))
net_spec[layer_name] = layer
if layer_name == up_to_layer:
break
if up_to_layer is None:
assert_raises(StopIteration)
return net_spec
def conv_fanout(input_len, kernel_len, conv_mode):
"""
Note: this is horrific and must be simplified.
:param input_len:
:param kernel_len:
:param conv_mode:
:return:
"""
if conv_mode == 'full':
return kernel_len * np.ones(input_len)
else:
if conv_mode == 'half':
conv_mode = 'same'
left_pad = kernel_len // 2 if conv_mode == 'same' else 0 if conv_mode == 'valid' else conv_mode if isinstance(
conv_mode, int) else bad_value(conv_mode)
right_pad = (
kernel_len - 1
) // 2 if conv_mode == 'same' else 0 if conv_mode == 'valid' else conv_mode if isinstance(
conv_mode, int) else bad_value(conv_mode)
full_range = np.arange(left_pad + input_len + right_pad)
max_fanout = np.minimum(
kernel_len, np.maximum(input_len - kernel_len + 1 + 2 * left_pad,
1))
fanout_over_full_range = np.minimum(
max_fanout, np.minimum(full_range + 1, full_range[::-1] + 1))
fanout = fanout_over_full_range[left_pad:len(full_range) - right_pad]
return fanout
def initialize_weight_matrix(n_in,
n_out,
mag='xavier',
base_dist='normal',
rng=None):
"""
Initialize a weight matrix
:param n_in: Number of input units
:param n_out: Number of output units
:param mag: The magnitude, or a string identifying how to calculate the magnitude.
String options can be:
'xavier-forward' - Best for preserving variance of a linear, tanh, or sigmoidal network across layers.
'xavier-both': - A compromize between preserving the variance of the forward, backward pass
'xavier-relu': - Best for preserving variance on the forward pass in a ReLU net.
:param base_dist: 'normal' or 'uniform', or a function taking (n_in, n_out) and returning a (n_in, n_out) array
:param rng: Random number generator or seed
:return: A shape (n_in, n_out) initial weight matrix.
"""
rng = get_rng(rng)
w_base = rng.randn(n_in, n_out) if base_dist == 'normal' else \
(np.rand(n_in, n_out) - 0.5)*np.sqrt(12) if base_dist=='uniform' else \
bad_value(base_dist)
mag_number = \
np.sqrt(2./(n_in+n_out)) if mag in ('xavier', 'xavier-both') else \
np.sqrt(1./n_in) if mag=='xavier-forward' else \
np.sqrt(2./n_in) if mag=='xavier-relu' else \
mag if isinstance(mag, numbers.Real) else \
bad_value(mag)
return w_base * mag_number
def cd_function(*input_signals):
wake_visible = input_signals if input_layers is None else up_path(*input_signals)
wake_hidden = propup(*wake_visible)
initial_hidden =[theano.shared(np.zeros(wh.tag.test_value.shape, dtype = theano.config.floatX), name = 'persistent_hidden_state') for wh in wake_hidden] \
if persistent else wake_hidden
gibbs_path = [(hidden_layers, visible_layers)] + [(visible_layers, hidden_layers), (hidden_layers, visible_layers)] * (n_gibbs-1)
sleep_visible = self.get_inference_function(hidden_layers, visible_layers, gibbs_path)(*initial_hidden)
sleep_hidden = propup(*sleep_visible)
all_params = sum([x.parameters for x in ([self.layers[i] for i in visible_layers]
+[self.layers[i] for i in hidden_layers]+[self.bridges[i, j] for i in visible_layers for j in hidden_layers])], [])
if method == 'free_energy':
cost = free_energy(*wake_visible).mean() - free_energy(*sleep_visible).mean()
elif method == 'energy':
cost = tt.mean(wake_visible.T.dot(wake_hidden) - sleep_visible.T.dot(sleep_hidden))
else:
bad_value(method)
optimizer(cost = cost, parameters = all_params, constants = wake_visible+sleep_visible)
if persistent:
for p, s in zip(initial_hidden, sleep_hidden):
add_update(p, s)
def initialize_weight_matrix(n_in, n_out, mag='xavier', base_dist='normal', scale=1., rng=None):
"""
Initialize a weight matrix
:param n_in: Number of input units
:param n_out: Number of output units
:param mag: The magnitude, or a string identifying how to calculate the magnitude.
String options can be:
'xavier-forward' - Best for preserving variance of a linear, tanh, or sigmoidal network across layers.
'xavier-both': - A compromize between preserving the variance of the forward, backward pass
'xavier-relu': - Best for preserving variance on the forward pass in a ReLU net.
:param base_dist: 'normal' or 'uniform', or a function taking (n_in, n_out) and returning a (n_in, n_out) array
:param rng: Random number generator or seed
:return: A shape (n_in, n_out) initial weight matrix.
"""
rng = get_rng(rng)
w_base = rng.randn(n_in, n_out) if base_dist == 'normal' else \
(rng.rand(n_in, n_out) - 0.5)*np.sqrt(12) if base_dist=='uniform' else \
bad_value(base_dist)
mag_number = \
np.sqrt(2./(n_in+n_out)) if mag in ('xavier', 'xavier-both') else \
np.sqrt(1./n_in) if mag=='xavier-forward' else \
np.sqrt(2./n_in) if mag=='xavier-relu' else \
mag if isinstance(mag, numbers.Real) else \
bad_value(mag)
return w_base * (mag_number*scale)
def sparse_nn_flop_count(activations, fanouts, mode = 'adds'):
assert len(activations)==len(fanouts)
if mode=='adds':
assert all(np.array_equal(np.round(a), a) for a in activations)
n_ops = sum(np.abs(s).sum()*fanout for s, fanout in izip_equal(activations, fanouts))
elif mode=='multiplyadds':
n_ops = 2 * sum(np.sum(s != 0) * fanout for s, fanout in izip_equal(activations, fanouts))
else:
bad_value(mode)
return n_ops
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index])
elif column=='Result':
return results[index]
else:
bad_value(column)
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args.items())
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index].items())
elif column=='Result':
return get_oneline_result_string(records[index])
else:
bad_value(column)
def sparse_nn_flop_count(activations, fanouts, mode='adds'):
assert len(activations) == len(fanouts)
if mode == 'adds':
assert all(np.array_equal(np.round(a), a) for a in activations)
n_ops = sum(
np.abs(s).sum() * fanout
for s, fanout in izip_equal(activations, fanouts))
elif mode == 'multiplyadds':
n_ops = 2 * sum(
np.sum(s != 0) * fanout
for s, fanout in izip_equal(activations, fanouts))
else:
bad_value(mode)
return n_ops
def __init__(self, region, mode, stride=None):
if isinstance(region, int):
region = (region, region)
assert mode in ('max', 'average')
if stride is None:
stride=region
elif isinstance(stride, int):
stride = (stride, stride)
elif isinstance(stride, tuple):
assert len(stride)==2
else:
bad_value(stride, "Expected None, and int, or a tuple of length 2. Not %s" % (stride, ))
self.region = region
self.stride = stride
self.mode = mode
def __init__(self,
x_dim,
z_dim,
encoder_hidden_sizes=[100],
decoder_hidden_sizes=[100],
hidden_activation='tanh',
w_init=lambda n_in, n_out: 0.1 * np.random.randn(n_in, n_out),
x_distribution='gaussian',
z_distribution='gaussian'):
self._n_observed_dim = x_dim
self._n_latent_dim = z_dim
self.p_net = DistributionMLP(input_size=z_dim,
hidden_sizes=encoder_hidden_sizes,
output_size=x_dim,
hidden_activation=hidden_activation,
w_init=w_init,
distribution=x_distribution)
self.q_net = DistributionMLP(input_size=x_dim,
hidden_sizes=decoder_hidden_sizes,
output_size=z_dim,
hidden_activation=hidden_activation,
w_init=w_init,
distribution=z_distribution)
self._prior = \
StandardUniformDistribution(z_dim) if z_distribution == 'bernoulli' else \
StandardNormalDistribution(z_dim) if z_distribution == 'gaussian' else \
bad_value(z_distribution)
def __init__(self, input_size, hidden_sizes, output_size, distribution = 'gaussian', hidden_activation = 'sig', w_init = lambda n_in, n_out: 0.01*np.random.randn(n_in, n_out)):
"""
:param input_size: The dimensionality of the input
:param hidden_sizes: A list indicating the sizes of each hidden layer.
:param output_size: The dimensionality of the output
:param distribution: The form of the output distribution (currently 'gaussian' or 'bernoulli')
:param hidden_activation: A string indicating the type of each hidden layer.
{'sig', 'tanh', 'rect-lin', 'lin', 'softmax'}
:param w_init: A function which, given input dims, output dims, returns an initial weight matrix
"""
all_layer_sizes = [input_size]+hidden_sizes
all_layer_activations = [hidden_activation] * len(hidden_sizes)
processing_chain = sum([[
FullyConnectedTransform(w = w_init(pre_size, post_size)),
get_named_activation_function(activation_fcn)
] for (pre_size, post_size), activation_fcn in zip(zip(all_layer_sizes[:-1], all_layer_sizes[1:]), all_layer_activations)
], [])
distribution_function = \
Branch(
FullyConnectedTransform(w = w_init(all_layer_sizes[-1], output_size)),
FullyConnectedTransform(w_init(all_layer_sizes[-1], output_size))) \
if distribution == 'gaussian' else \
Chain(FullyConnectedTransform(w = w_init(all_layer_sizes[-1], output_size)), get_named_activation_function('sig')) \
if distribution=='bernoulli' else \
bad_value(distribution)
self.distribution = distribution
self.chain = Chain(*processing_chain+[distribution_function])
def __init__(self, specied_path, default_smooth=False):
"""
:param specied_path: Can be defined in two ways:
(1) As a list of source/dest signals (with an optional third argument defining whether the pass should be "smooth". e.g.
[('vis', 'hid'), ('hid', 'ass'), ('ass', 'lab', True)]
[('vis', 'hid'), (('hid', 'lab'), 'ass)), ('ass', ('hid', 'lab')), ('hid', 'vis', True)]
(2) As a list of signals to compute in order:
['vis', 'hid', 'ass', 'lab']
:param default_smooth: For steps where smooth is not specified, define whether the pass should be smooth.
"""
assert isinstance(specied_path, list)
assert isinstance(default_smooth, bool)
spec_type = \
1 if all(isinstance(el, tuple) and len(el) in (2, 3) for el in specied_path) else \
2 if all(isinstance(el, (int, str)) for el in specied_path) else \
bad_value(specied_path, 'Could not interpret the path %s - see docstring of InferencePath for required format.')
if spec_type == 1:
path = []
for el in specied_path:
srcs = _tuplefy_singles(el[0])
for dest in _tuplefy_singles(el[1]):
is_smooth = el[2] if len(el) == 3 else default_smooth
assert isinstance(is_smooth, bool), 'The third element of the tuple in your specified path must be ' \
'a boolean indicating whether to do a smooth pass. It was %s' % (is_smooth, )
path.append((srcs, dest, is_smooth))
else:
path = [((src, ), dest, default_smooth)
for src, dest in zip(specied_path[:-1], specied_path[1:])]
self._path = path
def lookup_function(prisoner_a_choice, prisoner_b_choice):
total_utility = \
2 if prisoner_a_choice=='cooperate' and prisoner_b_choice=='cooperate' else \
3 if prisoner_a_choice != prisoner_b_choice else \
4 if prisoner_b_choice=='betray' and prisoner_a_choice=='betray' \
else bad_value((prisoner_a_choice, prisoner_b_choice))
return total_utility
def get_plot_from_data(data, mode, **plot_preference_kwargs):
return \
get_live_plot_from_data(data, **plot_preference_kwargs) if mode == 'live' else \
get_static_plot_from_data(data, **plot_preference_kwargs) if mode == 'static' else \
ImagePlot(**plot_preference_kwargs) if mode == 'image' else \
bad_value(mode, 'Unknown plot modee: %s' % (mode, ))
def lookup_function(prisoner_a_choice, prisoner_b_choice):
total_utility = \
2 if prisoner_a_choice=='cooperate' and prisoner_b_choice=='cooperate' else \
3 if prisoner_a_choice != prisoner_b_choice else \
4 if prisoner_b_choice=='betray' and prisoner_a_choice=='betray' \
else bad_value((prisoner_a_choice, prisoner_b_choice))
return total_utility
def temporalize(x, smoothing_steps, distance='L1'):
"""
:param x: An (n_samples, n_dims) dataset
:return: A (n_samples, ) array of indexes that can be used to shuffle the input for temporal smoothness.
"""
x_flat = x.reshape(x.shape[0], -1)
index_buffer = np.arange(1, smoothing_steps + 1)
next_sample_buffer = x_flat[1:smoothing_steps + 1].copy()
# Technically, we could do this without a next_sample_buffer (and only an index_buffer), but it would require
# repeatedly accessing a bunch of really scattered memory, so we do it this way.
shuffling_indices = np.zeros(len(x), dtype=int)
rectifier = np.abs if distance == 'L1' else np.square if distance == 'L2' else bad_value(
distance)
p = ProgressIndicator(len(x))
current_index = 0
for i in xrange(len(x)):
shuffling_indices[i] = current_index
closest = np.argmin(
rectifier(x_flat[current_index] - next_sample_buffer).sum(axis=1))
current_index = index_buffer[closest]
weve_aint_done_yet = i + smoothing_steps + 1 < len(x)
next_index = i + smoothing_steps + 1
next_sample_buffer[closest] = x_flat[
next_index] if weve_aint_done_yet else float('inf')
index_buffer[closest] = next_index if weve_aint_done_yet else -1
p(i)
return shuffling_indices
def get_plot_from_data(data, mode, **plot_preference_kwargs):
return \
get_live_plot_from_data(data, **plot_preference_kwargs) if mode == 'live' else \
get_static_plot_from_data(data, **plot_preference_kwargs) if mode == 'static' else \
ImagePlot(**plot_preference_kwargs) if mode == 'image' else \
bad_value(mode, 'Unknown plot modee: %s' % (mode, ))
def __init__(self,
w,
b=0,
normalize_minibatch=False,
scale=False,
use_bias=True):
"""
:param w: Initial weight value. Can be:
- A numpy array, in which case a shared variable is instantiated from this data.
- A symbolic variable that is either a shared variabe or descended from a shared variable.
This is used when there are shared parameters.
:param b: Can be:
- A numpy vector representing the initial bias on the hidden layer, where len(b) = w.shape[1]
- A scaler, which just initializes the full vector to this value
:param normalize_minibatch: Set to True to normalize over the minibatch. This has been shown to cause better optimization
:param scale: Set to True to include an scale term (per output). Generally this only makes sense if
normalize_minibatch is True.
:param use_bias: Use a bias term? Generally, the answer is "True", a bias term helps.
"""
self.w = create_shared_variable(w, name='w')
self.b = create_shared_variable(
b,
shape=w.shape[1] if w.ndim == 2 else
(w.shape[0], w.shape[2]) if w.ndim == 3 else bad_value(w.shape),
name='b')
self.log_scale = create_shared_variable(
0 if scale else None, shape=w.shape[1],
name='log_scale') if scale else None
self.normalizer = \
batch_normalize if normalize_minibatch is True else \
None if normalize_minibatch is False else \
normalize_minibatch
self._use_bias = use_bias
def get_execution_path(self, inference_path):
"""
Given a path defined in terms of variables int the factor graph, return a full inference that defines the order
in which to compute factors and sample variables.
:param varible_path: A list of 2-tuples, identifying the source, destination layers for the update.
:return: path: An OrderedDict<(tuple<*int>, int): function> indicating the path to take.
"""
if not isinstance(inference_path, InferencePath):
inference_path = InferencePath(inference_path)
path = OrderedDict()
for src_vars, dest_var, is_smooth in inference_path:
# Add the factor nodes pointing to the given variable.
for src_var in src_vars:
path[(src_var, ), factor_name(src_var, dest_var)] = \
self._factors[src_var, dest_var] if (src_var, dest_var) in self._factors else \
self._factors[dest_var, src_var].reverse if (dest_var, src_var) in self._factors else \
bad_value((src_var, dest_var), 'Factor %s does not exist in the graph: %s' % ((src_var, dest_var), self._factors.keys()))
# Add the variable node
path[tuple(
factor_name(src_var, dest_var) for src_var in src_vars
), dest_var] = self._variables[
dest_var].smooth if is_smooth else self._variables[dest_var]
return ExecutionPath(path)
def demo_rbm_mnist(
vis_activation = 'bernoulli',
hid_activation = 'bernoulli',
n_hidden = 500,
plot = True,
eta = 0.01,
optimizer = 'sgd',
w_init_mag = 0.001,
minibatch_size = 9,
persistent = False,
n_epochs = 100,
plot_interval = 100,
):
"""
In this demo we train an RBM on the MNIST input data (labels are ignored). We plot the state of a markov chanin
that is being simulaniously sampled from the RBM, and the parameters of the RBM.
What you see:
A plot will appear with 6 subplots. The subplots are as follows:
hidden-neg-chain: The activity of the hidden layer for each of the persistent CD chains for draewing negative samples.
visible-neg-chain: The probabilities of the visible activations corresponding to the state of hidden-neg-chain.
w: A subset of the weight vectors, reshaped to the shape of the input.
b: The bias of the hidden units.
b_rev: The bias of the visible units.
visible-sample: The probabilities of the visible samples drawin from an independent free-sampling chain (outside the
training function).
As learning progresses, visible-neg-chain and visible-sample should increasingly resemble the data.
"""
with EnableOmniscence():
# EnableOmniscence allows us to plot internal variables (by referencing the .locals() attribute of a symbolic function.. see plot_fcn below)
if is_test_mode():
n_epochs = 0.01
data = get_mnist_dataset(flat = True).training_set.input
rbm = simple_rbm(
visible_layer = StochasticNonlinearity(vis_activation),
bridge=FullyConnectedBridge(w = w_init_mag*np.random.randn(28*28, n_hidden).astype(theano.config.floatX), b=0, b_rev = 0),
hidden_layer = StochasticNonlinearity(hid_activation)
)
optimizer = \
SimpleGradientDescent(eta = eta) if optimizer == 'sgd' else \
AdaMax(alpha=eta) if optimizer == 'adamax' else \
bad_value(optimizer)
train_function = rbm.get_training_fcn(n_gibbs = 1, persistent = persistent, optimizer = optimizer).compile()
def plot_fcn():
lv = train_function.locals()
dbplot(lv['wake_visible'].reshape((-1, 28, 28)), 'visible-pos-chain')
dbplot(lv['sleep_visible'].reshape((-1, 28, 28)), 'visible-neg-chain')
for i, visible_data in enumerate(minibatch_iterate(data, minibatch_size=minibatch_size, n_epochs=n_epochs)):
train_function(visible_data)
if plot and i % plot_interval == 0:
plot_fcn()
def get_initial_state(self, x_init):
assert x_init.dim(
) == 2, 'x_init should have 2 dimensions: (batch_size, n_dims). Its shape is {}'.format(
x_init.size())
return torch.autograd.Variable(torch.zeros(1, len(x_init), self.n_hid), requires_grad=True) if self.rnn_type in ('elman', 'gru') else \
(torch.autograd.Variable(torch.zeros(1, len(x_init), self.n_hid), requires_grad=True), torch.autograd.Variable(torch.zeros(1, len(x_init), self.n_hid), requires_grad=True)) if self.rnn_type == 'lstm' else \
bad_value(self.rnn_type)
def get_archive(relative_path, url, force_extract=False, archive_type = None, force_download = False):
"""
Download a compressed archive and extract it into a folder.
:param relative_path: Local name for the extracted folder. (Zip file will be named this with the appropriate zip extension)
:param url: Url of the archive to download
:param force_extract: Force the zip file to re-extract (rather than just reusing the extracted folder)
:return: The full path to the extracted folder on your system.
"""
local_folder_path = get_artemis_data_path(relative_path)
assert archive_type in ('.tar.gz', '.zip', None)
if force_download:
shutil.rmtree(local_folder_path)
if not os.path.exists(local_folder_path) or force_download: # If the folder does not exist, download zip and extract.
# (We also check force download here to avoid a race condition)
response = urllib2.urlopen(url)
# Need to infer
if archive_type is None:
if url.endswith('.tar.gz'):
archive_type = '.tar.gz'
elif url.endswith('.zip'):
archive_type = '.zip'
else:
info = response.info()
try:
header = next(x for x in info.headers if x.startswith('Content-Disposition'))
original_file_name = next(x for x in header.split(';') if x.startswith('filename')).split('=')[-1].lstrip('"\'').rstrip('"\'')
archive_type = '.tar.gz' if original_file_name.endswith('.tar.gz') else '.zip' if original_file_name.endswith('.zip') else \
bad_value(original_file_name, 'Filename "%s" does not end with a familiar zip extension like .zip or .tar.gz' % (original_file_name, ))
except StopIteration:
raise Exception("Could not infer archive type from user argument, url-name, or file-header. Please specify archive type as either '.zip' or '.tar.gz'.")
print 'Downloading archive from url: "%s"...' % (url, )
data = response.read()
print '...Done.'
local_zip_path = local_folder_path + archive_type
make_file_dir(local_zip_path)
with open(local_zip_path, 'w') as f:
f.write(data)
force_extract = True
if force_extract:
if archive_type == '.tar.gz':
with tarfile.open(local_zip_path) as f:
f.extractall(local_folder_path)
elif archive_type == '.zip':
with ZipFile(local_zip_path) as f:
f.extractall(local_folder_path)
else:
raise Exception()
return local_folder_path
def __init__(self, threshold_freq, order=2, design='cheby1'):
"""
:param threshold_freq: Threshold frequency to filter out, as a fraction of sampling freq. E.g. 0.1
"""
self.b, self.a = \
butter(N=order, Wn=threshold_freq, btype='low') if design == 'butter' else \
cheby1(N=order, rp=0.1, Wn=threshold_freq, btype='low') if design == 'cheby1' else \
bad_value(design)
self.filter_state = lfilter_zi(b=self.b, a=self.a)
def __init__(self, threshold_freq, order=2, design='cheby1'):
"""
:param threshold_freq: Threshold frequency to filter out, as a fraction of sampling freq. E.g. 0.1
"""
self.b, self.a = \
butter(N=order, Wn=threshold_freq, btype='low') if design == 'butter' else \
cheby1(N=order, rp=0.1, Wn=threshold_freq, btype='low') if design == 'cheby1' else \
bad_value(design)
self.filter_state = lfilter_zi(b=self.b, a=self.a)
def minibatch_index_generator(n_samples,
minibatch_size,
n_epochs=1,
final_treatment='stop',
slice_when_possible=True):
"""
Generates the indices for minibatch-iteration.
:param n_samples: Number of samples in the data you want to iterate through
:param minibatch_size: Number of samples in the minibatch
:param n_epochs: Number of epochs to iterate for
:param final_treatment: How to terminate. Options are:
'stop': Stop when you can no longer get a complete minibatch
'truncate': Produce a runt-minibatch at the end.
:param slice_when_possible: Return slices, instead of indices, as long as the indexing does not wrap around. This
can be more efficient, since it avoids array copying, but you have to be careful not to modify your source array.
:yield: IIndices that you can use to slice arrays for minibatch iteration.
"""
if minibatch_size == SINGLE_MINIBATCH_SIZE:
for i in (xrange(n_samples * n_epochs)
if not np.isinf(n_epochs) else itertools.count(0)):
yield i % n_samples
return
true_minibatch_size = n_samples if minibatch_size == FULL_MINIBATCH_SIZE else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
remaining_samples = int(n_epochs *
n_samples) if not np.isinf(n_epochs) else np.inf
base_indices = np.arange(minibatch_size)
standard_indices = (
lambda: slice(i, i + minibatch_size)) if slice_when_possible else (
lambda: base_indices + i)
i = 0
while True:
next_i = i + true_minibatch_size
if remaining_samples < minibatch_size: # Final minibatch case
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
yield np.arange(i, i + remaining_samples) % n_samples
break
else:
raise Exception('Unknown final treatment: %s' %
final_treatment)
elif next_i < n_samples: # Standard case
segment = standard_indices()
else: # Wraparound case
segment = np.arange(i, next_i) % n_samples
next_i = next_i % n_samples
yield segment
i = next_i
remaining_samples -= minibatch_size
def iteration_info(n_samples, minibatch_size, test_epochs = None, n_epochs = 5):
"""
Create an iterator that keeps track of the state of minibatch iteration, and simplifies the scheduling of tests.
You can izip this iterator with one that returns your data.
:param n_samples: Number of samples in the dataset.
:param minibatch_size: Size of minibatches
:param test_epochs: Epochs on which you'd like to run tests. You can also enter
'every', which will test once-per-epoch,
'always', which will test on every iteration
'never', which will never test.
('every', 0.2), which will test at every 0.2 epochs (for example)
:yield: IterationInfo objects which contain info about the state of iteration.
"""
# next_text_point = 0 if test_epochs is not None and len(test_epochs)>0 else None
start_time = time.time()
n_samples = float(n_samples)
if minibatch_size==FULL_MINIBATCH_SIZE:
minibatch_size = n_samples
elif minibatch_size == SINGLE_MINIBATCH_SIZE:
minibatch_size = 1
elif not isinstance(minibatch_size, int):
raise Exception('Unexpected value for minibatch_size: {}'.format(minibatch_size))
if isinstance(test_epochs, str):
assert test_epochs in ('always', 'never', 'every'), "test_epochs={} is not valid".format(test_epochs)
elif isinstance(test_epochs, tuple):
assert len(test_epochs)==2, "If you pass in a tuple for test epochs, it should be in the form ('every', period). Get {}".format(test_epochs)
name, period = test_epochs
assert period > 0, 'Period must be a positive number, not {}'.format(period)
assert name == 'every'
elif n_epochs is None and isinstance(test_epochs, (list, tuple, np.ndarray)):
n_epochs = test_epochs[-1]
last_epoch = -float('inf')
for i in itertools.count(0):
epoch = i*minibatch_size/n_samples
test_now = (
True if test_epochs=='always' else
False if test_epochs=='never' else
np.floor(epoch)>np.floor(last_epoch) if test_epochs == 'every' else
bad_value(test_epochs)
) if isinstance(test_epochs, string_types) else \
np.floor(epoch/period) > np.floor(last_epoch/period) if isinstance(test_epochs, tuple) else \
False if test_epochs is None else \
np.searchsorted(test_epochs, epoch, side='right') > np.searchsorted(test_epochs, last_epoch, side='right')
info = IterationInfo(
iteration = i,
epoch = epoch,
sample = i*minibatch_size,
time = time.time()-start_time,
test_now = test_now,
done = epoch >= n_epochs if n_epochs is not None else False
)
yield info
last_epoch = epoch
def iteration_info(n_samples, minibatch_size, test_epochs = None, n_epochs = 5):
"""
Create an iterator that keeps track of the state of minibatch iteration, and simplifies the scheduling of tests.
You can izip this iterator with one that returns your data.
:param n_samples: Number of samples in the dataset.
:param minibatch_size: Size of minibatches
:param test_epochs: Epochs on which you'd like to run tests. You can also enter
'every', which will test once-per-epoch,
'always', which will test on every iteration
'never', which will never test.
('every', 0.2), which will test at every 0.2 epochs (for example)
:yield: IterationInfo objects which contain info about the state of iteration.
"""
# next_text_point = 0 if test_epochs is not None and len(test_epochs)>0 else None
start_time = time.time()
n_samples = float(n_samples)
if minibatch_size==FULL_MINIBATCH_SIZE:
minibatch_size = n_samples
elif minibatch_size == SINGLE_MINIBATCH_SIZE:
minibatch_size = 1
elif not isinstance(minibatch_size, int):
raise Exception('Unexpected value for minibatch_size: {}'.format(minibatch_size))
if isinstance(test_epochs, str):
assert test_epochs in ('always', 'never', 'every'), "test_epochs={} is not valid".format(test_epochs)
elif isinstance(test_epochs, tuple):
assert len(test_epochs)==2, "If you pass in a tuple for test epochs, it should be in the form ('every', period). Get {}".format(test_epochs)
name, period = test_epochs
assert period > 0, 'Period must be a positive number, not {}'.format(period)
assert name == 'every'
elif n_epochs is None and isinstance(test_epochs, (list, tuple, np.ndarray)):
n_epochs = test_epochs[-1]
last_epoch = -float('inf')
for i in itertools.count(0):
epoch = i*minibatch_size/n_samples
test_now = (
True if test_epochs=='always' else
False if test_epochs=='never' else
np.floor(epoch)>np.floor(last_epoch) if test_epochs == 'every' else
bad_value(test_epochs)
) if isinstance(test_epochs, string_types) else \
np.floor(epoch/period) > np.floor(last_epoch/period) if isinstance(test_epochs, tuple) else \
False if test_epochs is None else \
np.searchsorted(test_epochs, epoch, side='right') > np.searchsorted(test_epochs, last_epoch, side='right')
info = IterationInfo(
iteration = i,
epoch = epoch,
sample = i*minibatch_size,
time = time.time()-start_time,
test_now = test_now,
done = epoch >= n_epochs if n_epochs is not None else False
)
yield info
last_epoch = epoch
def plot_learning_curve(train_test_errors, scale='loglog'):
cumulative_loss = RunningAverage.batch(train_test_errors['online_errors'])
recent_loss = RecentRunningAverage.batch(train_test_errors['online_errors'])
plt_func = plt.loglog if scale =='loglog' else plt.plot if scale=='normal' else bad_value(scale)
plt_func(recent_loss, label = 'Recent Loss')
plt_func(cumulative_loss, label = 'Cumulative Loss')
plt.xlabel('t')
plt.ylabel('error')
plt.legend()
plt.grid()
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def load_ilsvrc_video(identifier,
size=None,
resize_mode='scale_crop',
cut_edges=True,
cut_edges_thresh=5):
"""
Load a file from the ILSVRC Dataset. The first time this is run, it will download an 8GB file, so be patient.
Note: If you are using the same videos repeatedly, and applying resizing, you may want to call this function as:
memoize_to_disk(load_ilsvrc_video)(identifier, size, ...)
This will save you time on future runs.
:param identifier: The file-name of the video, not including the extension. Eg: 'ILSVRC2015_train_00249001'
:param size:
:param cut_edges:
:param cut_edges_thresh:
:return:
"""
print('Downloading ILSVER2015... this may take a while...')
archive_folder_path = get_archive(
relative_path='data/ILSVRC2015',
url=
'http://vision.cs.unc.edu/ilsvrc2015/ILSVRC2015_VID_snippets_final.tar.gz'
)
print('Done.')
subpath = \
'ILSVRC2015/Data/VID/snippets/test' if 'test' in identifier else \
'ILSVRC2015/Data/VID/snippets/val' if 'val' in identifier else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0001/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0001/', identifier + '.mp4')) else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0002/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0002/', identifier + '.mp4')) else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0003/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0003/', identifier + '.mp4')) else \
bad_value(identifier, 'Could not find identifier: {}'.format(identifier, ))
print('Loading %s' % (identifier, ))
full_path = get_file_in_archive(
relative_path='data/ILSVRC2015',
subpath=os.path.join(subpath, identifier + '.mp4'),
url=
'http://vision.cs.unc.edu/ilsvrc2015/ILSVRC2015_VID_snippets_final.tar.gz'
)
video = smart_load_video(full_path,
size=size,
cut_edges=cut_edges,
resize_mode=resize_mode,
cut_edges_thresh=cut_edges_thresh)
print('Done.')
return video
def minibatch_index_generator(n_samples, minibatch_size, n_epochs = 1, final_treatment = 'stop', slice_when_possible = True):
"""
Generates the indices for minibatch-iteration.
:param n_samples: Number of samples in the data you want to iterate through
:param minibatch_size: Number of samples in the minibatch
:param n_epochs: Number of epochs to iterate for
:param final_treatment: How to terminate. Options are:
'stop': Stop when you can no longer get a complete minibatch
'truncate': Produce a runt-minibatch at the end.
:param slice_when_possible: Return slices, instead of indices, as long as the indexing does not wrap around. This
can be more efficient, since it avoids array copying, but you have to be careful not to modify your source array.
:yield: IIndices that you can use to slice arrays for minibatch iteration.
"""
if minibatch_size==SINGLE_MINIBATCH_SIZE:
for i in (xrange(n_samples*n_epochs) if not np.isinf(n_epochs) else itertools.count(0)):
yield i % n_samples
return
true_minibatch_size = n_samples if minibatch_size == FULL_MINIBATCH_SIZE else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
remaining_samples = int(n_epochs * n_samples) if not np.isinf(n_epochs) else np.inf
base_indices = np.arange(minibatch_size)
standard_indices = (lambda: slice(i, i+minibatch_size)) if slice_when_possible else (lambda: base_indices+i)
i = 0
while remaining_samples>0:
next_i = i + true_minibatch_size
if remaining_samples < minibatch_size: # Final minibatch case
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
yield np.arange(i, i+remaining_samples) % n_samples
break
else:
raise Exception('Unknown final treatment: %s' % final_treatment)
elif next_i < n_samples: # Standard case
segment = standard_indices()
else: # Wraparound case
segment = np.arange(i, next_i) % n_samples
next_i = next_i % n_samples
yield segment
i = next_i
remaining_samples -= minibatch_size
def _create_subplot(fig=None, layout=None):
if layout is None:
layout = _newplot_settings['layout']
if fig is None:
fig = plt.gcf()
n = len(fig.axes)
n_rows, n_cols = (1, n+1) if layout in ('h', 'horizontal') else (n+1, 1) if layout in ('v', 'vertical') else \
vector_length_to_tile_dims(n+1) if layout in ('g', 'grid') else bad_value(layout)
for i in range(n):
fig.axes[i].change_geometry(n_rows, n_cols, i + 1)
ax = fig.add_subplot(n_rows, n_cols, n + 1)
if not _newplot_settings['show_x']:
ax.get_xaxis().set_visible(False)
if not _newplot_settings['show_y']:
ax.get_yaxis().set_visible(False)
return ax
def _create_subplot(fig = None, layout = None, position = None, **subplot_args):
if layout is None:
layout = _newplot_settings['layout']
if fig is None:
fig = plt.gcf()
n = len(fig.axes)
n_rows, n_cols = (1, n+1) if layout in ('h', 'horizontal') else (n+1, 1) if layout in ('v', 'vertical') else \
vector_length_to_tile_dims(n+1) if layout in ('g', 'grid') else bad_value(layout)
for i in range(n):
fig.axes[i].change_geometry(n_rows, n_cols, i+1)
for arg in ('sharex', 'sharey'):
if isinstance(_newplot_settings[arg], plt.Axes):
subplot_args[arg]=_newplot_settings[arg]
ax = fig.add_subplot(n_rows, n_cols, n+1, **subplot_args)
if _newplot_settings['xlabel'] is not None:
ax.set_xlabel(_newplot_settings['xlabel'])
if _newplot_settings['ylabel'] is not None:
ax.set_ylabel(_newplot_settings['ylabel'])
if _newplot_settings['xlim'] is not None:
ax.set_xlim(_newplot_settings['xlim'])
if _newplot_settings['ylim'] is not None:
ax.set_ylim(_newplot_settings['ylim'])
if _newplot_settings['grid']:
plt.grid()
for arg in ('sharex', 'sharey'):
if _newplot_settings[arg] is True:
_newplot_settings[arg]=ax
if not _newplot_settings['show_x']:
ax.tick_params(axis='x', labelbottom='off')
# ax.get_xaxis().set_visible(False)
if not _newplot_settings['show_y']:
ax.tick_params(axis='y', labelleft='off')
# ax.get_yaxis().set_visible(False)
return ax
def __init__(self,
ws,
bs=None,
comp_weight=1e-6,
optimizer=None,
layerwise_scales=False,
parametrization='log',
hidden_activations='relu',
output_activation='softmax',
rng=None):
"""
Learns how to rescale the units to be an optimal rounding network.
:param ws: A list of (n_in, n_out) weight matrices
:param bs: A length of bias vectors (same length as ws)
:param comp_weight: The weight (lambda in the paper) given to computation
:param optimizer: The optimizer (an IGradientOptimizer object)
:param layerwise_scales: Make scales layerwise (as opposed to unitwise)
:param parametrization: What space to parametrize in ('log', 'direct', or 'softplus')
:param hidden_activations: Hidden activation functions (as a string, eg 'relu')
:param output_activation: Output activation function
:param rng: Random number generator or seed.
"""
if optimizer is None:
optimizer = get_named_optimizer('sgd', 0.01)
if bs is None:
bs = [np.zeros(w.shape[1]) for w in ws]
self.ws = [create_shared_variable(w) for w in ws]
self.bs = [create_shared_variable(b) for b in bs]
self.comp_weight = tt.constant(comp_weight, dtype=theano.config.floatX)
self.optimizer = optimizer
self.hidden_activations = hidden_activations
self.output_activation = output_activation
scale_dims = [()] * len(ws) if layerwise_scales else [
ws[0].shape[0]
] + [w.shape[1] for w in ws[:-1]]
self.k_params = \
[create_shared_variable(np.ones(d)) for d in scale_dims] if parametrization=='direct' else \
[create_shared_variable(np.zeros(d)) for d in scale_dims] if parametrization=='log' else \
[create_shared_variable(np.zeros(d)+np.exp(1)-1) for d in scale_dims] if parametrization=='softplus' else \
bad_value(parametrization)
self.parametrization = parametrization
self.rng = get_theano_rng(rng)
def iterator(data_collection):
"""
:param data_collection: A DataCollection object
:yield: A 2-tuple of (input_data, label_data)
"""
assert isinstance(data_collection, DataCollection)
i = 0
n_samples = data_collection.n_samples
total_samples = epochs * n_samples
true_minibatch_size = n_samples if minibatch_size == 'full' else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
if single_channel:
input_data = data_collection.input
target_data = data_collection.target
else:
input_data = data_collection.inputs
target_data = data_collection.targets
while i < total_samples:
next_i = i + true_minibatch_size
segment = np.arange(i, next_i) % n_samples
if next_i > total_samples:
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
next_i = total_samples
else:
raise Exception('Unknown final treatment: %s' %
final_treatment)
if single_channel:
input_minibatch = input_data[segment]
target_minibatch = target_data[segment]
else:
input_minibatch, = [d[segment] for d in input_data]
target_minibatch, = [d[segment] for d in target_data]
yield next_i, input_minibatch, target_minibatch
i = next_i
def iterator(data_collection):
"""
:param data_collection: A DataCollection object
:yield: A 2-tuple of (input_data, label_data)
"""
assert isinstance(data_collection, DataCollection)
i = 0
n_samples = data_collection.n_samples
total_samples = epochs * n_samples
true_minibatch_size = n_samples if minibatch_size == 'full' else \
minibatch_size if isinstance(minibatch_size, int) else \
bad_value(minibatch_size)
if single_channel:
input_data = data_collection.input
target_data = data_collection.target
else:
input_data = data_collection.inputs
target_data = data_collection.targets
while i < total_samples:
next_i = i + true_minibatch_size
segment = np.arange(i, next_i) % n_samples
if next_i > total_samples:
if final_treatment == 'stop':
break
elif final_treatment == 'truncate':
next_i = total_samples
else:
raise Exception('Unknown final treatment: %s' % final_treatment)
if single_channel:
input_minibatch = input_data[segment]
target_minibatch = target_data[segment]
else:
input_minibatch, = [d[segment] for d in input_data]
target_minibatch, = [d[segment] for d in target_data]
yield next_i, input_minibatch, target_minibatch
i = next_i
def load_ilsvrc_video(identifier, size = None, resize_mode='scale_crop', cut_edges=True, cut_edges_thresh=5):
"""
Load a file from the ILSVRC Dataset. The first time this is run, it will download an 8GB file, so be patient.
Note: If you are using the same videos repeatedly, and applying resizing, you may want to call this function as:
memoize_to_disk(load_ilsvrc_video)(identifier, size, ...)
This will save you time on future runs.
:param identifier: The file-name of the video, not including the extension. Eg: 'ILSVRC2015_train_00249001'
:param size:
:param cut_edges:
:param cut_edges_thresh:
:return:
"""
print ('Downloading ILSVER2015... this may take a while...')
archive_folder_path = get_archive(
relative_path='data/ILSVRC2015',
url='http://vision.cs.unc.edu/ilsvrc2015/ILSVRC2015_VID_snippets_final.tar.gz'
)
print ('Done.')
subpath = \
'ILSVRC2015/Data/VID/snippets/test' if 'test' in identifier else \
'ILSVRC2015/Data/VID/snippets/val' if 'val' in identifier else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0001/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0001/', identifier + '.mp4')) else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0002/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0002/', identifier + '.mp4')) else \
'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0003/' if os.path.exists(os.path.join(archive_folder_path, 'ILSVRC2015/Data/VID/snippets/train/ILSVRC2015_VID_train_0003/', identifier + '.mp4')) else \
bad_value(identifier, 'Could not find identifier: {}'.format(identifier, ))
print('Loading %s' % (identifier, ))
full_path = get_file_in_archive(
relative_path='data/ILSVRC2015',
subpath=os.path.join(subpath, identifier+'.mp4'),
url='http://vision.cs.unc.edu/ilsvrc2015/ILSVRC2015_VID_snippets_final.tar.gz'
)
video = smart_load_video(full_path, size=size, cut_edges=cut_edges, resize_mode=resize_mode, cut_edges_thresh=cut_edges_thresh)
print('Done.')
return video
def get_simple_constrcutor(cls, epsilons, quantizer, lambdas=None):
stepper = create_step_sizer(epsilons)
quantizer = \
SigmaDeltaQuantizer() if quantizer == 'sigma_delta' else \
StochasticQuantizer() if quantizer == 'stochastic' else \
ThresholdQuantizer() if quantizer == 'threshold' else \
SecondOrderSigmaDeltaQuantizer() if quantizer == 'second_order_sd' else \
bad_value(quantizer)
if lambdas is None:
encoder = quantizer
decoder = IdentityFunction()
else:
lambda_stepper = create_step_sizer(lambdas)
encoder = PredictiveEncoder(lambda_stepper=lambda_stepper,
quantizer=quantizer)
decoder = PredictiveDecoder(lambda_stepper=lambda_stepper)
return cls.get_partial_constructor(encoder=encoder,
decoder=decoder,
stepper=stepper)
def struct_to_layer(struct):
layer_type = struct[1][0]
layer_name = str(struct[0][0])
assert isinstance(layer_type, basestring)
if layer_type == 'conv':
w_orig = struct[2][0, 0] # (n_rows, n_cols, n_in_maps, n_out_maps)
w = w_orig.T.swapaxes(2, 3)
b = struct[2][0, 1][:, 0]
layer = ConvolverSpec(
w=w,
b=b,
mode='full' if layer_name.startswith('fc') else 'same'
if layer_name.startswith('conv') else bad_value(layer_name))
elif layer_type in ('relu', 'softmax'):
layer = NonlinearitySpec(layer_type)
elif layer_type == 'pool':
layer = PoolerSpec(region=tuple(struct[3][0].astype(int)),
stride=tuple(struct[4][0].astype(int)),
mode=struct[2][0])
else:
raise Exception("Don't know about this '%s' layer type." %
layer_type)
return layer_name, layer
def temporalize(x, smoothing_steps, distance='L1'):
"""
:param x: An (n_samples, n_dims) dataset
:return: A (n_samples, ) array of indexes that can be used to shuffle the input for temporal smoothness.
"""
x_flat = x.reshape(x.shape[0], -1)
index_buffer = np.arange(1, smoothing_steps+1)
next_sample_buffer = x_flat[1:smoothing_steps+1].copy()
# Technically, we could do this without a next_sample_buffer (and only an index_buffer), but it would require
# repeatedly accessing a bunch of really scattered memory, so we do it this way.
shuffling_indices = np.zeros(len(x), dtype=int)
rectifier = np.abs if distance=='L1' else np.square if distance=='L2' else bad_value(distance)
p=ProgressIndicator(len(x), name = 'Temporalize')
current_index = 0
for i in xrange(len(x)):
shuffling_indices[i] = current_index
closest = np.argmin(rectifier(x_flat[current_index]-next_sample_buffer).sum(axis=1))
current_index = index_buffer[closest]
weve_aint_done_yet = i+smoothing_steps+1 < len(x)
next_index = i+smoothing_steps+1
next_sample_buffer[closest] = x_flat[next_index] if weve_aint_done_yet else float('inf')
index_buffer[closest] = next_index if weve_aint_done_yet else -1
p()
return shuffling_indices
def conv_fanout(input_len, kernel_len, conv_mode):
"""
Note: this is horrific and must be simplified.
:param input_len:
:param kernel_len:
:param conv_mode:
:return:
"""
if conv_mode=='full':
return kernel_len*np.ones(input_len)
else:
if conv_mode=='half':
conv_mode='same'
left_pad = kernel_len // 2 if conv_mode == 'same' else 0 if conv_mode == 'valid' else conv_mode if isinstance(conv_mode, int) else bad_value(conv_mode)
right_pad = (kernel_len-1) // 2 if conv_mode == 'same' else 0 if conv_mode == 'valid' else conv_mode if isinstance(conv_mode, int) else bad_value(conv_mode)
full_range = np.arange(left_pad + input_len + right_pad)
max_fanout = np.minimum(kernel_len, np.maximum(input_len-kernel_len+1+2*left_pad, 1))
fanout_over_full_range = np.minimum(max_fanout, np.minimum(full_range+1, full_range[::-1]+1))
fanout = fanout_over_full_range[left_pad:len(full_range)-right_pad]
return fanout
def compare_predictors(dataset, online_predictors={}, offline_predictors={}, minibatch_size = 'full',
evaluation_function = 'mse', test_epochs = sqrtspace(0, 1, 10), report_test_scores = True,
test_on = 'training+test', test_batch_size = None, accumulators = None, online_test_callbacks = {}):
"""
DEPRECATED: use train_and_test_online_predictor instead.
Compare a set of predictors by running them on a dataset, and return the learning curves for each predictor.
:param dataset: A DataSet object
:param online_predictors: A dict<str:IPredictor> of online predictors. An online predictor is
sequentially fed minibatches of data and updates its parameters with each minibatch.
:param offline_predictors: A dict<str:object> of offline predictors. Offline predictors obey sklearn's
Estimator/Predictor interfaces - ie they methods
estimator = object.fit(data, targets) and
prediction = object.predict(data)
:param minibatch_size: Size of the minibatches to use for online predictors. Can be:
An int, in which case it represents the minibatch size for all classifiers.
A dict<str: int>, in which case you can set the minibatch size per-classifier.
In place of the int, you can put 'all' if you want to train on the whole dataset in each iteration.
:param test_epochs: Test points to use for online predictors. Can be:
A list of integers - in which case the classifier is after seeing this many samples.
A list of floats - in which case the classifier is tested after seeing this many epochs.
'always' - In which case a test is performed after every training step
The final test point determines the end of training.
:param evaluation_function: Function used to evaluate output of predictors
:param report_test_scores: Boolean indicating whether you'd like to report results online.
:param test_on: 'training', 'test', 'training+test'
:param test_batch_size: When the test set is too large to process in one step, use this to break it
up into chunks.
:param accumulators: A dict<str: accum_fcn>, where accum_fcn is a stateful-function of the form:
accmulated_output = accum_fcn(this_output)
Special case: accum_fcn can be 'avg' to make a running average.
:param online_test_callbacks: A dict<str: fcn> where fcn is a callback that takes an online
predictor as an argument. Useful for logging/plotting/debugging progress during training.
:return: An OrderedDict<LearningCurveData>
"""
all_keys = online_predictors.keys()+offline_predictors.keys()
assert len(all_keys) > 0, 'You have to give at least one predictor. Is that too much to ask?'
assert len(all_keys) == len(np.unique(all_keys)), "You have multiple predictors using the same names. Change that."
type_constructor_dict = OrderedDict(
[(k, ('offline', offline_predictors[k])) for k in sorted(offline_predictors.keys())] +
[(k, ('online', online_predictors[k])) for k in sorted(online_predictors.keys())]
)
minibatch_size = _pack_into_dict(minibatch_size, expected_keys=online_predictors.keys())
accumulators = _pack_into_dict(accumulators, expected_keys=online_predictors.keys())
online_test_callbacks = _pack_into_dict(online_test_callbacks, expected_keys=online_predictors.keys(), allow_subset=True)
test_epochs = np.array(test_epochs)
if isinstance(evaluation_function, str):
evaluation_function = get_evaluation_function(evaluation_function)
records = OrderedDict()
# Run the offline predictors
for predictor_name, (predictor_type, predictor) in type_constructor_dict.items():
print('%s\nRunning predictor %s\n%s' % ('='*20, predictor_name, '-'*20))
records[predictor_name] = \
assess_offline_predictor(
predictor=predictor,
dataset = dataset,
evaluation_function = evaluation_function,
report_test_scores = report_test_scores,
test_on = test_on,
test_batch_size = test_batch_size
) if predictor_type == 'offline' else \
assess_online_predictor(
predictor=predictor,
dataset = dataset,
evaluation_function = evaluation_function,
test_epochs = test_epochs,
accumulator = accumulators[predictor_name],
minibatch_size = minibatch_size[predictor_name],
report_test_scores = report_test_scores,
test_on = test_on,
test_batch_size = test_batch_size,
test_callback=online_test_callbacks[predictor_name] if predictor_name in online_test_callbacks else None
) if predictor_type == 'online' else \
bad_value(predictor_type)
print('Done!')
return records
def dataset_to_testing_sets(dataset, test_on = 'training+test'):
return \
{'Training': (dataset.training_set.input, dataset.training_set.target), 'Test': (dataset.test_set.input, dataset.test_set.target)} if test_on == 'training+test' else \
{'Test': (dataset.test_set.input, dataset.test_set.target)} if test_on == 'test' else \
{'Training': (dataset.training_set.input, dataset.training_set.target)} if test_on == 'training' else \
bad_value(test_on)
def get_archive(url, relative_path=None, force_extract=False, archive_type = None, use_cache=True):
"""
Download a compressed archive and extract it into a folder.
:param relative_path: Local name for the extracted folder. (Zip file will be named this with the appropriate zip extension)
:param url: Url of the archive to download
:param force_extract: Force the zip file to re-extract (rather than just reusing the extracted folder)
:return: The full path to the extracted folder on your system.
"""
if relative_path is None:
relative_path = get_unnamed_file_hash(url)
local_folder_path, _ = os.path.splitext(os.path.join(FILE_ROOT, relative_path))
assert archive_type in ('.tar.gz', '.zip', None)
if (not os.path.exists(local_folder_path)) or (not use_cache): # If the folder does not exist, download zip and extract.
# (We also check force download here to avoid a race condition)
if not use_cache and os.path.exists(local_folder_path):
shutil.rmtree(local_folder_path)
response = urlopen(url)
# Need to infer
if archive_type is None:
if url.endswith('.tar.gz'):
archive_type = '.tar.gz'
elif url.endswith('.zip'):
archive_type = '.zip'
else:
# info = response.info()
try:
# header = next(x for x in info.headers if x.startswith('Content-Disposition'))
header = response.headers['content-disposition']
original_file_name = next(x for x in header.split(';') if x.startswith('filename')).split('=')[-1].lstrip('"\'').rstrip('"\'')
archive_type = '.tar.gz' if original_file_name.endswith('.tar.gz') else '.zip' if original_file_name.endswith('.zip') else \
bad_value(original_file_name, 'Filename "%s" does not end with a familiar zip extension like .zip or .tar.gz' % (original_file_name, ))
except StopIteration:
raise Exception("Could not infer archive type from user argument, url-name, or file-header. Please specify archive type as either '.zip' or '.tar.gz'.")
print('Downloading archive from url: "%s"...' % (url, ))
data = response.read()
print('...Done.')
local_zip_path = local_folder_path + archive_type
if os.path.exists(local_zip_path):
if os.path.isdir(local_zip_path): # This shouldnt happen but may by accident.
rmtree(local_zip_path)
else:
os.remove(local_zip_path)
make_file_dir(local_zip_path)
with open(local_zip_path, 'wb') as f:
f.write(data)
force_extract = True
if force_extract:
if archive_type == '.tar.gz':
with tarfile.open(local_zip_path) as f:
f.extractall(local_folder_path)
elif archive_type == '.zip':
with ZipFile(local_zip_path) as f:
f.extractall(local_folder_path)
else:
raise Exception()
return local_folder_path
def _plot_last_data(self, data):
"""
:param data: Can be:
An array of y_data (x_data is assumed to be np.arange(data.shape[0])
A 2-tuple of (x_data, y_data)
"""
# Format all data as (list<x_data>, list<y_data>)
if isinstance(data, tuple) and len(data)==2:
x_data, y_data = data
else:
x_data = None
y_data = data
if isinstance(y_data, np.ndarray):
n_lines = 1 if y_data.ndim==1 else y_data.shape[1]
x_data = [np.arange(1, y_data.shape[0]+1) if self.x_axis_type=='log' else np.arange(y_data.shape[0])] * n_lines if x_data is None else [x_data] * n_lines if x_data.ndim==1 else x_data.T
# x_data = y_data.T if y_data.ndim==2 else y_data[None] if y_data.ndim==1 else bad_value(y_data.ndim)
y_data = y_data.T if y_data.ndim==2 else y_data[None] if y_data.ndim==1 else bad_value(y_data.ndim, "Line plot data must be 1D or 2D, not {}D".format(y_data.ndim))
else: # List of arrays
if all(d.ndim==0 for d in data): # Turn it into one line
x_data = np.arange(1, y_data.shape[0]+1) if self.x_axis_type=='log' else np.arange(y_data.shape[0])[None, :]
y_data = np.array(data)[None, :]
else: # List of arrays becomes a list of lines
x_data = [np.arange(1, y_data.shape[0]+1) if self.x_axis_type=='log' else np.arange(y_data.shape[0]) for d in y_data] if x_data is None else x_data
assert len(x_data)==len(y_data), "The number of lines in your x-data (%s) does not match the number in your y-data (%s)" % (len(x_data), len(y_data))
lower, upper = (np.nanmin(y_data) if self.y_bounds[0] is None else self.y_bounds[0], np.nanmax(y_data)+1e-9 if self.y_bounds[1] is None else self.y_bounds[1])
left, right = (np.nanmin(x_data) if self.x_bounds[0] is None else self.x_bounds[0], np.nanmax(x_data)+1e-9 if self.x_bounds[1] is None else self.x_bounds[1])
lower = lower if np.isfinite(lower) else 0
upper = upper if np.isfinite(upper) else lower+1e-9
left = left if np.isfinite(left) else 0
right = right if np.isfinite(right) else right+1e-9
if left==right:
right+=1e-9
# Expand x_bound:
delta = right-left if left-right >0 else 1e-9
left -= self.x_bound_extend[0]*delta
right += self.x_bound_extend[1]*delta
# Expand y_bound:
if self.y_axis_type=='lin':
delta = upper-lower if upper-lower >0 else 1e-9
lower -= self.y_bound_extend[0]*delta
upper += self.y_bound_extend[1]*delta
elif self.y_axis_type=='log':
lower *= (1-self.y_bound_extend[0])
upper *= (1+self.y_bound_extend[1])
else:
raise Exception(self.y_axis_type)
if self._plots is None:
ax = plt.gca()
if self.reset_color_cycle:
ax.set_color_cycle(None)
self._plots = []
ax.autoscale(enable=False)
ax.get_yaxis().get_major_formatter().set_useOffset(self.allow_axis_offset)
ax.get_xaxis().get_major_formatter().set_useOffset(self.allow_axis_offset)
if self.x_axis_type!='lin':
ax.set_xscale(self.x_axis_type)
if self.y_axis_type!='lin':
ax.set_yscale(self.y_axis_type)
if isinstance(self.plot_kwargs, dict):
plot_kwargs = [self.plot_kwargs]*len(x_data)
elif isinstance(self.plot_kwargs, (list, tuple)):
assert len(self.plot_kwargs)==len(x_data), "You provided a list of {0} plot kwargs, but {1} lines".format(len(self.plot_kwargs), len(x_data))
plot_kwargs = self.plot_kwargs
for i, (xd, yd, legend_entry) in enumerate(zip(x_data, y_data, self.legend_entries if self.legend_entries is not None else [None]*len(x_data))):
p, =plt.plot(xd, yd, label = legend_entry, **plot_kwargs[i])
self._plots.append(p)
_update_axes_bound(p.axes, (left, right), (lower, upper), self.axes_update_mode)
if self.add_end_markers:
colour = p.get_color()
self._end_markers.append((plt.plot(xd[[0]], yd[[0]], marker='.', markersize=20, color=colour)[0], plt.plot(xd[0], yd[0], marker='x', markersize=10, mew=4, color=colour)[0]))
if (self.make_legend is True) or (self.make_legend is None and (self.legend_entries is not None or len(y_data)>1)):
plt.legend(loc='best', framealpha=0.5, prop={'size':self.legend_entry_size})
# entries = [str(i) for i in xrange(len(y_data))] if self.legend_entries is None else self.legend_entries
# assert len(self._plots) == len(self.legend_entries), 'You have %s plots but you specified %s entries for the legend: %s' % (len(self._plots), len(entries), entries)
# handles, labels = plt.gca().get_legend_handles_labels()
# if len(handles)==0:
# plt.legend(handles + self._plots, labels+entries, loc='best', prop={'size':8})
# else:
# plt.gca().set_legend_handles_labels(handles + self._plots, labels+entries)
else:
for i, (p, xd, yd) in enumerate(zip(self._plots, x_data, y_data)):
p.set_xdata(xd)
p.set_ydata(yd)
_update_axes_bound(p.axes, (left, right), (lower, upper), self.axes_update_mode)
if self.add_end_markers:
self._end_markers[i][0].set_xdata(xd[[0]])
self._end_markers[i][0].set_ydata(yd[[0]])
self._end_markers[i][1].set_xdata(xd[[-1]])
self._end_markers[i][1].set_ydata(yd[[-1]])
