def __call__(self,
joined_array,
share_data=True,
transform_func=None,
check_types=True):
axis = joined_array.ndim - 1
pre_join_shapes = [
list(s[:axis]) + [np.prod(list(s[axis:]), dtype=int)]
for s in self.split_shapes
]
split_axis_ixs = np.cumsum([0] + [s_[-1] for s_ in pre_join_shapes],
axis=0)
if share_data:
x_split = [
joined_array[..., start:end].reshape(shape)
for (start, end, shape) in izip_equal(
split_axis_ixs[:-1], split_axis_ixs[1:], self.split_shapes)
]
else: # Note: this will raise an Error if the self.dim != 0, because the data is no longer contigious in memory.
x_split = [
joined_array[..., start:end].copy().reshape(shape)
for (start, end, shape) in izip_equal(
split_axis_ixs[:-1], split_axis_ixs[1:], self.split_shapes)
]
if transform_func is not None:
x_split = [transform_func(xs) for xs in x_split]
x_reassembled = fill_meta_object(self.meta_object,
(x for x in x_split),
check_types=check_types)
return x_reassembled
def test_dict_assignment():
a = Duck() # When assigning with a dict we first sort keys. Here we just verify that keys remain sorted
a[next, :] = {letter: number for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz', range(1, 27))}
a[next, :] = {letter: number for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz', range(27, 27+26))}
assert list(a[0].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[0].values()) == list(range(1, 27))
assert list(a[1].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[1].values()) == list(range(27, 27+26))
def sparse_nn_flop_count(activations, fanouts, mode = 'adds'):
assert len(activations)==len(fanouts)
if mode=='adds':
assert all(np.array_equal(np.round(a), a) for a in activations)
n_ops = sum(np.abs(s).sum()*fanout for s, fanout in izip_equal(activations, fanouts))
elif mode=='multiplyadds':
n_ops = 2 * sum(np.sum(s != 0) * fanout for s, fanout in izip_equal(activations, fanouts))
else:
bad_value(mode)
return n_ops
def eqprop_update(negative_acts,
positive_acts,
ws,
bs,
learning_rate,
beta,
bidirectional,
l2_loss=None,
gamma=0.5,
dy_squared=None):
n_samples = negative_acts[0].shape[0]
w_grads = [
-(pa_pre.float().cuda().t() @ pa_post.float().cuda() -
pa_pre.float().cuda().t() @ na_post.float().cuda()) /
float(n_samples) for na_pre, na_post, pa_pre, pa_post in izip_equal(
negative_acts[:-1], negative_acts[1:], positive_acts[:-1],
positive_acts[1:])
]
## AdaGrad
for i, dy in enumerate(w_grads):
if dy_squared[i] is None:
dy_squared[i] = dy * dy
else:
dy_squared[i] += dy * dy
w_grads[i] = dy / (torch.sqrt(dy_squared[i]) + 1e-7)
b_grads = [
-torch.mean(pa_post - na_post, dim=0) for pa_post, na_post in
izip_equal(positive_acts[1:], negative_acts[1:])
]
if l2_loss is not None:
w_grads = [(1 - l2_loss) * wg for wg in w_grads]
b_grads = [(1 - l2_loss) * bg for bg in b_grads]
if not isinstance(learning_rate, (list, tuple)):
learning_rate = [learning_rate] * len(ws)
new_ws = [
w - lr * w_grad
for w, w_grad, lr in izip_equal(ws, w_grads, learning_rate)
]
new_bs = [
b - lr * b_grad
for b, b_grad, lr in izip_equal(bs, b_grads, learning_rate)
]
return new_ws, new_bs, dy_squared
def sparse_nn_flop_count(activations, fanouts, mode='adds'):
assert len(activations) == len(fanouts)
if mode == 'adds':
assert all(np.array_equal(np.round(a), a) for a in activations)
n_ops = sum(
np.abs(s).sum() * fanout
for s, fanout in izip_equal(activations, fanouts))
elif mode == 'multiplyadds':
n_ops = 2 * sum(
np.sum(s != 0) * fanout
for s, fanout in izip_equal(activations, fanouts))
else:
bad_value(mode)
return n_ops
def __getitem__(self, ix):
if isinstance(ix, slice):
return DynamicSequence(list.__getitem__(self, ix))
elif isinstance(ix, UniversalCollection):
return self.__getitem__(ix.to_struct())
elif isinstance(ix, (list, tuple)):
arrix = np.array(ix)
if arrix.dtype == np.bool:
if len(arrix) != len(self):
raise InvalidKeyError(
'If you use boolean indices, the length ({} here) must match the length of the collection ({} here)'
.format(len(arrix), len(self)))
else:
return DynamicSequence(a
for a, b in izip_equal(self, arrix)
if b)
else:
return DynamicSequence((list.__getitem__(self, i) for i in ix))
else:
assert not isinstance(ix, bool), 'You cannot index with a boolean.'
try:
return list.__getitem__(self, ix)
except TypeError:
raise InvalidKeyError(
'You tried getting index "{}" from a {}, but {} object can only be indexed by ints, slices, numeric lists'
.format(ix,
type(self).__name__,
type(self).__name__))
except IndexError:
raise IndexError(
'Your index "{}" exceeds the range of this DynamicSequence of length {}'
.format(ix, len(self)))
def _filter_experiments(user_range, exp_record_dict):
if user_range in exp_record_dict:
is_in = [k==user_range for k in exp_record_dict]
else:
number_range = interpret_numbers(user_range)
if number_range is not None:
# experiment_ids = [experiment_list[i] for i in number_range]
is_in = [i in number_range for i in xrange(len(exp_record_dict))]
elif user_range == 'all':
# experiment_ids = experiment_list
is_in = [True]*len(exp_record_dict)
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
# experiment_ids = [exp_id for exp_id in experiment_list if phrase in exp_id]
is_in = [phrase in exp_id for exp_id in exp_record_dict]
elif user_range.startswith('1diff:'):
# select experiments whose arguments differ by one element from the selected experiments
base_range = user_range[len('1diff:'):]
base_range_exps = select_experiments(base_range, exp_record_dict) # list<experiment_id>
all_exp_args_hashes = {eid: set(compute_fixed_hash(a) for a in load_experiment(eid).get_args().items()) for eid in exp_record_dict} # dict<experiment_id : set<arg_hashes>>
# assert all_equal_length(all_exp_args_hashes.values()), 'All variants must have the same number of arguments' # Note: we diable this because we may have lists of experiments with different root functions.
is_in = [any(len(all_exp_args_hashes[eid].difference(all_exp_args_hashes[other_eid]))<=1 for other_eid in base_range_exps) for eid in exp_record_dict]
elif user_range.startswith('hasnot:'):
phrase = user_range[len('hasnot:'):]
# experiment_ids = [exp_id for exp_id in experiment_list if phrase not in exp_id]
is_in = [phrase not in exp_id for exp_id in exp_record_dict]
elif user_range in ('unfinished', 'invalid', 'corrupt'): # Return all experiments where all records are unfinished/invalid/corrupt
record_filters = _filter_records(user_range, exp_record_dict)
# experiment_ids = [exp_id for exp_id in experiment_list if len(record_filters[exp_id])]
is_in = [all(record_filters[exp_id]) for exp_id in exp_record_dict]
else:
raise Exception("Don't know how to use input '{}' to select experiments".format(user_range))
return OrderedDict((exp_id, exp_is_in) for exp_id, exp_is_in in izip_equal(exp_record_dict, is_in))
def __init__(self, ws, hidden_activation, output_activation, optimizer, loss, encdec, encdec_back = None,
grad_calc='xx', minibatch_size=1):
"""
:param ws:
:param encdec: A PDEncoderDecoder pair
:param hidden_activation:
:param output_activation:
:param optimizer:
:param loss:
:param enddec_back: Opt
:param grad_calc:
:param minibatch_size:
:param fwd_quantizer:
:param back_quantizer:
"""
if isinstance(encdec, dict):
encdec = PDEncoderDecoder(kp=encdec['kp'], kd=encdec['kd'], quantization=encdec['quantizer'])
if encdec_back is None:
encdec_back = encdec
elif isinstance(encdec_back, dict):
encdec_back = PDEncoderDecoder(kp=encdec_back['kp'], kd=encdec_back['kd'], quantization=encdec_back['quantizer'])
self.layers = [PDHerdingLayer(w, b=np.zeros(w.shape[1]), encdec=encdec if not callable(encdec) else encdec(), encdec_back=encdec_back if not callable(encdec_back) else encdec_back(),
nonlinearity=nonlinearity, grad_calc=grad_calc, minibatch_size=minibatch_size)
for w, nonlinearity in izip_equal(ws, [hidden_activation]*(len(ws)-1)+[output_activation])]
self.optimizer = optimizer
self.loss = get_named_cost_function(loss) if isinstance(loss, basestring) else loss
self.minibatch_size = minibatch_size
def train_scales(self, input_data, error_loss='L1'):
# Some tricky stuff in here.
# When computing the error grad, we need to make all future units pass-through (otherwise grad will be zero)
# When computing cost-grad, we:
# - Only consider the local cost (nans seemed to happen when we considered total cost)
# - Consider this layer to be pass-through (otherwise cost-grad will be zero)
layer_signals = self.compute_activations(input_data, do_round=True)
true_out = self.predict(input_data, do_round=False)
error_loss = get_error_loss(guess=layer_signals[-1]['output'],
truth=true_out,
loss_type=error_loss)
grads = []
for i, (kp, k, sigs) in enumerate(
izip_equal(self.k_params, self.get_scales(), layer_signals)):
error_grad = tt.grad(error_loss,
wrt=kp,
consider_constant=[
other_sigs['eta']
for other_sigs in layer_signals[i + 1:]
]) # PROBABLY NO GOOD
layer_comp_loss = tt.switch(
sigs['scaled_input'] > 0, sigs['spikes'],
-sigs['spikes']).sum() * self.ws[i].shape[1] # lets be safe
comp_grad = tt.grad(layer_comp_loss,
wrt=kp,
consider_constant=[sigs['eta']])
layer_grad = error_grad + self.comp_weight * comp_grad
grads.append(layer_grad)
self.optimizer.update_from_gradients(parameters=self.k_params,
gradients=grads)
def batchify_generator(generator_generator,
batch_size,
receive_input=False,
out_format='array'):
"""
Best understood by example:
Suppose we want to get batches of frames from video data. Where the batch[t][i] is the frame after batch[t-1][i].
e.g. Suppose we have 7 videos. In the following, each column represents a batch of data, and rows represent the
index within a batch.
-------vid-1---------|-------vid-5-------|--vid-7--
-----------vid-2-----------|--------vid-6---------|
-----vid-3-------|----------vid-4------------------
generator_genererator yields 7 generators, corresponding to each of the movies.
Each of those generators is a frame-generator, which produces the frames in a given video.
Here, we generate frames from each movie, and start a new movies whenever an old one stops, until there are no
new movies to start.
:param generator_generator: An generator which generates generators
:param batch_size: The size if the batch you want to yield
:param receive_input: Expect a "send" to this generatoer AFTER it yields. (see Python coroutines)
:param out_format: 'array' or 'tuple_of_arrays' currently supported.
:yield: An array consisting of batch_size of the outputs of the subgenerator, batched together.
"""
assert receive_input in (False, 'post'), 'pre-receive not yet implemented'
total = batch_size
assert out_format in ('array', 'tuple_of_arrays')
generators = [next(generator_generator) for _ in range(batch_size)]
while True:
items = []
for i in range(batch_size):
while True:
try:
items.append(next(generators[i]))
break
except StopIteration:
total += 1
generators[i] = next(
generator_generator
) # This will rais StopIteration when we're out of generators
if out_format == 'array':
output = np.array(items)
else:
output = tuple(np.array(x) for x in zip(*items))
if not receive_input:
yield output
elif receive_input == 'post':
received_signal = (yield output) # Assume in_format=='array'
for gen, sig in izip_equal(generators, received_signal):
gen.send(sig)
yield None
else:
raise Exception(receive_input)
def scaled_quantized_forward_pass(inputs, weights, scales = None, biases = None, hidden_activations='relu', output_activations = 'relu',
quantization_method = 'herd', rng=None):
"""
Return the activations from a forward pass of a ReLU net.
:param inputs: A (n_frames, n_dims_in) array
:param weights: A list of (n_dim_in, n_dim_out) weight matrices
:param biases: An optional (len(weights)) list of (w.shape[1]) biases for each weight matrix
:param hidden_activations: Indicates the hidden layer activation function
:param output_activations: Indicates the output layer activation function
:param quantization_method: The method for quantizing (see function: sequential_quantize)
:param rng: A random number generator or seed
:return: activations:
A len(weights)*3+1 list of (n_frames, n_dims) activations.
Elements [::3] will be a length(w)+1 list containing the input to each rounding unit, and the final output
Elements [1::3] will be the length(w) rounded "spike" signal.
Elements [2::3] will be the length(w) inputs to each nonlinearity
"""
rng = get_rng(rng)
activations = [inputs]
if biases is None:
biases = [0]*len(weights)
else:
assert len(biases)==len(weights)
if scales is None:
scales = [1.]*len(weights)
x = inputs # (n_samples, n_units)
for i, (w, b, k) in enumerate(izip_equal(weights, biases, scales)):
s = quantize(x*k, method=quantization_method, rng=rng)
u = (s/k).dot(w)+b
x = activation_function(u, output_activations if i==len(weights)-1 else hidden_activations)
activations += [s, u, x]
return activations
def run_multiple_experiments(experiments, prefixes = None, parallel = False, display_results=False, raise_exceptions=True, notes = (), run_args = {}):
"""
Run multiple experiments, optionally in parallel with multiprocessing.
:param experiments: A collection of experiments
:param parallel: Can be:
True/'all': Run in parallel with as many processes as CPUs
An integer indicating the number of processes to run
False/None Don't run in parallel.
:param raise_exceptions: Terminate exectution when one experiment fails.
:param run_args: Other args to pass to Experiment.run()
:return: A collection of experiment records.
"""
if parallel:
if parallel in (True, 'all'):
parallel = multiprocessing.cpu_count()
else:
assert isinstance(parallel, int)
experiment_identifiers = [ex.get_id() for ex in experiments]
if prefixes is None:
prefixes = range(len(experiment_identifiers))
prefixes = [s+': ' for s in equalize_string_lengths(prefixes, side='right')]
print ('Prefix key: \n'+'\n'.join('{}{}'.format(p, eid) for p, eid in izip_equal(prefixes, experiment_identifiers)))
target_func = partial(_parallel_run_target, notes=notes, raise_exceptions=raise_exceptions, **run_args)
p = multiprocessing.Pool(processes=parallel)
return p.map(target_func, zip(experiment_identifiers, prefixes))
else:
return [ex.run(raise_exceptions=raise_exceptions, display_results=display_results, notes=notes, **run_args) for ex in experiments]
def select_experiment_records_from_list(user_range, experiment_records):
return [
rec_id for rec_id, f in izip_equal(
experiment_records,
_filter_experiment_record_list(user_range, experiment_records))
if f
]
def forward_pass_and_cost(input_data,
weights,
biases=None,
**forward_pass_kwargs):
"""
Do a forward pass and return the computaitonal cost.
:param input_data: A (n_frames, n_dims_in) array
:param weights: A list of (n_dim_in, n_dim_out) weight matrices
:param biases: An optional (len(weights)) list of (w.shape[1]) biases for each weight matrix
:param hidden_activations: Indicates the hidden layer activation function
:param output_activations: Indicates the output layer activation function
:return: output, n_dense_flops, n_sparse_flops ... where:
output is a (input_data.shape[0], weights[-1].shape[1]) array of outputs
n_dense_flops is the total number of floating point operations (adds and multiples) required in the forward pass
n_sparse_flops is the total number of operations not-counting zero-activations.
"""
activations = forward_pass_activations(input_data=input_data,
weights=weights,
biases=biases,
**forward_pass_kwargs)
# dbplot(activations[2][:100], 'full-acts')
# full_flops_per_datapoint = [(w.shape[0]*w.shape[1] + (w.shape[0]-1)*w.shape[1]) for w in weights]
n_dense_flops = sum(w.shape[0] * w.shape[1] + (w.shape[0] - 1) * w.shape[1]
for w in weights) * input_data.shape[0]
layerwise_total_nonzero_acts = [(act != 0).sum()
for act in activations[:-1:2]]
n_sparse_flops = sum(
(actsum * w.shape[1] + (actsum - 1) * w.shape[1])
for actsum, w in izip_equal(layerwise_total_nonzero_acts, weights))
# n_sparse_flops =sum((act!=0).mean(axis=1).sum(axis=0)*full_layer_flops for act, full_layer_flops in izip_equal(activations[:-1:2], full_flops_per_datapoint))
return activations[-1], n_dense_flops, n_sparse_flops
def join_arrays_and_get_rebuild_func(arrays, axis=0, transform_func=None):
"""
Given a nested structure of arrays, join them into a single array by flattening dimensions from axis on
concatenating them. Return the joined array and a function which can take the joined array and reproduce the
original structure.
:param arrays: A possibly nested structure containing arrays which you want to join into a single array.
:param axis: Axis after which to flatten and join all arrays. The resulting array will be (dim+1) dimensional.
:param transform_func: Optionally, a function which you apply to every element in the nested struture of arrays first.
:return ndarray, ArrayStructRebuilder: The joined array, and the function which can be called to reconstruct
the structure from the joined array.
"""
meta_object = get_meta_object(arrays)
data_list = get_leaf_values(arrays)
if transform_func is not None:
data_list = [transform_func(d) for d in data_list]
split_shapes = [x_.shape for x_ in data_list]
pre_join_shapes = [
list(s[:axis]) + [np.prod(list(s[axis:]), dtype=int)]
for s in split_shapes
]
joined_arr = np.concatenate(list(
x_.reshape(s_) for x_, s_ in izip_equal(data_list, pre_join_shapes)),
axis=axis)
rebuild_function = ArrayStructRebuilder(split_shapes=split_shapes,
meta_object=meta_object)
return joined_arr, rebuild_function
def equilibriating_step(
params,
states: Sequence[np.ndarray],
x: np.ndarray,
fwd_states,
forward_deviation_cost,
zero_deviation_cost,
y=None,
y_pressure=0.5,
epsilon=0.5
) -> Sequence[np.ndarray]: # n_layers list of new layer states
assert len(states) == len(params)
state_grads = tt.grad(energy(params=params,
states=states,
fwd_states=fwd_states,
forward_deviation_cost=forward_deviation_cost,
zero_deviation_cost=zero_deviation_cost,
x=x,
beta=y_pressure,
y=y).sum(),
wrt=states)
new_states = [
tt.clip(s - epsilon * g, 0, 1)
for s, g in izip_equal(states, state_grads)
]
return new_states
def make_record_comparison_table(records, args_to_show=None, results_extractor = None, print_table = False, tablefmt='simple', reorder_by_args=False):
"""
Make a table comparing the arguments and results of different experiment records. You can use the output
of this function with the tabulate package to make a nice readable table.
:param records: A list of records whose results to compare
:param args_to_show: A list of arguments to show. If none, it will just show all arguments
that differ between experiments.
:param results_extractor: A dict<str->callable> where the callables take the result of the
experiment as an argument and return an entry in the table.
:param print_table: Optionally, import tabulate and print the table here and now.
:return: headers, rows
headers is a list of of headers for the top of the table
rows is a list of lists filling in the information.
example usage:
headers, rows = make_record_comparison_table(
record_ids = [experiment_id_to_latest_record_id(eid) for eid in [
'demo_fast_weight_mlp.multilayer_baseline.1epoch.version=mlp',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=1',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=20',
]],
results_extractor={
'Test': lambda result: result.get_best('test').score.get_score('test'),
'Train': lambda result: result.get_best('test').score.get_score('train'),
}
)
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt)
"""
args = [rec.get_args(ExpInfoFields.ARGS) for rec in records]
if args_to_show is None:
common, separate = separate_common_items(args)
args_to_show = [k for k, v in separate[0]]
if results_extractor is None:
results_extractor = {'Result': str}
elif callable(results_extractor):
results_extractor = {'Result': results_extractor}
else:
assert isinstance(results_extractor, dict)
headers = args_to_show + results_extractor.keys()
rows = []
for record, record_args in izip_equal(records, args):
arg_dict = dict(record_args)
args_vals = [arg_dict[k] for k in args_to_show]
results = record.get_result()
rows.append(args_vals+[f(results) for f in results_extractor.values()])
if reorder_by_args:
rows = sorted(rows)
if print_table:
import tabulate
print(tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt))
return headers, rows
def broadcast_into_meta_object(meta_object,
data_object,
is_container=is_primitive_container,
check_types=True):
"""
"Broadcast" the data object into the meta object. This puts the data into the structure of the meta-object.
E.g.
>>> broadcast_into_meta_object([int, int, int], 1)
[1, 1, 1]
>>> broadcast_into_meta_object([int, (int, int), int], (1, 2, 3))
[1, (2, 2), 3]
:param meta_object: A nested structure of types
:param data_object: A nested structure of data
:param is_container: A function that returns True if an object is considered to be in a container.
:return: A new data object with the structure of the meta object and the data of the data object.
"""
kwargs = dict(check_types=check_types, is_container=is_container)
if is_container(meta_object):
if isnamedtupleinstance(meta_object):
if isinstance(data_object, (list, tuple, set)):
return meta_object.__class__(
*(broadcast_into_meta_object(m, d, **kwargs)
for m, d in izip_equal(meta_object, data_object)))
else:
return meta_object.__class__(
*(broadcast_into_meta_object(m, data_object, **kwargs)
for m in meta_object))
elif isinstance(meta_object, (list, tuple, set)):
if isinstance(data_object, (list, tuple, set)):
return meta_object.__class__(
broadcast_into_meta_object(m, d, **kwargs)
for m, d in izip_equal(meta_object, data_object))
else:
return meta_object.__class__(
broadcast_into_meta_object(m, data_object, **kwargs)
for m in meta_object)
else:
raise NotImplementedError('Dict iteration not supported yet.')
else:
if check_types:
assert isinstance(
data_object, meta_object
), "Data object {} does not have type of meta-object: {}".format(
data_object, meta_object)
return data_object
def search_func(fixed):
n_calls_to_make = n_calls if n_calls is not None else 3 if is_test_mode() else 100
this_objective = partial(objective, **fixed)
for iter_info in parameter_search(this_objective, n_calls=n_calls_to_make, space=space, **search_params):
info = dict(names=list(space.keys()), x_iters =iter_info.x_iters, func_vals=iter_info.func_vals, score = iter_info.func_vals, x=iter_info.x, fun=iter_info.fun)
latest_info = {name: val for name, val in izip_equal(info['names'], iter_info.x_iters[-1])}
print('Latest: {}, Score: {:.3g}'.format(latest_info, iter_info.func_vals[-1]))
yield info
def compare(records):
record_experiment_ids_uniquified = uniquify_duplicates(
rec.get_experiment_id() for rec in records)
comparison_function(
OrderedDict(
(unique_rid, rec.get_result())
for unique_rid, rec in izip_equal(
record_experiment_ids_uniquified, records)))
def test_dict_assignment():
a = Duck(
) # When assigning with a dict we first sort keys. Here we just verify that keys remain sorted
a[next, :] = {
letter: number
for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz',
range(1, 27))
}
a[next, :] = {
letter: number
for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz',
range(27, 27 + 26))
}
assert list(a[0].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[0].values()) == list(range(1, 27))
assert list(a[1].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[1].values()) == list(range(27, 27 + 26))
def _bitwise(op, filter_set_1, filter_set_2):
assert op in ('and', 'or')
filter_set_3 = filter_set_1.copy()
for k in filter_set_1.keys():
filter_set_3[k] = [
(a or b) if op == 'or' else (a and b)
for a, b in izip_equal(filter_set_1[k], filter_set_2[k])
]
return filter_set_3
def make_record_comparison_table(records, args_to_show=None, results_extractor = None, print_table = False):
"""
Make a table comparing the arguments and results of different experiment records. You can use the output
of this function with the tabulate package to make a nice readable table.
:param records: A list of records whose results to compare
:param args_to_show: A list of arguments to show. If none, it will just show all arguments
that differ between experiments.
:param results_extractor: A dict<str->callable> where the callables take the result of the
experiment as an argument and return an entry in the table.
:param print_table: Optionally, import tabulate and print the table here and now.
:return: headers, rows
headers is a list of of headers for the top of the table
rows is a list of lists filling in the information.
example usage:
headers, rows = make_record_comparison_table(
record_ids = [experiment_id_to_latest_record_id(eid) for eid in [
'demo_fast_weight_mlp.multilayer_baseline.1epoch.version=mlp',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=1',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=20',
]],
results_extractor={
'Test': lambda result: result.get_best('test').score.get_score('test'),
'Train': lambda result: result.get_best('test').score.get_score('train'),
}
)
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt)
"""
args = [rec.info.get_field(ExpInfoFields.ARGS) for rec in records]
if args_to_show is None:
common, separate = separate_common_items(args)
args_to_show = [k for k, v in separate[0]]
if results_extractor is None:
results_extractor = {'Result': str}
elif callable(results_extractor):
results_extractor = {'Result': results_extractor}
else:
assert isinstance(results_extractor, dict)
headers = args_to_show + results_extractor.keys()
rows = []
for record, record_args in izip_equal(records, args):
arg_dict = dict(record_args)
args_vals = [arg_dict[k] for k in args_to_show]
results = record.get_result()
rows.append(args_vals+[f(results) for f in results_extractor.values()])
if print_table:
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt='simple')
return headers, rows
def eqprop_update(negative_acts,
positive_acts,
ws,
bs,
learning_rate,
beta,
bidirectional,
l2_loss=None):
n_samples = negative_acts[0].shape[0]
w_grads = [
-np.transpose(na_pre) @ (pa_post - na_post) / n_samples
for na_pre, pa_post, na_post in izip_equal(
negative_acts[:-1], positive_acts[1:], negative_acts[1:])
]
if bidirectional:
w_grads = w_grads[:1] + [
wg - np.transpose(pa_pre - na_pre) @ na_post / n_samples
for wg, pa_pre, na_pre, na_post in izip_equal(
w_grads[1:], positive_acts[1:-1], negative_acts[1:-1],
negative_acts[2:])
]
b_grads = [
-np.mean(pa_post - na_post, axis=0) for pa_post, na_post in izip_equal(
positive_acts[1:], negative_acts[1:])
]
if l2_loss is not None:
w_grads = [(1 - l2_loss) * wg for wg in w_grads]
b_grads = [(1 - l2_loss) * bg for bg in b_grads]
if not isinstance(learning_rate, (list, tuple)):
learning_rate = [learning_rate] * len(ws)
new_ws = [
w - lr / beta * w_grad
for w, w_grad, lr in izip_equal(ws, w_grads, learning_rate)
]
new_bs = [
b - lr / beta * b_grad
for b, b_grad, lr in izip_equal(bs, b_grads, learning_rate)
]
return new_ws, new_bs
def eqprop_update(negative_acts,
positive_acts,
ws,
bs,
learning_rate,
beta,
bidirectional,
l2_loss=None):
n_samples = negative_acts[0].shape[0]
if bidirectional is False:
w_grads = [
-na_pre.t() @ (pa_post - na_post) / float(n_samples)
for na_pre, pa_post, na_post in izip_equal(
negative_acts[:-1], positive_acts[1:], negative_acts[1:])
]
elif bidirectional is True:
w_grads = [
-na_pre.t() @ (pa_post - na_post) / float(n_samples)
for na_pre, pa_post, na_post in izip_equal(
negative_acts[:-1], positive_acts[1:], negative_acts[1:])
]
if bidirectional:
w_grads = w_grads[:1] + [
wg - (pa_pre - na_pre).t() @ na_post / n_samples
for wg, pa_pre, na_pre, na_post in izip_equal(
w_grads[1:], positive_acts[1:-1], negative_acts[1:-1],
negative_acts[2:])
]
else:
assert bidirectional == 'full'
w_grads = [
-(pa_pre.t() @ pa_post - na_pre.t() @ na_post) / float(n_samples)
for na_pre, na_post, pa_pre, pa_post in izip_equal(
negative_acts[:-1], negative_acts[1:], positive_acts[:-1],
positive_acts[1:])
]
b_grads = [
-torch.mean(pa_post - na_post, dim=0) for pa_post, na_post in
izip_equal(positive_acts[1:], negative_acts[1:])
]
if l2_loss is not None:
w_grads = [(1 - l2_loss) * wg for wg in w_grads]
b_grads = [(1 - l2_loss) * bg for bg in b_grads]
if not isinstance(learning_rate, (list, tuple)):
learning_rate = [learning_rate] * len(ws)
new_ws = [
w - lr / beta * w_grad
for w, w_grad, lr in izip_equal(ws, w_grads, learning_rate)
]
new_bs = [
b - lr / beta * b_grad
for b, b_grad, lr in izip_equal(bs, b_grads, learning_rate)
]
return new_ws, new_bs
def draw_norm_y_axis(x_position,
lims,
scale='lin',
axis_thickness=2,
n_intermediates=3,
tickwidth=0.1,
axiscolor='k',
ticklabel_format='{:.3g}',
tick_round_grid=40):
"""
Draw a y-axis in a Parallel Coordinates plot
:param x_position: Position in x to draw the axis
:param lims: The (min, max) limit of the y-axis
:param scale: Not implemented for now, just leave at 'lin'. (Todo: implement 'log')
:param axis_thickness: Thickness of the axis line
:param n_intermediates: Number of ticks to put in between ends of axis
:param tickwidth: Width of tick lines
:param axiscolor: Color of axis
:param ticklabel_format: Format for string ticklabel numbers
:param tick_round_grid: Round ticks to a grid with this number of points, or None to not do this. (Causes nicer axis labels)
:return: The handels for the (<vertical line>, <ticks>, <tick labels>)
"""
assert scale == 'lin', 'For now'
lower, upper = lims
vertical_line_handel = plt.axvline(x=x_position,
ymin=0,
ymax=1,
linewidth=axis_thickness,
color=axiscolor)
y_axisticks = np.linspace(0, 1, n_intermediates + 2)
y_trueticks = y_axisticks * (upper - lower) + lower
if tick_round_grid is not None:
# spacing = (upper - lower)/tick_round_grid
spacing = 10**np.round(np.log10((upper - lower) / tick_round_grid))
y_trueticks = np.round(y_trueticks / spacing) * spacing
y_axisticks = (y_trueticks - y_trueticks[0]) / (y_trueticks[-1] -
y_trueticks[0])
y_labels = [ticklabel_format.format(y) for y in y_trueticks]
tick_label_handels = [
plt.text(x=x_position + tickwidth / 2.,
y=y,
s=ylab,
color='k',
bbox=dict(boxstyle="square",
fc=(1., 1., 1., 0.5),
ec=(0, 0, 0, 0.)))
for y, ylab in izip_equal(y_axisticks, y_labels)
]
tick_handels = axhlines(ys=y_axisticks,
lims=(x_position - tickwidth / 2.,
x_position + tickwidth / 2.),
linewidth=axis_thickness,
color=axiscolor,
zorder=4)
return vertical_line_handel, tick_handels, tick_label_handels
def deepstr(obj,
memo=None,
array_print_threhold=8,
array_summary_threshold=10000,
indent=' ',
float_format=''):
"""
A recursive, readable print of a data structure.
"""
if memo is None:
memo = set()
if id(obj) in memo:
return "<{type} at {loc}> already listed".format(
type=type(obj).__name__, loc=hex(id(obj)))
memo.add(id(obj))
if isinstance(obj, np.ndarray):
string_desc = arraystr(obj,
print_threshold=array_print_threhold,
summary_threshold=array_summary_threshold)
elif isinstance(obj, (list, tuple, set, dict)):
kwargs = dict(memo=memo,
array_print_threhold=array_print_threhold,
array_summary_threshold=array_summary_threshold,
indent=indent,
float_format=float_format)
if isinstance(obj, (list, tuple)):
keys, values = [str(i) for i in xrange(len(obj))], obj
elif isinstance(obj, dict):
keys, values = obj.keys(), obj.values()
elif isinstance(obj, set):
keys, values = ['- '] * len(obj), obj
else:
raise Exception('Should never be here')
max_indent = max(len(k) for k in keys) if len(keys) > 0 else 0
elements = [
'{k}: {v}'.format(k=k,
v=' ' * (max_indent - len(k)) +
indent_string(deepstr(v, **kwargs),
indent=' ' * max_indent,
include_first=False))
for k, v in izip_equal(keys, values)
]
string_desc = '<{type} at {id}>\n'.format(
type=type(obj).__name__, id=hex(id(obj))) + indent_string(
'\n'.join(elements), indent=indent)
return string_desc
elif isinstance(obj, float):
string_desc = '{{:{}}}'.format(float_format).format(obj)
else:
string_desc = str(obj)
return string_desc
def energy(params, states, x, beta=0., y=None):
energy_per_sample = sum(
.5 * (s_post**2).sum(axis=1) -
tt.batched_dot(rho(s_pre).dot(w), rho(s_post)) - rho(s_post).dot(b)
for s_pre, s_post, (w, b) in izip_equal([x] +
states[:-1], states, params))
if y is not None:
energy_per_sample = energy_per_sample + beta * (
(states[-1] - y)**2).sum(axis=1)
return energy_per_sample
def batchify_generator(generator_generator, batch_size, receive_input=False, out_format ='array'):
"""
Best understood by example:
Suppose we want to get batches of frames from video data. Where the batch[t][i] is the frame after batch[t-1][i].
e.g. Suppose we have 7 videos. In the following, each column represents a batch of data, and rows represent the
index within a batch.
-------vid-1---------|-------vid-5-------|--vid-7--
-----------vid-2-----------|--------vid-6---------|
-----vid-3-------|----------vid-4------------------
generator_genererator yields 7 generators, corresponding to each of the movies.
Each of those generators is a frame-generator, which produces the frames in a given video.
Here, we generate frames from each movie, and start a new movies whenever an old one stops, until there are no
new movies to start.
:param generator_generator: An generator which generates generators
:param batch_size: The size if the batch you want to yield
:param receive_input: Expect a "send" to this generatoer AFTER it yields. (see Python coroutines)
:param out_format: 'array' or 'tuple_of_arrays' currently supported.
:yield: An array consisting of batch_size of the outputs of the subgenerator, batched together.
"""
assert receive_input in (False, 'post'), 'pre-receive not yet implemented'
total = batch_size
assert out_format in ('array', 'tuple_of_arrays')
generators = [next(generator_generator) for _ in range(batch_size)]
while True:
items = []
for i in range(batch_size):
while True:
try:
items.append(next(generators[i]))
break
except StopIteration:
total+=1
generators[i] = next(generator_generator) # This will rais StopIteration when we're out of generators
if out_format=='array':
output= np.array(items)
else:
output = tuple(np.array(x) for x in zip(*items))
if not receive_input:
yield output
elif receive_input=='post':
received_signal = (yield output) # Assume in_format=='array'
for gen, sig in izip_equal(generators, received_signal):
gen.send(sig)
yield None
else:
raise Exception(receive_input)
def get_grads(disconnect):
forward_states = forward_pass(params=forward_params,
x=x,
nonlinearity=nonlinearity,
disconnect_grads=disconnect)
loss = sum(
(.5 * (fs - es)**2).sum(axis=1)
for fs, es in izip_equal(forward_states, eq_states)).mean(axis=0)
weight_params = [w for w, b in forward_params]
flat_param_grads = tt.grad(loss, wrt=weight_params)
return flat_param_grads
def join(self, x):
if self._split_axis_sizes is None:
self._nested_type = NestedType.from_data(x)
data_list = self._nested_type.get_leaves(x)
self._split_shapes = [x_.size() for x_ in data_list]
self._pre_join_shapes = [list(x_.size()[:self.dim])+[np.prod(list(x_.size()[self.dim:]))] for x_ in data_list]
self._split_axis_ixs = torch.cumsum(torch.IntTensor([0] + [s_[-1] for s_ in self._pre_join_shapes]), dim=0)
else:
data_list = self._nested_type.get_leaves(x)
return torch.cat(list(x_.view(*s_) for x_, s_ in izip_equal(data_list, self._pre_join_shapes)), dim=self.dim)
def select_last_record_of_experiments(user_range, exp_record_dict):
experiments = select_experiments(user_range=user_range,
exp_record_dict=exp_record_dict)
records = [
load_experiment(ex).get_latest_record(only_completed=True,
err_if_none=False)
for ex in experiments
]
if None in records:
print('WARNING: Experiments {} have no completed records.',
[e for e, r in izip_equal(experiments, records) if r is None])
return records
def __init__(self,
ws,
hidden_activation,
output_activation,
optimizer,
loss,
encdec,
encdec_back=None,
grad_calc='xx',
minibatch_size=1):
"""
:param ws:
:param encdec: A PDEncoderDecoder pair
:param hidden_activation:
:param output_activation:
:param optimizer:
:param loss:
:param enddec_back: Opt
:param grad_calc:
:param minibatch_size:
:param fwd_quantizer:
:param back_quantizer:
"""
if isinstance(encdec, dict):
encdec = PDEncoderDecoder(kp=encdec['kp'],
kd=encdec['kd'],
quantization=encdec['quantizer'])
if encdec_back is None:
encdec_back = encdec
elif isinstance(encdec_back, dict):
encdec_back = PDEncoderDecoder(
kp=encdec_back['kp'],
kd=encdec_back['kd'],
quantization=encdec_back['quantizer'])
self.layers = [
PDHerdingLayer(w,
b=np.zeros(w.shape[1]),
encdec=encdec if not callable(encdec) else encdec(),
encdec_back=encdec_back
if not callable(encdec_back) else encdec_back(),
nonlinearity=nonlinearity,
grad_calc=grad_calc,
minibatch_size=minibatch_size)
for w, nonlinearity in izip_equal(ws, [hidden_activation] *
(len(ws) - 1) +
[output_activation])
]
self.optimizer = optimizer
self.loss = get_named_cost_function(loss) if isinstance(
loss, basestring) else loss
self.minibatch_size = minibatch_size
def remove_common_results_prefix(results_dict):
"""
Remove the common prefix for the results you are comparing.
:param results_dict: An OrderedDict of experiment Results
:return: An OrderedDict of results with the common beginnings of the keys truncated.
"""
# TODO: Fix this so that it splits correctly, not just on '.', which is not necessarily a separator.
assert isinstance(dict, OrderedDict), 'Expecting an OrderedDict of <experiment_name -> result>'
split_keys = [k.split('.') for k in results_dict.keys()]
trimmed_keys = remove_common_prefix(split_keys)
return OrderedDict((k, v) for k, v in izip_equal(trimmed_keys, results_dict.values()))
def initialize_params(layer_sizes: Sequence[int],
initial_weight_scale=1.,
rng=None) -> Sequence[LayerParams]:
rng = get_rng(rng)
ws = [
uniform(low=-initial_weight_scale * (6. / (n_pre + n_post))**.5,
high=(6. / (n_pre + n_post))**.5,
size=(n_pre, n_post))
for n_pre, n_post in izip_equal(layer_sizes[:-1], layer_sizes[1:])
]
bs = [torch.zeros(n_post) for n_post in layer_sizes[1:]]
return _params_vals_to_params(ws, bs)
def get_cost_and_output(self, x, scales, reset_state = True):
if reset_state:
for v in self.named_layers.values():
if isinstance(v, (ScaledHerdingLayer, TemporalIntegrator, TemporalDifference, OnceAddConvLayer)):
v.reset()
for scale_value, scaled_rounder in izip_equal(scales, [layer for layer in self.named_layers.values() if isinstance(layer, (ScaledRoundingLayer, ScaledHerdingLayer))]):
scaled_rounder.set_scale(scale_value)
# First: Reset states and OnceAddConvLayers
comp_costs, outputs = zip(*[self._func(xi[None]) for xi in x])
return np.concatenate(comp_costs), np.concatenate(outputs)
def remove_common_results_prefix(results_dict):
"""
Remove the common prefix for the results you are comparing.
:param results_dict: An OrderedDict of experiment Results
:return: An OrderedDict of results with the common beginnings of the keys truncated.
"""
# TODO: Fix this so that it splits correctly, not just on '.', which is not necessarily a separator.
assert isinstance(dict, OrderedDict), 'Expecting an OrderedDict of <experiment_name -> result>'
split_keys = [k.split('.') for k in results_dict.keys()]
trimmed_keys = remove_common_prefix(split_keys)
return OrderedDict((k, v) for k, v in izip_equal(trimmed_keys, results_dict.values()))
def cleave(self, x_flat, share_data = True):
"""
Split (or cleave) a variable into its component parts..
:param x_flat:
:param share_data:
:return:
"""
if share_data:
x_split = [x_flat[..., start:end].view(shape) for (start, end, shape) in izip_equal(self._split_axis_ixs[:-1], self._split_axis_ixs[1:], self._split_shapes)]
else: # Note: this will raise an Error if the self.dim != 0, because the data is no longer contigious in memory.
x_split = [x_flat[..., start:end].clone().view(shape) for (start, end, shape) in izip_equal(self._split_axis_ixs[:-1], self._split_axis_ixs[1:], self._split_shapes)]
x_reassembled = self._nested_type.expand_from_leaves(x_split, check_types=False)
return x_reassembled
def to_array_and_keys(self):
keys = self._first_element_keys()
first = True
key_indices = tuple(range(len(k)) for k in keys)
for numeric_ix, key_ix in izip_equal(itertools.product(*key_indices), itertools.product(*keys)):
if first:
this_data = np.array(self[key_ix])
arr = np.zeros(tuple(len(k) for k in keys)+this_data.shape, dtype=this_data.dtype)
arr[numeric_ix] = this_data
first = False
else:
arr[numeric_ix] = self[key_ix]
return keys, arr
def train_one(self, x, target):
x, target = tt.unbroadcast(x, 0), tt.unbroadcast(target, 0) # F'ing scan
states = {}
for layer in self.layers:
x, layer_state = layer.forward_pass_and_state(x, count_ops=True)
states[layer]=layer_state
loss = self.loss(x, target)
param_grad_pairs = []
grad = None
for layer in self.layers[::-1]:
grad, param_grads = layer.backward_pass(state=states[layer], grad=grad, cost = loss, count_ops=True)
loss = None
param_grad_pairs += list(izip_equal(layer.parameters, param_grads))
all_params, all_param_grads = zip(*param_grad_pairs)
self.optimizer.update_from_gradients(parameters=all_params, gradients=all_param_grads)
return create_constant(0.) # scan demands some return
def _filter_experiments(user_range, exp_record_dict, return_is_in = False):
if user_range.startswith('~'):
is_in = _filter_experiments(user_range=user_range[1:], exp_record_dict=exp_record_dict, return_is_in=True)
is_in = [not r for r in is_in]
else:
if user_range in exp_record_dict:
is_in = [k==user_range for k in exp_record_dict]
else:
number_range = interpret_numbers(user_range)
if number_range is not None:
is_in = [i in number_range for i in xrange(len(exp_record_dict))]
elif user_range == 'all':
is_in = [True]*len(exp_record_dict)
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
is_in = [phrase in exp_id for exp_id in exp_record_dict]
elif user_range.startswith('1diff:'):
base_range = user_range[len('1diff:'):]
base_range_exps = select_experiments(base_range, exp_record_dict) # list<experiment_id>
all_exp_args_hashes = {eid: set(compute_fixed_hash(a) for a in load_experiment(eid).get_args().items()) for eid in exp_record_dict} # dict<experiment_id : set<arg_hashes>>
is_in = [any(len(all_exp_args_hashes[eid].difference(all_exp_args_hashes[other_eid]))<=1 for other_eid in base_range_exps) for eid in exp_record_dict]
elif user_range.startswith('hasnot:'):
phrase = user_range[len('hasnot:'):]
is_in = [phrase not in exp_id for exp_id in exp_record_dict]
elif user_range in ('unfinished', 'invalid', 'corrupt'): # Return all experiments where all records are unfinished/invalid/corrupt
record_filters = _filter_records(user_range, exp_record_dict)
is_in = [all(record_filters[exp_id]) for exp_id in exp_record_dict]
elif user_range == 'started':
is_in = [has_experiment_record(exp_id) for exp_id in exp_record_dict]
else:
raise RecordSelectionError("Don't know how to use input '{}' to select experiments".format(user_range))
if return_is_in:
return is_in
else:
return OrderedDict((exp_id, exp_is_in) for exp_id, exp_is_in in izip_equal(exp_record_dict, is_in))
def arrayify_axis(self, axis, subkeys = (), inplace=False):
"""
Take a particular axis (depth) of this object, concatenate it into an array for each leaf node. This requires
that all values along this axis have the same structure. Best understood by the following example:
a = Duck()
a[0, 'x'] = 1
a[0, 'y'] = 2
a[1, 'x'] = 3
a[1, 'y'] = 4
b = a.arrayify_axis(axis=0)
assert np.array_equal(b['x'], [1, 3])
assert np.array_equal(b['y'], [2, 4])
:param axis: The depth at which you'd like to concatenate.
:param subkeys: A sub-branch of struct to do this along (ie. only do it on branch ('subkey', 'subsubkey'))
:param inplace: Do the operation in-place (ie change this object (saves you from copying a new object))
:return: A new Duck, with the given axis shifted to the end.
"""
b = Duck.from_struct(self.to_struct()) if not inplace else self
if not isinstance(subkeys, tuple):
subkeys = (subkeys,)
indexing_keys = list(self[subkeys]._first_element_keys())
indexing_keys[axis - len(subkeys)] = (slice(None),)
indexing_keys = tuple(indexing_keys)
assigning_keys = list(indexing_keys)
del assigning_keys[axis - len(subkeys)]
if len(assigning_keys)==0:
assert len(indexing_keys)==1
substruct = np.array(self[subkeys+indexing_keys[0]])
else:
substruct = Duck()
for src_key, dest_key in izip_equal(itertools.product(*indexing_keys), itertools.product(*assigning_keys)):
substruct[dest_key] = np.array(self[subkeys + src_key])
b[subkeys] = substruct
return b
def deepstr(obj, memo=None, array_print_threhold = 8, array_summary_threshold=10000, max_expansion = None, indent =' ', float_format = ''):
"""
A recursive, readable print of a data structure.
"""
if memo is None:
memo = set()
if id(obj) in memo:
return "<{type} at {loc}> already listed".format(type=type(obj).__name__, loc=hex(id(obj)))
memo.add(id(obj))
if isinstance(obj, np.ndarray):
string_desc = arraystr(obj, print_threshold=array_print_threhold, summary_threshold=array_summary_threshold)
elif isinstance(obj, (list, tuple, set, dict)):
kwargs = dict(memo=memo, array_print_threhold=array_print_threhold, max_expansion=max_expansion, array_summary_threshold=array_summary_threshold, indent=indent, float_format=float_format)
if isinstance(obj, (list, tuple)):
keys, values = [str(i) for i in xrange(len(obj))], obj
elif isinstance(obj, dict):
keys, values = obj.keys(), obj.values()
elif isinstance(obj, set):
keys, values = ['- ']*len(obj), obj
else:
raise Exception('Should never be here')
max_indent = max(len(str(k)) for k in keys) if len(keys)>0 else 0
if max_expansion is not None and len(keys)>max_expansion:
elements = ['{k}: {v}'.format(k=k, v=' '*(max_indent-len(str(k))) + indent_string(deepstr(v, **kwargs), indent=' '*max_indent, include_first=False)) for k, v in izip_equal(keys[:max_expansion-1]+[keys[-1]], values[:max_expansion-1]+[values[-1]])]
elements.insert(-1, '... Skipping {} of {} elements ...'.format(len(keys)-len(elements), len(keys)))
else:
elements = ['{k}: {v}'.format(k=k, v=' '*(max_indent-len(str(k))) + indent_string(deepstr(v, **kwargs), indent=' '*max_indent, include_first=False)) for k, v in izip_equal(keys, values)]
string_desc = '<{type} at {id}>\n'.format(type = type(obj).__name__, id=hex(id(obj))) + indent_string('\n'.join(elements), indent=indent)
return string_desc
elif isinstance(obj, float):
string_desc = '{{:{}}}'.format(float_format).format(obj)
else:
string_desc = str(obj)
return string_desc
def items(self):
for item, full_path in izip_equal(self, self.values()):
yield item, full_path
def compare(records):
record_experiment_ids_uniquified = uniquify_duplicates(rec.get_experiment_id() for rec in records)
comparison_function(OrderedDict((unique_rid, rec.get_result()) for unique_rid, rec in izip_equal(record_experiment_ids_uniquified, records)))
def get_experiment_list_str(self, exp_record_dict):
headers = {
'full': [ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION, ExpRecordDisplayFields.STATUS, ExpRecordDisplayFields.ARGS_CHANGED, ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES],
'results': [ExpRecordDisplayFields.RESULT_STR]
}[self.view_mode]
if self.remove_prefix:
deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys())
exp_record_dict = OrderedDict((k, v) for k, v in zip(deprefixed_ids, exp_record_dict.values()))
row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows
header_names = [h.value for h in headers]
def remove_notes_if_no_notes(_record_rows, _record_headers):
notes_column_index = _record_headers.index(ExpRecordDisplayFields.NOTES.value) if ExpRecordDisplayFields.NOTES.value in _record_headers else None
# Remove the notes column if there are no notes!
if notes_column_index is not None and all(row[notes_column_index]=='' or row[notes_column_index]=='-' for row in _record_rows):
new_rows = []
for row in _record_rows:
new_rows.append(row[:notes_column_index]+row[notes_column_index+1:])
new_headers = _record_headers[:notes_column_index]+_record_headers[notes_column_index+1:]
else:
new_rows = _record_rows
new_headers = _record_headers
return new_rows, new_headers
if self.display_format=='nested': # New Display mode
if self.show_args:
_, argdiff = separate_common_items([load_experiment(ex).get_args().items() for ex in exp_record_dict])
argdiff = {k: args for k, args in izip_equal(exp_record_dict.keys(), argdiff)}
# Build a list of experiments and a list of records.
full_headers = ['#']+header_names
record_rows = []
experiment_rows = []
experiment_row_ixs = []
counter = 1 # Start at 2 because record table has the headers.
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
experiment_row_ixs.append(counter)
exp_identifier = exp_id if not self.show_args else ','.join('{}={}'.format(k, v) for k, v in argdiff[exp_id])
experiment_rows.append([i, exp_identifier])
for j, record_id in enumerate(record_ids):
record_rows.append([j]+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
counter+=1
record_rows, full_headers = remove_notes_if_no_notes(record_rows, full_headers)
# Merge the experiments table and record table.
if self.table_package=='tabulate':
record_table_rows = tabulate(record_rows, headers=full_headers, tablefmt="pipe").split('\n')
del record_table_rows[1] # Get rid of that silly line.
experiment_table_rows = tabulate(experiment_rows, numalign='left').split('\n')[1:-1] # First and last are just borders
longest_row = max(max(len(r) for r in record_table_rows), max(len(r) for r in experiment_table_rows)+4) if len(experiment_table_rows)>0 and len(record_table_rows)>0 else 0
record_table_rows = [r if len(r)==longest_row else r[:-1] + ' '*(longest_row-len(r)) + r[-1] for r in record_table_rows]
experiment_table_rows = [('=' if i==0 else '-')*longest_row+'\n'+r + ' '*(longest_row-len(r)-1)+'|' for i, r in enumerate(experiment_table_rows)]
all_rows = [surround_with_header('Experiments', width=longest_row, char='=')] + (insert_at(record_table_rows, experiment_table_rows, indices=experiment_row_ixs) if len(experiment_table_rows)>0 else ['<No non-root Experiments>']) + ['='*longest_row]
table = '\n'.join(all_rows)
else:
raise NotImplementedError(self.table_package)
elif self.display_format=='flat': # Display First record on same row
full_headers = ['E#', 'R#', 'Experiment']+header_names
rows = []
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
if len(record_ids)==0:
rows.append([str(i), '', exp_id, '<No Records>'] + ['-']*(len(headers)-1))
else:
for j, record_id in enumerate(record_ids):
rows.append([str(i) if j==0 else '', j, exp_id if j==0 else '']+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
assert len(rows[0])==len(full_headers)
rows, full_headers = remove_notes_if_no_notes(rows, full_headers)
if self.table_package == 'pretty_table':
from prettytable.prettytable import PrettyTable
table = str(PrettyTable(rows, field_names=full_headers, align='l', max_table_width=self.max_width))
elif self.table_package == 'tabulate':
table = tabulate(rows, headers=full_headers)
else:
raise NotImplementedError(self.table_package)
else:
raise NotImplementedError(self.display_format)
return table
def _select_record_ids_from_filters(filters, exp_record_dict):
return OrderedDict((k, [record_id for record_id, f in izip_equal(exp_record_dict[k], filters[k]) if f]) for k in exp_record_dict.keys())
def _bitwise_and(a, b):
return [a_ and b_ for a_, b_ in izip_equal(a, b)]
def forward_pass_and_cost(input_data, weights, biases = None, **forward_pass_kwargs):
"""
Do a forward pass and return the computaitonal cost.
:param input_data: A (n_frames, n_dims_in) array
:param weights: A list of (n_dim_in, n_dim_out) weight matrices
:param biases: An optional (len(weights)) list of (w.shape[1]) biases for each weight matrix
:param hidden_activations: Indicates the hidden layer activation function
:param output_activations: Indicates the output layer activation function
:return: output, n_dense_flops, n_sparse_flops ... where:
output is a (input_data.shape[0], weights[-1].shape[1]) array of outputs
n_dense_flops is the total number of floating point operations (adds and multiples) required in the forward pass
n_sparse_flops is the total number of operations not-counting zero-activations.
"""
activations = forward_pass_activations(input_data=input_data, weights=weights, biases=biases, **forward_pass_kwargs)
# dbplot(activations[2][:100], 'full-acts')
# full_flops_per_datapoint = [(w.shape[0]*w.shape[1] + (w.shape[0]-1)*w.shape[1]) for w in weights]
n_dense_flops = sum(w.shape[0]*w.shape[1]+(w.shape[0]-1)*w.shape[1] for w in weights)*input_data.shape[0]
layerwise_total_nonzero_acts = [(act!=0).sum() for act in activations[:-1:2]]
n_sparse_flops = sum((actsum*w.shape[1] + (actsum-1)*w.shape[1]) for actsum, w in izip_equal(layerwise_total_nonzero_acts, weights))
# n_sparse_flops =sum((act!=0).mean(axis=1).sum(axis=0)*full_layer_flops for act, full_layer_flops in izip_equal(activations[:-1:2], full_flops_per_datapoint))
return activations[-1], n_dense_flops, n_sparse_flops
def _bitwise_or(a, b):
return [a_ or b_ for a_, b_ in izip_equal(a, b)]
def select_experiment_records_from_list(user_range, experiment_records):
return [rec_id for rec_id, f in izip_equal(experiment_records, _filter_experiment_record_list(user_range, experiment_records)) if f]
