def test_minibatch_process():
x = np.random.randn(5, 3)
mat = np.random.randn(3, 4)
def func(x):
return x.dot(mat)
y1 = func(x)
assert y1.shape==(5, 4)
y2 = minibatch_process(func, minibatch_size=2, mb_args=(x, ))
assert np.allclose(y1, y2) # weird numpy rounding makes it not exactly equal
def test_minibatch_process():
x = np.random.randn(5, 3)
mat = np.random.randn(3, 4)
def func(x):
return x.dot(mat)
y1 = func(x)
assert y1.shape==(5, 4)
y2 = minibatch_process(func, minibatch_size=2, mb_args=(x, ))
assert np.allclose(y1, y2) # weird numpy rounding makes it not exactly equal
def assess_prediction_functions(test_pairs, functions, costs, print_results=False, prediction_minibatches = None):
"""
:param test_pairs: A list<pair_name, (x, y)>, where x, y are equal-length vectors representing the samples in a dataset.
Eg. [('training', (x_train, y_train)), ('test', (x_test, y_test))]
:param functions: A list<function_name, function> of functions for computing the forward pass.
:param costs: A list<(cost_name, cost_function)> or dict<cost_name: cost_function> of cost functions, where cost_function has the form:
cost = cost_fcn(guess, y), where cost is a scalar, and guess is the output of the prediction function given one
of the inputs (x) in test_pairs.
:param prediction_minibatches: Size of minibatches to predict in.
:return: A ModelTestScore object
"""
if isinstance(test_pairs, DataSet):
test_pairs = _dataset_to_test_pair(test_pairs)
assert isinstance(test_pairs, list)
assert all(len(_)==2 for _ in test_pairs)
assert all(len(pair)==2 for name, pair in test_pairs)
if isinstance(functions, dict):
functions = functions.items()
if callable(functions):
functions = [(functions.__name__ if hasattr(functions, '__name__') else None, functions)]
else:
assert all(callable(f) for name, f in functions)
if callable(costs):
costs = [(costs.__name__, costs)]
elif isinstance(costs, string_types):
costs = [(costs, get_evaluation_function(costs))]
elif isinstance(costs, dict):
costs = costs.items()
else:
costs = [(cost, get_evaluation_function(cost)) if isinstance(cost, string_types) else (cost.__name__, cost) if callable(cost) else cost for cost in costs]
assert all(callable(cost) for name, cost in costs)
results = ModelTestScore()
for test_pair_name, (x, y) in test_pairs:
for function_name, function in functions:
if prediction_minibatches is None:
predictions = function(x)
else:
predictions = minibatch_process(function, minibatch_size=prediction_minibatches, mb_args=(x, ))
for cost_name, cost_function in costs:
results[test_pair_name, function_name, cost_name] = cost_function(predictions, y)
if print_results:
print(results.get_table())
return results
def assess_prediction_functions(test_pairs, functions, costs, print_results=False, prediction_minibatches = None):
"""
:param test_pairs: A list<pair_name, (x, y)>, where x, y are equal-length vectors representing the samples in a dataset.
Eg. [('training', (x_train, y_train)), ('test', (x_test, y_test))]
:param functions: A list<function_name, function> of functions for computing the forward pass.
:param costs: A list<(cost_name, cost_function)> or dict<cost_name: cost_function> of cost functions, where cost_function has the form:
cost = cost_fcn(guess, y), where cost is a scalar, and guess is the output of the prediction function given one
of the inputs (x) in test_pairs.
:param prediction_minibatches: Size of minibatches to predict in.
:return: A ModelTestScore object
"""
if isinstance(test_pairs, DataSet):
test_pairs = _dataset_to_test_pair(test_pairs)
assert isinstance(test_pairs, list)
assert all(len(_)==2 for _ in test_pairs)
assert all(len(pair)==2 for name, pair in test_pairs)
if isinstance(functions, dict):
functions = functions.items()
if callable(functions):
functions = [(functions.__name__ if hasattr(functions, '__name__') else None, functions)]
else:
assert all(callable(f) for name, f in functions)
if callable(costs):
costs = [(costs.__name__, costs)]
elif isinstance(costs, string_types):
costs = [(costs, get_evaluation_function(costs))]
elif isinstance(costs, dict):
costs = costs.items()
else:
costs = [(cost, get_evaluation_function(cost)) if isinstance(cost, string_types) else (cost.__name__, cost) if callable(cost) else cost for cost in costs]
assert all(callable(cost) for name, cost in costs)
results = ModelTestScore()
for test_pair_name, (x, y) in test_pairs:
for function_name, function in functions:
if prediction_minibatches is None:
predictions = function(x)
else:
predictions = minibatch_process(function, minibatch_size=prediction_minibatches, mb_args=(x, ))
for cost_name, cost_function in costs:
results[test_pair_name, function_name, cost_name] = cost_function(predictions, y)
if print_results:
print(results.get_table())
return results