def test_minibatch_process(): x = np.random.randn(5, 3) mat = np.random.randn(3, 4) def func(x): return x.dot(mat) y1 = func(x) assert y1.shape==(5, 4) y2 = minibatch_process(func, minibatch_size=2, mb_args=(x, )) assert np.allclose(y1, y2) # weird numpy rounding makes it not exactly equal
def assess_prediction_functions(test_pairs, functions, costs, print_results=False, prediction_minibatches = None): """ :param test_pairs: A list<pair_name, (x, y)>, where x, y are equal-length vectors representing the samples in a dataset. Eg. [('training', (x_train, y_train)), ('test', (x_test, y_test))] :param functions: A list<function_name, function> of functions for computing the forward pass. :param costs: A list<(cost_name, cost_function)> or dict<cost_name: cost_function> of cost functions, where cost_function has the form: cost = cost_fcn(guess, y), where cost is a scalar, and guess is the output of the prediction function given one of the inputs (x) in test_pairs. :param prediction_minibatches: Size of minibatches to predict in. :return: A ModelTestScore object """ if isinstance(test_pairs, DataSet): test_pairs = _dataset_to_test_pair(test_pairs) assert isinstance(test_pairs, list) assert all(len(_)==2 for _ in test_pairs) assert all(len(pair)==2 for name, pair in test_pairs) if isinstance(functions, dict): functions = functions.items() if callable(functions): functions = [(functions.__name__ if hasattr(functions, '__name__') else None, functions)] else: assert all(callable(f) for name, f in functions) if callable(costs): costs = [(costs.__name__, costs)] elif isinstance(costs, string_types): costs = [(costs, get_evaluation_function(costs))] elif isinstance(costs, dict): costs = costs.items() else: costs = [(cost, get_evaluation_function(cost)) if isinstance(cost, string_types) else (cost.__name__, cost) if callable(cost) else cost for cost in costs] assert all(callable(cost) for name, cost in costs) results = ModelTestScore() for test_pair_name, (x, y) in test_pairs: for function_name, function in functions: if prediction_minibatches is None: predictions = function(x) else: predictions = minibatch_process(function, minibatch_size=prediction_minibatches, mb_args=(x, )) for cost_name, cost_function in costs: results[test_pair_name, function_name, cost_name] = cost_function(predictions, y) if print_results: print(results.get_table()) return results