def measure_goodness(**kwargs): """ [WIP] Test the compressed model against the original model. :param kwargs: :return: """ original_model = load_model(kwargs.get("model"), **kwargs) kwargs['prefix'] = kwargs['prefix'] + "_compressed" compressed_model = load_model(kwargs.get("model"), **kwargs) loss_original = original_model.test() loss_compressed = compressed_model.test() print(loss_original, loss_compressed) # Fine tune print("Fine tuning") compressed_model.train() loss_compressed_fine_tuned = compressed_model.test() print(loss_compressed_fine_tuned) compressed_model.prefix += "_fine_tuned" compressed_model.save()
def calc_eigs(**kwargs): """ Calculates the inter layer covariance and corresponding eigen values and stores them as 'inter_layer_covariance'. Dependency: train_model :param kwargs: :return: """ model_wrapper = odin.model_wrapper = load_model(kwargs.get("model"), **kwargs) co.calc_inter_layer_covariance(model_wrapper=model_wrapper, **kwargs) datastore = model_wrapper.get_group("inter_layer_covariance") cov = datastore["cov"] eigen_values = datastore["eigen_values"] if kwargs["plot"]: for i, eigs in enumerate(eigen_values): l = len(eigs) eigs = abs(eigs) oplt.plot_eigen_values(eigs, title="%s layer %d (%d)" % (model_wrapper.model_name, i, l)) oplt.save("eigs(%d)" % i) oplt.plot_matrix(cov[i], title="Covariance (%d)" % i) oplt.save("cov(%d)" % i) return cov, eigen_values
def calc_index_set(**kwargs): """ [WIP] Calculate the new architecture given already computed layer widths. Dependency: calc_dof :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) method = "Newton-CG" r_dof = model_wrapper.get_group("range_test") # bounds = r_dof["rho_range"] layer_widths_list = r_dof["all_layer_widths"] # lambdas = r_dof["all_lambdas"] data = model_wrapper.get_group("inter_layer_covariance") cov_list = data["cov"] # eigen_values = data["eigen_values"] loss_before = [] loss_after = [] num = kwargs.get("num") or 1 layer_widths = layer_widths_list[num] # TODO this is not what we want logging.info("Finding the new architecture for %s" % layer_widths) print("Finding the new architecture for %s" % layer_widths) architecture.calculate_index_set(model_wrapper=model_wrapper, layer_widths=layer_widths, cov_list=cov_list)
def range_test_plot(**kwargs): """ Plot the result of range test dependency: range_test :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) data = model_wrapper.get_group("range_test", experiment=kwargs.get("experiment")) rho_range = data["rho_range"] all_lambdas = data["all_lambdas"] all_layer_widths = data["all_layer_widths"] n_layers = len(all_lambdas) fig = oplt.figure() # oplt.suptitle(r"Different layer widths for $\rho \in [0.01,0.15]$") oplt.subplot(2, 1, 1) oplt.plot(rho_range, all_lambdas) oplt.labels(r"$\rho$", "$\lambda_{\ell}$") oplt.legend([r"$\lambda_{%d}$" % l for l in range(n_layers)]) oplt.subplot(2, 1, 2) oplt.plot(rho_range, all_layer_widths) oplt.labels(r"$\rho$", "$\hat{m}_{\ell}$") oplt.legend([r"$\hat{m}_{%d}$" % l for l in range(n_layers)]) oplt.save("range_test", experiment=kwargs.get("experiment")) oplt.show() return fig
def calc_dof(**kwargs): """ This computes the layer widths for a range of hyper parameters for the given model. Dependency: calc_eigs :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) l_opt = lambda_param.LambdaOptimizer(model_wrapper) method = "Newton-CG" line = co.xp.linspace(5.0, 100.0, num=100) lambdas, layer_widths, success = l_opt.range_lambda_dof(line, method=method) co.store_elements(model_wrapper=model_wrapper, experiment=kwargs.get("experiment"), elements={ "lambdas": lambdas, "bounds": line, "layer_widths": layer_widths, "success": success }, group_name="range_dof_%s" % method)
def test_lambda_optimizer(**kwargs): """ :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) datastore = model_wrapper.get_group("inter_layer_covariance") cov = datastore["cov"] eigen_values = datastore["eigen_values"] l_opt = lambda_param.LambdaOptimizer(model_wrapper) w_opt = compress.CovarianceOptimizer(model_wrapper) lambdas, layer_widths = l_opt.optimize(method="Newton-CG", rho=1) lambdas, layer_widths = l_opt.optimize(method="SLSQP", bound=80) result = l_opt.result_text() print(l_opt.current_result) logging.info(result) print(result) w_result = w_opt.compress() print(w_result)
def create_compressed_network(**kwargs): """ Dependecy: calc_index_set :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) r_dof = model_wrapper.get_group("range_test") layer_widths_list = r_dof["all_layer_widths"] data = model_wrapper.get_group("inter_layer_covariance") cov_list = data["cov"] model = architecture.transfer_to_architecture(model_wrapper=model_wrapper, cov_list=cov_list) model.save()
def test_model(**kwargs): """ Test the specified model :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) scores = model_wrapper.test(**kwargs) if len(scores) >= 2: print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) else: print('Test score:', scores) return scores
def range_test(**kwargs): """ Test LambdaOptimizer over a range Dependency: calc_eigs :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) l_opt = lambda_param.LambdaOptimizer(model_wrapper) method = "Newton-CG" num = kwargs.get("num") or 1000 max_rho = kwargs.get("max") or 0.15 regularizing = kwargs.get("regularizing") rho_range = co.xp.linspace(0.01, max_rho, num=num) all_lambdas = [] all_layer_widths = [] bar = ChargingBar("Testing a range of rho on LambdaOptimizer", max=num) for rho in rho_range: lambdas, layer_widths = l_opt.optimize(rho=rho, method=method, debug=kwargs.get('verbose'), regularizing=regularizing) if kwargs['verbose']: print("rho=", rho, layer_widths, lambdas) all_lambdas.append(lambdas) all_layer_widths.append(layer_widths) bar.next() bar.finish() co.store_elements(model_wrapper=model_wrapper, group_name="range_test", experiment=kwargs.get("experiment"), elements={ "rho_range": rho_range, "all_lambdas": all_lambdas, "all_layer_widths": all_layer_widths }) if kwargs.get('plot'): range_test_plot(**kwargs)
def plot_dof(**kwargs): """ Makes a plot of layer widths given different hyper parameters Dependency: calc_dof :param kwargs: :return: """ model_wrapper = load_model(kwargs.get("model"), **kwargs) # lambdas, optimal = l_opt.optimize(bound=0.1, debug=True) # l_arr, layer_widths = l_opt.n_sphere_lambda_dof() # lambda_param.plot_lambdas(l_arr, layer_widths, optimal=optimal, prefix="sphere") method = "Newton-CG" r_dof = model_wrapper.get_group("range_dof_%s" % method) bounds = r_dof["bounds"] layer_widths = r_dof["layer_widths"] lambdas = r_dof["lambdas"] lambda_param.plot_lambdas(lambdas, bounds, layer_widths, prefix=method)
def train_model(**kwargs): """ Train the specified model. Commandline arguments will be passed to the training function. :param kwargs: :return: """ # Remove None values kwargs = {a: b for a, b in kwargs.items() if b is not None} kwargs['new_model'] = True model_wrapper = load_model(kwargs.get("model"), **kwargs) model_wrapper.train(**kwargs) model_wrapper.save() if model_wrapper.history: model_wrapper.put_group("training_history", {"history": model_wrapper.history.history}) if kwargs["plot"]: oplt.plot_model_history(model_wrapper) oplt.save("loss") oplt.show() return model_wrapper