# write predator graph predator_graph_data = [[data_point[0], data_point[2]] for data_point in datas[i]] predator_graph_data = unite_data(predator_graph_data, model_predator_data1) predator_graph_data = unite_data(predator_graph_data, model_predator_data2) export_array_array_to_csv(predator_graph_data, output_folder + "/" + single_data_set_predator_graph_format.format(i+1)) # write the error graph file single_data_errors_data = [[i] + [single_data_errors[j][i] for j in range(2*len(data_file_names))] for i in range(single_data_set_num_steps+1)] export_array_array_to_csv(single_data_errors_data, output_folder + "/" + single_data_set_errors_out) # run both the normal and modified gradient descent algorithms on all datasets together new_parameters1 = find_minimal_parameters(datas, model, initial_parameters, epsilon1, step_size1, all_data_sets_num_steps, use_improved_descent=False, y_factor=y_factor, error_out=all_data_sets_errors[0], debug_mod=debug_mod) new_parameters2 = find_minimal_parameters(datas, model, initial_parameters, epsilon2, step_size2, all_data_sets_num_steps, use_improved_descent=True, y_factor=y_factor, error_out=all_data_sets_errors[1], debug_mod=debug_mod, minimal_second_gradient=minimal_second_gradient) # write to the output file content string result_file_content += "Using all data:\n" result_file_content += "Normal gradient descent found the following parameters that gave error {}:\n {}\n".format( all_data_sets_errors[0][-1], str(new_parameters1) ) result_file_content += "Improved gradient descent found the following parameters that gave error {}:\n {}\n".format( all_data_sets_errors[1][-1], str(new_parameters2) ) result_file_content += "\n"
def analise(name, model, initial_parameters, data_file_names, analysis_parameters, debug_mod=1000, model_graph_dt=0.01, output_folder_location=""): # some definitions step_size11 = analysis_parameters["step_size11"] step_size12 = analysis_parameters["step_size11"] step_size21 = analysis_parameters["step_size21"] step_size22 = analysis_parameters["step_size22"] epsilon1 = analysis_parameters["epsilon1"] epsilon2 = analysis_parameters["epsilon2"] minimal_second_gradient = analysis_parameters["minimal_second_gradient"] e1 = analysis_parameters["e1"] e2 = analysis_parameters["e2"] single_data_set_num_steps = analysis_parameters[ "single_data_set_num_steps"] all_data_sets_num_steps = analysis_parameters["all_data_sets_num_steps"] output_folder = name + "AnalysisResults" if output_folder_location == "" else \ output_folder_location + os.path.sep + name + "AnalysisResults" log_file = name + "_log.txt" # create an empty string for the contents of the result_file to be contained in result_file_content = "" # create an empty array to store the names of all of the generated csv files csv_file_names = [] # load data from files datas = [] for file_name in data_file_names: datas.append(get_data_from_csv(file_name)) # create empty arrays for the errors and parameters to be stored in single_data_errors = [[] for _ in range(2 * len(datas))] all_data_sets_errors = [[], []] single_data_parameters = [[] for _ in range(2 * len(datas))] all_data_parameters = [[], []] # create the output folder if it doesn't exist already: if not os.path.exists(output_folder): os.mkdir(output_folder) # run both the normal and modified gradient descent algorithms on all datasets seperately for i in range(len(datas)): new_parameters1 = find_minimal_parameters( [datas[i]], model, initial_parameters, epsilon1, step_size11, single_data_set_num_steps, use_improved_descent=False, error_out=single_data_errors[2 * i], parameter_out=single_data_parameters[2 * i], debug_mod=debug_mod, e1=e1, e2=e2) new_parameters2 = find_minimal_parameters( [datas[i]], model, initial_parameters, epsilon2, step_size21, single_data_set_num_steps, use_improved_descent=True, error_out=single_data_errors[2 * i + 1], parameter_out=single_data_parameters[2 * i + 1], minimal_second_gradient=minimal_second_gradient, debug_mod=debug_mod, e1=e1, e2=e2) # write to the output file content string result_file_content += "Using data from {}:\n".format( data_file_names[i]) result_file_content += "Normal gradient descent found the following parameters that gave error {}:\n {}\n".format( min(single_data_errors[2 * i]), str(new_parameters1)) result_file_content += "Improved gradient descent found the following parameters that gave error {}:\n {}\n".format( min(single_data_errors[2 * i + 1]), str(new_parameters2)) result_file_content += "\n" # write some files that are used to create graphs to visually compare the model csv_file_name = output_folder + "/" + single_data_set_graph_format.format( i + 1) write_comparison_csv(csv_file_name, datas[i], model, [new_parameters1, new_parameters2], model_dt=model_graph_dt, headers=[ "D" + str(i + 1), "{} {} {}".format(name, "D" + str(i + 1), "NGD"), "{} {} {}".format(name, "D" + str(i + 1), "AGD") ]) csv_file_names.append(csv_file_name) # write the error graph file single_data_errors_data = [ [i0] + [single_data_errors[j][i0] for j in range(2 * len(datas))] for i0 in range(single_data_set_num_steps + 1) ] csv_file_name = output_folder + "/" + single_data_set_errors_out headers = [""] for i in range(len(datas)): headers.append("{} D{} NGD".format(name, str(i + 1))) headers.append("{} D{} AGD".format(name, str(i + 1))) export_array_array_to_csv(single_data_errors_data, csv_file_name, headers=headers) csv_file_names.append(csv_file_name) # write the parameters file single_data_parameters_data = [[None] + conc([[ "{}_{}_{}".format(str(k // 2), str(k % 2 + 1), key) for key in initial_parameters.keys() ] for k in range(2 * len(datas))])] single_data_parameters_data += [ [i0] + conc([single_data_parameters[j][i0] for j in range(2 * len(datas))]) for i0 in range(single_data_set_num_steps + 1) ] csv_file_name = output_folder + "/" + single_data_set_parameters_out export_array_array_to_csv(single_data_parameters_data, csv_file_name) del single_data_parameters_data del single_data_parameters # run both the normal and modified gradient descent algorithms on all datasets together new_parameters1 = find_minimal_parameters( datas, model, initial_parameters, epsilon1, step_size12, all_data_sets_num_steps, use_improved_descent=False, parameter_out=all_data_parameters[0], error_out=all_data_sets_errors[0], debug_mod=debug_mod, e1=e1, e2=e2) new_parameters2 = find_minimal_parameters( datas, model, initial_parameters, epsilon2, step_size22, all_data_sets_num_steps, use_improved_descent=True, parameter_out=all_data_parameters[1], error_out=all_data_sets_errors[1], debug_mod=debug_mod, minimal_second_gradient=minimal_second_gradient, e1=e1, e2=e2) # write to the output file content string result_file_content += "Using all data:\n" result_file_content += "Normal gradient descent found the following parameters that gave error {}:\n {}\n".format( min(all_data_sets_errors[0]), str(new_parameters1)) result_file_content += "Improved gradient descent found the following parameters that gave error {}:\n {}\n".format( min(all_data_sets_errors[1]), str(new_parameters2)) result_file_content += "\n" # write some files that are used to create graphs to visually compare the model for i in range(len(datas)): csv_file_name = output_folder + "/" + all_data_sets_graph_format.format( i + 1) write_comparison_csv(csv_file_name, datas[i], model, [new_parameters1, new_parameters2], model_dt=model_graph_dt, headers=[ "D" + str(i + 1), "{} {} {}".format(name, "AD", "NGD"), "{} {} {}".format(name, "D" + str(i + 1), "AGD") ]) csv_file_names.append(csv_file_name) # write the second error graph file all_data_sets_errors_data = [ [i] + [all_data_sets_errors[0][i], all_data_sets_errors[1][i]] for i in range(all_data_sets_num_steps + 1) ] csv_file_name = output_folder + "/" + all_data_sets_errors_out export_array_array_to_csv(all_data_sets_errors_data, csv_file_name, headers=["", name + " AD NGD", name + " AD AGD"]) csv_file_names.append(csv_file_name) # write the second parameters file all_data_parameters_data = [[None] + conc( [["{}_{}".format(str(k), key) for key in initial_parameters.keys()] for k in range(2)])] all_data_parameters_data += [[i0] + all_data_parameters[0][i0] + all_data_parameters[1][i0] for i0 in range(all_data_sets_num_steps + 1)] csv_file_name = output_folder + "/" + all_data_sets_parameters_out export_array_array_to_csv(all_data_parameters_data, csv_file_name) del all_data_parameters_data del all_data_parameters # write to the result excel file excel_writer = pd.ExcelWriter(output_folder + os.path.sep + resulting_data_excel_format.format(name)) for file_name in csv_file_names: df = pd.read_csv(file_name, sep=";", decimal=",", header=0) df.to_excel(excel_writer, os.path.splitext(ntpath.basename(file_name))[0], header=True, index=False) excel_writer.save() # write to the result file result_file = open(output_folder + r"/" + results_out, "w") result_file.write(result_file_content) result_file.close() # write to log file log_results(name, analysis_parameters, result_file_content, log_file)
m_values = [ 1e-8, 0.8e-7, 0.9e-7, 0.98e-7, 0.99e-7, 1e-7, 1.01e-7, 1.02e-7, 1.1e-7, 1.2e-7, 1e-6, 1e-5, 1e-4 ] data_set_num = 2 parameters = fetch_parameters("../CLVAnalysisResults", False, min_step, "AGD", data_set_num) errors = [[] for _ in m_values] for i, m in enumerate(m_values): find_minimal_parameters([datas[data_set_num - 1]], model, parameters, epsilon, step_size, max_step - min_step, debug_mod=debug_mod, e1=e1, e2=e2, minimal_second_gradient=m, error_out=errors[i], use_improved_descent=True) lines = [[i] + [errors[j][i] for j in range(len(errors))] for i in range(max_step - min_step + 1)] export_array_array_to_csv(lines, "test_result6.csv", [""] + ["m={:.2E}".format(m) for m in m_values])