def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir( this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' result = do_pca(cma_args.n_components, cma_args.n_comp_to_use, traj_params_dir_name, intermediate_data_dir, proj=False, origin="mean_param", use_IPCA=cma_args.use_IPCA, chunk_size=cma_args.chunk_size, reuse=True) logger.debug("after pca") final_pcs = result["first_n_pcs"] all_param_iterator = get_allinone_concat_df(dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) plane_angles_vs_final_plane_along_the_way = [] ipca = IncrementalPCA(n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up for chunk in all_param_iterator: logger.log(f"currently at {all_param_iterator._currow}") ipca.partial_fit(chunk.values) first_n_pcs = ipca.components_[:cma_args.n_comp_to_use] assert final_pcs.shape[0] == first_n_pcs.shape[0] plane_angle = cal_angle_between_nd_planes(first_n_pcs, final_pcs) plane_angles_vs_final_plane_along_the_way.append(plane_angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) plane_angles_vs_final_plane_plot_dir = get_plane_angles_vs_final_plane_along_the_way_plot_dir(plot_dir, cma_args.n_comp_to_use) if not os.path.exists(plane_angles_vs_final_plane_plot_dir): os.makedirs(plane_angles_vs_final_plane_plot_dir) angles_plot_name = f"plane_angles_vs_final_plane_plot_dir " plot_2d(plane_angles_vs_final_plane_plot_dir, angles_plot_name, np.arange(len(plane_angles_vs_final_plane_along_the_way)), plane_angles_vs_final_plane_along_the_way, "num of chunks", "angle with diff in degrees", False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) all_grads_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size, index="grads") angles_with_pc1_along_the_way = [] grad_vs_final_min_current_param = [] ipca = IncrementalPCA(1) # for sparse PCA to speed up for chunk in all_param_iterator: logger.log(f"currently at {all_param_iterator._currow}") target_direction = final_params - chunk.values[-1] ipca.partial_fit(chunk.values) angle_with_pc1 = cal_angle(target_direction, ipca.components_[0]) angles_with_pc1_along_the_way.append(angle_with_pc1) grads = all_grads_iterator.__next__().values for i, grad in enumerate(grads): grad_angle = cal_angle(grad, final_params - chunk.values[i]) grad_vs_final_min_current_param.append(grad_angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"final - current VS so far pc1" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size}" plot_2d(plot_dir, angles_plot_name, np.arange(len(angles_with_pc1_along_the_way)), angles_with_pc1_along_the_way, "num of chunks", "angle with diff in degrees", False) grad_vs_current_plot_name = f"##final - current param VS current grad" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size}" plot_2d(plot_dir, grad_vs_current_plot_name, np.arange(len(grad_vs_final_min_current_param)), grad_vs_final_min_current_param, "num of chunks", "angle with diff in degrees", False)
plot_next_n_dir = f"{plot_dir}/next_n" this_run_dir = get_dir_path_for_this_run(args) traj_params_dir_name = get_full_params_dir(this_run_dir) if os.path.exists(plot_next_n_dir): import shutil shutil.rmtree(plot_next_n_dir) os.makedirs(plot_next_n_dir) logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_concat_params = pd.read_csv(final_file, header=None).values[0] tic = time.time() concat_matrix_diff = get_allinone_concat_df( dir_name=traj_params_dir_name, final_concat_params=final_concat_params) toc = time.time() print('\nElapsed time getting the full concat diff took {:.2f} s\n'.format( toc - tic)) check_interval = len(concat_matrix_diff) // args.even_check_point_num check_points = [ i for i in range(check_interval, len(concat_matrix_diff) - 1, check_interval) ] for check_index in check_points: pca = PCA(n_components=args.n_components) tic = time.time() pca.fit(concat_matrix_diff[:check_index])
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params ''' ========================================================================================== get the pc vectors ========================================================================================== ''' result = do_pca(cma_args.n_components, cma_args.n_comp_to_use, traj_params_dir_name, intermediate_data_dir, proj=False, origin="mean_param", use_IPCA=cma_args.use_IPCA, chunk_size=cma_args.chunk_size, reuse=True) logger.debug("after pca") final_plane = result["first_n_pcs"] count_file = get_full_param_traj_file_path(traj_params_dir_name, "total_num_dumped") total_num = pd.read_csv(count_file, header=None).values[0] all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) unduped_angles_along_the_way = [] duped_angles_along_the_way = [] diff_along = [] unweighted_pc1_vs_V_angles = [] duped_pc1_vs_V_angles = [] pc1_vs_V_diffs = [] unweighted_ipca = IncrementalPCA( n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up all_matrix_buffer = [] try: i = -1 for chunk in all_param_iterator: i += 1 if i >= 2: break chunk = chunk.values unweighted_ipca.partial_fit(chunk) unweighted_angle = cal_angle_between_nd_planes( final_plane, unweighted_ipca.components_[:cma_args.n_comp_to_use]) unweighted_pc1_vs_V_angle = postize_angle( cal_angle_between_nd_planes(V, unweighted_ipca.components_[0])) unweighted_pc1_vs_V_angles.append(unweighted_pc1_vs_V_angle) #TODO ignore 90 or 180 for now if unweighted_angle > 90: unweighted_angle = 180 - unweighted_angle unduped_angles_along_the_way.append(unweighted_angle) np.testing.assert_almost_equal( cal_angle_between_nd_planes( unweighted_ipca.components_[:cma_args.n_comp_to_use][0], final_plane[0]), cal_angle( unweighted_ipca.components_[:cma_args.n_comp_to_use][0], final_plane[0])) all_matrix_buffer.extend(chunk) weights = gen_weights(all_matrix_buffer, Funcs[cma_args.func_index_to_use]) logger.log(f"currently at {all_param_iterator._currow}") # ipca = PCA(n_components=1) # for sparse PCA to speed up # ipca.fit(duped_in_so_far) wpca = WPCA(n_components=cma_args.n_comp_to_use ) # for sparse PCA to speed up tic = time.time() wpca.fit(all_matrix_buffer, weights=weights) toc = time.time() logger.debug( f"WPCA of {len(all_matrix_buffer)} data took {toc - tic} secs " ) duped_angle = cal_angle_between_nd_planes( final_plane, wpca.components_[:cma_args.n_comp_to_use]) duped_pc1_vs_V_angle = postize_angle( cal_angle_between_nd_planes(V, wpca.components_[0])) duped_pc1_vs_V_angles.append(duped_pc1_vs_V_angle) pc1_vs_V_diffs.append(duped_pc1_vs_V_angle - unweighted_pc1_vs_V_angle) #TODO ignore 90 or 180 for now if duped_angle > 90: duped_angle = 180 - duped_angle duped_angles_along_the_way.append(duped_angle) diff_along.append(unweighted_angle - duped_angle) finally: plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"WPCA" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(duped_angles_along_the_way)), duped_angles_along_the_way, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"Not WPCA exponential 2" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(unduped_angles_along_the_way)), unduped_angles_along_the_way, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"Not WPCA - WPCA diff_along exponential 2," \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(diff_along)), diff_along, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"PC1 VS VWPCA PC1 VS V" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(duped_pc1_vs_V_angles)), duped_pc1_vs_V_angles, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"PC1 VS VNot WPCA PC1 VS V" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(unweighted_pc1_vs_V_angles)), unweighted_pc1_vs_V_angles, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"PC1 VS VNot WPCA - WPCA diff PC1 VS V" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(pc1_vs_V_diffs)), pc1_vs_V_diffs, "num of chunks", "angle with diff in degrees", False) del all_matrix_buffer import gc gc.collect()
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) angles_along_the_way = [] latest_thetas = deque(maxlen=cma_args.deque_len) for chunk in all_param_iterator: pca = PCA( n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up if chunk.shape[0] < cma_args.n_comp_to_use: logger.log("skipping too few data") continue latest_thetas.extend(chunk.values) logger.log(f"currently at {all_param_iterator._currow}") pca.fit(latest_thetas) pcs = pca.components_[:cma_args.n_comp_to_use] angle = cal_angle(V, pcs[0]) angles_along_the_way.append(angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"lastest angles algone the way start start n_comp_used :{cma_args.n_comp_to_use} dim space of mean pca plane, " \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(angles_along_the_way)), angles_along_the_way, "num of chunks", "angle with diff in degrees", False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) angles_along_the_way = [] ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up for chunk in all_param_iterator: if all_param_iterator._currow <= cma_args.pc1_chunk_size * cma_args.skipped_chunks: logger.log( f"skipping: currow: {all_param_iterator._currow} skip threshold {cma_args.pc1_chunk_size * cma_args.skipped_chunks}" ) continue logger.log(f"currently at {all_param_iterator._currow}") ipca.partial_fit(chunk.values) angle = cal_angle(V, ipca.components_[0]) #TODO ignore 90 or 180 for now if angle > 90: angle = 180 - angle angles_along_the_way.append(angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"skipped angles algone the way skipped {cma_args.skipped_chunks}" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(angles_along_the_way)), angles_along_the_way, "num of chunks", "angle with diff in degrees", False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir( this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params all_param_iterator = get_allinone_concat_df(dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) angles_along_the_way = [] grad_vs_Vs = [] ipca = IncrementalPCA(n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up last_chunk_last = start_params for chunk in all_param_iterator: chunk = chunk.values if chunk.shape[0] < cma_args.n_comp_to_use: logger.log("skipping too few data") continue for i in range(chunk.shape[0]): if i == 0: grad = chunk[i] - last_chunk_last else: grad = chunk[i] - chunk[i-1] grad_angle = cal_angle(grad, V) grad_vs_Vs.append(grad_angle) last_chunk_last = chunk[-1] logger.log(f"currently at {all_param_iterator._currow}") ipca.partial_fit(chunk) angle = cal_angle(V, ipca.components_[0]) if angle > 90: angle = 180 - angle angles_along_the_way.extend([angle]*chunk.shape[0]) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # # #TODO ignore negative for now # angles_along_the_way = np.array(angles_along_the_way) # if angles_along_the_way[-1] > 90: # angles_along_the_way = 180 - angles_along_the_way assert len(angles_along_the_way) == len(grad_vs_Vs) angles_plot_name = f"in so far update direction and pc1 vs final - start " \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d_2(plot_dir, angles_plot_name, np.arange(len(grad_vs_Vs)),grad_vs_v=grad_vs_Vs, pc1_vs_V=angles_along_the_way, xlabel="num of chunks", ylabel="angle with diff in degrees", show=False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params all_grads_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size, index="grads") all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) angles_along_the_way = [] grad_vs_pull = [] pc1s = [] ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up i = 1 for chunk in all_param_iterator: logger.log(f"currently at {all_param_iterator._currow}") ipca.partial_fit(chunk.values) pc1 = ipca.components_[0] if i % 2 == 0: pc1 = -pc1 angle = cal_angle(V, pc1) angles_along_the_way.append(angle) pc1s.append(pc1) current_grad = all_grads_iterator.__next__().values[-1] current_param = chunk.values[-1] delta = unit_vector(current_param - start_params) pull_dir = V - delta pull_dir_vs_grad = cal_angle(pull_dir, current_grad) grad_vs_pull.append(pull_dir_vs_grad) i += 1 plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) first_n_pc1_vs_V_plot_dir = get_first_n_pc1_vs_V_plot_dir( plot_dir, cma_args.pc1_chunk_size) if not os.path.exists(first_n_pc1_vs_V_plot_dir): os.makedirs(first_n_pc1_vs_V_plot_dir) angles_plot_name = f"angles algone the way dim space of mean pca plane " plot_2d(first_n_pc1_vs_V_plot_dir, angles_plot_name, np.arange(len(angles_along_the_way)), angles_along_the_way, "num of chunks", "angle with diff in degrees", False) grad_vs_pull_plot_name = f"grad vs V - delta_theta" plot_2d(first_n_pc1_vs_V_plot_dir, grad_vs_pull_plot_name, np.arange(len(grad_vs_pull)), grad_vs_pull, "num of chunks", "angle in degrees", False) pcpca = PCA(n_components=min(len(pc1s), 100)) pcpca.fit(pc1s) logger.log(pcpca.explained_variance_ratio_) logger.log(cal_angle_plane(V, pcpca.components_[:2])) np.savetxt(f"{first_n_pc1_vs_V_plot_dir}/pcs_pcs.txt", pcpca.explained_variance_ratio_, delimiter=',') np.savetxt( f"{first_n_pc1_vs_V_plot_dir}/pcs_V_vs_pcapca_first_2_comp_plane.txt", np.array([cal_angle_plane(V, pcpca.components_[:2])]), delimiter=',') i = 0 for angle in angles_along_the_way: if angle > 90: i += 1 np.savetxt(f"{first_n_pc1_vs_V_plot_dir}/num of angles bigger than 90.txt", np.array([i]), delimiter=',')
def main(n_comp_start=2, do_eval=True): import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir( this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' from stable_baselines.low_dim_analysis.common import \ calculate_projection_errors, plot_2d, get_allinone_concat_df, calculate_num_axis_to_explain origin = "mean_param" ratio_threshold = 0.99 consec_threshold = 5 error_threshold = 0.05 tic = time.time() all_param_matrix = get_allinone_concat_df(dir_name=traj_params_dir_name).values toc = time.time() print('\nElapsed time getting the chunk concat diff took {:.2f} s\n' .format(toc - tic)) n_comps = min(cma_args.n_comp_to_use, cma_args.chunk_size) num_to_explains = [] deviates = [] for i in range(0, len(all_param_matrix), cma_args.chunk_size): if i + cma_args.chunk_size >= len(all_param_matrix): break chunk = all_param_matrix[i:i + cma_args.chunk_size] pca = PCA(n_components=n_comps) # for sparse PCA to speed up pca.fit(chunk) num, explained = calculate_num_axis_to_explain(pca, ratio_threshold) num_to_explains.append(num) pcs_components = pca.components_ num_to_deviate = 0 consec = 0 for j in range(i + cma_args.chunk_size, len(all_param_matrix)): errors = calculate_projection_errors(pca.mean_, pcs_components, all_param_matrix[j], num) if errors[0] >= error_threshold: consec += 1 if consec >= consec_threshold: break num_to_deviate += 1 deviates.append(num_to_deviate) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) deviate_plot_name = f"num of steps to deviates from this plane chunk_size: {cma_args.chunk_size} ratio_threshold: {ratio_threshold} consec_threshold: {consec_threshold}error_threshold: {error_threshold}, " plot_2d(plot_dir, deviate_plot_name, np.arange(len(deviates)), deviates, "num of chunks", "num of steps to deviates from this plane", False) num_to_explain_plot_name = f"num to explain chunk_size: {cma_args.chunk_size} " plot_2d(plot_dir, num_to_explain_plot_name, np.arange(len(num_to_explains)), num_to_explains, "num of chunks", "num_to_explains", False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) save_dir = get_save_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] V = final_params - start_params pcs_components = np.loadtxt(get_pcs_filename( intermediate_dir=intermediate_data_dir, n_comp=cma_args.num_comp_to_load), delimiter=',') smallest_error_angle = postize_angle(cal_angle(V, pcs_components[0])) logger.log(f"@@@@@@@@@@@@ {smallest_error_angle}") curr_angles = [] all_param_iterator = get_allinone_concat_df( dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size) ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up inside_final_cone = [] for chunk in all_param_iterator: # for param in chunk.values: logger.log(f"currently at {all_param_iterator._currow}") ipca.partial_fit(chunk.values) angle = postize_angle(cal_angle(V, ipca.components_[0])) param = chunk.values[-1] curr_angle = cal_angle(param - start_params, ipca.components_[0]) curr_angle = postize_angle(curr_angle) curr_angle_final = cal_angle(param - start_params, pcs_components[0]) inside_final_cone.append(curr_angle_final - smallest_error_angle) curr_angles.append(curr_angle - angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"$$$curr_angles$$$" plot_2d(plot_dir, angles_plot_name, np.arange(len(curr_angles)), curr_angles, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"inside final cone?" plot_2d(plot_dir, angles_plot_name, np.arange(len(inside_final_cone)), inside_final_cone, "num of chunks", "angle with diff in degrees", False)
def main(): # requires n_comp_to_use, pc1_chunk_size import sys logger.log(sys.argv) common_arg_parser = get_common_parser() cma_args, cma_unknown_args = common_arg_parser.parse_known_args() this_run_dir = get_dir_path_for_this_run(cma_args) traj_params_dir_name = get_full_params_dir(this_run_dir) intermediate_data_dir = get_intermediate_data_dir(this_run_dir) if not os.path.exists(intermediate_data_dir): os.makedirs(intermediate_data_dir) ''' ========================================================================================== get the pc vectors ========================================================================================== ''' logger.log("grab final params") final_file = get_full_param_traj_file_path(traj_params_dir_name, "final") final_params = pd.read_csv(final_file, header=None).values[0] logger.log("grab start params") start_file = get_full_param_traj_file_path(traj_params_dir_name, "start") start_params = pd.read_csv(start_file, header=None).values[0] count_file = get_full_param_traj_file_path(traj_params_dir_name, "total_num_dumped") total_num = pd.read_csv(count_file, header=None).values[0] V = final_params - start_params all_thetas_downsampled = get_allinone_concat_df( dir_name=traj_params_dir_name).values[::2] unduped_angles_along_the_way = [] duped_angles_along_the_way = [] diff_along = [] num = 2 #TODO hardcode! undup_ipca = PCA(n_components=1) # for sparse PCA to speed up all_matrix_buffer = [] for chunk in all_param_iterator: chunk = chunk.values undup_ipca.partial_fit(chunk) unduped_angle = cal_angle(V, undup_ipca.components_[0]) #TODO ignore 90 or 180 for now if unduped_angle > 90: unduped_angle = 180 - unduped_angle unduped_angles_along_the_way.append(unduped_angle) all_matrix_buffer.extend(chunk) weights = gen_weights(all_param_iterator._currow, total_num) duped_in_so_far = dup_so_far_buffer(all_matrix_buffer, last_percentage, num) logger.log( f"currently at {all_param_iterator._currow}, last_pecentage: {last_percentage}" ) # ipca = PCA(n_components=1) # for sparse PCA to speed up # ipca.fit(duped_in_so_far) ipca = WPCA( n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up for i in range(0, len(duped_in_so_far), cma_args.chunk_size): logger.log( f"partial fitting: i : {i} len(duped_in_so_far): {len(duped_in_so_far)}" ) if i + cma_args.chunk_size > len(duped_in_so_far): ipca.partial_fit(duped_in_so_far[i:]) else: ipca.partial_fit(duped_in_so_far[i:i + cma_args.chunk_size]) duped_angle = cal_angle(V, ipca.components_[0]) #TODO ignore 90 or 180 for now if duped_angle > 90: duped_angle = 180 - duped_angle duped_angles_along_the_way.append(duped_angle) diff_along.append(unduped_angle - duped_angle) plot_dir = get_plot_dir(cma_args) if not os.path.exists(plot_dir): os.makedirs(plot_dir) angles_plot_name = f"duped exponential 2, num dup: {num}" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(duped_angles_along_the_way)), duped_angles_along_the_way, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"unduped exponential 2, num dup: {num}" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(unduped_angles_along_the_way)), unduped_angles_along_the_way, "num of chunks", "angle with diff in degrees", False) angles_plot_name = f"undup - dup diff_along exponential 2, num dup: {num}" \ f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} " plot_2d(plot_dir, angles_plot_name, np.arange(len(diff_along)), diff_along, "num of chunks", "angle with diff in degrees", False) del all_matrix_buffer import gc gc.collect()