def initialize_directories(dock=None): """ Initialize directories for procedure's product files. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: (dict<str>): collection of paths to directories for procedure's files """ # Define paths to directories. path_permutation = os.path.join(dock, "permutation") path_genes = os.path.join(path_permutation, "genes") # Remove previous files to avoid version or batch confusion. utility.remove_directory(path=path_permutation) # Initialize directories. utility.create_directory(path_permutation) utility.create_directory(path_genes) # Collect information. paths = dict() paths["permutation"] = path_permutation paths["genes"] = path_genes # Return information. return paths
def write_product_chunk(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_structure = os.path.join(dock, "structure") utility.create_directory(path_structure) path_data_pairs_text = os.path.join( path_structure, "data_pairs_sort_format_pandas.txt.gz") # Write information to file. information["data_chunk"].to_csv( path_or_buf=path_data_pairs_text, columns=None, sep="\t", na_rep="", header=False, index=False, mode="a", compression="gzip", ) pass
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_selection = os.path.join(dock, "selection") utility.create_directory(path_selection) path_tissues = os.path.join(path_selection, "tissues.pickle") path_patients = os.path.join(path_selection, "patients.pickle") path_tissues_samples = os.path.join(path_selection, "tissues_samples.pickle") path_patients_samples = os.path.join(path_selection, "patients_samples.pickle") # Write information to file. with open(path_tissues, "wb") as file_product: pickle.dump(information["tissues"], file_product) with open(path_patients, "wb") as file_product: pickle.dump(information["patients"], file_product) with open(path_tissues_samples, "wb") as file_product: pickle.dump(information["tissues_patients_samples"], file_product) with open(path_patients_samples, "wb") as file_product: pickle.dump(information["patients_tissues_samples"], file_product)
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_organization = os.path.join(dock, "organization") utility.create_directory(path_organization) path_imputation = os.path.join(path_organization, "data_gene_signal_imputation.pickle") path_aggregation = os.path.join(path_organization, "data_gene_signal_aggregation.pickle") path_log = os.path.join(path_organization, "data_gene_signal_log.pickle") # Write information to file. with open(path_imputation, "wb") as file_product: pickle.dump(information["data_gene_signal_tissue_median"], file_product) with open(path_aggregation, "wb") as file_product: pickle.dump(information["data_gene_signal_aggregation"], file_product) with open(path_log, "wb") as file_product: pickle.dump(information["data_gene_signal_log"], file_product)
def visualize_graph_ring(adj,n,dirname_parent): fig,ax = plt.subplots() # Scatter plot all the center points t = np.arange(0,2*np.pi,2*np.pi/n) x = np.cos(t) y = np.sin(t) plt.scatter(x,y,s=50,alpha=1.0,zorder=21) plt.scatter(x[0],y[0],s=60,alpha=1.0,zorder=110,marker='s') # Highlight the reference node # Draw edge lines linecolor = (0.1,0.1,0.1) lines = [] for i in range(n): for j in range(i+1,n): if adj[i,j] > 0: line = ((x[i],y[i]),(x[j],y[j])) lines.append(line) linecol = LineCollection(lines,linewidths=1,alpha=0.5,colors=linecolor,zorder=10) ax.add_collection(linecol) # Plot options plt.axis('scaled') plt.axis('equal') plt.axis('off') plt.ion() plt.show() dirname_out = os.path.join(dirname_parent,'network_images') create_directory(dirname_out) filename_out = 'network.png' path_out = os.path.join(dirname_out,filename_out) plt.savefig(path_out,dpi=300) plt.close()
def initialize_directories(dock=None): """ Initialize directories for procedure's product files. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: (dict<str>): collection of paths to directories for procedure's files """ # Collect paths. paths = dict() # Define paths to directories. paths["dock"] = dock paths["function"] = os.path.join(paths["dock"], "function") # Remove previous files to avoid version or batch confusion. utility.remove_directory(path=paths["function"]) utility.create_directory(path=paths["function"]) # Return information. return paths
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_expecto = os.path.join(dock, "expecto") utility.create_directory(path_expecto) path_data_feather = os.path.join(path_expecto, "data_expecto.feather") path_data_pickle = os.path.join(path_expecto, "data_expecto.pickle") # Write information to file. information["data_expecto"].to_feather(fname=path_data_feather, ) pandas.to_pickle( information["data_expecto"], path=path_data_pickle, #compression="gzip", ) pass
def execute_procedure(dock=None): """ Function to execute module's main behavior. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: """ # Define path. directories = ["access"] #path_local = os.path.join(os.sep, *directories) path_local = os.path.join(dock, *directories) utility.create_directory(path_local) # Define reference. reference = define_source() # Download files. download_files(reference=reference, path_local=path_local) # Extract files. extract_files(reference=reference, path_local=path_local) pass
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_selection = os.path.join(dock, "selection") utility.create_directory(path_selection) path_tissues = os.path.join(path_selection, "tissues.pickle") path_persons = os.path.join(path_selection, "persons.pickle") path_gene_signal = os.path.join(path_selection, "data_gene_signal.pickle") # Write information to file. with open(path_tissues, "wb") as file_product: pickle.dump(information["tissues"], file_product) with open(path_persons, "wb") as file_product: pickle.dump(information["persons"], file_product) pandas.to_pickle(information["data_gene_signal"], path_gene_signal) pass
def write_product_gene( gene=None, information=None, paths=None, ): """ Writes product information to file. arguments: gene (str): identifier of single gene for which to execute the process. information (object): information to write to file paths (dict<str>): collection of paths to directories for procedure's files raises: returns: """ # Specify directories and files. path_gene = os.path.join(paths["genes"], gene) utility.create_directory(path_gene) path_permutations = os.path.join(path_gene, "permutations.pickle") # Write information to file. with open(path_permutations, "wb") as file_product: pickle.dump(information["permutations"], file_product) pass
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_measurement = os.path.join(dock, "measurement") utility.create_directory(path_measurement) path_gene_count = os.path.join(path_measurement, "data_gene_count.pickle") path_gene_signal = os.path.join(path_measurement, "data_gene_signal.pickle") # Write information to file. pandas.to_pickle(information["data_gene_count"], path_gene_count) pandas.to_pickle(information["data_gene_signal"], path_gene_signal) pass
def plot_results(SS1,SS2,chist_data,dirname_in): img_dirname_out = os.path.join(dirname_in,'analysis_plots') create_directory(img_dirname_out) c1hist_on_SS1 = np.copy(chist_data[0]) c2hist_on_SS1 = np.copy(chist_data[1]) c1hist_on_SS2 = np.copy(chist_data[2]) c2hist_on_SS2 = np.copy(chist_data[3]) ccare1 = SS1.ccare ccare2 = SS2.ccare # Normalize and shift c1hist_on_SS1 /= ccare1 c2hist_on_SS1 /= ccare2 c1hist_on_SS2 /= ccare1 c2hist_on_SS2 /= ccare2 c1hist_on_SS1 -= 1 c2hist_on_SS1 -= 1 c1hist_on_SS2 -= 1 c2hist_on_SS2 -= 1 plotfun = plt.semilogy marker1 = 'o' marker2 = 'v' ms = 8 color1 = 'tab:blue' color2 = 'tab:red' lw1 = 3 lw2 = 3 fs = 14 fig1 = plt.figure(figsize=(4.5,2.5)) plotfun(c1hist_on_SS1,color=color1,linewidth=lw1,marker=marker1,markersize=ms,markevery=500) plotfun(c2hist_on_SS1,color=color2,linewidth=lw2,marker=marker2,markersize=ms,markevery=500) plt.xlabel('Iteration',fontsize=fs) plt.ylabel('Normalized cost diff.',fontsize=fs) plt.legend(['LQRm cost (with noise)','LQR cost (no noise)']) filename_out = 'plot_lqrm_cost_vs_iteration_suspension' path_out = os.path.join(img_dirname_out,filename_out) plt.savefig(path_out,dpi=300,bbox_inches='tight') fig2 = plt.figure(figsize=(4.5,2.5)) plotfun(c1hist_on_SS2,color=color1,linewidth=lw1,marker=marker1,markersize=ms,markevery=1200) plotfun(c2hist_on_SS2,color=color2,linewidth=lw2,marker=marker2,markersize=ms,markevery=1200) plt.xlabel('Iteration',fontsize=fs) plt.ylabel('Normalized cost diff.',fontsize=fs) plt.legend(['LQRm cost (with noise)','LQR cost (no noise)'],loc='right') filename_out = 'plot_lqr_cost_vs_iteration_suspension' path_out = os.path.join(img_dirname_out,filename_out) plt.savefig(path_out,dpi=300,bbox_inches='tight')
def write_product_sets(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_tissue = os.path.join(dock, "tissue") utility.create_directory(path_tissue) path_sets = os.path.join(path_tissue, "sets") # Remove previous files since they change from run to run. utility.remove_directory(path=path_sets) utility.create_directory(path_sets) # Iterate on sets. for set in information["sets"]: # Access information. tissue = set["tissue"] data_sample = set["sample"] data_gene = set["gene"] # Specify directories and files. path_sample = os.path.join( path_sets, (tissue + "_samples.tsv") ) path_gene = os.path.join( path_sets, (tissue + "_genes.tsv") ) # Write information to file. data_sample.to_csv( path_or_buf=path_sample, sep="\t", header=True, index=True, ) data_gene.to_csv( path_or_buf=path_gene, sep="\t", header=True, index=True, ) pass pass
def plot_sparsity_data(SS0, sparsity_data, dirname_in, fig_list=[], ax_list=[]): rws = [] spars = [] Ks = [] cs = [] ccare = SS0.ccare for s in sparsity_data: rws.append(s[0]) spars.append(s[1]) Ks.append(s[2]) cs.append(s[3]) fs = 16 fig_idx_list = range(2) img_dirname_out = os.path.join(dirname_in, 'analysis_plots') create_directory(img_dirname_out) for fig_idx in fig_idx_list: fig, ax = plt.subplots() if not fig_list: fig_list.append(fig) else: fig_list[fig_idx] = fig if fig_idx == 0: plt.plot(rws, spars, 'o-') plt.xlabel('Regularization weight', fontsize=fs) plt.ylabel('Sparsity', fontsize=fs) filename_out = 'plot_sparsity_vs_regweight' elif fig_idx == 1: plt.plot(spars, cs, 'o-') plt.plot(spars, np.ones_like(spars) * ccare, 'r--') plt.xlabel('Sparsity', fontsize=fs) plt.ylabel('LQR cost', fontsize=fs) filename_out = 'plot_lqr_cost_vs_sparsity' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight') return fig_list, ax_list
def initialize_directories(dock=None): """ Initialize directories for procedure's product files. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: (dict<str>): collection of paths to directories for procedure's files """ # Collect paths. paths = dict() # Define paths to directories. paths["dock"] = dock paths["stratification"] = os.path.join(paths["dock"], "stratification") # Remove previous files to avoid version or batch confusion. utility.remove_directory(path=paths["stratification"]) utility.create_directory(path=paths["stratification"]) # Define paths for cohorts of persons. cohorts = list() cohorts.append("selection") cohorts.append("respiration") cohorts.append("ventilation") for cohort in cohorts: paths[cohort] = dict() paths[cohort]["component"] = os.path.join(paths["stratification"], cohort, "component") paths[cohort]["regression"] = os.path.join(paths["stratification"], cohort, "regression") paths[cohort]["summary"] = os.path.join(paths["stratification"], cohort, "summary") # Initialize directories. utility.create_directories(path=paths[cohort]["component"]) utility.create_directories(path=paths[cohort]["regression"]) utility.create_directories(path=paths[cohort]["summary"]) # Return information. return paths
def write_product( information=None, path_directory=None, ): """ Writes product information to file. arguments: information (dict): information to write to file path_directory (str): path to directory for product directories and files raises: returns: """ # Specify directories and files. utility.create_directories(path_directory) path_genes = os.path.join(path_directory, "genes.pickle") path_genes_samples_signals = os.path.join(path_directory, "genes_samples_signals.pickle") # Write information to file. with open(path_genes, "wb") as file_product: pickle.dump(information["genes"], file_product) with open(path_genes_samples_signals, "wb") as file_product: pickle.dump(information["genes_samples_signals"], file_product) # Write separate files for genes. # Specify directories and files. path_collection = os.path.join(path_directory, "collection") utility.create_directory(path_collection) # Iterate on genes. for gene in information["genes"]: # Specify directories and files. path_gene = os.path.join(path_collection, (gene + ".pickle")) # Write information to file. pandas.to_pickle(information["genes_samples_signals"][gene], path_gene) pass pass
def initialize_directories(dock=None): """ Initialize directories for procedure's product files. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: (dict<str>): collection of paths to directories for procedure's files """ # Collect paths. paths = dict() # Define paths to directories. paths["dock"] = dock paths["candidacy"] = os.path.join(paths["dock"], "candidacy") # Remove previous files to avoid version or batch confusion. utility.remove_directory(path=paths["candidacy"]) utility.create_directory(path=paths["candidacy"]) # Define paths for cohorts of persons. cohorts = list() cohorts.append("selection") cohorts.append("respiration") cohorts.append("ventilation") for cohort in cohorts: paths[cohort] = dict() paths[cohort]["threshold"] = os.path.join( paths["candidacy"], cohort, "threshold" ) # Define paths for groups of genes by their distributions. paths[cohort]["distribution"] = dict() paths[cohort]["distribution"]["multimodal"] = os.path.join( paths["candidacy"], cohort, "distribution", "multimodal" ) paths[cohort]["distribution"]["unimodal"] = os.path.join( paths["candidacy"], cohort, "distribution", "unimodal" ) paths[cohort]["distribution"]["nonmultimodal"] = os.path.join( paths["candidacy"], cohort, "distribution", "nonmultimodal" ) paths[cohort]["distribution"]["any"] = os.path.join( paths["candidacy"], cohort, "distribution", "any" ) # Initialize directories. utility.create_directories(path=paths[cohort]["threshold"]) utility.create_directories( path=paths[cohort]["distribution"]["any"] ) utility.create_directories( path=paths[cohort]["distribution"]["multimodal"] ) utility.create_directories( path=paths[cohort]["distribution"]["unimodal"] ) utility.create_directories( path=paths[cohort]["distribution"]["nonmultimodal"] ) # Return information. return paths
printout('%f' % np.mean(error_norm_all), textfile) # printout('standard deviation of raw gradient estimate, entrywise',textfile) # printout('%f' % np.std(G_est_all,0),textfile) printout('average time per gradient estimate (s)', textfile) printout("%.3f" % ((t_end - t_start) / n_iterc), textfile) printout('', textfile) return G_act, G_est_all, error_angle_all, error_scale_all, error_norm_all ############################################################################### # Main ############################################################################### timestr = str(time()).replace('.', 'p') dirname_out = timestr create_directory(dirname_out) textfilename_only = "noiseless.txt" textfile_out = os.path.join(dirname_out, textfilename_only) textfile = open(textfile_out, "w+") data_noiseless = gradient_estimate_variance(noise=False, textfile=textfile) textfile.close() filename_only = 'data_noiseless.pickle' filename_out = os.path.join(dirname_out, filename_only) pickle_export(dirname_out, filename_out, data_noiseless) textfilename_only = "noisy.txt" textfile_out = os.path.join(dirname_out, textfilename_only) textfile = open(textfile_out, "w+") data_noisy = gradient_estimate_variance(noise=True, textfile=textfile) textfile.close()
def pickle_export(dirname_out, filename_out, data): create_directory(dirname_out) pickle_on = open(filename_out, "wb") pickle.dump(data, pickle_on) pickle_on.close()
def write_product(dock=None, information=None): """ Writes product information to file. arguments: dock (str): path to root or dock directory for source and product directories and files. information (object): information to write to file. raises: returns: """ # Specify directories and files. path_tissue = os.path.join(dock, "tissue") utility.create_directory(path_tissue) path_gene_factor = os.path.join( path_tissue, "data_gene_signal_factor.pickle" ) path_component_variance_all = os.path.join( path_tissue, "data_component_variance_all.pickle" ) path_factor_component_all = os.path.join( path_tissue, "data_factor_component_all.pickle" ) path_component_variance_few = os.path.join( path_tissue, "data_component_variance_few.pickle" ) path_factor_component_few = os.path.join( path_tissue, "data_factor_component_few.pickle" ) # Write information to file. if False: pandas.to_pickle( information["data_gene_signal_factor"], path_gene_factor ) pass pandas.to_pickle( information["data_component_variance_all"], path_component_variance_all ) pandas.to_pickle( information["data_factor_component_all"], path_factor_component_all ) pandas.to_pickle( information["data_component_variance_few"], path_component_variance_few ) pandas.to_pickle( information["data_factor_component_few"], path_factor_component_few ) if False: information["data_component_variance"].to_csv( path_or_buf=path_component_variance_text, sep="\t", header=True, index=False, ) pass
def plot_data(all_dict, dirname_in): img_dirname_out = os.path.join(dirname_in,'analysis_plots') create_directory(img_dirname_out) nSS = len(all_dict['gradient']['costnorm']) lw = 3 fs = 14 markerdict = {'gradient': 'o', 'gradient_model_free': 's', 'natural_gradient': 'v', 'gauss_newton': '^'} linestyle_dict = {'gradient': ':', 'gradient_model_free': '-', 'natural_gradient': '-.', 'gauss_newton': '--'} ms = 8 pmax = 90 pmin = 10 figsize = (6, 3) leg_str_dict = {'gradient_model_free': 'Gradient (model-free)', 'gradient': 'Gradient', 'natural_gradient': 'Natural gradient', 'gauss_newton': 'Gauss-Newton'} quant_list = ['costnorm','gradnorm'] for quant in quant_list: leg_str_list = [] xlab = 'Iteration' if quant=='costnorm': ylab = 'Normalized cost diff.' elif quant=='gradnorm': ylab = 'Fro. norm of gradient' plt.figure(figsize=figsize) # plt.xscale('log') plt.yscale('log') plt.xlabel(xlab,fontsize=fs) plt.ylabel(ylab,fontsize=fs) for step_direction in list(leg_str_dict.keys()): leg_str_list.append(leg_str_dict[step_direction]) max_iters = all_dict[step_direction][quant][0].size quant_all = np.zeros([nSS,max_iters]) for i in range(nSS): quant_all[i] = all_dict[step_direction][quant][i] quant_mean = quant_all.mean(axis=0) # max_iters_partial = 20 max_iters_partial = max_iters x = np.arange(max_iters_partial)+1 upr = np.percentile(quant_all,pmax,axis=0)[0:max_iters_partial] lwr = np.percentile(quant_all,pmin,axis=0)[0:max_iters_partial] mid = quant_mean[0:max_iters_partial] plt.fill_between(x,lwr,upr,alpha=0.3) plt.plot(x, mid, linewidth=lw, linestyle=linestyle_dict[step_direction]) if max_iters_partial <= 20: plt.xticks(ticks=1+np.arange(max_iters_partial)) # Position the legend to custom location plt.legend(leg_str_list, loc='right', bbox_to_anchor=(0.97, 0.35)) # plt.legend(leg_str_list, loc='right', bbox_to_anchor=(0.97, 0.65)) # plt.show() filename_out = 'plot_'+quant+'_vs_iteration_random' path_out = os.path.join(img_dirname_out,filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight')
def plot_data(all_dict, dirname_in): img_dirname_out = os.path.join(dirname_in, 'analysis_plots') create_directory(img_dirname_out) nSS = len(all_dict['gradient']['costnorm']) lw = 3 fs = 14 markerdict = { 'gradient': 'o', 'natural_gradient': 'v', 'gauss_newton': '^' } ms = 8 pmax = 100 pmin = 0 quant_list = ['costnorm', 'gradnorm'] for quant in quant_list: plt.figure(figsize=(4.5, 2.5)) plt.yscale('log') xlab = 'Iteration' plt.xlabel(xlab, fontsize=fs) if quant == 'costnorm': ylab = 'Normalized cost diff.' elif quant == 'gradnorm': ylab = 'Fro. norm of gradient' plt.ylabel(ylab, fontsize=fs) for step_direction in all_dict: max_iters = all_dict[step_direction][quant][0].size quant_all = np.zeros([nSS, max_iters]) for i in range(nSS): quant_all[i] = all_dict[step_direction][quant][i] quant_mean = quant_all.mean(axis=0) x = np.arange(max_iters) + 1 upr = np.percentile(quant_all, pmax, axis=0) lwr = np.percentile(quant_all, pmin, axis=0) mid = quant_mean plt.fill_between(x, lwr, upr, alpha=0.5) plt.plot(x, mid, linewidth=lw, marker=markerdict[step_direction], markersize=ms, markevery=499) leg = ['Gradient', 'Natural gradient', 'Gauss-Newton'] plt.legend(leg, loc='lower right') plt.show() filename_out = 'plot_' + quant + '_vs_iteration_random' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight') plt.figure(figsize=(4.5, 2.5)) plt.yscale('log') plt.xlabel(xlab, fontsize=fs) plt.ylabel(ylab, fontsize=fs) for step_direction in all_dict: max_iters = all_dict[step_direction][quant][0].size quant_all = np.zeros([nSS, max_iters]) for i in range(nSS): quant_all[i] = all_dict[step_direction][quant][i] quant_mean = quant_all.mean(axis=0) max_iters_partial = 10 x = np.arange(max_iters_partial) + 1 upr = np.percentile(quant_all, pmax, axis=0)[0:max_iters_partial] lwr = np.percentile(quant_all, pmin, axis=0)[0:max_iters_partial] mid = quant_mean[0:max_iters_partial] plt.fill_between(x, lwr, upr, alpha=0.5) plt.plot(x, mid, linewidth=lw, marker=markerdict[step_direction], markersize=ms) plt.xticks(ticks=1 + np.arange(max_iters_partial)) leg = ['Gradient', 'Natural gradient', 'Gauss-Newton'] plt.legend(leg, bbox_to_anchor=[0.5, 0.55], loc='center left') plt.show() filename_out = 'plot_' + quant + '_vs_iteration_random_zoom' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight')
def plot_results(SS1, SS2, chist_data, dirname_in): img_dirname_out = os.path.join(dirname_in, 'analysis_plots') create_directory(img_dirname_out) c1hist_on_SS1 = np.copy(chist_data[0]) c2hist_on_SS1 = np.copy(chist_data[1]) c1hist_on_SS2 = np.copy(chist_data[2]) c2hist_on_SS2 = np.copy(chist_data[3]) ccare1 = SS1.ccare ccare2 = SS2.ccare # Normalize and shift c1hist_on_SS1 /= ccare1 c2hist_on_SS1 /= ccare2 c1hist_on_SS2 /= ccare1 c2hist_on_SS2 /= ccare2 c1hist_on_SS1 -= 1 c2hist_on_SS1 -= 1 c1hist_on_SS2 -= 1 c2hist_on_SS2 -= 1 plotfun = plt.semilogy marker1 = 'o' marker2 = 'v' ms = 8 color1 = 'tab:blue' color2 = 'tab:red' lw1 = 3 lw2 = 3 fs = 14 figsize = (5.5, 2) xlim_lwr = -0.5 xlim_upr = np.max([ x.size for x in [c1hist_on_SS1, c1hist_on_SS2, c2hist_on_SS1, c2hist_on_SS2] ]) fig1 = plt.figure(figsize=figsize) plotfun(c1hist_on_SS1, color=color1, linewidth=lw1, linestyle='-') plotfun(c1hist_on_SS2, color=color2, linewidth=lw1, linestyle='--') plt.xlabel('Iteration', fontsize=fs) plt.ylabel('Relative cost error', fontsize=fs) plt.xlim(xlim_lwr, xlim_upr) plt.legend(['$K_m$', '$K_\ell$'], fontsize=fs) filename_out = 'plot_lqrm_cost_vs_iteration_suspension' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight') fig2 = plt.figure(figsize=figsize) plotfun(c2hist_on_SS1[0:7400], color=color1, linewidth=lw2, linestyle='-') plotfun(c2hist_on_SS2[0:7400], color=color2, linewidth=lw2, linestyle='--') plt.xlabel('Iteration', fontsize=fs) plt.ylabel('Relative cost error', fontsize=fs) plt.xlim(xlim_lwr, xlim_upr) plt.legend(['$K_m$', '$K_\ell$'], loc='right', fontsize=fs) filename_out = 'plot_lqr_cost_vs_iteration_suspension' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight')
def routine_gen(): folderstr = 'systems' timestr = str(time()).replace('.', 'p') dirname_in = os.path.join(folderstr, timestr) create_directory(dirname_in) nSS = 20 # Number of independent runs/systems all_dict = { 'gradient': { 'costnorm': [], 'gradnorm': [] }, 'natural_gradient': { 'costnorm': [], 'gradnorm': [] }, 'gauss_newton': { 'costnorm': [], 'gradnorm': [] } } for i in range(nSS): SS = gen_system_mult(n=10, m=10, safety_margin=0.3, noise='olmss_weak', mult_noise_method='random', SStype='random', seed=-1, saveSS=False) # Policy gradient setup K0_method = 'zero' K0 = set_initial_gains(SS, K0_method=K0_method) step_direction_dict = { 'gradient': { 'eta': 1e-5, 'max_iters': 5000 }, 'natural_gradient': { 'eta': 1e-4, 'max_iters': 2000 }, 'gauss_newton': { 'eta': 1 / 2, 'max_iters': 10 } } sleep(1) for step_direction in step_direction_dict: SS.setK(K0) t_start = time() eta = step_direction_dict[step_direction]['eta'] max_iters = step_direction_dict[step_direction]['max_iters'] PGO = policy_gradient_setup(SS, eta, step_direction, max_iters) t_end = time() print('Initialization completed after %.3f seconds' % (t_end - t_start)) SS, histlist = run_policy_gradient(SS, PGO) costnorm = (histlist[2] / SS.ccare) - 1 gradnorm = la.norm(histlist[1], ord='fro', axis=(0, 1)) all_dict[step_direction]['costnorm'].append(costnorm) all_dict[step_direction]['gradnorm'].append(gradnorm) filename_out = 'monte_carlo_all_dict.pickle' path_out = os.path.join(dirname_in, filename_out) pickle_export(dirname_in, path_out, all_dict) plot_data(all_dict, dirname_in)
def traverse_sparsity(SS, PGO, optiongroup, optiongroup_dir): # Sparsity traversal settings sparsity_required = 0.95 sparse_thresh = 0.001 plt.ioff() img_folder = 'sparsity_images' img_dirname_out = os.path.join(optiongroup_dir, img_folder) create_directory(img_dirname_out) filename_out_pre = 'sparsity_are' plot_sparse(img_dirname_out, filename_out_pre, SS.Kare, SS.ccare, sparse_thresh, PGO, are_flag=True) regweight_ratio = np.sqrt(2) if optiongroup == 'proximal_gradient_GN_PI': eta_ratio = 1 else: eta_ratio = (1 / regweight_ratio)**np.sqrt( regweight_ratio) # empirically works well sparsity_data = [] img_pattern = 'sparsity%02d' iterc = 0 iterc_max = 18 stop = False sparsity_prev = 0 sparsity_max = 0 while not stop: # Policy gradient t_start = time() SS, hist_list = run_policy_gradient(SS, PGO) t_end = time() filename_out = 'system_%d_regweight_%.3f.pickle' % (iterc, PGO.regweight) filename_out = filename_out.replace('.', 'p') path_out = os.path.join(optiongroup_dir, filename_out) pickle_export(optiongroup_dir, path_out, SS) # Plotting filename_out_pre = img_pattern % iterc ax_im, ax_im_bw, ax_hist, img, img_bw, cbar, sparsity = plot_sparse( img_dirname_out, filename_out_pre, SS.K, SS.c, sparse_thresh, PGO) plt.close('all') sparsity_data.append( [PGO.regweight, sparsity, SS.K, SS.c, t_end - t_start, hist_list]) if sparsity > sparsity_required: stop = True # if sparsity < sparsity_prev: # stop = True # if sparsity_max > 0.60 and sparsity < 0.05: # stop = True if sparsity_max > 0.60 and sparsity < sparsity_prev: stop = True if iterc >= iterc_max - 1: stop = True PGO.eta *= eta_ratio PGO.regweight *= regweight_ratio sparsity_prev = sparsity sparsity_max = np.max([sparsity, sparsity_max]) iterc += 1 # input("Press [enter] to continue.") # vidname = 'sparsity_evolution' # vidsave(img_folder,img_pattern,vidname) filename_out = 'sparsity_data.pickle' path_out = os.path.join(optiongroup_dir, filename_out) pickle_export(optiongroup_dir, path_out, sparsity_data) return sparsity_data
def plot_comparisons(SS0, sparsity_data_all, dirname_in, fig_list=[], ax_list=[]): img_dirname_out = os.path.join(dirname_in, 'comparison_plots') create_directory(img_dirname_out) nfigs = 5 fig_idx_list = range(nfigs) ax_idx_list = range(nfigs) if not fig_list: fig_list = [None] * len(fig_idx_list) if not ax_list: ax_list = [None] * len(ax_idx_list) legend_strs = [] for key in sparsity_data_all.keys(): legend_strs.append(key.replace('_', ' ').capitalize()) marker_list = ['o', 'v', 'x'] fs = 14 fsleg = 12 xlab_xoffs = 0.50 xlab_yoffs = -0.05 for fig_idx in fig_idx_list: # fig,ax = plt.subplots(figsize=[6,3]) fig, ax = plt.subplots(figsize=[6, 6]) fig_list[fig_idx] = fig ax_list[fig_idx] = ax if fig_idx == 0: for i, optiongroup in enumerate(sparsity_data_all.keys()): nr = len(sparsity_data_all[optiongroup]) regweights = [] walltimes = [] for r in range(nr): regweights.append(sparsity_data_all[optiongroup][r][0]) walltimes.append(sparsity_data_all[optiongroup][r][4]) plt.semilogx(regweights, walltimes, marker=marker_list[i]) plt.xlabel('Regularization weight', fontsize=fs) ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs) plt.ylabel('Compute time (s)', fontsize=fs) plt.legend(legend_strs, prop={'size': fsleg}) filename_out = 'plot_comparison_walltime_vs_regweight' if fig_idx == 1: for i, optiongroup in enumerate(sparsity_data_all.keys()): nr = len(sparsity_data_all[optiongroup]) regweights = [] itercounts = [] for r in range(nr): regweights.append(sparsity_data_all[optiongroup][r][0]) itercounts.append( len(sparsity_data_all[optiongroup][r][5][2])) plt.semilogx(regweights, itercounts, marker=marker_list[i]) plt.xlabel('Regularization weight', fontsize=fs) plt.ylabel('Number of iterations', fontsize=fs) ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs) plt.legend(legend_strs, prop={'size': fsleg}) filename_out = 'plot_comparison_itercount_vs_regweight' if fig_idx == 2: for i, optiongroup in enumerate(sparsity_data_all.keys()): nr = len(sparsity_data_all[optiongroup]) regweights = [] objvals = [] for r in range(nr): regweights.append(sparsity_data_all[optiongroup][r][0]) objvals.append(sparsity_data_all[optiongroup][r][5][3][-1]) plt.semilogx(regweights, objvals, marker=marker_list[i]) plt.xlabel('Regularization weight', fontsize=fs) plt.ylabel('Objective value', fontsize=fs) ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs) plt.legend(legend_strs, prop={'size': fsleg}) filename_out = 'plot_comparison_objval_vs_regweight' if fig_idx == 3: for i, optiongroup in enumerate(sparsity_data_all.keys()): nr = len(sparsity_data_all[optiongroup]) regweights = [] sps = [] for r in range(nr): regweights.append(sparsity_data_all[optiongroup][r][0]) sps.append(sparsity_data_all[optiongroup][r][1]) plt.semilogx(regweights, sps, marker=marker_list[i]) plt.xlabel('Regularization weight', fontsize=fs) plt.ylabel('Sparsity', fontsize=fs) ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs) plt.legend(legend_strs, prop={'size': fsleg}) filename_out = 'plot_comparison_sparsity_vs_regweight' if fig_idx == 4: YSCALE = 1 spsmin = 1 spsmax = 0 for i, optiongroup in enumerate(sparsity_data_all.keys()): nr = len(sparsity_data_all[optiongroup]) sps = [] cs = [] for r in range(nr): sps.append(sparsity_data_all[optiongroup][r][1]) cs.append(YSCALE * sparsity_data_all[optiongroup][r][3]) plt.plot(sps, cs, marker=marker_list[i]) spsmin = np.min([spsmin, np.min(sps)]) spsmax = np.max([spsmax, np.max(sps)]) plt.plot([spsmin, spsmax], [YSCALE * SS0.ccare, YSCALE * SS0.ccare], 'r--') plt.xlabel('Sparsity', fontsize=fs) plt.ylabel('LQRm cost', fontsize=fs) ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs) legend_strs.append('Zero regularization') plt.legend(legend_strs, prop={'size': fsleg}) filename_out = 'plot_comparison_LQR_cost_vs_sparsity' path_out = os.path.join(img_dirname_out, filename_out) plt.savefig(path_out, dpi=300, bbox_inches='tight') return fig_list, ax_list
def execute_procedure(dock=None): """ Function to execute module's main behavior. arguments: dock (str): path to root or dock directory for source and product directories and files raises: returns: """ # Remove previous files. # Specify directories and files. path_metric = os.path.join(dock, "metric") utility.create_directory(path_metric) path_figure = os.path.join(path_metric, "figure") utility.remove_directory(path=path_figure) # Read source information from file. #source = read_source(dock=dock) utility.print_terminal_partition(level=1) print("Test of metrics of modality.") # Unimodal normal distribution. utility.print_terminal_partition(level=2) print("Simulation on 1,000,000 random values with a unimodal normal " + "distribution.") print("Expectations for unimodal normal distribution...") print("skewness = 0") print("kurtosis = 0") print("bimodality coefficient < 0.55") print("dip statistic < 0.05") utility.print_terminal_partition(level=3) # Generate random values with a normal distribution. series = generate_random_values_normal(mean=1.0, deviation=3.0, count=1000000, method="random") report_metrics(name="unimodality", series=series, dock=dock) utility.print_terminal_partition(level=3) # Bimodal normal distribution 1. utility.print_terminal_partition(level=2) print("Simulation on 1,000,000 random values with a bimodal normal " + "distribution.") print("Expectations for bimodal normal distribution...") print("skewness = ?") print("kurtosis = ?") print("bimodality coefficient > 0.55") print("dip statistic > 0.05") utility.print_terminal_partition(level=3) # Generate random values with a normal distribution. series_one = generate_random_values_normal(mean=1.0, deviation=1.0, count=500000, method="random") series_two = generate_random_values_normal(mean=5.0, deviation=2.0, count=500000, method="random") #series_one.extend(series_two) series = series_one + series_two report_metrics(name="bimodality_1", series=series, dock=dock) utility.print_terminal_partition(level=3) # Bimodal normal distribution 2. utility.print_terminal_partition(level=2) print("Simulation on 1,000,000 random values with a bimodal normal " + "distribution.") print("Expectations for bimodal normal distribution...") print("skewness = ?") print("kurtosis = ?") print("bimodality coefficient > 0.55") print("dip statistic > 0.05") utility.print_terminal_partition(level=3) # Generate random values with a normal distribution. series_one = generate_random_values_normal(mean=1.0, deviation=1.0, count=500000, method="random") series_two = generate_random_values_normal(mean=10.0, deviation=2.0, count=500000, method="random") #series_one.extend(series_two) series = series_one + series_two report_metrics(name="bimodality_2", series=series, dock=dock) utility.print_terminal_partition(level=3) # Bimodal normal distribution 3. utility.print_terminal_partition(level=2) print("Simulation on 1,000,000 random values with a bimodal normal " + "distribution.") print("Expectations for bimodal normal distribution...") print("skewness = ?") print("kurtosis = ?") print("bimodality coefficient > 0.55") print("dip statistic > 0.05") utility.print_terminal_partition(level=3) # Generate random values with a normal distribution. series_one = generate_random_values_normal(mean=1.0, deviation=1.0, count=100000, method="random") series_two = generate_random_values_normal(mean=10.0, deviation=2.0, count=900000, method="random") #series_one.extend(series_two) series = series_one + series_two report_metrics(name="bimodality_3", series=series, dock=dock) utility.print_terminal_partition(level=3) # Compile information. information = {} #Write product information to file. #write_product(dock=dock, information=information) pass
# save final output as csv if os.path.exists("data/final_output/"): remove_directory("data/final_output/") logger.info("Save final output as csv") final_df.repartition(1).write.format("csv").save( os.path.join(DATA_PATH, "final_output")) self.unpersist() if __name__ == '__main__': START_TIME = datetime.datetime.now() logger = logging.getLogger("Cogo labs assessment") # pylint: disable=C0103 create_directory("logs/") create_directory("data/") file_handler = logging.FileHandler("logs/main.log", mode="w") # pylint: disable=C0103 format_handler = logging.Formatter(fmt="%(asctime)s - %(name)s - " "%(levelname)s - %(message)s") # pylint: disable=C0103 file_handler.setFormatter(format_handler) logger.addHandler(file_handler) args = parse_args() # pylint: disable=C0103 if args.log_level == "INFO": logger.setLevel(logging.INFO) else: logger.setLevel(logging.DEBUG) try: logger.info("Process started at %s", START_TIME) logger.info("Creating application instance") app = App(args)
SStype='random') # timestr = '1556656014p3178775_n50_olmss_vec1' # SS = load_system(timestr) check_olmss(SS) optiongroup_list = ['gradient', 'subgradient', 'proximal_gradient'] # optiongroup_list = ['gradient'] # optiongroup_list = ['subgradient'] # optiongroup_list = ['proximal_gradient'] for optiongroup in optiongroup_list: optiongroup_dir = os.path.join(SS.dirname, optiongroup) create_directory(optiongroup_dir) # Policy gradient setup t_start = time() K0_method = 'are' set_initial_gains(SS, K0_method=K0_method) PGO = policy_gradient_setup(SS, optiongroup) filename_out = 'policy_gradient_options.pickle' path_out = os.path.join(optiongroup_dir, filename_out) pickle_export(optiongroup_dir, path_out, PGO) t_end = time() print('Initialization completed after %.3f seconds' % (t_end - t_start)) # Sparsity traversal traverse_sparsity(SS, PGO, optiongroup, optiongroup_dir)