def initialize_directories(dock=None):
"""
Initialize directories for procedure's product files.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
(dict<str>): collection of paths to directories for procedure's files
"""
# Define paths to directories.
path_permutation = os.path.join(dock, "permutation")
path_genes = os.path.join(path_permutation, "genes")
# Remove previous files to avoid version or batch confusion.
utility.remove_directory(path=path_permutation)
# Initialize directories.
utility.create_directory(path_permutation)
utility.create_directory(path_genes)
# Collect information.
paths = dict()
paths["permutation"] = path_permutation
paths["genes"] = path_genes
# Return information.
return paths
def write_product_chunk(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_structure = os.path.join(dock, "structure")
utility.create_directory(path_structure)
path_data_pairs_text = os.path.join(
path_structure, "data_pairs_sort_format_pandas.txt.gz")
# Write information to file.
information["data_chunk"].to_csv(
path_or_buf=path_data_pairs_text,
columns=None,
sep="\t",
na_rep="",
header=False,
index=False,
mode="a",
compression="gzip",
)
pass
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_selection = os.path.join(dock, "selection")
utility.create_directory(path_selection)
path_tissues = os.path.join(path_selection, "tissues.pickle")
path_patients = os.path.join(path_selection, "patients.pickle")
path_tissues_samples = os.path.join(path_selection,
"tissues_samples.pickle")
path_patients_samples = os.path.join(path_selection,
"patients_samples.pickle")
# Write information to file.
with open(path_tissues, "wb") as file_product:
pickle.dump(information["tissues"], file_product)
with open(path_patients, "wb") as file_product:
pickle.dump(information["patients"], file_product)
with open(path_tissues_samples, "wb") as file_product:
pickle.dump(information["tissues_patients_samples"], file_product)
with open(path_patients_samples, "wb") as file_product:
pickle.dump(information["patients_tissues_samples"], file_product)
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_organization = os.path.join(dock, "organization")
utility.create_directory(path_organization)
path_imputation = os.path.join(path_organization,
"data_gene_signal_imputation.pickle")
path_aggregation = os.path.join(path_organization,
"data_gene_signal_aggregation.pickle")
path_log = os.path.join(path_organization, "data_gene_signal_log.pickle")
# Write information to file.
with open(path_imputation, "wb") as file_product:
pickle.dump(information["data_gene_signal_tissue_median"],
file_product)
with open(path_aggregation, "wb") as file_product:
pickle.dump(information["data_gene_signal_aggregation"], file_product)
with open(path_log, "wb") as file_product:
pickle.dump(information["data_gene_signal_log"], file_product)
def visualize_graph_ring(adj,n,dirname_parent):
fig,ax = plt.subplots()
# Scatter plot all the center points
t = np.arange(0,2*np.pi,2*np.pi/n)
x = np.cos(t)
y = np.sin(t)
plt.scatter(x,y,s=50,alpha=1.0,zorder=21)
plt.scatter(x[0],y[0],s=60,alpha=1.0,zorder=110,marker='s') # Highlight the reference node
# Draw edge lines
linecolor = (0.1,0.1,0.1)
lines = []
for i in range(n):
for j in range(i+1,n):
if adj[i,j] > 0:
line = ((x[i],y[i]),(x[j],y[j]))
lines.append(line)
linecol = LineCollection(lines,linewidths=1,alpha=0.5,colors=linecolor,zorder=10)
ax.add_collection(linecol)
# Plot options
plt.axis('scaled')
plt.axis('equal')
plt.axis('off')
plt.ion()
plt.show()
dirname_out = os.path.join(dirname_parent,'network_images')
create_directory(dirname_out)
filename_out = 'network.png'
path_out = os.path.join(dirname_out,filename_out)
plt.savefig(path_out,dpi=300)
plt.close()
def initialize_directories(dock=None):
"""
Initialize directories for procedure's product files.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
(dict<str>): collection of paths to directories for procedure's files
"""
# Collect paths.
paths = dict()
# Define paths to directories.
paths["dock"] = dock
paths["function"] = os.path.join(paths["dock"], "function")
# Remove previous files to avoid version or batch confusion.
utility.remove_directory(path=paths["function"])
utility.create_directory(path=paths["function"])
# Return information.
return paths
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_expecto = os.path.join(dock, "expecto")
utility.create_directory(path_expecto)
path_data_feather = os.path.join(path_expecto, "data_expecto.feather")
path_data_pickle = os.path.join(path_expecto, "data_expecto.pickle")
# Write information to file.
information["data_expecto"].to_feather(fname=path_data_feather, )
pandas.to_pickle(
information["data_expecto"],
path=path_data_pickle,
#compression="gzip",
)
pass
def execute_procedure(dock=None):
"""
Function to execute module's main behavior.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
"""
# Define path.
directories = ["access"]
#path_local = os.path.join(os.sep, *directories)
path_local = os.path.join(dock, *directories)
utility.create_directory(path_local)
# Define reference.
reference = define_source()
# Download files.
download_files(reference=reference, path_local=path_local)
# Extract files.
extract_files(reference=reference, path_local=path_local)
pass
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_selection = os.path.join(dock, "selection")
utility.create_directory(path_selection)
path_tissues = os.path.join(path_selection, "tissues.pickle")
path_persons = os.path.join(path_selection, "persons.pickle")
path_gene_signal = os.path.join(path_selection, "data_gene_signal.pickle")
# Write information to file.
with open(path_tissues, "wb") as file_product:
pickle.dump(information["tissues"], file_product)
with open(path_persons, "wb") as file_product:
pickle.dump(information["persons"], file_product)
pandas.to_pickle(information["data_gene_signal"], path_gene_signal)
pass
def write_product_gene(
gene=None,
information=None,
paths=None,
):
"""
Writes product information to file.
arguments:
gene (str): identifier of single gene for which to execute the process.
information (object): information to write to file
paths (dict<str>): collection of paths to directories for procedure's
files
raises:
returns:
"""
# Specify directories and files.
path_gene = os.path.join(paths["genes"], gene)
utility.create_directory(path_gene)
path_permutations = os.path.join(path_gene, "permutations.pickle")
# Write information to file.
with open(path_permutations, "wb") as file_product:
pickle.dump(information["permutations"], file_product)
pass
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_measurement = os.path.join(dock, "measurement")
utility.create_directory(path_measurement)
path_gene_count = os.path.join(path_measurement, "data_gene_count.pickle")
path_gene_signal = os.path.join(path_measurement,
"data_gene_signal.pickle")
# Write information to file.
pandas.to_pickle(information["data_gene_count"], path_gene_count)
pandas.to_pickle(information["data_gene_signal"], path_gene_signal)
pass
def plot_results(SS1,SS2,chist_data,dirname_in):
img_dirname_out = os.path.join(dirname_in,'analysis_plots')
create_directory(img_dirname_out)
c1hist_on_SS1 = np.copy(chist_data[0])
c2hist_on_SS1 = np.copy(chist_data[1])
c1hist_on_SS2 = np.copy(chist_data[2])
c2hist_on_SS2 = np.copy(chist_data[3])
ccare1 = SS1.ccare
ccare2 = SS2.ccare
# Normalize and shift
c1hist_on_SS1 /= ccare1
c2hist_on_SS1 /= ccare2
c1hist_on_SS2 /= ccare1
c2hist_on_SS2 /= ccare2
c1hist_on_SS1 -= 1
c2hist_on_SS1 -= 1
c1hist_on_SS2 -= 1
c2hist_on_SS2 -= 1
plotfun = plt.semilogy
marker1 = 'o'
marker2 = 'v'
ms = 8
color1 = 'tab:blue'
color2 = 'tab:red'
lw1 = 3
lw2 = 3
fs = 14
fig1 = plt.figure(figsize=(4.5,2.5))
plotfun(c1hist_on_SS1,color=color1,linewidth=lw1,marker=marker1,markersize=ms,markevery=500)
plotfun(c2hist_on_SS1,color=color2,linewidth=lw2,marker=marker2,markersize=ms,markevery=500)
plt.xlabel('Iteration',fontsize=fs)
plt.ylabel('Normalized cost diff.',fontsize=fs)
plt.legend(['LQRm cost (with noise)','LQR cost (no noise)'])
filename_out = 'plot_lqrm_cost_vs_iteration_suspension'
path_out = os.path.join(img_dirname_out,filename_out)
plt.savefig(path_out,dpi=300,bbox_inches='tight')
fig2 = plt.figure(figsize=(4.5,2.5))
plotfun(c1hist_on_SS2,color=color1,linewidth=lw1,marker=marker1,markersize=ms,markevery=1200)
plotfun(c2hist_on_SS2,color=color2,linewidth=lw2,marker=marker2,markersize=ms,markevery=1200)
plt.xlabel('Iteration',fontsize=fs)
plt.ylabel('Normalized cost diff.',fontsize=fs)
plt.legend(['LQRm cost (with noise)','LQR cost (no noise)'],loc='right')
filename_out = 'plot_lqr_cost_vs_iteration_suspension'
path_out = os.path.join(img_dirname_out,filename_out)
plt.savefig(path_out,dpi=300,bbox_inches='tight')
def write_product_sets(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_tissue = os.path.join(dock, "tissue")
utility.create_directory(path_tissue)
path_sets = os.path.join(path_tissue, "sets")
# Remove previous files since they change from run to run.
utility.remove_directory(path=path_sets)
utility.create_directory(path_sets)
# Iterate on sets.
for set in information["sets"]:
# Access information.
tissue = set["tissue"]
data_sample = set["sample"]
data_gene = set["gene"]
# Specify directories and files.
path_sample = os.path.join(
path_sets, (tissue + "_samples.tsv")
)
path_gene = os.path.join(
path_sets, (tissue + "_genes.tsv")
)
# Write information to file.
data_sample.to_csv(
path_or_buf=path_sample,
sep="\t",
header=True,
index=True,
)
data_gene.to_csv(
path_or_buf=path_gene,
sep="\t",
header=True,
index=True,
)
pass
pass
def plot_sparsity_data(SS0,
sparsity_data,
dirname_in,
fig_list=[],
ax_list=[]):
rws = []
spars = []
Ks = []
cs = []
ccare = SS0.ccare
for s in sparsity_data:
rws.append(s[0])
spars.append(s[1])
Ks.append(s[2])
cs.append(s[3])
fs = 16
fig_idx_list = range(2)
img_dirname_out = os.path.join(dirname_in, 'analysis_plots')
create_directory(img_dirname_out)
for fig_idx in fig_idx_list:
fig, ax = plt.subplots()
if not fig_list:
fig_list.append(fig)
else:
fig_list[fig_idx] = fig
if fig_idx == 0:
plt.plot(rws, spars, 'o-')
plt.xlabel('Regularization weight', fontsize=fs)
plt.ylabel('Sparsity', fontsize=fs)
filename_out = 'plot_sparsity_vs_regweight'
elif fig_idx == 1:
plt.plot(spars, cs, 'o-')
plt.plot(spars, np.ones_like(spars) * ccare, 'r--')
plt.xlabel('Sparsity', fontsize=fs)
plt.ylabel('LQR cost', fontsize=fs)
filename_out = 'plot_lqr_cost_vs_sparsity'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
return fig_list, ax_list
def initialize_directories(dock=None):
"""
Initialize directories for procedure's product files.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
(dict<str>): collection of paths to directories for procedure's files
"""
# Collect paths.
paths = dict()
# Define paths to directories.
paths["dock"] = dock
paths["stratification"] = os.path.join(paths["dock"], "stratification")
# Remove previous files to avoid version or batch confusion.
utility.remove_directory(path=paths["stratification"])
utility.create_directory(path=paths["stratification"])
# Define paths for cohorts of persons.
cohorts = list()
cohorts.append("selection")
cohorts.append("respiration")
cohorts.append("ventilation")
for cohort in cohorts:
paths[cohort] = dict()
paths[cohort]["component"] = os.path.join(paths["stratification"],
cohort, "component")
paths[cohort]["regression"] = os.path.join(paths["stratification"],
cohort, "regression")
paths[cohort]["summary"] = os.path.join(paths["stratification"],
cohort, "summary")
# Initialize directories.
utility.create_directories(path=paths[cohort]["component"])
utility.create_directories(path=paths[cohort]["regression"])
utility.create_directories(path=paths[cohort]["summary"])
# Return information.
return paths
def write_product(
information=None,
path_directory=None,
):
"""
Writes product information to file.
arguments:
information (dict): information to write to file
path_directory (str): path to directory for product directories and
files
raises:
returns:
"""
# Specify directories and files.
utility.create_directories(path_directory)
path_genes = os.path.join(path_directory, "genes.pickle")
path_genes_samples_signals = os.path.join(path_directory,
"genes_samples_signals.pickle")
# Write information to file.
with open(path_genes, "wb") as file_product:
pickle.dump(information["genes"], file_product)
with open(path_genes_samples_signals, "wb") as file_product:
pickle.dump(information["genes_samples_signals"], file_product)
# Write separate files for genes.
# Specify directories and files.
path_collection = os.path.join(path_directory, "collection")
utility.create_directory(path_collection)
# Iterate on genes.
for gene in information["genes"]:
# Specify directories and files.
path_gene = os.path.join(path_collection, (gene + ".pickle"))
# Write information to file.
pandas.to_pickle(information["genes_samples_signals"][gene], path_gene)
pass
pass
def initialize_directories(dock=None):
"""
Initialize directories for procedure's product files.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
(dict<str>): collection of paths to directories for procedure's files
"""
# Collect paths.
paths = dict()
# Define paths to directories.
paths["dock"] = dock
paths["candidacy"] = os.path.join(paths["dock"], "candidacy")
# Remove previous files to avoid version or batch confusion.
utility.remove_directory(path=paths["candidacy"])
utility.create_directory(path=paths["candidacy"])
# Define paths for cohorts of persons.
cohorts = list()
cohorts.append("selection")
cohorts.append("respiration")
cohorts.append("ventilation")
for cohort in cohorts:
paths[cohort] = dict()
paths[cohort]["threshold"] = os.path.join(
paths["candidacy"], cohort, "threshold"
)
# Define paths for groups of genes by their distributions.
paths[cohort]["distribution"] = dict()
paths[cohort]["distribution"]["multimodal"] = os.path.join(
paths["candidacy"], cohort, "distribution", "multimodal"
)
paths[cohort]["distribution"]["unimodal"] = os.path.join(
paths["candidacy"], cohort, "distribution", "unimodal"
)
paths[cohort]["distribution"]["nonmultimodal"] = os.path.join(
paths["candidacy"], cohort, "distribution", "nonmultimodal"
)
paths[cohort]["distribution"]["any"] = os.path.join(
paths["candidacy"], cohort, "distribution", "any"
)
# Initialize directories.
utility.create_directories(path=paths[cohort]["threshold"])
utility.create_directories(
path=paths[cohort]["distribution"]["any"]
)
utility.create_directories(
path=paths[cohort]["distribution"]["multimodal"]
)
utility.create_directories(
path=paths[cohort]["distribution"]["unimodal"]
)
utility.create_directories(
path=paths[cohort]["distribution"]["nonmultimodal"]
)
# Return information.
return paths
printout('%f' % np.mean(error_norm_all), textfile)
# printout('standard deviation of raw gradient estimate, entrywise',textfile)
# printout('%f' % np.std(G_est_all,0),textfile)
printout('average time per gradient estimate (s)', textfile)
printout("%.3f" % ((t_end - t_start) / n_iterc), textfile)
printout('', textfile)
return G_act, G_est_all, error_angle_all, error_scale_all, error_norm_all
###############################################################################
# Main
###############################################################################
timestr = str(time()).replace('.', 'p')
dirname_out = timestr
create_directory(dirname_out)
textfilename_only = "noiseless.txt"
textfile_out = os.path.join(dirname_out, textfilename_only)
textfile = open(textfile_out, "w+")
data_noiseless = gradient_estimate_variance(noise=False, textfile=textfile)
textfile.close()
filename_only = 'data_noiseless.pickle'
filename_out = os.path.join(dirname_out, filename_only)
pickle_export(dirname_out, filename_out, data_noiseless)
textfilename_only = "noisy.txt"
textfile_out = os.path.join(dirname_out, textfilename_only)
textfile = open(textfile_out, "w+")
data_noisy = gradient_estimate_variance(noise=True, textfile=textfile)
textfile.close()
def pickle_export(dirname_out, filename_out, data):
create_directory(dirname_out)
pickle_on = open(filename_out, "wb")
pickle.dump(data, pickle_on)
pickle_on.close()
def write_product(dock=None, information=None):
"""
Writes product information to file.
arguments:
dock (str): path to root or dock directory for source and product
directories and files.
information (object): information to write to file.
raises:
returns:
"""
# Specify directories and files.
path_tissue = os.path.join(dock, "tissue")
utility.create_directory(path_tissue)
path_gene_factor = os.path.join(
path_tissue, "data_gene_signal_factor.pickle"
)
path_component_variance_all = os.path.join(
path_tissue, "data_component_variance_all.pickle"
)
path_factor_component_all = os.path.join(
path_tissue, "data_factor_component_all.pickle"
)
path_component_variance_few = os.path.join(
path_tissue, "data_component_variance_few.pickle"
)
path_factor_component_few = os.path.join(
path_tissue, "data_factor_component_few.pickle"
)
# Write information to file.
if False:
pandas.to_pickle(
information["data_gene_signal_factor"],
path_gene_factor
)
pass
pandas.to_pickle(
information["data_component_variance_all"],
path_component_variance_all
)
pandas.to_pickle(
information["data_factor_component_all"],
path_factor_component_all
)
pandas.to_pickle(
information["data_component_variance_few"],
path_component_variance_few
)
pandas.to_pickle(
information["data_factor_component_few"],
path_factor_component_few
)
if False:
information["data_component_variance"].to_csv(
path_or_buf=path_component_variance_text,
sep="\t",
header=True,
index=False,
)
pass
def plot_data(all_dict, dirname_in):
img_dirname_out = os.path.join(dirname_in,'analysis_plots')
create_directory(img_dirname_out)
nSS = len(all_dict['gradient']['costnorm'])
lw = 3
fs = 14
markerdict = {'gradient': 'o',
'gradient_model_free': 's',
'natural_gradient': 'v',
'gauss_newton': '^'}
linestyle_dict = {'gradient': ':',
'gradient_model_free': '-',
'natural_gradient': '-.',
'gauss_newton': '--'}
ms = 8
pmax = 90
pmin = 10
figsize = (6, 3)
leg_str_dict = {'gradient_model_free': 'Gradient (model-free)',
'gradient': 'Gradient',
'natural_gradient': 'Natural gradient',
'gauss_newton': 'Gauss-Newton'}
quant_list = ['costnorm','gradnorm']
for quant in quant_list:
leg_str_list = []
xlab = 'Iteration'
if quant=='costnorm':
ylab = 'Normalized cost diff.'
elif quant=='gradnorm':
ylab = 'Fro. norm of gradient'
plt.figure(figsize=figsize)
# plt.xscale('log')
plt.yscale('log')
plt.xlabel(xlab,fontsize=fs)
plt.ylabel(ylab,fontsize=fs)
for step_direction in list(leg_str_dict.keys()):
leg_str_list.append(leg_str_dict[step_direction])
max_iters = all_dict[step_direction][quant][0].size
quant_all = np.zeros([nSS,max_iters])
for i in range(nSS):
quant_all[i] = all_dict[step_direction][quant][i]
quant_mean = quant_all.mean(axis=0)
# max_iters_partial = 20
max_iters_partial = max_iters
x = np.arange(max_iters_partial)+1
upr = np.percentile(quant_all,pmax,axis=0)[0:max_iters_partial]
lwr = np.percentile(quant_all,pmin,axis=0)[0:max_iters_partial]
mid = quant_mean[0:max_iters_partial]
plt.fill_between(x,lwr,upr,alpha=0.3)
plt.plot(x, mid, linewidth=lw, linestyle=linestyle_dict[step_direction])
if max_iters_partial <= 20:
plt.xticks(ticks=1+np.arange(max_iters_partial))
# Position the legend to custom location
plt.legend(leg_str_list, loc='right', bbox_to_anchor=(0.97, 0.35))
# plt.legend(leg_str_list, loc='right', bbox_to_anchor=(0.97, 0.65))
#
plt.show()
filename_out = 'plot_'+quant+'_vs_iteration_random'
path_out = os.path.join(img_dirname_out,filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
def plot_data(all_dict, dirname_in):
img_dirname_out = os.path.join(dirname_in, 'analysis_plots')
create_directory(img_dirname_out)
nSS = len(all_dict['gradient']['costnorm'])
lw = 3
fs = 14
markerdict = {
'gradient': 'o',
'natural_gradient': 'v',
'gauss_newton': '^'
}
ms = 8
pmax = 100
pmin = 0
quant_list = ['costnorm', 'gradnorm']
for quant in quant_list:
plt.figure(figsize=(4.5, 2.5))
plt.yscale('log')
xlab = 'Iteration'
plt.xlabel(xlab, fontsize=fs)
if quant == 'costnorm':
ylab = 'Normalized cost diff.'
elif quant == 'gradnorm':
ylab = 'Fro. norm of gradient'
plt.ylabel(ylab, fontsize=fs)
for step_direction in all_dict:
max_iters = all_dict[step_direction][quant][0].size
quant_all = np.zeros([nSS, max_iters])
for i in range(nSS):
quant_all[i] = all_dict[step_direction][quant][i]
quant_mean = quant_all.mean(axis=0)
x = np.arange(max_iters) + 1
upr = np.percentile(quant_all, pmax, axis=0)
lwr = np.percentile(quant_all, pmin, axis=0)
mid = quant_mean
plt.fill_between(x, lwr, upr, alpha=0.5)
plt.plot(x,
mid,
linewidth=lw,
marker=markerdict[step_direction],
markersize=ms,
markevery=499)
leg = ['Gradient', 'Natural gradient', 'Gauss-Newton']
plt.legend(leg, loc='lower right')
plt.show()
filename_out = 'plot_' + quant + '_vs_iteration_random'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
plt.figure(figsize=(4.5, 2.5))
plt.yscale('log')
plt.xlabel(xlab, fontsize=fs)
plt.ylabel(ylab, fontsize=fs)
for step_direction in all_dict:
max_iters = all_dict[step_direction][quant][0].size
quant_all = np.zeros([nSS, max_iters])
for i in range(nSS):
quant_all[i] = all_dict[step_direction][quant][i]
quant_mean = quant_all.mean(axis=0)
max_iters_partial = 10
x = np.arange(max_iters_partial) + 1
upr = np.percentile(quant_all, pmax, axis=0)[0:max_iters_partial]
lwr = np.percentile(quant_all, pmin, axis=0)[0:max_iters_partial]
mid = quant_mean[0:max_iters_partial]
plt.fill_between(x, lwr, upr, alpha=0.5)
plt.plot(x,
mid,
linewidth=lw,
marker=markerdict[step_direction],
markersize=ms)
plt.xticks(ticks=1 + np.arange(max_iters_partial))
leg = ['Gradient', 'Natural gradient', 'Gauss-Newton']
plt.legend(leg, bbox_to_anchor=[0.5, 0.55], loc='center left')
plt.show()
filename_out = 'plot_' + quant + '_vs_iteration_random_zoom'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
def plot_results(SS1, SS2, chist_data, dirname_in):
img_dirname_out = os.path.join(dirname_in, 'analysis_plots')
create_directory(img_dirname_out)
c1hist_on_SS1 = np.copy(chist_data[0])
c2hist_on_SS1 = np.copy(chist_data[1])
c1hist_on_SS2 = np.copy(chist_data[2])
c2hist_on_SS2 = np.copy(chist_data[3])
ccare1 = SS1.ccare
ccare2 = SS2.ccare
# Normalize and shift
c1hist_on_SS1 /= ccare1
c2hist_on_SS1 /= ccare2
c1hist_on_SS2 /= ccare1
c2hist_on_SS2 /= ccare2
c1hist_on_SS1 -= 1
c2hist_on_SS1 -= 1
c1hist_on_SS2 -= 1
c2hist_on_SS2 -= 1
plotfun = plt.semilogy
marker1 = 'o'
marker2 = 'v'
ms = 8
color1 = 'tab:blue'
color2 = 'tab:red'
lw1 = 3
lw2 = 3
fs = 14
figsize = (5.5, 2)
xlim_lwr = -0.5
xlim_upr = np.max([
x.size
for x in [c1hist_on_SS1, c1hist_on_SS2, c2hist_on_SS1, c2hist_on_SS2]
])
fig1 = plt.figure(figsize=figsize)
plotfun(c1hist_on_SS1, color=color1, linewidth=lw1, linestyle='-')
plotfun(c1hist_on_SS2, color=color2, linewidth=lw1, linestyle='--')
plt.xlabel('Iteration', fontsize=fs)
plt.ylabel('Relative cost error', fontsize=fs)
plt.xlim(xlim_lwr, xlim_upr)
plt.legend(['$K_m$', '$K_\ell$'], fontsize=fs)
filename_out = 'plot_lqrm_cost_vs_iteration_suspension'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
fig2 = plt.figure(figsize=figsize)
plotfun(c2hist_on_SS1[0:7400], color=color1, linewidth=lw2, linestyle='-')
plotfun(c2hist_on_SS2[0:7400], color=color2, linewidth=lw2, linestyle='--')
plt.xlabel('Iteration', fontsize=fs)
plt.ylabel('Relative cost error', fontsize=fs)
plt.xlim(xlim_lwr, xlim_upr)
plt.legend(['$K_m$', '$K_\ell$'], loc='right', fontsize=fs)
filename_out = 'plot_lqr_cost_vs_iteration_suspension'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
def routine_gen():
folderstr = 'systems'
timestr = str(time()).replace('.', 'p')
dirname_in = os.path.join(folderstr, timestr)
create_directory(dirname_in)
nSS = 20 # Number of independent runs/systems
all_dict = {
'gradient': {
'costnorm': [],
'gradnorm': []
},
'natural_gradient': {
'costnorm': [],
'gradnorm': []
},
'gauss_newton': {
'costnorm': [],
'gradnorm': []
}
}
for i in range(nSS):
SS = gen_system_mult(n=10,
m=10,
safety_margin=0.3,
noise='olmss_weak',
mult_noise_method='random',
SStype='random',
seed=-1,
saveSS=False)
# Policy gradient setup
K0_method = 'zero'
K0 = set_initial_gains(SS, K0_method=K0_method)
step_direction_dict = {
'gradient': {
'eta': 1e-5,
'max_iters': 5000
},
'natural_gradient': {
'eta': 1e-4,
'max_iters': 2000
},
'gauss_newton': {
'eta': 1 / 2,
'max_iters': 10
}
}
sleep(1)
for step_direction in step_direction_dict:
SS.setK(K0)
t_start = time()
eta = step_direction_dict[step_direction]['eta']
max_iters = step_direction_dict[step_direction]['max_iters']
PGO = policy_gradient_setup(SS, eta, step_direction, max_iters)
t_end = time()
print('Initialization completed after %.3f seconds' %
(t_end - t_start))
SS, histlist = run_policy_gradient(SS, PGO)
costnorm = (histlist[2] / SS.ccare) - 1
gradnorm = la.norm(histlist[1], ord='fro', axis=(0, 1))
all_dict[step_direction]['costnorm'].append(costnorm)
all_dict[step_direction]['gradnorm'].append(gradnorm)
filename_out = 'monte_carlo_all_dict.pickle'
path_out = os.path.join(dirname_in, filename_out)
pickle_export(dirname_in, path_out, all_dict)
plot_data(all_dict, dirname_in)
def traverse_sparsity(SS, PGO, optiongroup, optiongroup_dir):
# Sparsity traversal settings
sparsity_required = 0.95
sparse_thresh = 0.001
plt.ioff()
img_folder = 'sparsity_images'
img_dirname_out = os.path.join(optiongroup_dir, img_folder)
create_directory(img_dirname_out)
filename_out_pre = 'sparsity_are'
plot_sparse(img_dirname_out,
filename_out_pre,
SS.Kare,
SS.ccare,
sparse_thresh,
PGO,
are_flag=True)
regweight_ratio = np.sqrt(2)
if optiongroup == 'proximal_gradient_GN_PI':
eta_ratio = 1
else:
eta_ratio = (1 / regweight_ratio)**np.sqrt(
regweight_ratio) # empirically works well
sparsity_data = []
img_pattern = 'sparsity%02d'
iterc = 0
iterc_max = 18
stop = False
sparsity_prev = 0
sparsity_max = 0
while not stop:
# Policy gradient
t_start = time()
SS, hist_list = run_policy_gradient(SS, PGO)
t_end = time()
filename_out = 'system_%d_regweight_%.3f.pickle' % (iterc,
PGO.regweight)
filename_out = filename_out.replace('.', 'p')
path_out = os.path.join(optiongroup_dir, filename_out)
pickle_export(optiongroup_dir, path_out, SS)
# Plotting
filename_out_pre = img_pattern % iterc
ax_im, ax_im_bw, ax_hist, img, img_bw, cbar, sparsity = plot_sparse(
img_dirname_out, filename_out_pre, SS.K, SS.c, sparse_thresh, PGO)
plt.close('all')
sparsity_data.append(
[PGO.regweight, sparsity, SS.K, SS.c, t_end - t_start, hist_list])
if sparsity > sparsity_required:
stop = True
# if sparsity < sparsity_prev:
# stop = True
# if sparsity_max > 0.60 and sparsity < 0.05:
# stop = True
if sparsity_max > 0.60 and sparsity < sparsity_prev:
stop = True
if iterc >= iterc_max - 1:
stop = True
PGO.eta *= eta_ratio
PGO.regweight *= regweight_ratio
sparsity_prev = sparsity
sparsity_max = np.max([sparsity, sparsity_max])
iterc += 1
# input("Press [enter] to continue.")
# vidname = 'sparsity_evolution'
# vidsave(img_folder,img_pattern,vidname)
filename_out = 'sparsity_data.pickle'
path_out = os.path.join(optiongroup_dir, filename_out)
pickle_export(optiongroup_dir, path_out, sparsity_data)
return sparsity_data
def plot_comparisons(SS0,
sparsity_data_all,
dirname_in,
fig_list=[],
ax_list=[]):
img_dirname_out = os.path.join(dirname_in, 'comparison_plots')
create_directory(img_dirname_out)
nfigs = 5
fig_idx_list = range(nfigs)
ax_idx_list = range(nfigs)
if not fig_list:
fig_list = [None] * len(fig_idx_list)
if not ax_list:
ax_list = [None] * len(ax_idx_list)
legend_strs = []
for key in sparsity_data_all.keys():
legend_strs.append(key.replace('_', ' ').capitalize())
marker_list = ['o', 'v', 'x']
fs = 14
fsleg = 12
xlab_xoffs = 0.50
xlab_yoffs = -0.05
for fig_idx in fig_idx_list:
# fig,ax = plt.subplots(figsize=[6,3])
fig, ax = plt.subplots(figsize=[6, 6])
fig_list[fig_idx] = fig
ax_list[fig_idx] = ax
if fig_idx == 0:
for i, optiongroup in enumerate(sparsity_data_all.keys()):
nr = len(sparsity_data_all[optiongroup])
regweights = []
walltimes = []
for r in range(nr):
regweights.append(sparsity_data_all[optiongroup][r][0])
walltimes.append(sparsity_data_all[optiongroup][r][4])
plt.semilogx(regweights, walltimes, marker=marker_list[i])
plt.xlabel('Regularization weight', fontsize=fs)
ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs)
plt.ylabel('Compute time (s)', fontsize=fs)
plt.legend(legend_strs, prop={'size': fsleg})
filename_out = 'plot_comparison_walltime_vs_regweight'
if fig_idx == 1:
for i, optiongroup in enumerate(sparsity_data_all.keys()):
nr = len(sparsity_data_all[optiongroup])
regweights = []
itercounts = []
for r in range(nr):
regweights.append(sparsity_data_all[optiongroup][r][0])
itercounts.append(
len(sparsity_data_all[optiongroup][r][5][2]))
plt.semilogx(regweights, itercounts, marker=marker_list[i])
plt.xlabel('Regularization weight', fontsize=fs)
plt.ylabel('Number of iterations', fontsize=fs)
ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs)
plt.legend(legend_strs, prop={'size': fsleg})
filename_out = 'plot_comparison_itercount_vs_regweight'
if fig_idx == 2:
for i, optiongroup in enumerate(sparsity_data_all.keys()):
nr = len(sparsity_data_all[optiongroup])
regweights = []
objvals = []
for r in range(nr):
regweights.append(sparsity_data_all[optiongroup][r][0])
objvals.append(sparsity_data_all[optiongroup][r][5][3][-1])
plt.semilogx(regweights, objvals, marker=marker_list[i])
plt.xlabel('Regularization weight', fontsize=fs)
plt.ylabel('Objective value', fontsize=fs)
ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs)
plt.legend(legend_strs, prop={'size': fsleg})
filename_out = 'plot_comparison_objval_vs_regweight'
if fig_idx == 3:
for i, optiongroup in enumerate(sparsity_data_all.keys()):
nr = len(sparsity_data_all[optiongroup])
regweights = []
sps = []
for r in range(nr):
regweights.append(sparsity_data_all[optiongroup][r][0])
sps.append(sparsity_data_all[optiongroup][r][1])
plt.semilogx(regweights, sps, marker=marker_list[i])
plt.xlabel('Regularization weight', fontsize=fs)
plt.ylabel('Sparsity', fontsize=fs)
ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs)
plt.legend(legend_strs, prop={'size': fsleg})
filename_out = 'plot_comparison_sparsity_vs_regweight'
if fig_idx == 4:
YSCALE = 1
spsmin = 1
spsmax = 0
for i, optiongroup in enumerate(sparsity_data_all.keys()):
nr = len(sparsity_data_all[optiongroup])
sps = []
cs = []
for r in range(nr):
sps.append(sparsity_data_all[optiongroup][r][1])
cs.append(YSCALE * sparsity_data_all[optiongroup][r][3])
plt.plot(sps, cs, marker=marker_list[i])
spsmin = np.min([spsmin, np.min(sps)])
spsmax = np.max([spsmax, np.max(sps)])
plt.plot([spsmin, spsmax],
[YSCALE * SS0.ccare, YSCALE * SS0.ccare], 'r--')
plt.xlabel('Sparsity', fontsize=fs)
plt.ylabel('LQRm cost', fontsize=fs)
ax.xaxis.set_label_coords(xlab_xoffs, xlab_yoffs)
legend_strs.append('Zero regularization')
plt.legend(legend_strs, prop={'size': fsleg})
filename_out = 'plot_comparison_LQR_cost_vs_sparsity'
path_out = os.path.join(img_dirname_out, filename_out)
plt.savefig(path_out, dpi=300, bbox_inches='tight')
return fig_list, ax_list
def execute_procedure(dock=None):
"""
Function to execute module's main behavior.
arguments:
dock (str): path to root or dock directory for source and product
directories and files
raises:
returns:
"""
# Remove previous files.
# Specify directories and files.
path_metric = os.path.join(dock, "metric")
utility.create_directory(path_metric)
path_figure = os.path.join(path_metric, "figure")
utility.remove_directory(path=path_figure)
# Read source information from file.
#source = read_source(dock=dock)
utility.print_terminal_partition(level=1)
print("Test of metrics of modality.")
# Unimodal normal distribution.
utility.print_terminal_partition(level=2)
print("Simulation on 1,000,000 random values with a unimodal normal " +
"distribution.")
print("Expectations for unimodal normal distribution...")
print("skewness = 0")
print("kurtosis = 0")
print("bimodality coefficient < 0.55")
print("dip statistic < 0.05")
utility.print_terminal_partition(level=3)
# Generate random values with a normal distribution.
series = generate_random_values_normal(mean=1.0,
deviation=3.0,
count=1000000,
method="random")
report_metrics(name="unimodality", series=series, dock=dock)
utility.print_terminal_partition(level=3)
# Bimodal normal distribution 1.
utility.print_terminal_partition(level=2)
print("Simulation on 1,000,000 random values with a bimodal normal " +
"distribution.")
print("Expectations for bimodal normal distribution...")
print("skewness = ?")
print("kurtosis = ?")
print("bimodality coefficient > 0.55")
print("dip statistic > 0.05")
utility.print_terminal_partition(level=3)
# Generate random values with a normal distribution.
series_one = generate_random_values_normal(mean=1.0,
deviation=1.0,
count=500000,
method="random")
series_two = generate_random_values_normal(mean=5.0,
deviation=2.0,
count=500000,
method="random")
#series_one.extend(series_two)
series = series_one + series_two
report_metrics(name="bimodality_1", series=series, dock=dock)
utility.print_terminal_partition(level=3)
# Bimodal normal distribution 2.
utility.print_terminal_partition(level=2)
print("Simulation on 1,000,000 random values with a bimodal normal " +
"distribution.")
print("Expectations for bimodal normal distribution...")
print("skewness = ?")
print("kurtosis = ?")
print("bimodality coefficient > 0.55")
print("dip statistic > 0.05")
utility.print_terminal_partition(level=3)
# Generate random values with a normal distribution.
series_one = generate_random_values_normal(mean=1.0,
deviation=1.0,
count=500000,
method="random")
series_two = generate_random_values_normal(mean=10.0,
deviation=2.0,
count=500000,
method="random")
#series_one.extend(series_two)
series = series_one + series_two
report_metrics(name="bimodality_2", series=series, dock=dock)
utility.print_terminal_partition(level=3)
# Bimodal normal distribution 3.
utility.print_terminal_partition(level=2)
print("Simulation on 1,000,000 random values with a bimodal normal " +
"distribution.")
print("Expectations for bimodal normal distribution...")
print("skewness = ?")
print("kurtosis = ?")
print("bimodality coefficient > 0.55")
print("dip statistic > 0.05")
utility.print_terminal_partition(level=3)
# Generate random values with a normal distribution.
series_one = generate_random_values_normal(mean=1.0,
deviation=1.0,
count=100000,
method="random")
series_two = generate_random_values_normal(mean=10.0,
deviation=2.0,
count=900000,
method="random")
#series_one.extend(series_two)
series = series_one + series_two
report_metrics(name="bimodality_3", series=series, dock=dock)
utility.print_terminal_partition(level=3)
# Compile information.
information = {}
#Write product information to file.
#write_product(dock=dock, information=information)
pass
# save final output as csv
if os.path.exists("data/final_output/"):
remove_directory("data/final_output/")
logger.info("Save final output as csv")
final_df.repartition(1).write.format("csv").save(
os.path.join(DATA_PATH, "final_output"))
self.unpersist()
if __name__ == '__main__':
START_TIME = datetime.datetime.now()
logger = logging.getLogger("Cogo labs assessment") # pylint: disable=C0103
create_directory("logs/")
create_directory("data/")
file_handler = logging.FileHandler("logs/main.log", mode="w") # pylint: disable=C0103
format_handler = logging.Formatter(fmt="%(asctime)s - %(name)s - "
"%(levelname)s - %(message)s") # pylint: disable=C0103
file_handler.setFormatter(format_handler)
logger.addHandler(file_handler)
args = parse_args() # pylint: disable=C0103
if args.log_level == "INFO":
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.DEBUG)
try:
logger.info("Process started at %s", START_TIME)
logger.info("Creating application instance")
app = App(args)
SStype='random')
# timestr = '1556656014p3178775_n50_olmss_vec1'
# SS = load_system(timestr)
check_olmss(SS)
optiongroup_list = ['gradient', 'subgradient', 'proximal_gradient']
# optiongroup_list = ['gradient']
# optiongroup_list = ['subgradient']
# optiongroup_list = ['proximal_gradient']
for optiongroup in optiongroup_list:
optiongroup_dir = os.path.join(SS.dirname, optiongroup)
create_directory(optiongroup_dir)
# Policy gradient setup
t_start = time()
K0_method = 'are'
set_initial_gains(SS, K0_method=K0_method)
PGO = policy_gradient_setup(SS, optiongroup)
filename_out = 'policy_gradient_options.pickle'
path_out = os.path.join(optiongroup_dir, filename_out)
pickle_export(optiongroup_dir, path_out, PGO)
t_end = time()
print('Initialization completed after %.3f seconds' %
(t_end - t_start))
# Sparsity traversal
traverse_sparsity(SS, PGO, optiongroup, optiongroup_dir)
