def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
result = do_pca(cma_args.n_components, cma_args.n_comp_to_use, traj_params_dir_name, intermediate_data_dir,
proj=False,
origin="mean_param", use_IPCA=cma_args.use_IPCA, chunk_size=cma_args.chunk_size, reuse=True)
logger.debug("after pca")
final_pcs = result["first_n_pcs"]
all_param_iterator = get_allinone_concat_df(dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size)
plane_angles_vs_final_plane_along_the_way = []
ipca = IncrementalPCA(n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up
for chunk in all_param_iterator:
logger.log(f"currently at {all_param_iterator._currow}")
ipca.partial_fit(chunk.values)
first_n_pcs = ipca.components_[:cma_args.n_comp_to_use]
assert final_pcs.shape[0] == first_n_pcs.shape[0]
plane_angle = cal_angle_between_nd_planes(first_n_pcs, final_pcs)
plane_angles_vs_final_plane_along_the_way.append(plane_angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
plane_angles_vs_final_plane_plot_dir = get_plane_angles_vs_final_plane_along_the_way_plot_dir(plot_dir, cma_args.n_comp_to_use)
if not os.path.exists(plane_angles_vs_final_plane_plot_dir):
os.makedirs(plane_angles_vs_final_plane_plot_dir)
angles_plot_name = f"plane_angles_vs_final_plane_plot_dir "
plot_2d(plane_angles_vs_final_plane_plot_dir, angles_plot_name, np.arange(len(plane_angles_vs_final_plane_along_the_way)), plane_angles_vs_final_plane_along_the_way, "num of chunks", "angle with diff in degrees", False)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(args)
plot_dir_alg = get_plot_dir(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir,
params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
if not os.path.exists(plot_dir_alg):
os.makedirs(plot_dir_alg)
final_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
def make_env():
env_out = gym.make(args.env)
env_out = bench.Monitor(env_out,
logger.get_dir(),
allow_early_resets=True)
return env_out
env = DummyVecEnv([make_env])
if args.normalize:
env = VecNormalize(env)
model = PPO2.load(f"{save_dir}/ppo2") # this also loads V function
model.set_pi_from_flat(final_params)
if args.normalize:
env.load_running_average(save_dir)
obz_tensor = model.act_model.fake_input_tensor
some_neuron = model.act_model.policy_neurons[2][-1]
grads = tf.gradients(tf.math.negative(some_neuron), obz_tensor)
grads = list(zip(grads, obz_tensor))
trainer = tf.train.AdamOptimizer(learning_rate=0.01, epsilon=1e-5)
train_op = trainer.apply_gradients(grads)
for i in range(10000):
obz, _ = model.sess.run([obz_tensor, train_op])
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
run_nums = cma_args.run_nums_to_check
run_nums = [int(run_num) for run_num in run_nums.split(":")]
final_params_list = []
start_params_list = []
for run_num in run_nums:
cma_args.run_num = run_num
if os.path.exists(get_dir_path_for_this_run(cma_args)):
this_run_dir = get_dir_path_for_this_run(cma_args)
plot_dir_alg = get_plot_dir(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(
this_run_dir, params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
if not os.path.exists(plot_dir_alg):
os.makedirs(plot_dir_alg)
start_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_start")
start_params = pd.read_csv(start_file, header=None).values[0]
final_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
final_params_list.append(final_params)
start_params_list.append(start_params)
cma_args.run_num += 1
final_params_distances = []
for i in range(len(final_params_list)):
for j in range(i + 1, len(final_params_list)):
final_params_distances.append(
LA.norm(final_params_list[i] - final_params_list[j], ord=2))
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
np.savetxt(f"{plot_dir}/final_params_distances.txt",
final_params_distances,
delimiter=",")
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import do_pca, plot_2d
origin = "mean_param"
result = do_pca(cma_args.n_components,
cma_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=False,
origin=origin,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size)
final_params = result["final_concat_params"]
all_pcs = result["pcs_components"]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
angles = []
for pc in all_pcs:
angles.append(cal_angle(pc, final_params - start_params))
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"angles with final - start start n_comp:{all_pcs.shape[0]} dim space of mean pca plane, "
plot_2d(plot_dir, angles_plot_name, np.arange(all_pcs.shape[0]), angles,
"num of pcs", "angle with diff", False)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
pcs_components = np.loadtxt(
get_pcs_filename(intermediate_dir=intermediate_data_dir, n_comp=cma_args.num_comp_to_load), delimiter=',')
angle = cal_angle(V, pcs_components[0])
logger.log(f"@@@@@@@@@@@@ {angle}")
def main(origin="final_param"):
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
cma_args = {
"alg": 'ppo2',
"env": "DartHopper-v1",
"num_timesteps": 5000,
"normalize": True,
"n_steps": 2048,
"nminibatches": 32,
"run_num": 0
}
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import do_pca
result = do_pca(cma_args.n_components,
cma_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=True,
origin=origin,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
result = do_pca(cma_args.n_components,
cma_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=False,
origin="mean_param",
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size,
reuse=True)
logger.debug("after pca")
final_plane = result["first_n_pcs"]
count_file = get_full_param_traj_file_path(traj_params_dir_name,
"total_num_dumped")
total_num = pd.read_csv(count_file, header=None).values[0]
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
unduped_angles_along_the_way = []
duped_angles_along_the_way = []
diff_along = []
unweighted_pc1_vs_V_angles = []
duped_pc1_vs_V_angles = []
pc1_vs_V_diffs = []
unweighted_ipca = IncrementalPCA(
n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up
all_matrix_buffer = []
try:
i = -1
for chunk in all_param_iterator:
i += 1
if i >= 2:
break
chunk = chunk.values
unweighted_ipca.partial_fit(chunk)
unweighted_angle = cal_angle_between_nd_planes(
final_plane,
unweighted_ipca.components_[:cma_args.n_comp_to_use])
unweighted_pc1_vs_V_angle = postize_angle(
cal_angle_between_nd_planes(V, unweighted_ipca.components_[0]))
unweighted_pc1_vs_V_angles.append(unweighted_pc1_vs_V_angle)
#TODO ignore 90 or 180 for now
if unweighted_angle > 90:
unweighted_angle = 180 - unweighted_angle
unduped_angles_along_the_way.append(unweighted_angle)
np.testing.assert_almost_equal(
cal_angle_between_nd_planes(
unweighted_ipca.components_[:cma_args.n_comp_to_use][0],
final_plane[0]),
cal_angle(
unweighted_ipca.components_[:cma_args.n_comp_to_use][0],
final_plane[0]))
all_matrix_buffer.extend(chunk)
weights = gen_weights(all_matrix_buffer,
Funcs[cma_args.func_index_to_use])
logger.log(f"currently at {all_param_iterator._currow}")
# ipca = PCA(n_components=1) # for sparse PCA to speed up
# ipca.fit(duped_in_so_far)
wpca = WPCA(n_components=cma_args.n_comp_to_use
) # for sparse PCA to speed up
tic = time.time()
wpca.fit(all_matrix_buffer, weights=weights)
toc = time.time()
logger.debug(
f"WPCA of {len(all_matrix_buffer)} data took {toc - tic} secs "
)
duped_angle = cal_angle_between_nd_planes(
final_plane, wpca.components_[:cma_args.n_comp_to_use])
duped_pc1_vs_V_angle = postize_angle(
cal_angle_between_nd_planes(V, wpca.components_[0]))
duped_pc1_vs_V_angles.append(duped_pc1_vs_V_angle)
pc1_vs_V_diffs.append(duped_pc1_vs_V_angle -
unweighted_pc1_vs_V_angle)
#TODO ignore 90 or 180 for now
if duped_angle > 90:
duped_angle = 180 - duped_angle
duped_angles_along_the_way.append(duped_angle)
diff_along.append(unweighted_angle - duped_angle)
finally:
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"WPCA" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(duped_angles_along_the_way)),
duped_angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
angles_plot_name = f"Not WPCA exponential 2" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(unduped_angles_along_the_way)),
unduped_angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
angles_plot_name = f"Not WPCA - WPCA diff_along exponential 2," \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name, np.arange(len(diff_along)),
diff_along, "num of chunks", "angle with diff in degrees",
False)
angles_plot_name = f"PC1 VS VWPCA PC1 VS V" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(duped_pc1_vs_V_angles)), duped_pc1_vs_V_angles,
"num of chunks", "angle with diff in degrees", False)
angles_plot_name = f"PC1 VS VNot WPCA PC1 VS V" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(unweighted_pc1_vs_V_angles)),
unweighted_pc1_vs_V_angles, "num of chunks",
"angle with diff in degrees", False)
angles_plot_name = f"PC1 VS VNot WPCA - WPCA diff PC1 VS V" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name, np.arange(len(pc1_vs_V_diffs)),
pc1_vs_V_diffs, "num of chunks", "angle with diff in degrees",
False)
del all_matrix_buffer
import gc
gc.collect()
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
# origin = "final_param"
origin = cma_args.origin
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
cma_run_num, cma_intermediate_data_dir = generate_run_dir(get_cma_and_then_ppo_run_dir,
intermediate_dir=intermediate_data_dir,
n_comp=cma_args.n_comp_to_use,
cma_steps=cma_args.cma_num_timesteps
)
# cma_intermediate_data_dir = get_cma_and_then_ppo_run_dir(intermediate_dir = intermediate_data_dir,
# n_comp = cma_args.n_comp_to_use,
# cma_steps = cma_args.cma_num_timesteps, run_num=0)
best_theta_file_name = "best theta from cma"
# if not os.path.exists(f"{cma_intermediate_data_dir}/{best_theta_file_name}.csv") or \
# not os.path.exists(f"{cma_intermediate_data_dir}/opt_mean_path.csv"):
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
proj_or_not = (cma_args.n_comp_to_use == 2)
result = do_pca(cma_args.n_components, cma_args.n_comp_to_use, traj_params_dir_name, intermediate_data_dir,
proj=proj_or_not,
origin=origin, use_IPCA=cma_args.use_IPCA, chunk_size=cma_args.chunk_size, reuse=True)
logger.debug("after pca")
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import plot_contour_trajectory, gen_subspace_coords,do_eval_returns, \
do_proj_on_first_n
if origin=="final_param":
origin_param = result["final_concat_params"]
else:
origin_param = result["mean_param"]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
starting_coord = do_proj_on_first_n(start_params, result["first_n_pcs"], origin_param)
# starting_coord = np.random.rand(1, cma_args.n_comp_to_use)
# starting_coord = (1/2*np.max(xcoordinates_to_eval), 1/2*np.max(ycoordinates_to_eval)) # use mean
assert result["first_n_pcs"].shape[0] == cma_args.n_comp_to_use
mean_rets, min_rets, max_rets, opt_path, opt_path_mean, best_theta = do_cma(cma_args, result["first_n_pcs"],
origin_param, save_dir, starting_coord, cma_args.cma_var)
np.savetxt(f"{cma_intermediate_data_dir}/opt_mean_path.csv", opt_path_mean, delimiter=',')
np.savetxt(f"{cma_intermediate_data_dir}/{best_theta_file_name}.csv", best_theta, delimiter=',')
episode_returns, conti_ppo_full_param_traj_dir_path = do_ppo(args=cma_args, start_theta=best_theta, parent_this_run_dir=cma_intermediate_data_dir, full_space_save_dir=save_dir)
np.savetxt(f"{cma_intermediate_data_dir}/ppo part returns.csv", episode_returns, delimiter=',')
plot_dir = get_plot_dir(cma_args)
cma_and_then_ppo_plot_dir = get_cma_and_then_ppo_plot_dir(plot_dir, cma_args.n_comp_to_use,
cma_run_num, cma_num_steps=cma_args.cma_num_timesteps,
ppo_num_steps=cma_args.ppo_num_timesteps,
origin=origin)
if not os.path.exists(cma_and_then_ppo_plot_dir):
os.makedirs(cma_and_then_ppo_plot_dir)
if cma_args.n_comp_to_use <= 2:
proj_coords = result["proj_coords"]
assert proj_coords.shape[1] == 2
xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(cma_args, np.vstack((proj_coords, opt_path_mean)).T)
eval_returns = do_eval_returns(cma_args, intermediate_data_dir, result["first_n_pcs"], origin_param,
xcoordinates_to_eval, ycoordinates_to_eval, save_dir, pca_center=origin, reuse=False)
plot_contour_trajectory(cma_and_then_ppo_plot_dir, f"{origin}_origin_eval_return_contour_plot", xcoordinates_to_eval,
ycoordinates_to_eval, eval_returns, proj_coords[:, 0], proj_coords[:, 1],
result["explained_variance_ratio"][:2],
num_levels=25, show=False, sub_alg_path=opt_path_mean)
ret_plot_name = f"cma return on {cma_args.n_comp_to_use} dim space of real pca plane, " \
f"explained {np.sum(result['explained_variance_ratio'][:cma_args.n_comp_to_use])}"
plot_cma_returns(cma_and_then_ppo_plot_dir, ret_plot_name, mean_rets, min_rets, max_rets, show=False)
final_ppo_ep_name = f"final episodes returns after CMA"
plot_2d(cma_and_then_ppo_plot_dir, final_ppo_ep_name, np.arange(len(episode_returns)),
episode_returns, "num episode", "episode returns", False)
#
# if cma_args.n_comp_to_use == 2:
# proj_coords = result["proj_coords"]
# assert proj_coords.shape[1] == 2
#
# xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(cma_args, np.vstack((proj_coords, opt_path_mean)).T)
#
# eval_returns = do_eval_returns(cma_args, intermediate_data_dir, result["first_n_pcs"], origin_param,
# xcoordinates_to_eval, ycoordinates_to_eval, save_dir, pca_center=origin, reuse=False)
#
# plot_contour_trajectory(cma_and_then_ppo_plot_dir, f"{origin}_origin_eval_return_contour_plot", xcoordinates_to_eval,
# ycoordinates_to_eval, eval_returns, proj_coords[:, 0], proj_coords[:, 1],
# result["explained_variance_ratio"][:2],
# num_levels=25, show=False, sub_alg_path=opt_path_mean)
skip_rows = lambda x: x%2 == 0
conti_ppo_params = get_allinone_concat_df(conti_ppo_full_param_traj_dir_path, index=0, skip_rows=skip_rows).values
opt_mean_path_in_old_basis = [mean_projected_param.dot(result["first_n_pcs"]) + result["mean_param"] for mean_projected_param in opt_path_mean]
distance_to_final = [LA.norm(opt_mean - result["final_concat_params"], ord=2) for opt_mean in np.vstack((opt_mean_path_in_old_basis, conti_ppo_params))]
distance_to_final_plot_name = f"distance_to_final over generations "
plot_2d(cma_and_then_ppo_plot_dir, distance_to_final_plot_name, np.arange(len(distance_to_final)), distance_to_final, "num generation", "distance_to_final", False)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
angles_along_the_way = []
latest_thetas = deque(maxlen=cma_args.deque_len)
for chunk in all_param_iterator:
pca = PCA(
n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up
if chunk.shape[0] < cma_args.n_comp_to_use:
logger.log("skipping too few data")
continue
latest_thetas.extend(chunk.values)
logger.log(f"currently at {all_param_iterator._currow}")
pca.fit(latest_thetas)
pcs = pca.components_[:cma_args.n_comp_to_use]
angle = cal_angle(V, pcs[0])
angles_along_the_way.append(angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"lastest angles algone the way start start n_comp_used :{cma_args.n_comp_to_use} dim space of mean pca plane, " \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name, np.arange(len(angles_along_the_way)),
angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
pcs_components = np.loadtxt(get_pcs_filename(
intermediate_dir=intermediate_data_dir,
n_comp=cma_args.num_comp_to_load),
delimiter=',')
smallest_error_angle = postize_angle(cal_angle(V, pcs_components[0]))
logger.log(f"@@@@@@@@@@@@ {smallest_error_angle}")
curr_angles = []
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up
inside_final_cone = []
for chunk in all_param_iterator:
# for param in chunk.values:
logger.log(f"currently at {all_param_iterator._currow}")
ipca.partial_fit(chunk.values)
angle = postize_angle(cal_angle(V, ipca.components_[0]))
param = chunk.values[-1]
curr_angle = cal_angle(param - start_params, ipca.components_[0])
curr_angle = postize_angle(curr_angle)
curr_angle_final = cal_angle(param - start_params, pcs_components[0])
inside_final_cone.append(curr_angle_final - smallest_error_angle)
curr_angles.append(curr_angle - angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"$$$curr_angles$$$"
plot_2d(plot_dir, angles_plot_name, np.arange(len(curr_angles)),
curr_angles, "num of chunks", "angle with diff in degrees", False)
angles_plot_name = f"inside final cone?"
plot_2d(plot_dir, angles_plot_name, np.arange(len(inside_final_cone)),
inside_final_cone, "num of chunks", "angle with diff in degrees",
False)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
all_grads_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size,
index="grads")
angles_with_pc1_along_the_way = []
grad_vs_final_min_current_param = []
ipca = IncrementalPCA(1) # for sparse PCA to speed up
for chunk in all_param_iterator:
logger.log(f"currently at {all_param_iterator._currow}")
target_direction = final_params - chunk.values[-1]
ipca.partial_fit(chunk.values)
angle_with_pc1 = cal_angle(target_direction, ipca.components_[0])
angles_with_pc1_along_the_way.append(angle_with_pc1)
grads = all_grads_iterator.__next__().values
for i, grad in enumerate(grads):
grad_angle = cal_angle(grad, final_params - chunk.values[i])
grad_vs_final_min_current_param.append(grad_angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"final - current VS so far pc1" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size}"
plot_2d(plot_dir, angles_plot_name,
np.arange(len(angles_with_pc1_along_the_way)),
angles_with_pc1_along_the_way, "num of chunks",
"angle with diff in degrees", False)
grad_vs_current_plot_name = f"##final - current param VS current grad" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size}"
plot_2d(plot_dir, grad_vs_current_plot_name,
np.arange(len(grad_vs_final_min_current_param)),
grad_vs_final_min_current_param, "num of chunks",
"angle with diff in degrees", False)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
origin = "mean_param"
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
cma_run_num, cma_intermediate_data_dir = generate_run_dir(
get_cma_returns_dirname,
intermediate_dir=intermediate_data_dir,
n_comp=cma_args.n_comp_to_use)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_param = pd.read_csv(final_file, header=None).values[0]
final_pca = IncrementalPCA(n_components=2) # for sparse PCA to speed up
theta_file = get_full_param_traj_file_path(traj_params_dir_name, 0)
concat_df = pd.read_csv(theta_file, header=None, chunksize=10000)
tic = time.time()
for chunk in concat_df:
logger.log(f"currnet at : {concat_df._currow}")
if chunk.shape[0] < 2:
logger.log(f"last column too few: {chunk.shape[0]}")
continue
final_pca.partial_fit(chunk.values)
toc = time.time()
logger.log(
'\nElapsed time computing the chunked PCA {:.2f} s\n'.format(toc -
tic))
logger.log(final_pca.explained_variance_ratio_)
pcs_components = final_pca.components_
first_2_pcs = pcs_components[:2]
mean_param = final_pca.mean_
origin_param = mean_param
theta_file = get_full_param_traj_file_path(traj_params_dir_name, 0)
concat_df = pd.read_csv(theta_file, header=None, chunksize=10000)
proj_coords = do_proj_on_first_n_IPCA(concat_df, first_2_pcs, origin_param)
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import plot_contour_trajectory, gen_subspace_coords,do_eval_returns, \
get_allinone_concat_df, do_proj_on_first_n
from stable_baselines.ppo2.run_mujoco import eval_return
last_proj_coord = do_proj_on_first_n(final_param, first_2_pcs,
origin_param)
starting_coord = last_proj_coord
tic = time.time()
#TODO better starting locations, record how many samples,
logger.log(f"CMAES STARTING :{starting_coord}")
es = cma.CMAEvolutionStrategy(starting_coord, 5)
total_num_of_evals = 0
total_num_timesteps = 0
mean_rets = []
min_rets = []
max_rets = []
eval_returns = None
optimization_path = []
while total_num_timesteps < cma_args.cma_num_timesteps and not es.stop():
solutions = es.ask()
optimization_path.extend(solutions)
thetas = [
np.matmul(coord, first_2_pcs) + origin_param for coord in solutions
]
logger.log(
f"current time steps num: {total_num_timesteps} total time steps: {cma_args.cma_num_timesteps}"
)
eval_returns = Parallel(n_jobs=cma_args.cores_to_use) \
(delayed(eval_return)(cma_args, save_dir, theta, cma_args.eval_num_timesteps, i) for
(i, theta) in enumerate(thetas))
mean_rets.append(np.mean(eval_returns))
min_rets.append(np.min(eval_returns))
max_rets.append(np.max(eval_returns))
total_num_of_evals += len(eval_returns)
total_num_timesteps += cma_args.eval_num_timesteps * len(eval_returns)
logger.log(f"current eval returns: {str(eval_returns)}")
logger.log(f"total timesteps so far: {total_num_timesteps}")
negative_eval_returns = [-r for r in eval_returns]
es.tell(solutions, negative_eval_returns)
es.logger.add() # write data to disc to be plotted
es.disp()
toc = time.time()
logger.log(
f"####################################CMA took {toc-tic} seconds")
es_logger = es.logger
if not hasattr(es_logger, 'xmean'):
es_logger.load()
n_comp_used = first_2_pcs.shape[0]
optimization_path_mean = np.vstack(
(starting_coord, es_logger.xmean[:, 5:5 + n_comp_used]))
dump_rows_write_csv(cma_intermediate_data_dir, optimization_path_mean,
"opt_mean_path")
plot_dir = get_plot_dir(cma_args)
cma_plot_dir = get_cma_plot_dir(plot_dir,
cma_args.n_comp_to_use,
cma_run_num,
origin=origin)
if not os.path.exists(cma_plot_dir):
os.makedirs(cma_plot_dir)
ret_plot_name = f"cma return on {cma_args.n_comp_to_use} dim space of real pca plane, " \
f"explained {np.sum(final_pca.explained_variance_ratio_[:2])}"
plot_cma_returns(cma_plot_dir,
ret_plot_name,
mean_rets,
min_rets,
max_rets,
show=False)
assert proj_coords.shape[1] == 2
xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(
cma_args,
np.vstack((proj_coords, optimization_path_mean)).T)
from stable_baselines.ppo2.run_mujoco import eval_return
thetas_to_eval = [
origin_param + x * first_2_pcs[0] + y * first_2_pcs[1]
for y in ycoordinates_to_eval for x in xcoordinates_to_eval
]
tic = time.time()
eval_returns = Parallel(n_jobs=-1, max_nbytes='100M') \
(delayed(eval_return)(cma_args, save_dir, theta, cma_args.eval_num_timesteps, i) for (i, theta) in
enumerate(thetas_to_eval))
toc = time.time()
logger.log(
f"####################################1st version took {toc-tic} seconds"
)
plot_contour_trajectory(
cma_plot_dir,
f"cma redo___{origin}_origin_eval_return_contour_plot",
xcoordinates_to_eval,
ycoordinates_to_eval,
eval_returns,
proj_coords[:, 0],
proj_coords[:, 1],
final_pca.explained_variance_ratio_,
num_levels=25,
show=False,
sub_alg_path=optimization_path_mean.T)
opt_mean_path_in_old_basis = [
mean_projected_param.dot(first_2_pcs) + mean_param
for mean_projected_param in optimization_path_mean
]
distance_to_final = [
LA.norm(opt_mean - final_param, ord=2)
for opt_mean in opt_mean_path_in_old_basis
]
distance_to_final_plot_name = f"cma redo distance_to_final over generations "
plot_2d(cma_plot_dir, distance_to_final_plot_name,
np.arange(len(distance_to_final)), distance_to_final,
"num generation", "distance_to_final", False)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(args)
plot_dir_alg = get_plot_dir(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir,
params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
if not os.path.exists(plot_dir_alg):
os.makedirs(plot_dir_alg)
final_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
fig = plt.figure(figsize=(12, 12))
ax = fig.gca()
ax.axis('off')
preload_neuron_values_list = preload_neurons(args, save_dir, final_params,
args.eval_num_timesteps)
obz_norm, latent_norm, dist_norm = give_normalizers(
preload_neuron_values_list)
latent_cmap = plt.get_cmap("Oranges")
obz_cmap = plt.get_cmap("Blues")
dist_cmap = plt.get_cmap("Greys")
neuron_values_gen = neuron_values_generator(args, save_dir, final_params,
args.eval_num_timesteps)
left, right, bottom, top = 0.1, 0.9, 0.1, 0.9
result_artists = []
def init():
try:
first_neurons = neuron_values_gen.__next__()
except StopIteration:
return
layer_sizes = [layer.shape[1] for layer in first_neurons]
first_neurons = np.array(
[neuron_value.reshape(-1) for neuron_value in first_neurons])
v_spacing = (top - bottom) / float(max(layer_sizes))
h_spacing = (right - left) / float(len(layer_sizes) - 1)
# Nodes
for n, neuron_layer_value in enumerate(first_neurons):
neuron_layer_value = neuron_layer_value.reshape(-1)
layer_size = len(neuron_layer_value)
layer_top = v_spacing * (layer_size - 1) / 2. + (top + bottom) / 2.
for m, neuron_value in enumerate(neuron_layer_value):
if n == 0:
# obz
circle = plt.Circle(
(n * h_spacing + left, layer_top - m * v_spacing),
v_spacing / 4.,
color=obz_cmap(obz_norm(neuron_value)),
ec='k',
zorder=4)
elif n >= len(first_neurons) - 2:
# dist
circle = plt.Circle(
(n * h_spacing + left, layer_top - m * v_spacing),
v_spacing / 4.,
color=dist_cmap(dist_norm(neuron_value)),
ec='k',
zorder=4)
else:
#latent
circle = plt.Circle(
(n * h_spacing + left, layer_top - m * v_spacing),
v_spacing / 4.,
color=latent_cmap(latent_norm(neuron_value)),
ec='k',
zorder=4)
ax.add_artist(circle)
result_artists.append(circle)
return result_artists
def update_neuron(neuron_values):
num_of_obz_neurons = neuron_values[0].reshape(-1).shape[0]
num_of_dist_neurons = np.concatenate(neuron_values[-2:],
axis=1).shape[1]
neuron_values = np.concatenate(neuron_values,
axis=1).reshape(-1).ravel()
for i, neuron_value in enumerate(neuron_values):
if i < num_of_obz_neurons:
# obz
result_artists[i].set_color(obz_cmap(obz_norm(neuron_value)))
elif i >= neuron_values.shape[0] - num_of_dist_neurons:
# dist
result_artists[i].set_color(dist_cmap(dist_norm(neuron_value)))
else:
#latent
result_artists[i].set_color(
latent_cmap(latent_norm(neuron_value)))
# plt.draw()
return result_artists
rot_animation = FuncAnimation(fig,
update_neuron,
frames=neuron_values_gen,
init_func=init,
interval=3)
plt.show()
print(f"~~~~~~~~~~~~~~~~~~~~~~saving to {plot_dir_alg}/neuron_vis.pdf")
file_path = f"{plot_dir_alg}/neuron_firing.gif"
if os.path.isfile(file_path):
os.remove(file_path)
rot_animation.save(file_path, dpi=80, writer='imagemagick')
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
count_file = get_full_param_traj_file_path(traj_params_dir_name,
"total_num_dumped")
total_num = pd.read_csv(count_file, header=None).values[0]
V = final_params - start_params
all_thetas_downsampled = get_allinone_concat_df(
dir_name=traj_params_dir_name).values[::2]
unduped_angles_along_the_way = []
duped_angles_along_the_way = []
diff_along = []
num = 2 #TODO hardcode!
undup_ipca = PCA(n_components=1) # for sparse PCA to speed up
all_matrix_buffer = []
for chunk in all_param_iterator:
chunk = chunk.values
undup_ipca.partial_fit(chunk)
unduped_angle = cal_angle(V, undup_ipca.components_[0])
#TODO ignore 90 or 180 for now
if unduped_angle > 90:
unduped_angle = 180 - unduped_angle
unduped_angles_along_the_way.append(unduped_angle)
all_matrix_buffer.extend(chunk)
weights = gen_weights(all_param_iterator._currow, total_num)
duped_in_so_far = dup_so_far_buffer(all_matrix_buffer, last_percentage,
num)
logger.log(
f"currently at {all_param_iterator._currow}, last_pecentage: {last_percentage}"
)
# ipca = PCA(n_components=1) # for sparse PCA to speed up
# ipca.fit(duped_in_so_far)
ipca = WPCA(
n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up
for i in range(0, len(duped_in_so_far), cma_args.chunk_size):
logger.log(
f"partial fitting: i : {i} len(duped_in_so_far): {len(duped_in_so_far)}"
)
if i + cma_args.chunk_size > len(duped_in_so_far):
ipca.partial_fit(duped_in_so_far[i:])
else:
ipca.partial_fit(duped_in_so_far[i:i + cma_args.chunk_size])
duped_angle = cal_angle(V, ipca.components_[0])
#TODO ignore 90 or 180 for now
if duped_angle > 90:
duped_angle = 180 - duped_angle
duped_angles_along_the_way.append(duped_angle)
diff_along.append(unduped_angle - duped_angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"duped exponential 2, num dup: {num}" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(duped_angles_along_the_way)),
duped_angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
angles_plot_name = f"unduped exponential 2, num dup: {num}" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name,
np.arange(len(unduped_angles_along_the_way)),
unduped_angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
angles_plot_name = f"undup - dup diff_along exponential 2, num dup: {num}" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name, np.arange(len(diff_along)), diff_along,
"num of chunks", "angle with diff in degrees", False)
del all_matrix_buffer
import gc
gc.collect()
def main(n_comp_start=2, do_eval=True):
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir,
params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import do_pca, get_projected_data_in_old_basis, \
calculate_projection_errors, plot_2d
origin = "mean_param"
result = do_pca(cma_args.n_components,
cma_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=False,
origin=origin,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size)
final_params = result["final_concat_params"]
all_pcs = result["pcs_components"]
mean_param = result["mean_param"]
projected = []
projection_errors = []
for num_pcs in range(n_comp_start, all_pcs.shape[0] + 1):
projected.append(
get_projected_data_in_old_basis(mean_param, all_pcs, final_params,
num_pcs))
proj_to_n_pcs_error = calculate_projection_errors(
mean_param, all_pcs, final_params, num_pcs)
assert len(proj_to_n_pcs_error) == 1
projection_errors.extend(proj_to_n_pcs_error)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
if do_eval:
from stable_baselines.ppo2.run_mujoco import eval_return
thetas_to_eval = projected
tic = time.time()
eval_returns = Parallel(n_jobs=cma_args.cores_to_use, max_nbytes='100M') \
(delayed(eval_return)(cma_args, save_dir, theta, cma_args.eval_num_timesteps, i) for (i, theta) in
enumerate(thetas_to_eval))
toc = time.time()
logger.log(
f"####################################1st version took {toc-tic} seconds"
)
np.savetxt(get_projected_finals_eval_returns_filename(
intermediate_dir=intermediate_data_dir,
n_comp_start=n_comp_start,
np_comp_end=all_pcs.shape[0],
pca_center=origin),
eval_returns,
delimiter=',')
ret_plot_name = f"final project performances on start: {n_comp_start} end:{all_pcs.shape[0]} dim space of mean pca plane, "
plot_final_project_returns_returns(plot_dir,
ret_plot_name,
eval_returns,
n_comp_start,
all_pcs.shape[0],
show=False)
error_plot_name = f"final project errors on start: {n_comp_start} end:{all_pcs.shape[0]} dim space of mean pca plane, "
plot_2d(plot_dir, error_plot_name,
np.arange(n_comp_start, all_pcs.shape[0] + 1), projection_errors,
"num of pcs", "projection error", False)
def run_experiment_with_trained(augment_num_timesteps, linear_co_threshold, augment_seed, augment_run_num, network_size,
policy_env, policy_num_timesteps, policy_run_num, policy_seed, eval_seed, eval_run_num, learning_rate,
additional_note, result_dir, keys_to_include, metric_param, linear_top_vars_list=None,
linear_correlation_neuron_list=None, visualize=False, lagrangian_inds_to_include=None,
neurons_inds_to_include=None, use_lagrangian=True):
trained_model = None
if not use_lagrangian:
with tf.variable_scope("trained_model"):
common_arg_parser = get_common_parser()
trained_args, cma_unknown_args = common_arg_parser.parse_known_args()
trained_args.env = policy_env
trained_args.seed = policy_seed
trained_args.num_timesteps = policy_num_timesteps
trained_args.run_num = policy_run_num
trained_this_run_dir = get_dir_path_for_this_run(trained_args)
trained_traj_params_dir_name = get_full_params_dir(trained_this_run_dir)
trained_save_dir = get_save_dir(trained_this_run_dir)
trained_final_file = get_full_param_traj_file_path(trained_traj_params_dir_name, "pi_final")
trained_final_params = pd.read_csv(trained_final_file, header=None).values[0]
trained_model = PPO2.load(f"{trained_save_dir}/ppo2", seed=augment_seed)
trained_model.set_pi_from_flat(trained_final_params)
args = AttributeDict()
args.normalize = True
args.num_timesteps = augment_num_timesteps
args.run_num = augment_run_num
args.alg = "ppo2"
args.seed = augment_seed
logger.log(f"#######TRAIN: {args}")
# non_linear_global_dict
timestamp = get_time_stamp('%Y_%m_%d_%H_%M_%S')
experiment_label = f"learning_rate_{learning_rate}timestamp_{timestamp}_augment_num_timesteps{augment_num_timesteps}" \
f"_top_num_to_include{linear_co_threshold.start}_{linear_co_threshold.stop}" \
f"_augment_seed{augment_seed}_augment_run_num{augment_run_num}_network_size{network_size}" \
f"_policy_num_timesteps{policy_num_timesteps}_policy_run_num{policy_run_num}_policy_seed{policy_seed}" \
f"_eval_seed{eval_seed}_eval_run_num{eval_run_num}_additional_note_{additional_note}"
if policy_env == "DartWalker2d-v1":
entry_point = 'gym.envs.dart:DartWalker2dEnv_aug_input'
elif policy_env == "DartHopper-v1":
entry_point = 'gym.envs.dart:DartHopperEnv_aug_input'
elif policy_env == "DartHalfCheetah-v1":
entry_point = 'gym.envs.dart:DartHalfCheetahEnv_aug_input'
elif policy_env == "DartSnake7Link-v1":
entry_point = 'gym.envs.dart:DartSnake7LinkEnv_aug_input'
else:
raise NotImplemented()
this_run_dir = get_experiment_path_for_this_run(entry_point, args.num_timesteps, args.run_num,
args.seed, learning_rate=learning_rate, top_num_to_include=linear_co_threshold,
result_dir=result_dir, network_size=network_size)
full_param_traj_dir_path = get_full_params_dir(this_run_dir)
log_dir = get_log_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
create_dir_remove(this_run_dir)
create_dir_remove(full_param_traj_dir_path)
create_dir_remove(save_dir)
create_dir_remove(log_dir)
logger.configure(log_dir)
linear_top_vars_list_wanted_to_print = []
if (use_lagrangian and lagrangian_inds_to_include is None) or (not use_lagrangian and neurons_inds_to_include is None):
# note this is only linear
if linear_top_vars_list is None or linear_correlation_neuron_list is None:
linear_top_vars_list, linear_correlation_neuron_list = read_linear_top_var(policy_env, policy_num_timesteps, policy_run_num, policy_seed, eval_seed,
eval_run_num, additional_note, metric_param=metric_param)
lagrangian_inds_to_include, neurons_inds_to_include, linear_top_vars_list_wanted_to_print = \
get_wanted_lagrangians_and_neurons(keys_to_include, linear_top_vars_list, linear_correlation_neuron_list, linear_co_threshold)
with open(f"{log_dir}/lagrangian_inds_to_include.json", 'w') as fp:
json.dump(lagrangian_inds_to_include, fp)
with open(f"{log_dir}/linear_top_vars_list_wanted_to_print.json", 'w') as fp:
json.dump(linear_top_vars_list_wanted_to_print, fp)
with open(f"{log_dir}/neurons_inds_to_include.json", 'w') as fp:
json.dump(neurons_inds_to_include, fp)
args.env = f'{experiment_label}_{entry_point}-v1'
if not use_lagrangian:
register(
id=args.env,
entry_point=entry_point,
max_episode_steps=1000,
kwargs={"lagrangian_inds_to_include": None, "trained_model": trained_model,
"neurons_inds_to_include": neurons_inds_to_include}
)
else:
register(
id=args.env,
entry_point=entry_point,
max_episode_steps=1000,
kwargs={"lagrangian_inds_to_include": lagrangian_inds_to_include, "trained_model": None,
"neurons_inds_to_include": None}
)
def make_env():
env_out = gym.make(args.env)
env_out.env.visualize = visualize
env_out = bench.Monitor(env_out, logger.get_dir(), allow_early_resets=True)
return env_out
env = DummyVecEnv([make_env])
walker_env = env.envs[0].env.env
walker_env.disableViewer = not visualize
if args.normalize:
env = VecNormalize(env)
policy = MlpPolicy
set_global_seeds(args.seed)
walker_env.seed(args.seed)
num_dof = walker_env.robot_skeleton.ndofs
show_M_matrix(num_dof, lagrangian_inds_to_include, linear_co_threshold, log_dir)
# extra run info I added for my purposes
run_info = {"run_num": args.run_num,
"env_id": args.env,
"full_param_traj_dir_path": full_param_traj_dir_path}
layers = [network_size, network_size]
policy_kwargs = {"net_arch" : [dict(vf=layers, pi=layers)]}
model = PPO2(policy=policy, env=env, n_steps=4096, nminibatches=64, lam=0.95, gamma=0.99,
noptepochs=10,
ent_coef=0.0, learning_rate=learning_rate, cliprange=0.2, optimizer='adam', policy_kwargs=policy_kwargs,
seed=args.seed)
model.tell_run_info(run_info)
model.learn(total_timesteps=args.num_timesteps, seed=args.seed)
model.save(f"{save_dir}/ppo2")
if args.normalize:
env.save_running_average(save_dir)
return log_dir
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(args)
plot_dir_alg = get_plot_dir(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(plot_dir_alg):
os.makedirs(plot_dir_alg)
final_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
def make_env():
env_out = gym.make(args.env)
env_out = bench.Monitor(env_out,
logger.get_dir(),
allow_early_resets=True)
return env_out
env = DummyVecEnv([make_env])
# env_out = gym.make(args.env)
# env_out = bench.Monitor(env_out, logger.get_dir(), allow_early_resets=True)
if args.normalize:
env = VecNormalize(env)
# policy = MlpPolicy
model = PPO2.load(f"{save_dir}/ppo2") # this also loads V function
# model = PPO2(policy=policy, env=env, n_steps=args.n_steps, nminibatches=args.nminibatches, lam=0.95, gamma=0.99, noptepochs=10,
# ent_coef=0.0, learning_rate=3e-4, cliprange=0.2, optimizer=args.optimizer)
model.set_pi_from_flat(final_params)
if args.normalize:
env.load_running_average(save_dir)
sk = env.venv.envs[0].env.env.robot_skeleton
lagrangian_values = {}
obs = np.zeros((env.num_envs, ) + env.observation_space.shape)
obs[:] = env.reset()
# env = VecVideoRecorder(env, "./",
# record_video_trigger=lambda x: x == 0, video_length=3000,
# name_prefix="3000000agent-{}".format(args.env))
lagrangian_values["M"] = [sk.M.reshape((-1, 1))]
lagrangian_values["COM"] = [sk.C.reshape((-1, 1))]
lagrangian_values["Coriolis"] = [sk.c.reshape((-1, 1))]
lagrangian_values["q"] = [sk.q.reshape((-1, 1))]
lagrangian_values["dq"] = [sk.dq.reshape((-1, 1))]
contact_values = {}
neuron_values = model.give_neuron_values(obs)
layer_values_list = [[neuron_value.reshape((-1, 1))]
for neuron_value in neuron_values]
env.render()
ep_infos = []
steps_to_first_done = 0
first_done = False
for _ in range(3000):
actions = model.step(obs)[0]
# yield neuron_values
obs, rew, done, infos = env.step(actions)
if done and not first_done:
first_done = True
if not first_done:
steps_to_first_done += 1
neuron_values = model.give_neuron_values(obs)
for i, layer in enumerate(neuron_values):
layer_values_list[i].append(layer.reshape((-1, 1)))
fill_contacts_jac_dict(infos[0]["contacts"],
contact_dict=contact_values,
neuron_values=neuron_values)
lagrangian_values["M"].append(sk.M.reshape((-1, 1)))
lagrangian_values["q"].append(sk.q.reshape((-1, 1)))
lagrangian_values["dq"].append(sk.dq.reshape((-1, 1)))
lagrangian_values["COM"].append(sk.C.reshape((-1, 1)))
lagrangian_values["Coriolis"].append(sk.c.reshape((-1, 1)))
env.render()
# time.sleep(1)
for info in infos:
maybe_ep_info = info.get('episode')
if maybe_ep_info is not None:
ep_infos.append(maybe_ep_info)
# env.render()
done = done.any()
if done:
episode_rew = safe_mean([ep_info['r'] for ep_info in ep_infos])
print(f'episode_rew={episode_rew}')
obs = env.reset()
#Hstack into a big matrix
lagrangian_values["M"] = np.hstack(lagrangian_values["M"])
lagrangian_values["COM"] = np.hstack(lagrangian_values["COM"])
lagrangian_values["Coriolis"] = np.hstack(lagrangian_values["Coriolis"])
lagrangian_values["q"] = np.hstack(lagrangian_values["q"])
lagrangian_values["dq"] = np.hstack(lagrangian_values["dq"])
for contact_body_name, l in contact_values.items():
body_contact_dict = contact_values[contact_body_name]
for name, l in body_contact_dict.items():
body_contact_dict[name] = np.hstack(body_contact_dict[name])
layer_values_list = [
np.hstack(layer_list) for layer_list in layer_values_list
][1:-2] # drop variance
# plt.scatter(lagrangian_values["M"][15], layer_values_list[1][2])
# plt.scatter(lagrangian_values["M"][11], layer_values_list[0][63])
out_dir = f"/home/panda-linux/PycharmProjects/low_dim_update_dart/low_dim_update_stable/neuron_vis/plots_{args.env}_{args.num_timesteps}"
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
all_weights = model.get_all_weight_values()
for ind, weights in enumerate(all_weights):
fname = f"{out_dir}/weights_layer_{ind}.txt"
np.savetxt(fname, weights)
PLOT_CUTOFF = steps_to_first_done
plot_everything(lagrangian_values, layer_values_list, out_dir, PLOT_CUTOFF)
scatter_the_linear_significant_ones(lagrangian_values,
layer_values_list,
threshold=0.6,
out_dir=out_dir)
scatter_the_nonlinear_significant_but_not_linear_ones(
lagrangian_values,
layer_values_list,
linear_threshold=0.3,
nonlinear_threshold=0.6,
out_dir=out_dir)
#
# contact_dicts = {}
# for contact_body_name, l in contact_values.items():
# body_contact_dict = contact_values[contact_body_name]
#
#
# contact_dicts[contact_body_name] = {}
#
# build_dict = contact_dicts[contact_body_name]
#
# build_dict["body"] = {}
# build_dict["layer"] = {}
# for name, l in body_contact_dict.items():
# for i in range(len(l)):
#
# if name == contact_body_name:
# build_dict["body"][f"{contact_body_name}_{i}"] = l[i]
# else:
# build_dict["layer"][f"layer_{name}_neuron_{i}"] = l[i]
#
# body_contact_df = pd.DataFrame.from_dict(build_dict["body"], "index")
# layer_contact_df = pd.DataFrame.from_dict(build_dict["layer"], "index")
# body_contact_df.to_csv(f"{data_dir}/{contact_body_name}_contact.txt", sep='\t')
# layer_contact_df.to_csv(f"{data_dir}/{contact_body_name}_layers.txt", sep='\t')
# #TO CSV format
# data_dir = f"/home/panda-linux/PycharmProjects/low_dim_update_dart/mictools/examples/neuron_vis_data{args.env}_time_steps_{args.num_timesteps}"
# if os.path.exists(data_dir):
# shutil.rmtree(data_dir)
#
# os.makedirs(data_dir)
#
# for contact_body_name, d in contact_dicts.items():
#
# build_dict = d
#
# body_contact_df = pd.DataFrame.from_dict(build_dict["body"], "index")
# layer_contact_df = pd.DataFrame.from_dict(build_dict["layer"], "index")
#
# body_contact_df.to_csv(f"{data_dir}/{contact_body_name}_contact.txt", sep='\t')
# layer_contact_df.to_csv(f"{data_dir}/{contact_body_name}_layers.txt", sep='\t')
#
#
#
# neurons_dict = {}
# for layer_index in range(len(layer_values_list)):
# for neuron_index in range(len(layer_values_list[layer_index])):
# neurons_dict[f"layer_{layer_index}_neuron_{neuron_index}"] = layer_values_list[layer_index][neuron_index]
#
# for i in range(len(lagrangian_values["COM"])):
# neurons_dict[f"COM_index_{i}"] = lagrangian_values["COM"][i]
#
# neuron_df = pd.DataFrame.from_dict(neurons_dict, "index")
#
#
#
# lagrangian_dict = {}
# for k,v in lagrangian_values.items():
# for i in range(len(v)):
# lagrangian_dict[f"{k}_index_{i}"] = v[i]
#
# lagrangian_df = pd.DataFrame.from_dict(lagrangian_dict, "index")
#
#
# neuron_df.to_csv(f"{data_dir}/neurons.txt", sep='\t')
# lagrangian_df.to_csv(f"{data_dir}/lagrangian.txt", sep='\t')
# cor = {}
# best_cor = {}
# cor["M"] = get_correlations(lagrangian_values["M"], layer_values_list)
# best_cor["M"] = [np.max(np.abs(cor_m)) for cor_m in cor["M"]]
#
#
# cor["COM"] = get_correlations(lagrangian_values["COM"], layer_values_list)
# best_cor["COM"] = [np.max(np.abs(cor_m)) for cor_m in cor["COM"]]
#
# cor["Coriolis"] = get_correlations(lagrangian_values["Coriolis"], layer_values_list)
# best_cor["Coriolis"] = [np.max(np.abs(cor_m)) for cor_m in cor["Coriolis"]]
# best_cor["Coriolis_argmax"] = [np.argmax(np.abs(cor_m)) for cor_m in cor["Coriolis"]]
#
#
#
#
# ncor = {}
# nbest_cor = {}
# ncor["M"] = get_normalized_correlations(lagrangian_values["M"], layer_values_list)
# nbest_cor["M"] = [np.max(np.abs(cor_m)) for cor_m in ncor["M"]]
#
#
# ncor["COM"] = get_normalized_correlations(lagrangian_values["COM"], layer_values_list)
# nbest_cor["COM"] = [np.max(np.abs(cor_m)) for cor_m in ncor["COM"]]
#
# ncor["Coriolis"] = get_normalized_correlations(lagrangian_values["Coriolis"], layer_values_list)
# nbest_cor["Coriolis"] = [np.max(np.abs(cor_m)) for cor_m in ncor["Coriolis"]]
# nbest_cor["Coriolis_argmax"] = [np.argmax(np.abs(cor_m)) for cor_m in ncor["Coriolis"]]
#
#
#
#
#
# lin_reg = {"perm_1":{}, "perm_2":{}}
# best_lin_reg = {"perm_1":{}, "perm_2":{}}
# lin_reg["perm_1"]["M"], best_lin_reg["perm_1"]["M"] = get_results("M", lagrangian_values, layer_values_list, perm_num=1)
# lin_reg["perm_2"]["M"], best_lin_reg["perm_2"]["M"] = get_results("M", lagrangian_values, layer_values_list, perm_num=2)
# lin_reg["perm_1"]["COM"], best_lin_reg["perm_1"]["COM"] = get_results("COM", lagrangian_values, layer_values_list, perm_num=1)
# lin_reg["perm_2"]["COM"], best_lin_reg["perm_2"]["COM"] = get_results("COM", lagrangian_values, layer_values_list, perm_num=2)
#
#
# lin_reg_1["M"] = get_linear_regressions_1_perm(lagrangian_values["M"], layer_values_list)
# lin_reg_2["M"] = get_linear_regressions_2_perm(lagrangian_values["M"], layer_values_list)
# best_lin_reg_2["M"] = []
# for lin_l in lin_reg_2["M"]:
# if lin_l == []:
# best_lin_reg_2["M"].append([])
# else:
# best_lin_reg_2["M"].append(lin_l[np.argmin(lin_l[:,0])])
#
# best_lin_reg_1["M"] = []
# for lin_l in lin_reg_1["M"]:
# if lin_l == []:
# best_lin_reg_1["M"].append([])
# else:
# best_lin_reg_1["M"].append(lin_l[np.argmin(lin_l[:,0])])
# best_lin_reg_1["M"] = np.array(best_lin_reg_1["M"])
# best_lin_reg_2["M"] = np.array(best_lin_reg_2["M"])
#
#
# lin_reg_1["M"].dump("lin_reg_1_M.txt")
# lin_reg_2["M"].dump("lin_reg_2_M.txt")
# best_lin_reg_1["M"].dump("best_lin_reg_1_M.txt")
# best_lin_reg_2["M"].dump("best_lin_reg_2_M.txt")
#
# lin_reg_1["COM"] = get_linear_regressions_1_perm(lagrangian_values["COM"], layer_values_list)
# lin_reg_2["COM"] = get_linear_regressions_2_perm(lagrangian_values["COM"], layer_values_list)
# best_lin_reg_2["COM"] = []
# for lin_l in lin_reg_2["COM"]:
# if lin_l == []:
# best_lin_reg_2["COM"].append([])
# else:
# best_lin_reg_2["COM"].append(lin_l[np.argmin(lin_l[:, 0])])
#
# best_lin_reg_1["COM"] = []
# for lin_l in lin_reg_1["COM"]:
# if lin_l == []:
# best_lin_reg_1["COM"].append([])
# else:
# best_lin_reg_1["COM"].append(lin_l[np.argmin(lin_l[:, 0])])
#
#
# best_lin_reg_1["COM"] = np.array(best_lin_reg_1["M"])
# best_lin_reg_2["COM"] = np.array(best_lin_reg_2["M"])
# lin_reg_1["COM"].dump("lin_reg_1_COM.txt")
# lin_reg_2["COM"].dump("lin_reg_2_COM.txt")
# best_lin_reg_1["COM"].dump("best_lin_reg_1_COM.txt")
# best_lin_reg_2["COM"].dump("best_lin_reg_2_COM.txt")
pass
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
all_grads_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size,
index="grads")
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
angles_along_the_way = []
grad_vs_pull = []
pc1s = []
ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up
i = 1
for chunk in all_param_iterator:
logger.log(f"currently at {all_param_iterator._currow}")
ipca.partial_fit(chunk.values)
pc1 = ipca.components_[0]
if i % 2 == 0:
pc1 = -pc1
angle = cal_angle(V, pc1)
angles_along_the_way.append(angle)
pc1s.append(pc1)
current_grad = all_grads_iterator.__next__().values[-1]
current_param = chunk.values[-1]
delta = unit_vector(current_param - start_params)
pull_dir = V - delta
pull_dir_vs_grad = cal_angle(pull_dir, current_grad)
grad_vs_pull.append(pull_dir_vs_grad)
i += 1
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
first_n_pc1_vs_V_plot_dir = get_first_n_pc1_vs_V_plot_dir(
plot_dir, cma_args.pc1_chunk_size)
if not os.path.exists(first_n_pc1_vs_V_plot_dir):
os.makedirs(first_n_pc1_vs_V_plot_dir)
angles_plot_name = f"angles algone the way dim space of mean pca plane "
plot_2d(first_n_pc1_vs_V_plot_dir, angles_plot_name,
np.arange(len(angles_along_the_way)), angles_along_the_way,
"num of chunks", "angle with diff in degrees", False)
grad_vs_pull_plot_name = f"grad vs V - delta_theta"
plot_2d(first_n_pc1_vs_V_plot_dir, grad_vs_pull_plot_name,
np.arange(len(grad_vs_pull)), grad_vs_pull, "num of chunks",
"angle in degrees", False)
pcpca = PCA(n_components=min(len(pc1s), 100))
pcpca.fit(pc1s)
logger.log(pcpca.explained_variance_ratio_)
logger.log(cal_angle_plane(V, pcpca.components_[:2]))
np.savetxt(f"{first_n_pc1_vs_V_plot_dir}/pcs_pcs.txt",
pcpca.explained_variance_ratio_,
delimiter=',')
np.savetxt(
f"{first_n_pc1_vs_V_plot_dir}/pcs_V_vs_pcapca_first_2_comp_plane.txt",
np.array([cal_angle_plane(V, pcpca.components_[:2])]),
delimiter=',')
i = 0
for angle in angles_along_the_way:
if angle > 90:
i += 1
np.savetxt(f"{first_n_pc1_vs_V_plot_dir}/num of angles bigger than 90.txt",
np.array([i]),
delimiter=',')
def main(n_comp_start=2, do_eval=True):
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import \
calculate_projection_errors, plot_2d, get_allinone_concat_df, calculate_num_axis_to_explain
origin = "mean_param"
ratio_threshold = 0.99
consec_threshold = 5
error_threshold = 0.05
tic = time.time()
all_param_matrix = get_allinone_concat_df(dir_name=traj_params_dir_name).values
toc = time.time()
print('\nElapsed time getting the chunk concat diff took {:.2f} s\n'
.format(toc - tic))
n_comps = min(cma_args.n_comp_to_use, cma_args.chunk_size)
num_to_explains = []
deviates = []
for i in range(0, len(all_param_matrix), cma_args.chunk_size):
if i + cma_args.chunk_size >= len(all_param_matrix):
break
chunk = all_param_matrix[i:i + cma_args.chunk_size]
pca = PCA(n_components=n_comps) # for sparse PCA to speed up
pca.fit(chunk)
num, explained = calculate_num_axis_to_explain(pca, ratio_threshold)
num_to_explains.append(num)
pcs_components = pca.components_
num_to_deviate = 0
consec = 0
for j in range(i + cma_args.chunk_size, len(all_param_matrix)):
errors = calculate_projection_errors(pca.mean_, pcs_components, all_param_matrix[j], num)
if errors[0] >= error_threshold:
consec += 1
if consec >= consec_threshold:
break
num_to_deviate += 1
deviates.append(num_to_deviate)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
deviate_plot_name = f"num of steps to deviates from this plane chunk_size: {cma_args.chunk_size} ratio_threshold: {ratio_threshold} consec_threshold: {consec_threshold}error_threshold: {error_threshold}, "
plot_2d(plot_dir, deviate_plot_name, np.arange(len(deviates)), deviates, "num of chunks", "num of steps to deviates from this plane", False)
num_to_explain_plot_name = f"num to explain chunk_size: {cma_args.chunk_size} "
plot_2d(plot_dir, num_to_explain_plot_name, np.arange(len(num_to_explains)), num_to_explains, "num of chunks", "num_to_explains", False)
def visualize_policy_and_collect_COM(seed, run_num, policy_env,
policy_num_timesteps, policy_seed,
policy_run_num):
logger.log(sys.argv)
common_arg_parser = get_common_parser()
args, cma_unknown_args = common_arg_parser.parse_known_args()
args.env = policy_env
args.seed = policy_seed
args.num_timesteps = policy_num_timesteps
args.run_num = policy_run_num
this_run_dir = get_dir_path_for_this_run(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
final_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
def make_env():
env_out = gym.make(args.env)
env_out.env.disableViewer = False
env_out = bench.Monitor(env_out,
logger.get_dir(),
allow_early_resets=True)
env_out.seed(seed)
return env_out
env = DummyVecEnv([make_env])
if args.normalize:
env = VecNormalize(env)
model = PPO2.load(f"{save_dir}/ppo2", seed=seed)
model.set_pi_from_flat(final_params)
if args.normalize:
env.load_running_average(save_dir)
sk = env.venv.envs[0].env.env.robot_skeleton
lagrangian_values = {}
obs = np.zeros((env.num_envs, ) + env.observation_space.shape)
obs[:] = env.reset()
plot_dir = get_plot_dir(policy_env=args.env,
policy_num_timesteps=policy_num_timesteps,
policy_run_num=policy_run_num,
policy_seed=policy_seed,
eval_seed=seed,
eval_run_num=run_num,
additional_note="")
if os.path.exists(plot_dir):
shutil.rmtree(plot_dir)
os.makedirs(plot_dir)
env = VecVideoRecorder(env,
plot_dir,
record_video_trigger=lambda x: x == 0,
video_length=3000,
name_prefix="3000000agent-{}".format(args.env))
lagrangian_values["M"] = [sk.M.reshape((-1, 1))]
lagrangian_values["COM"] = [sk.C.reshape((-1, 1))]
lagrangian_values["Coriolis"] = [sk.c.reshape((-1, 1))]
lagrangian_values["q"] = [sk.q.reshape((-1, 1))]
lagrangian_values["dq"] = [sk.dq.reshape((-1, 1))]
contact_values = {}
neuron_values = model.give_neuron_values(obs)
raw_layer_values_list = [[neuron_value.reshape((-1, 1))]
for neuron_value in neuron_values]
env.render()
ep_infos = []
steps_to_first_done = 0
first_done = False
# epi_rew = 0
for _ in range(3000):
actions = model.step(obs)[0]
# yield neuron_values
obs, rew, done, infos = env.step(actions)
# epi_rew+= rew[0]
if done and not first_done:
first_done = True
if not first_done:
steps_to_first_done += 1
neuron_values = model.give_neuron_values(obs)
for i, layer in enumerate(neuron_values):
raw_layer_values_list[i].append(layer.reshape((-1, 1)))
# fill_contacts_jac_dict(infos[0]["contacts"], contact_dict=contact_values, neuron_values=neuron_values)
lagrangian_values["M"].append(sk.M.reshape((-1, 1)))
lagrangian_values["q"].append(sk.q.reshape((-1, 1)))
lagrangian_values["dq"].append(sk.dq.reshape((-1, 1)))
lagrangian_values["COM"].append(sk.C.reshape((-1, 1)))
lagrangian_values["Coriolis"].append(sk.c.reshape((-1, 1)))
# env.render()
# time.sleep(1)
for info in infos:
maybe_ep_info = info.get('episode')
if maybe_ep_info is not None:
ep_infos.append(maybe_ep_info)
env.render()
done = done.any()
if done:
episode_rew = safe_mean([ep_info['r'] for ep_info in ep_infos])
print(f'episode_rew={episode_rew}')
# print(f'episode_rew={epi_rew}')
# epi_rew = 0
obs = env.reset()
#Hstack into a big matrix
lagrangian_values["M"] = np.hstack(lagrangian_values["M"])
lagrangian_values["COM"] = np.hstack(lagrangian_values["COM"])
lagrangian_values["Coriolis"] = np.hstack(lagrangian_values["Coriolis"])
lagrangian_values["q"] = np.hstack(lagrangian_values["q"])
lagrangian_values["dq"] = np.hstack(lagrangian_values["dq"])
# for contact_body_name, l in contact_values.items():
# body_contact_dict = contact_values[contact_body_name]
# for name, l in body_contact_dict.items():
# body_contact_dict[name] = np.hstack(body_contact_dict[name])
input_values = np.hstack(raw_layer_values_list[0])
layers_values = [
np.hstack(layer_list) for layer_list in raw_layer_values_list
][1:-2] # drop variance and inputs
for i, com in enumerate(lagrangian_values["COM"]):
plt.figure()
plt.plot(np.arange(len(com)), com)
plt.xlabel("time")
plt.ylabel(f"COM{i}")
plt.savefig(f"{plot_dir}/COM{i}.jpg")
plt.close()
wr = csv.writer(fp)
wr.writerow(data)
def get_distances(concat):
return [LA.norm(concat[i + 1] - concat[i]) for i in range(len(concat) - 1)]
if __name__ == '__main__':
import time
import os
#TODO save your shits to files!!!
from stable_baselines.low_dim_analysis.common_parser import get_common_parser
parser = get_common_parser()
args = parser.parse_args()
threads_or_None = 'threads' if args.use_threads else None
logger.log(f"THREADS OR NOT: {threads_or_None}")
plot_dir = get_plot_dir(args)
plot_next_n_dir = f"{plot_dir}/next_n"
this_run_dir = get_dir_path_for_this_run(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
if os.path.exists(plot_next_n_dir):
import shutil
shutil.rmtree(plot_next_n_dir)
os.makedirs(plot_next_n_dir)
def eval_trained_policy_and_collect_data(eval_seed, eval_run_num, policy_env, policy_num_timesteps, policy_seed, policy_run_num, additional_note):
logger.log(sys.argv)
common_arg_parser = get_common_parser()
args, cma_unknown_args = common_arg_parser.parse_known_args()
args.env = policy_env
args.seed = policy_seed
args.num_timesteps = policy_num_timesteps
args.run_num = policy_run_num
this_run_dir = get_dir_path_for_this_run(args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
final_file = get_full_param_traj_file_path(traj_params_dir_name, "pi_final")
final_params = pd.read_csv(final_file, header=None).values[0]
def make_env():
env_out = gym.make(args.env)
env_out = bench.Monitor(env_out, logger.get_dir(), allow_early_resets=True)
env_out.seed(eval_seed)
return env_out
env = DummyVecEnv([make_env])
running_env = env.envs[0].env.env
set_global_seeds(eval_seed)
running_env.seed(eval_seed)
if args.normalize:
env = VecNormalize(env)
model = PPO2.load(f"{save_dir}/ppo2", seed=eval_seed)
model.set_pi_from_flat(final_params)
if args.normalize:
env.load_running_average(save_dir)
# is it necessary?
running_env = env.venv.envs[0].env.env
lagrangian_values = {}
obs = np.zeros((env.num_envs,) + env.observation_space.shape)
obs[:] = env.reset()
# env = VecVideoRecorder(env, "./",
# record_video_trigger=lambda x: x == 0, video_length=3000,
# name_prefix="3000000agent-{}".format(args.env))
#init lagrangian values
for lagrangian_key in lagrangian_keys:
flat_array = running_env.get_lagrangian_flat_array(lagrangian_key)
lagrangian_values[lagrangian_key] = [flat_array]
neuron_values = model.give_neuron_values(obs)
raw_layer_values_list = [[neuron_value.reshape((-1,1))] for neuron_value in neuron_values]
# env.render()
ep_infos = []
steps_to_first_done = 0
first_done = False
for _ in range(30000):
actions = model.step(obs)[0]
# yield neuron_values
obs, rew, done, infos = env.step(actions)
if done and not first_done:
first_done = True
if not first_done:
steps_to_first_done += 1
neuron_values = model.give_neuron_values(obs)
for i, layer in enumerate(neuron_values):
raw_layer_values_list[i].append(layer.reshape((-1,1)))
# fill_contacts_jac_dict(infos[0]["contacts"], contact_dict=contact_values, neuron_values=neuron_values)
# filling lagrangian values
for lagrangian_key in lagrangian_keys:
flat_array = running_env.get_lagrangian_flat_array(lagrangian_key)
lagrangian_values[lagrangian_key].append(flat_array)
# env.render()
# time.sleep(1)
for info in infos:
maybe_ep_info = info.get('episode')
if maybe_ep_info is not None:
ep_infos.append(maybe_ep_info)
# env.render()
done = done.any()
if done:
episode_rew = safe_mean([ep_info['r'] for ep_info in ep_infos])
print(f'episode_rew={episode_rew}')
obs = env.reset()
#Hstack into a big matrix
for lagrangian_key in lagrangian_keys:
lagrangian_values[lagrangian_key] = np.hstack(lagrangian_values[lagrangian_key])
# for contact_body_name, l in contact_values.items():
# body_contact_dict = contact_values[contact_body_name]
# for name, l in body_contact_dict.items():
# body_contact_dict[name] = np.hstack(body_contact_dict[name])
input_values = np.hstack(raw_layer_values_list[0])
layers_values = [np.hstack(layer_list) for layer_list in raw_layer_values_list][1:-2]# drop variance and inputs
data_dir = get_data_dir(policy_env=args.env, policy_num_timesteps=policy_num_timesteps, policy_run_num=policy_run_num
, policy_seed=policy_seed, eval_seed=eval_seed, eval_run_num=eval_run_num, additional_note=additional_note)
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
os.makedirs(data_dir)
lagrangian_values_fn = f"{data_dir}/lagrangian.pickle"
with open(lagrangian_values_fn, 'wb') as handle:
pickle.dump(lagrangian_values, handle, protocol=pickle.HIGHEST_PROTOCOL)
input_values_fn = f"{data_dir}/input_values.npy"
layers_values_fn = f"{data_dir}/layer_values.npy"
np.save(input_values_fn, input_values)
np.save(layers_values_fn, layers_values)
all_weights = model.get_all_weight_values()
for ind, weights in enumerate(all_weights):
fname = f"{data_dir}/weights_layer_{ind}.txt"
np.savetxt(fname, weights)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
# origin = "final_param"
origin = cma_args.origin
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
cma_run_num, cma_intermediate_data_dir = generate_run_dir(
get_cma_returns_dirname,
intermediate_dir=intermediate_data_dir,
n_comp=cma_args.n_comp_to_use)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
proj_or_not = (cma_args.n_comp_to_use == 2)
result = do_pca(cma_args.n_components,
cma_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=proj_or_not,
origin=origin,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size,
reuse=False)
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import plot_contour_trajectory, gen_subspace_coords,do_eval_returns\
, do_proj_on_first_n
if origin == "final_param":
origin_param = result["final_concat_params"]
else:
origin_param = result["mean_param"]
final_param = result["final_concat_params"]
last_proj_coord = do_proj_on_first_n(final_param, result["first_n_pcs"],
origin_param)
starting_coord = last_proj_coord
logger.log(f"CMA STASRTING CORRD: {starting_coord}")
# starting_coord = (1/2*np.max(xcoordinates_to_eval), 1/2*np.max(ycoordinates_to_eval)) # use mean
assert result["first_n_pcs"].shape[0] == cma_args.n_comp_to_use
mean_rets, min_rets, max_rets, opt_path, opt_path_mean = do_cma(
cma_args, result["first_n_pcs"], origin_param, save_dir,
starting_coord, cma_args.cma_var)
dump_rows_write_csv(cma_intermediate_data_dir, opt_path_mean,
"opt_mean_path")
plot_dir = get_plot_dir(cma_args)
cma_plot_dir = get_cma_plot_dir(plot_dir, cma_args.n_comp_to_use,
cma_run_num, origin)
if not os.path.exists(cma_plot_dir):
os.makedirs(cma_plot_dir)
ret_plot_name = f"cma return on {cma_args.n_comp_to_use} dim space of real pca plane, " \
f"explained {np.sum(result['explained_variance_ratio'][:cma_args.n_comp_to_use])}"
plot_cma_returns(cma_plot_dir,
ret_plot_name,
mean_rets,
min_rets,
max_rets,
show=False)
if cma_args.n_comp_to_use == 2:
proj_coords = result["proj_coords"]
assert proj_coords.shape[1] == 2
xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(
cma_args,
np.vstack((proj_coords, opt_path_mean)).T)
eval_returns = do_eval_returns(cma_args,
intermediate_data_dir,
result["first_n_pcs"],
origin_param,
xcoordinates_to_eval,
ycoordinates_to_eval,
save_dir,
pca_center=origin,
reuse=False)
plot_contour_trajectory(cma_plot_dir,
f"{origin}_origin_eval_return_contour_plot",
xcoordinates_to_eval,
ycoordinates_to_eval,
eval_returns,
proj_coords[:, 0],
proj_coords[:, 1],
result["explained_variance_ratio"][:2],
num_levels=25,
show=False,
sub_alg_path=opt_path_mean)
opt_mean_path_in_old_basis = [
mean_projected_param.dot(result["first_n_pcs"]) + result["mean_param"]
for mean_projected_param in opt_path_mean
]
distance_to_final = [
LA.norm(opt_mean - final_param, ord=2)
for opt_mean in opt_mean_path_in_old_basis
]
distance_to_final_plot_name = f"distance_to_final over generations "
plot_2d(cma_plot_dir, distance_to_final_plot_name,
np.arange(len(distance_to_final)), distance_to_final,
"num generation", "distance_to_final", False)
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir( this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
all_param_iterator = get_allinone_concat_df(dir_name=traj_params_dir_name, use_IPCA=True, chunk_size=cma_args.pc1_chunk_size)
angles_along_the_way = []
grad_vs_Vs = []
ipca = IncrementalPCA(n_components=cma_args.n_comp_to_use) # for sparse PCA to speed up
last_chunk_last = start_params
for chunk in all_param_iterator:
chunk = chunk.values
if chunk.shape[0] < cma_args.n_comp_to_use:
logger.log("skipping too few data")
continue
for i in range(chunk.shape[0]):
if i == 0:
grad = chunk[i] - last_chunk_last
else:
grad = chunk[i] - chunk[i-1]
grad_angle = cal_angle(grad, V)
grad_vs_Vs.append(grad_angle)
last_chunk_last = chunk[-1]
logger.log(f"currently at {all_param_iterator._currow}")
ipca.partial_fit(chunk)
angle = cal_angle(V, ipca.components_[0])
if angle > 90:
angle = 180 - angle
angles_along_the_way.extend([angle]*chunk.shape[0])
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
#
# #TODO ignore negative for now
# angles_along_the_way = np.array(angles_along_the_way)
# if angles_along_the_way[-1] > 90:
# angles_along_the_way = 180 - angles_along_the_way
assert len(angles_along_the_way) == len(grad_vs_Vs)
angles_plot_name = f"in so far update direction and pc1 vs final - start " \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d_2(plot_dir, angles_plot_name, np.arange(len(grad_vs_Vs)),grad_vs_v=grad_vs_Vs, pc1_vs_V=angles_along_the_way,
xlabel="num of chunks", ylabel="angle with diff in degrees", show=False)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
origin_name = "final_param"
this_run_dir = get_dir_path_for_this_run(cma_args)
plot_dir_alg = get_plot_dir(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir,
params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
if not os.path.exists(plot_dir_alg):
os.makedirs(plot_dir_alg)
start_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_start")
start_params = pd.read_csv(start_file, header=None).values[0]
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
pca_indexes = cma_args.other_pca_index
pca_indexes = [int(pca_index) for pca_index in pca_indexes.split(":")]
n_comp_to_project_on = pca_indexes
result = do_pca(n_components=cma_args.n_components,
traj_params_dir_name=traj_params_dir_name,
intermediate_data_dir=intermediate_data_dir,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size,
reuse=True)
logger.debug("after pca")
if origin_name == "final_param":
origin_param = result["final_params"]
elif origin_name == "start_param":
origin_param = start_params
else:
origin_param = result["mean_param"]
proj_coords = project(result["pcs_components"],
pcs_slice=n_comp_to_project_on,
origin_name=origin_name,
origin_param=origin_param,
IPCA_chunk_size=cma_args.chunk_size,
traj_params_dir_name=traj_params_dir_name,
intermediate_data_dir=intermediate_data_dir,
n_components=cma_args.n_components,
reuse=True)
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
other_pcs_plot_dir = get_other_pcs_plane_plot_dir(plot_dir_alg,
pca_indexes)
if not os.path.exists(other_pcs_plot_dir):
os.makedirs(other_pcs_plot_dir)
plot_3d_trajectory_path_only(
other_pcs_plot_dir,
f"{pca_indexes}_final_origin_3d_path_plot",
proj_coords,
explained_ratio=result["explained_variance_ratio"][pca_indexes])
def main():
# requires n_comp_to_use, pc1_chunk_size
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
logger.log("grab final params")
final_file = get_full_param_traj_file_path(traj_params_dir_name, "final")
final_params = pd.read_csv(final_file, header=None).values[0]
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name, "start")
start_params = pd.read_csv(start_file, header=None).values[0]
V = final_params - start_params
all_param_iterator = get_allinone_concat_df(
dir_name=traj_params_dir_name,
use_IPCA=True,
chunk_size=cma_args.pc1_chunk_size)
angles_along_the_way = []
ipca = IncrementalPCA(n_components=1) # for sparse PCA to speed up
for chunk in all_param_iterator:
if all_param_iterator._currow <= cma_args.pc1_chunk_size * cma_args.skipped_chunks:
logger.log(
f"skipping: currow: {all_param_iterator._currow} skip threshold {cma_args.pc1_chunk_size * cma_args.skipped_chunks}"
)
continue
logger.log(f"currently at {all_param_iterator._currow}")
ipca.partial_fit(chunk.values)
angle = cal_angle(V, ipca.components_[0])
#TODO ignore 90 or 180 for now
if angle > 90:
angle = 180 - angle
angles_along_the_way.append(angle)
plot_dir = get_plot_dir(cma_args)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
angles_plot_name = f"skipped angles algone the way skipped {cma_args.skipped_chunks}" \
f"cma_args.pc1_chunk_size: {cma_args.pc1_chunk_size} "
plot_2d(plot_dir, angles_plot_name, np.arange(len(angles_along_the_way)),
angles_along_the_way, "num of chunks",
"angle with diff in degrees", False)
def main():
import sys
logger.log(sys.argv)
common_arg_parser = get_common_parser()
cma_args, cma_unknown_args = common_arg_parser.parse_known_args()
# origin = "final_param"
origin_name = cma_args.origin
this_run_dir = get_dir_path_for_this_run(cma_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir,
params_scope="pi")
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
pca_indexes = cma_args.other_pca_index
pca_indexes = [int(pca_index) for pca_index in pca_indexes.split(":")]
cma_run_num, cma_intermediate_data_dir = generate_run_dir(
get_cma_and_then_ppo_run_dir,
intermediate_dir=intermediate_data_dir,
pca_indexes=pca_indexes,
cma_steps=cma_args.cma_num_timesteps)
best_theta_file_name = "best theta from cma"
start_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_start")
start_params = pd.read_csv(start_file, header=None).values[0]
# if not os.path.exists(f"{cma_intermediate_data_dir}/{best_theta_file_name}.csv") or \
# not os.path.exists(f"{cma_intermediate_data_dir}/opt_mean_path.csv"):
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
result = do_pca(n_components=cma_args.n_components,
traj_params_dir_name=traj_params_dir_name,
intermediate_data_dir=intermediate_data_dir,
use_IPCA=cma_args.use_IPCA,
chunk_size=cma_args.chunk_size,
reuse=True)
logger.debug("after pca")
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
from stable_baselines.low_dim_analysis.common import plot_contour_trajectory, gen_subspace_coords, do_eval_returns, \
do_proj_on_first_n
logger.log("grab start params")
start_file = get_full_param_traj_file_path(traj_params_dir_name,
"pi_start")
start_params = pd.read_csv(start_file, header=None).values[0]
if origin_name == "final_param":
origin_param = result["final_concat_params"]
elif origin_name == "start_param":
origin_param = start_params
else:
origin_param = result["mean_param"]
pcs = result["pcs_components"]
pcs_to_use = pcs[pca_indexes]
starting_coord = do_proj_on_first_n(start_params, pcs_to_use, origin_param)
plot_dir = get_plot_dir(cma_args)
cma_and_then_ppo_plot_dir = get_cma_and_then_ppo_plot_dir(
plot_dir,
pca_indexes,
cma_run_num,
cma_num_steps=cma_args.cma_num_timesteps,
ppo_num_steps=cma_args.ppo_num_timesteps,
origin=origin_name)
# starting_coord = (1/2*np.max(xcoordinates_to_eval), 1/2*np.max(ycoordinates_to_eval)) # use mean
mean_rets, min_rets, max_rets, opt_path, opt_path_mean, best_pi_theta = do_cma(
cma_args, pcs_to_use, origin_param, save_dir, starting_coord,
cma_args.cma_var)
# np.savetxt(f"{cma_intermediate_data_dir}/opt_mean_path.csv", opt_path_mean, delimiter=',')
np.savetxt(f"{cma_intermediate_data_dir}/{best_theta_file_name}.csv",
best_pi_theta,
delimiter=',')
ret_plot_name = f"cma return on {pca_indexes} dim space of real pca plane, " \
f"explained {np.sum(result['explained_variance_ratio'][pca_indexes])}"
plot_cma_returns(cma_and_then_ppo_plot_dir,
ret_plot_name,
mean_rets,
min_rets,
max_rets,
show=False)
vf_final_file = get_full_param_traj_file_path(traj_params_dir_name,
"vf_final")
vf_final_params = pd.read_csv(vf_final_file, header=None).values[0]
episode_returns, conti_ppo_full_param_traj_dir_path = do_ppo(
args=cma_args,
start_pi_theta=best_pi_theta,
parent_this_run_dir=cma_intermediate_data_dir,
full_space_save_dir=save_dir,
vf_final_params=vf_final_params)
# dump_row_write_csv(cma_intermediate_data_dir, episode_returns, "ppo part returns")
np.savetxt(f"{cma_intermediate_data_dir}/ppo part returns.csv",
episode_returns,
delimiter=",")
if not os.path.exists(cma_and_then_ppo_plot_dir):
os.makedirs(cma_and_then_ppo_plot_dir)
conti_ppo_params = get_allinone_concat_df(
conti_ppo_full_param_traj_dir_path).values
if len(pca_indexes) <= 2:
pcs_to_plot_contour = pca_indexes
if len(pcs_to_plot_contour) == 1:
if pcs_to_plot_contour[0] + 1 < cma_args.n_components:
pcs_to_plot_contour.append(pcs_to_plot_contour[0] + 1)
else:
pcs_to_plot_contour.append(pcs_to_plot_contour[0] - 1)
proj_coords = project(result["pcs_components"],
pcs_slice=pcs_to_plot_contour,
origin_name=origin_name,
origin_param=origin_param,
IPCA_chunk_size=cma_args.chunk_size,
traj_params_dir_name=traj_params_dir_name,
intermediate_data_dir=intermediate_data_dir,
n_components=cma_args.n_components,
reuse=False)
assert proj_coords.shape[1] == 2
if len(pca_indexes) == 1:
opt_path_mean_2d = np.hstack(
(opt_path_mean, np.zeros((1, len(opt_path_mean))).T))
else:
opt_path_mean_2d = opt_path_mean
xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(
cma_args,
np.vstack((proj_coords, opt_path_mean_2d)).T)
projected_after_ppo_params = do_proj_on_first_n(
conti_ppo_params, pcs[pcs_to_plot_contour], origin_param)
full_path = np.vstack((opt_path_mean_2d, projected_after_ppo_params))
eval_returns = do_eval_returns(cma_args,
intermediate_data_dir,
pcs[pcs_to_plot_contour],
origin_param,
xcoordinates_to_eval,
ycoordinates_to_eval,
save_dir,
pca_center=origin_name,
reuse=False)
plot_contour_trajectory(
cma_and_then_ppo_plot_dir,
f"{origin_name}_origin_eval_return_contour_plot",
xcoordinates_to_eval,
ycoordinates_to_eval,
eval_returns,
proj_coords[:, 0],
proj_coords[:, 1],
result["explained_variance_ratio"][pcs_to_plot_contour],
num_levels=25,
show=False,
sub_alg_path=full_path)
final_ppo_ep_name = f"final episodes returns CMA PPO"
plot_2d(cma_and_then_ppo_plot_dir, final_ppo_ep_name,
np.arange(len(episode_returns)), episode_returns, "num episode",
"episode returns", False)
opt_mean_path_in_old_basis = [
mean_projected_param.dot(pcs_to_use) + result["mean_param"]
for mean_projected_param in opt_path_mean
]
distance_to_final = [
LA.norm(opt_mean - result["final_params"], ord=2)
for opt_mean in opt_mean_path_in_old_basis
]
distance_to_final_plot_name = f"distance_to_final over generations of CMA "
plot_2d(cma_and_then_ppo_plot_dir, distance_to_final_plot_name,
np.arange(len(distance_to_final)), distance_to_final,
"num generation", "distance_to_final", False)
distance_to_final_ppo = [
LA.norm(opt_mean - result["final_params"], ord=2)
for opt_mean in conti_ppo_params
]
distance_to_final_plot_name = f"distance_to_final over generations of PPO"
plot_2d(cma_and_then_ppo_plot_dir, distance_to_final_plot_name,
np.arange(len(distance_to_final_ppo)), distance_to_final_ppo,
"num generation", "distance_to_final", False)
def main():
import sys
logger.log(sys.argv)
ppos_arg_parser = get_common_parser()
ppos_args, ppos_unknown_args = ppos_arg_parser.parse_known_args()
full_space_alg = ppos_args.alg
# origin = "final_param"
origin = ppos_args.origin
this_run_dir = get_dir_path_for_this_run(ppos_args)
traj_params_dir_name = get_full_params_dir(this_run_dir)
intermediate_data_dir = get_intermediate_data_dir(this_run_dir)
save_dir = get_save_dir(this_run_dir)
if not os.path.exists(intermediate_data_dir):
os.makedirs(intermediate_data_dir)
ppos_run_num, ppos_intermediate_data_dir = generate_run_dir(
get_ppos_returns_dirname,
intermediate_dir=intermediate_data_dir,
n_comp=ppos_args.n_comp_to_use)
'''
==========================================================================================
get the pc vectors
==========================================================================================
'''
proj_or_not = (ppos_args.n_comp_to_use == 2)
result = do_pca(ppos_args.n_components,
ppos_args.n_comp_to_use,
traj_params_dir_name,
intermediate_data_dir,
proj=proj_or_not,
origin=origin,
use_IPCA=ppos_args.use_IPCA,
chunk_size=ppos_args.chunk_size)
'''
==========================================================================================
eval all xy coords
==========================================================================================
'''
if origin == "final_param":
origin_param = result["final_concat_params"]
else:
origin_param = result["mean_param"]
final_param = result["final_concat_params"]
last_proj_coord = do_proj_on_first_n(final_param, result["first_n_pcs"],
origin_param)
if origin == "final_param":
back_final_param = low_dim_to_old_basis(last_proj_coord,
result["first_n_pcs"],
origin_param)
assert np.testing.assert_almost_equal(back_final_param, final_param)
starting_coord = last_proj_coord
logger.log(f"PPOS STASRTING CORRD: {starting_coord}")
# starting_coord = (1/2*np.max(xcoordinates_to_eval), 1/2*np.max(ycoordinates_to_eval)) # use mean
assert result["first_n_pcs"].shape[0] == ppos_args.n_comp_to_use
eprews, moving_ave_rewards, optimization_path = do_ppos(
ppos_args, result, intermediate_data_dir, origin_param)
ppos_args.alg = full_space_alg
plot_dir = get_plot_dir(ppos_args)
ppos_plot_dir = get_ppos_plot_dir(plot_dir, ppos_args.n_comp_to_use,
ppos_run_num)
if not os.path.exists(ppos_plot_dir):
os.makedirs(ppos_plot_dir)
ret_plot_name = f"cma return on {ppos_args.n_comp_to_use} dim space of real pca plane, " \
f"explained {np.sum(result['explained_variance_ratio'][:ppos_args.n_comp_to_use])}"
plot_ppos_returns(ppos_plot_dir,
ret_plot_name,
moving_ave_rewards,
show=False)
if ppos_args.n_comp_to_use == 2:
proj_coords = result["proj_coords"]
assert proj_coords.shape[1] == 2
xcoordinates_to_eval, ycoordinates_to_eval = gen_subspace_coords(
ppos_args,
np.vstack((proj_coords, optimization_path)).T)
eval_returns = do_eval_returns(ppos_args,
intermediate_data_dir,
result["first_n_pcs"],
origin_param,
xcoordinates_to_eval,
ycoordinates_to_eval,
save_dir,
pca_center=origin)
plot_contour_trajectory(ppos_plot_dir,
"end_point_origin_eval_return_contour_plot",
xcoordinates_to_eval,
ycoordinates_to_eval,
eval_returns,
proj_coords[:, 0],
proj_coords[:, 1],
result["explained_variance_ratio"][:2],
num_levels=25,
show=False,
sub_alg_path=optimization_path)
opt_mean_path_in_old_basis = [
low_dim_to_old_basis(projected_opt_params, result["first_n_pcs"],
origin_param)
for projected_opt_params in optimization_path
]
distance_to_final = [
LA.norm(opt_mean - final_param, ord=2)
for opt_mean in opt_mean_path_in_old_basis
]
distance_to_final_plot_name = f"distance_to_final over generations "
plot_2d(ppos_plot_dir, distance_to_final_plot_name,
np.arange(len(distance_to_final)), distance_to_final,
"num generation", "distance_to_final", False)
