def oracles():
df_oracle = pd.DataFrame()
seeds = list(range(2))
body_types = [300, 400, 500, 600]
num_tested_bodies = 16
if False:
for body_type in body_types:
for tested in range(num_tested_bodies):
body = body_type + tested
for seed in seeds:
path = f"output_data/tensorboard_oracles/model-{body}-sd{seed}/PPO_1"
print(f"Loading {path}")
df = tflog2pandas(path)
df_body = df[df["metric"] ==
f"eval/{body}_mean_reward"].copy()
if df_body.shape[0] != 62:
print(df_body.shape)
continue
df_body["body"] = template(body)
df_body["seed"] = seed
df_oracle = df_oracle.append(df_body)
df_oracle["method"] = "oracle"
df_oracle.to_pickle("output_data/tmp/df_oracle")
else:
df_oracle = pd.read_pickle("output_data/tmp/df_oracle")
print(df_oracle)
return df_oracle
def same_topology_get_oracles(tensorboard_path, read_cache=False):
method = "oracles"
df_results = pd.DataFrame()
seeds = list(range(2))
body_types = [
300,400,500,600
]
num_tested_bodies = 16
try:
if not read_cache:
raise Exception("not_read_cache")
df_results = pd.read_pickle(f"output_data/tmp/same_topology_{method}")
except Exception as e:
if str(e)!="not_read_cache" and not isinstance(e, FileNotFoundError):
raise e
for body_type in body_types:
for tested in range(num_tested_bodies):
body = body_type + tested
for seed in seeds:
path = f"{tensorboard_path}/model-{body}-sd{seed}/PPO_1"
print(f"Loading {path}")
df = tflog2pandas(path)
df_body = df[df["metric"]==f"eval/{body}_mean_reward"].copy()
if df_body.shape[0]!=312:
print(df_body.shape)
raise Exception("Data is not complete.")
df_body["body"] = template(body)
df_body["seed"] = seed
df_results = df_results.append(df_body)
df_results["method"] = "oracle"
df_results["num_bodies"] = 1
print(f"Saving file output_data/tmp/same_topology_{method}...")
df_results.to_pickle(f"output_data/tmp/same_topology_{method}")
return df_results
def get_wrapper_class():
args = common.args
body_set = set()
if len(args.train_bodies) > 0:
for body in args.train_bodies:
body_set.add(gym_interface.template(body).capitalize())
else:
for body in args.test_bodies:
body_set.add(gym_interface.template(body).capitalize())
assert len(body_set) == 2, "Need two types of bodies to apply this wrapper"
classname = ''.join(sorted(body_set))
print(classname)
classname = f"{classname}Case{common.args.wrapper_case}" # e.g. Walker2DWalkerArmsCase1
return getattr(sys.modules[__name__], classname)
def __init__(self, env):
super().__init__(env)
self.debug = 1
self.colored = False
self.enable_mutant_wrapper = True
if common.args.disable_reordering: # This is for test time.
self.enable_mutant_wrapper = False
assert gym_interface.template(env.robot.robot_id)=="walkerarms", "MutantWrapper only support walkerarms for now."
self.default_joint_order = {"arm_joint":0, "arm_left_joint":1, "thigh_joint":2, "leg_joint":3, "foot_joint":4, "thigh_left_joint":5, "leg_left_joint":6, "foot_left_joint":7}
self.default_foot_order = {"foot":0, "foot_left":1}
self.action_realign_idx = []
self.obs_realign_idx = [0,1,2,3,4,5,6,7]
env.reset()
print(f"{env.robot.robot_id} > init > joints:")
for i,j in enumerate(self.robot.ordered_joints):
print(f"\t{j.joint_name}")
self.obs_realign_idx.append(8+self.default_joint_order[j.joint_name]*2)
self.obs_realign_idx.append(8+self.default_joint_order[j.joint_name]*2+1)
self.action_realign_idx.append(self.default_joint_order[j.joint_name])
print(f"{env.robot.robot_id} > init > feet:")
for f in self.robot.foot_list:
print(f"\t{f}")
self.obs_realign_idx.append(24+self.default_foot_order[f])
# obs ------------ F() ------> abs_obs
# abs_action --- F^{-1}() ---> action
self.obs_realign_idx = np.argsort(self.obs_realign_idx).tolist()
self.realign_idx = self.action_realign_idx
if not self.enable_mutant_wrapper:
# reset everything
self.realign_idx = sorted(self.realign_idx)
self.action_realign_idx = sorted(self.action_realign_idx)
self.obs_realign_idx = sorted(self.obs_realign_idx)
def read_df(body):
dfs = []
for seed in seeds:
folder = f"{tb_folder}/model-{body}-sd{seed}/PPO_1"
print(f"Loading {folder} ...")
df = tflog2pandas(folder)
if df.shape[0] < 2697:
print(f"Data not complete! Skip!")
continue
df = df[df["metric"] == f"eval/{body}_mean_reward"]
max_value = df["value"].max()
final_value = df.iloc[-1, df.columns.get_loc("value")]
df = pd.DataFrame(
{
"body": template(body),
"body_id": body,
"max_value": max_value,
"final_value": final_value,
"seed": seed,
},
index=[body])
dfs.append(df)
if len(dfs) > 0:
return pd.concat(dfs)
return None
def same_topology_get_results(tensorboard_path, body_arr, seed, method, read_cache=False):
df_results = pd.DataFrame()
str_body_arr = '-'.join(str(x) for x in body_arr)
stacked_training_bodies = np.array(body_arr)
exp = [
(stacked_training_bodies + 300).tolist(),
(stacked_training_bodies + 400).tolist(),
(stacked_training_bodies + 500).tolist(),
(stacked_training_bodies + 600).tolist(),
]
seeds = [seed]
need_retrain = []
try:
if not read_cache:
raise Exception("not_read_cache")
df_results = pd.read_pickle(f"output_data/tmp/same_topology_{str_body_arr}_sd{seed}_{method}")
except Exception as e:
if str(e)!="not_read_cache" and not isinstance(e, FileNotFoundError):
raise e
for bodies in exp:
filename = "-".join([str(x) for x in bodies])
for seed in seeds:
if method=="joints_only":
str_method = "-ra-ph-pfc"
elif method=="aligned":
str_method = "-"
elif method=="general_joints_feetcontact":
str_method = "-ra"
elif method=="joints_feetcontact":
str_method = "-ra-ph"
else:
str_method = f"-{method}"
path = f"{tensorboard_path}/model-{filename}{str_method}-sd{seed}/PPO_1"
if not os.path.exists(path):
path = f"{tensorboard_path}/model-{filename}-{method}-sd{seed}/PPO_1"
if not os.path.exists(path):
raise Exception(f"Path not found. {path}")
print(f"Loading {path}")
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"]==f"eval/{body}_mean_reward"].copy()
if df_body.shape[0]<62:
print(df_body.shape)
need_retrain.append({
"bodies": bodies,
"seed": seed,
"method": method,
})
# raise Exception("Data is not complete.")
df_body["body"] = template(body)
df_body["seed"] = seed
df_body["method"] = method
df_body["num_bodies"] = len(body_arr)
df_results = df_results.append(df_body)
df_results.to_pickle(f"output_data/tmp/same_topology_{str_body_arr}_sd{seed}_{method}")
return df_results
def diff_topologies_get_results(tensorboard_path,
body_arr,
seed,
case_id,
read_cache=False):
df_results = pd.DataFrame()
str_body_arr = '-'.join(str(x) for x in body_arr)
exp = [body_arr]
seeds = [seed]
need_retrain = []
try:
if not read_cache:
raise Exception("not_read_cache")
df_results = pd.read_pickle(
f"output_data/tmp/diff_topology_{str_body_arr}_sd{seed}_case{case_id}"
)
except Exception as e:
if str(e) != "not_read_cache" and not isinstance(e, FileNotFoundError):
raise e
for bodies in exp:
filename = "-".join([str(x) for x in bodies])
for seed in seeds:
str_method = f"-Walker2DHopperCase{case_id}"
path = f"{tensorboard_path}/model-{filename}{str_method}-sd{seed}/PPO_1"
if not os.path.exists(path):
raise Exception(f"Path not found. {path}")
print(f"Loading {path}")
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"] ==
f"eval/{body}_mean_reward"].copy()
if df_body.shape[0] < 12:
print(df_body.shape)
need_retrain.append({
"bodies": bodies,
"seed": seed,
"case": case_id,
})
# raise Exception("Data is not complete.")
df_body["body"] = template(body)
df_body["seed"] = seed
df_body["case"] = case_id
df_body["num_bodies"] = len(body_arr)
df_results = df_results.append(df_body)
df_results.to_pickle(
f"output_data/tmp/diff_topology_{str_body_arr}_sd{seed}_case{case_id}"
)
return df_results
def ph_values():
df_ph_values = pd.DataFrame()
stacked_training_bodies = np.arange(start=0, stop=16)
exp = [
(stacked_training_bodies + 300).tolist(),
(stacked_training_bodies + 400).tolist(),
(stacked_training_bodies + 500).tolist(),
(stacked_training_bodies + 600).tolist(),
]
print(exp)
seeds = list(range(10))
methods = ["joints_feet", "joints"]
need_retrain = []
if False:
for bodies in exp:
filename = "-".join([str(x) for x in bodies])
for method in methods:
for seed in seeds:
if method == "joints":
str_method = "-ra-ph-pfc"
elif method == "joints_feet":
str_method = "-ra-ph"
path = f"output_data/tensorboard/model-{filename}{str_method}-sd{seed}/PPO_1"
print(f"Loading {path}")
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"] ==
f"eval/{body}_mean_reward"].copy()
if df_body.shape[0] != 62:
print(df_body.shape)
need_retrain.append({
"bodies": bodies,
"seed": seed,
"method": method,
})
break
df_body["body"] = template(body)
df_body["seed"] = seed
df_body["method"] = method
# df_body["filename"] = filename
df_ph_values = df_ph_values.append(df_body)
df_ph_values.to_pickle("output_data/tmp/df_ph_values")
else:
df_ph_values = pd.read_pickle("output_data/tmp/df_ph_values")
print(df_ph_values)
return df_ph_values
def read_df(body):
dfs = []
for seed in [0, 1, 2]:
folder = f"output_data/tensorboard_oracle/model-{body}-caseWalker2DHopperWrapper-sd{seed}/PPO_1"
print(f"Loading {folder} ...")
df = tflog2pandas(folder)
df = df[df["metric"] == f"eval/{body}_mean_reward"]
max_value = df["value"].max()
final_value = df.iloc[-1, df.columns.get_loc("value")]
df = pd.DataFrame(
{
"body": template(body),
"body_id": body,
"max_value": max_value,
"final_value": final_value,
"seed": seed,
},
index=[body])
dfs.append(df)
return pd.concat(dfs)
def read_df(body):
dfs = []
for seed in [0, 1]:
folder = f"output_data/tensorboard_oracle_random_bodies/model-{body}-sd{seed}/PPO_1"
print(f"Loading {folder} ...")
df = tflog2pandas(folder)
if df.shape[0] != 1353:
return None # Fly-away bug causes the job to abort
df = df[df["metric"] == f"eval/{body}_mean_reward"]
max_value = df["value"].max()
final_value = df.iloc[-1, df.columns.get_loc("value")]
df = pd.DataFrame(
{
"body": template(body),
"body_id": body,
"max_value": max_value,
"final_value": final_value,
"seed": seed,
},
index=[body])
dfs.append(df)
return pd.concat(dfs)
return pd.concat(dfs)
dfs = []
for body in np.arange(start=100, stop=200):
df = read_df(body)
if df is not None:
dfs.append(df)
df = pd.concat(dfs)
print(df)
df.to_pickle("output_data/tmp/oracle_1xx_df")
df = pd.read_pickle("output_data/tmp/oracle_1xx_df")
for body_type in [100]:
df_one_type = df[df["body"] == template(body_type)]
df_one_type = df_one_type.groupby("body_id").mean()
df_one_type = df_one_type.sort_values(by="max_value", ascending=False)
print(df_one_type.head(20))
selected = df_one_type.head(20).index.tolist()
start_id = body_type
for s in selected:
print(
f"cp output_data/bodies/{s}.xml ../input_data/bodies/{start_id}.xml"
)
print(
f"cp output_data/tmp/model-{s}-sd0.zip output_data/models/model-{start_id}-sd0.zip"
)
print(
lightPosition=[10, -10, 10])
p.resetDebugVisualizerCamera(3, 0, -20, [0, 0, 0.5])
linux.fullscreen()
M2 = "0,1,2,3,4,5,6,7::1,2,0,3,4,5,6,7::1,2,3,0,4,5,6,7::1,7,3,4,5,0,6,2::1,2,3,4,5,6,0,7::4,0,2,3,1,5,6,7::2,0,1,3,4,5,6,7::2,3,0,1,4,5,6,7"
M0 = "0,1,2,3,4,5,6,7::1,2,0,3,4,5,6,7::1,2,3,0,4,5,6,7::1,2,3,4,5,0,6,7::1,2,3,4,5,6,0,7::1,0,2,3,4,5,6,7::2,0,1,3,4,5,6,7::2,3,0,1,4,5,6,7"
M32 = "4,3,2,1,7,5,6,0::1,2,4,5,0,3,6,7::1,2,3,7,6,5,4,0::1,2,0,4,5,3,6,7::2,1,4,3,0,6,5,7::2,1,3,5,0,4,6,7::2,1,0,4,5,3,6,7::0,1,4,3,6,7,2,5"
M4 = "5,1,2,3,4,0,6,7::1,2,0,5,4,3,6,7::1,2,3,0,4,5,6,7::1,2,4,3,5,0,6,7::1,2,3,4,5,6,0,7::1,0,2,3,4,5,6,7::2,0,1,3,4,5,6,7::2,3,0,1,4,5,6,7"
TWO_ROBOTS_1 = "0,1,2,3,4,5,6,7::0,1,2,3,4,5,6,7"
TWO_ROBOTS_2 = "0,1,2,3,4,5,6,7::7,6,5,4,3,2,1,0"
NoColor = "0,0,0,0,0,0,0,0::0,0,0,0,0,0,0,0"
arrangement = NoColor.split("::")
arrangement = [x.split(",") for x in arrangement]
for body in [500, 900]:
p.resetDebugVisualizerCamera(
3 if gym_interface.template(body) == 'ant'
or gym_interface.template(body) == 'walkerarms' else 2, 0, -20,
[0, 0, 0.5])
p.resetSimulation()
time.sleep(0.1)
filename = os.path.join(pybullet_data.getDataPath(), "plane_stadium.sdf")
(floor, ) = p.loadSDF(filename)
filename = f"../input_data/bodies/{body}.xml"
(robot, ) = p.loadMJCF(filename)
p.removeBody(floor)
print(f"\n\nbody {body}\n\n")
_reset_link_color(p, robot, arrangement[0 if body == 500 else 1])
def all_values():
df_max_values = pd.DataFrame()
df_all_values = pd.DataFrame()
stacked_training_bodies = np.arange(start=0, stop=16)
exp = [
(stacked_training_bodies + 300).tolist(),
(stacked_training_bodies + 400).tolist(),
(stacked_training_bodies + 500).tolist(),
(stacked_training_bodies + 600).tolist(),
]
print(exp)
seeds = list(range(10))
methods = ["align", "random"]
need_retrain = []
if False:
for bodies in exp:
filename = "-".join([str(x) for x in bodies])
for method in methods:
for seed in seeds:
if method == "align":
str_method = ""
elif method == "misalign":
str_method = "-mis"
elif method == "random":
str_method = "-ra"
path = f"output_data/tensorboard/model-{filename}{str_method}-sd{seed}/PPO_1"
print(f"Loading {path}")
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"] ==
f"eval/{body}_mean_reward"].copy()
if df_body.shape[0] != 62:
print(df_body.shape)
need_retrain.append({
"bodies": bodies,
"seed": seed,
"method": method,
})
break
df_max_values = df_max_values.append(
{
"body": template(body),
"seed": seed,
"method": method,
"max_value": df_body["value"].max(),
"filename": filename,
},
ignore_index=True)
df_body["body"] = template(body)
df_body["seed"] = seed
df_body["method"] = method
df_body["filename"] = filename
df_all_values = df_all_values.append(df_body)
df_all_values.to_pickle("output_data/tmp/df_all_values")
else:
df_all_values = pd.read_pickle("output_data/tmp/df_all_values")
print(df_all_values) #[68448 rows x 7 columns]
return df_all_values
A = vanilla4_results["max_value_399"]
B = vanilla4_results["max_value_499"]
C = vanilla4_results["max_value_599"]
D = vanilla4_results["max_value_699"]
vanilla4_results["combined_value"] = 2*A + B + C + 0.5*D
vanilla4_results["combined_rank"] = vanilla4_results["combined_value"].rank()
vanilla4_results = vanilla4_results.sort_values(by="combined_rank", ascending=False)
print(vanilla4_results.head(10))
print(vanilla4_results.tail(10))
fig, axes = plt.subplots(ncols=2, nrows=2, sharey=True)
axes = axes.flatten()
for ax, body in zip(axes,bodies):
g = sns.histplot(data=vanilla4_results, x=f"max_value_{body}", ax=ax)
g.set(xlabel="Max Value")
g.set_title(template(body))
plt.tight_layout()
plt.savefig("output_data/plots/distribution_vanilla4_500_exps.png")
plt.close()
plt.figure()
g = sns.histplot(data=vanilla4_results, x="combined_value")
g.set(xlabel="Combined Value = 2A + B + C + 0.5D")
plt.savefig("output_data/plots/distribution_vanilla4_combined.png")
plt.close()
print("The best alignment:")
print(vanilla4_results.iloc[0]["custom_alignment"])
print("The worst alignment:")
print(f"Loading {path}")
df = tflog2pandas(path)
df["body"] = body
df["seed"] = seed
df["stackframe"] = stackframe
df = df[df["metric"] == f"eval/{body}_mean_reward"]
print(df.shape)
# print(df.head())
dfs.append(df)
df = pd.concat(dfs)
df.to_pickle(cache_filename)
print(df.shape)
fig, axes = plt.subplots(nrows=2, ncols=2, sharey=True, figsize=[10, 10])
axes = axes.flatten()
for idx, body in enumerate(bodies):
sns.lineplot(ax=axes[idx],
data=df[df["body"] == body],
x="step",
y="value",
hue="stackframe",
style="stackframe",
markers=True,
dashes=False).set_title(template(body))
plt.legend()
plt.tight_layout()
plt.savefig("output_data/plots/0.png")
plt.show()
str_method = ""
elif method == "misalign":
str_method = "-mis"
elif method == "random":
str_method = "-ra"
path = f"output_data/tensorboard/model-{filename}{str_method}-sd{seed}/PPO_1"
print(f"Loading {path}")
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"] == f"eval/{body}_mean_reward"]
print(df_body.shape)
# print(df_body.columns)
print(body, df_body["value"].max())
df_all = df_all.append(
{
"body": template(body),
"seed": seed,
"method": method,
"max_value": df_body["value"].max(),
"filename": filename,
},
ignore_index=True)
print(df_all)
sns.boxplot(hue="method", y="max_value", data=df_all, x="body")
plt.savefig("output_data/plots/2.png")
plt.close()
g = sns.FacetGrid(df_all, row="filename", col="body", height=3, aspect=1.5)
g.map(sns.barplot, "method", "max_value", order=methods)
g.set_xticklabels(rotation=20)
df = tflog2pandas(path)
for body in bodies:
df_body = df[df["metric"] ==
f"eval/{body}_mean_reward"].copy()
if df_body.shape[0] != 62:
print(df_body.shape)
need_retrain.append({
"bodies": bodies,
"seed": seed,
"method": method,
})
break
df_max_values = df_max_values.append(
{
"body": template(body),
"seed": seed,
"method": method,
"max_value": df_body["value"].max(),
"filename": filename,
},
ignore_index=True)
df_body["body"] = template(body)
df_body["seed"] = seed
df_body["method"] = method
df_body["filename"] = filename
df_all_values = df_all_values.append(df_body)
df_all_values.to_pickle("output_data/tmp/df_all_values")
else:
