def __init__(self, env):
super().__init__(env)
self.obs_header_length = self.obs_length - self.max_num_joint * 2 - self.max_num_feet
assert common.args.preserve_header or not common.args.preserve_feet_contact, "preserve_feet_contact should only be used when preserve_header is used."
if common.args.preserve_header:
if common.args.preserve_feet_contact:
self.align_idx = list(range(self.max_num_joint * 2))
else:
self.align_idx = list(
range(self.max_num_joint * 2 + self.max_num_feet))
else:
self.align_idx = list(range(self.obs_length))
with temp_seed(common.seed):
np.random.shuffle(self.align_idx)
with temp_seed(self.robot.robot_id):
np.random.shuffle(self.align_idx)
if common.args.random_even_same_body:
with temp_seed(env.rank):
np.random.shuffle(self.align_idx)
print(f"Random Wrapper: {self.align_idx}")
def one_exp(num_bodies=2, pick_bodies_from=[], method="aligned", seed=0, repeat=10):
global g_all_jobs, g_ge
max_num_joints = gym_interface.get_max_num_joints(pick_bodies_from[:1])
with seeds.temp_seed(seed):
run_seeds = np.random.randint(low=0, high=10000, size=[repeat])
for run_seed in run_seeds:
with seeds.temp_seed(run_seed):
train_on_bodies = np.random.choice(pick_bodies_from, size=[num_bodies], replace=False)
print("\n# train on bodies: ", train_on_bodies)
str_body_selected = ','.join([str(x) for x in train_on_bodies])
custom_alignment = "" # default is aligned
if method=="randomized":
custom_alignment = g_ge.construct_random_alignment(num_bodies=num_bodies, max_joints=max_num_joints, seed=run_seed)
assert g_ge.check_not_in_alignment_history(custom_alignment), "Not lucky! Not a new alignment!"
print(f"\n# ==> Alignment : ", custom_alignment)
str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest()
print(f"# {str_md5}")
print(f"sbatch -J $exp_name $submit_to python 1.train.py --seed={run_seed} -f=$exp_name/{num_bodies}_{method}/ --train_bodies={str_body_selected} --topology_wrapper=CustomAlignWrapper --custom_alignment={custom_alignment} --custom_align_max_joints={max_num_joints}")
job = {
"train_on_bodies": train_on_bodies,
"seed": seed,
"custom_alignment": custom_alignment,
"str_md5": str_md5,
"run_seed": run_seed,
}
g_all_jobs.append(job)
def one_exp():
global g_current_exp_id, g_all_jobs
str_body_selected = ','.join([str(x) for x in vanilla_bodies])
assert check_not_in_body_history(
str_body_selected), "Not lucky! Not a new set of bodies!"
print("\n# train on bodies: ", vanilla_bodies)
for seed in range(40):
if seed == 0:
# I have a good candidate from exp_019
custom_alignment = "1,5,0,4,6,7,3,2::3,7,0,6,2,4,5,1::5,4,7,0,1,6,2,3::3,7,5,4,0,6,2,1"
else:
custom_alignment = construct_custom_alignment()
assert check_not_in_alignment_history(
custom_alignment), "Not lucky! Not a new alignment!"
print(f"\n# ==> Alignment {seed} : ", custom_alignment)
str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest()
print(f"# {str_md5}")
with seeds.temp_seed(seed):
run_seeds = np.random.randint(low=0, high=10000, size=[5])
for run_seed in run_seeds:
seed = g_current_exp_id
print(
f"sbatch -J vanilla4 submit.sh python 1.train.py --train_steps=5e6 --seed={run_seed} --train_bodies={str_body_selected} --test_bodies={str_body_selected} --topology_wrapper=CustomAlignWrapper --custom_alignment={custom_alignment} --tensorboard=tensorboard_vanilla4 --custom_align_max_joints=8"
)
g_all_jobs[g_current_exp_id] = {
"vanilla_bodies": vanilla_bodies,
"custom_alignment": custom_alignment,
"seed": seed,
"str_md5": str_md5,
"run_seed": run_seed,
}
g_current_exp_id += 1
def construct_random_alignment(self, num_bodies=4, max_joints=8, seed=0):
orders = []
with seeds.temp_seed(seed):
order = np.arange(max_joints)
for i in range(num_bodies):
np.random.shuffle(order)
orders.append(",".join([str(x) for x in order]))
ret = "::".join(orders)
return ret
def construct_custom_alignment():
global g_seed_construct_custom_alignment
g_seed_construct_custom_alignment += 1
orders = []
with seeds.temp_seed(g_seed_construct_custom_alignment):
order = np.arange(8)
for i in range(4):
np.random.shuffle(order)
orders.append(",".join([str(x) for x in order]))
ret = "::".join(orders)
return ret
def make_body(body, mutate_seed=0):
with open(f"{body_template_folder}/{body}.xml", "r") as f:
_content = f.read()
tempalte_content = string.Template(_content)
data = variables[f"body-{body}"].copy()
# body specific calculation
if body==400:
data["torso_posterior"] = - data["torso_length"] / 2
data["torso_anterior"] = data["torso_length"] / 2
old_data = data.copy()
for key in old_data:
if "_angle" in key:
part_name = key[:-len("_angle")]
data[f"{part_name}_x"] = data[f"{part_name}_length"] * math.cos(data[f"{part_name}_angle"]/180*math.pi)
data[f"{part_name}_z"] = data[f"{part_name}_length"] * math.sin(data[f"{part_name}_angle"]/180*math.pi)
data["neck_x"] = data["torso_anterior"] + data["head_x"]
elif body==600:
data["foot_posterior"] = - data["foot_length"] / 3
data["foot_anterior"] = data["foot_posterior"] + data["foot_length"]
elif body==800:
data["torso_pos_arm"] = data["torso_length"] / 2
# mutate
if mutate_seed>0:
largest_noise = 1.4 # Most likely the mutation is within this range (99.7% three deviation)
with seeds.temp_seed(body+mutate_seed):
mutate_noise = np.random.normal(loc=0, scale=1/3, size=len(data))
mutate_noise = largest_noise**mutate_noise
for idx,key in enumerate(data.keys()):
data[key] *= mutate_noise[idx]
# print(f"{key}: {data[key]} *= {mutate_noise[idx]}")
# calculate height
fixed_height = 0.1
if body in [300,600,700,800]:
data["height"] = data["torso_length"] + data["thigh_length"] + data["leg_length"] + data["foot_thickness"] + fixed_height
elif body==400:
data["height"] = max(-data["bthigh_z"] - data["bshin_z"] - data["bfoot_z"] + data["bfoot_thickness"], -data["fthigh_z"] - data["fshin_z"] - data["ffoot_z"] + data["ffoot_thickness"]) + fixed_height
elif body==500:
data["height"] = data["foot_length"] + fixed_height + 0.2
# keep two decimals
cleaned_data = {}
for key in data:
cleaned_data[key] = int(data[key] * 100)/100
_content = tempalte_content.safe_substitute(cleaned_data)
assert "$" not in _content, "some variable not substituted."
with open(f"{body_folder}/{body+mutate_seed}.xml", "w") as f:
print(_content, file=f)
def ga_select_parent(self, population, k=5):
# Select two parents
# Tournament Selection: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_parent_selection.htm
# k-Way tournament
# small k means less selection pressure and more randomness
with seeds.temp_seed(int(time.time())):
selected = np.random.randint(low=0, high=len(population), size=[k])
fitnesses = []
for i in range(k):
fitnesses.append(population[selected[i]]["fitness"])
best = np.argsort(fitnesses)[::-1]
self.log.record(f"best: {selected[best]}")
return population[selected[best[0]]]
def construct_custom_alignment():
global g_seed_construct_custom_alignment
g_seed_construct_custom_alignment += 1
orders = []
with seeds.temp_seed(g_seed_construct_custom_alignment):
order = np.arange(custom_align_max_joints)
for i in range(8):
np.random.shuffle(order)
orders.append(",".join([str(x) for x in order]))
assert 16 % len(orders) == 0
orders = orders * (16 // len(orders))
ret = "::".join(orders)
return ret
def construct_custom_alignment(seed, num_joints, num_bodies, num_envs):
assert num_envs%num_bodies==0
repeat_times = num_envs//num_bodies
orders = []
with seeds.temp_seed(seed):
order = np.arange(num_joints)
for i in range(num_bodies):
np.random.shuffle(order)
orders.append(",".join([str(x) for x in order]))
unique_alignment = "::".join(orders)
repeated_alignments = []
for i in range(repeat_times):
repeated_alignments.append(unique_alignment)
ret = "::".join(repeated_alignments)
return ret
def ga_mutate(alignments, n=2, seed=0):
# Mutate individual
# Swap Mutation: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_mutation.htm
# small n means less mutation and less randomness
offspring = copy.deepcopy(alignments)
with seeds.temp_seed(seed):
target_sequence_ids = np.random.randint(low=0, high=len(offspring), size=[n])
target_positions = []
for i in range(n):
target_pos = np.random.choice(np.arange(start=0, stop=len(offspring[0])), size=2)
target_positions.append(target_pos)
for i in range(n):
offspring[target_sequence_ids[i]][target_positions[i][0]], offspring[target_sequence_ids[i]][target_positions[i][1]] = \
offspring[target_sequence_ids[i]][target_positions[i][1]], offspring[target_sequence_ids[i]][target_positions[i][0]]
return offspring
def evoke_master(self, individual):
# create lock
while os.path.exists(f"{self.database_path}.lock"):
with seeds.temp_seed(int(time.time())):
l = np.random.randint(low=1, high=30)
self.log.record(f"sleep for {l} sec...")
time.sleep(l)
open(f"{self.database_path}.lock", "w").close()
# !! entering critical section
self.is_in_critical_section = True
data = self._evoke_master(individual)
self.is_in_critical_section = True
# !! leaving critical section
os.remove(f"{self.database_path}.lock")
return data
def generate_jobs(mutants, label=""):
global g_current_exp_id, g_all_jobs
for offspring_alignment in mutants:
# print(offspring_alignment,run_seed)
str_md5 = hashlib.md5(offspring_alignment.encode()).hexdigest()
print(f"\n# {str_md5}")
print(f"# {offspring_alignment}")
with seeds.temp_seed(g_current_exp_id):
run_seeds = np.random.randint(low=0, high=1000000, size=[5])
for run_seed in run_seeds:
cmd = f"sbatch -J vanilla4_manual_ga submit.sh python 1.train.py --custom_alignment={offspring_alignment} --train_steps=5e6 --seed={run_seed} --train_bodies=399,499,599,699 --test_bodies=399,499,599,699 --topology_wrapper=CustomAlignWrapper --tensorboard=tensorboard_{exp_name} --custom_align_max_joints=8"
print(cmd)
g_all_jobs[g_current_exp_id] = {
# "vanilla_bodies": vanilla_bodies,
"custom_alignment": offspring_alignment,
"seed": g_current_exp_id,
"str_md5": str_md5,
"run_seed": run_seed,
"label": label,
}
g_current_exp_id += 1
def ga_mutate(self, parent, n=2):
# Mutate individual
# Swap Mutation: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_mutation.htm
# small n means less mutation and less randomness
offspring = copy.deepcopy(parent)
offspring["parent"] = parent["id"]
with seeds.temp_seed(int(time.time())):
target_sequence_ids = np.random.randint(low=0,
high=len(
offspring["geno"]),
size=[n])
target_positions = []
for i in range(n):
target_pos = np.random.choice(np.arange(
start=0, stop=len(offspring["geno"][0])),
size=2)
target_positions.append(target_pos)
for i in range(n):
offspring["geno"][target_sequence_ids[i]][target_positions[i][0]], offspring["geno"][target_sequence_ids[i]][target_positions[i][1]] = \
offspring["geno"][target_sequence_ids[i]][target_positions[i][1]], offspring["geno"][target_sequence_ids[i]][target_positions[i][0]]
self.log.record(f"parent geno {parent['geno']}")
self.log.record(f"offspring geno {offspring['geno']}")
return offspring
def __init__(self, env):
# assert not isinstance(env, WalkerWrapper), "Don't padding zero before realign"
super().__init__(env)
self.realign_method = common.args.realign_method
self.num_feet = len(env.robot.foot_list)
self.num_joints = env.action_space.shape[0]
self.length_general = 8
self.length_joints = self.num_joints * 2
self.length_feet = self.num_feet
# print(self.num_feet, self.num_joints)
print(self.realign_method)
if self.realign_method == "aligned":
self.realign_length = 0
elif self.realign_method == "general_only":
self.realign_length = self.length_general
elif self.realign_method == "joints_only":
self.realign_length = self.length_joints
elif self.realign_method == "feetcontact_only":
self.realign_length = self.length_feet
elif self.realign_method == "general_joints":
self.realign_length = self.length_general+self.length_joints
elif self.realign_method == "general_feetcontact":
self.realign_length = self.length_general+self.length_feet
elif self.realign_method == "joints_feetcontact":
self.realign_length = self.length_joints+self.length_feet
elif self.realign_method == "general_joints_feetcontact":
self.realign_length = self.length_general+self.length_joints+self.length_feet
else:
raise NotImplementedError
self.realign_idx = np.arange(self.realign_length)
with temp_seed(common.seed):
np.random.shuffle(self.realign_idx)
with temp_seed(self.robot.robot_id):
np.random.shuffle(self.realign_idx)
if common.args.random_even_same_body:
with temp_seed(env.rank):
np.random.shuffle(self.realign_idx)
if self.realign_method == "aligned":
self.realign_idx = np.arange(start=0,stop=self.length_general+self.length_joints+self.length_feet)
elif self.realign_method == "general_only":
self.realign_idx = np.concatenate((
self.realign_idx,
np.arange(start=self.length_general, stop=self.length_general+self.length_joints+self.length_feet)
))
elif self.realign_method == "joints_only":
self.realign_idx = np.concatenate((
np.arange(start=0, stop=self.length_general),
self.realign_idx + self.length_general,
np.arange(start=self.length_general+self.length_joints, stop=self.length_general+self.length_joints+self.length_feet)
))
elif self.realign_method == "feetcontact_only":
self.realign_idx = np.concatenate((
np.arange(start=0, stop=self.length_general+self.length_joints),
self.realign_idx + self.length_general + self.length_joints,
))
elif self.realign_method == "general_joints":
self.realign_idx = np.concatenate((
self.realign_idx,
np.arange(start=self.length_general+self.length_joints, stop=self.length_general+self.length_joints+self.length_feet),
))
elif self.realign_method == "general_feetcontact":
for i, v in enumerate(self.realign_idx):
if v>=self.length_general:
self.realign_idx[i] = v+self.length_joints
self.realign_idx = np.concatenate((
self.realign_idx[:self.length_general],
np.arange(start=self.length_general, stop=self.length_general+self.length_joints),
self.realign_idx[self.length_general:]
))
elif self.realign_method == "joints_feetcontact":
self.realign_idx = np.concatenate((
np.arange(start=0, stop=self.length_general),
self.realign_idx + self.length_general
))
elif self.realign_method == "general_joints_feetcontact":
self.realign_idx = self.realign_idx
else:
raise NotImplementedError
print(self.realign_idx)
print(f"# default alignment for 1xx: {str_meaninful_alignment}")
str_md5 = hashlib.md5(str_meaninful_alignment.encode()).hexdigest()
print(f"# {str_md5}")
str_md5 = hashlib.md5("::".join([str_meaninful_alignment] *
2).encode()).hexdigest()
print(f"# {str_md5}")
num_jobs = 0
all_jobs = []
for num_mutate in [128]:
for seed in range(40):
new_alignments = ga_mutate(alignments, n=num_mutate, seed=seed)
custom_alignment = get_str_alignments(new_alignments)
print(f"\n# mutate {num_mutate} alignment for 1xx: {custom_alignment}")
str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest()
print(f"# {str_md5}")
with seeds.temp_seed(seed):
run_seeds = np.random.randint(low=0, high=10000, size=[5])
for run_seed in run_seeds:
cmd = f"sbatch -J re_search_1xx submit.sh python 1.train.py --seed={run_seed} --custom_alignment={custom_alignment} --tensorboard=tensorboard/1xx_mutate_{num_mutate}_rerun --topology_wrapper=CustomAlignWrapper --custom_align_max_joints=8 --train_bodies=100,101,102,103,104,105,106,107 --test_bodies=100,101,102,103,104,105,106,107 --train_steps=1e7"
print(cmd)
job = {
"id": num_jobs,
"num_mutate": num_mutate,
"body_seed": seed,
"str_md5": str_md5,
"custom_alignment": custom_alignment,
"run_seed": run_seed,
}
all_jobs.append(job)
num_jobs += 1
print(f"# Total jobs: {num_jobs}.")
def __init__(self, env):
super().__init__(env)
self.align_idx = list(range(self.obs_length))
with temp_seed(self.robot.robot_id):
np.random.shuffle(self.align_idx)
print(f"Random Wrapper: {self.align_idx}")
def __init__(self, env: WalkerWrapper):
"""Before using this wrapper, first wrap with WalkerWrapper"""
super().__init__(env)
self.realign_method = common.args.realign_method
print(self.realign_method)
if self.realign_method == "general_only":
self.realign_length = 8
elif self.realign_method == "joints_only":
self.realign_length = 16
elif self.realign_method == "feetcontact_only":
self.realign_length = 6
elif self.realign_method == "general_joints":
self.realign_length = 8 + 16
elif self.realign_method == "general_feetcontact":
self.realign_length = 8 + 6
elif self.realign_method == "joints_feetcontact":
self.realign_length = 16 + 6
elif self.realign_method == "general_joints_feetcontact":
self.realign_length = 8 + 16 + 6
else:
raise NotImplementedError
self.realign_idx = np.arange(self.realign_length)
with temp_seed(common.seed):
np.random.shuffle(self.realign_idx)
with temp_seed(self.robot.robot_id):
np.random.shuffle(self.realign_idx)
if common.args.random_even_same_body:
with temp_seed(env.rank):
np.random.shuffle(self.realign_idx)
if self.realign_method == "general_only":
self.realign_idx = np.concatenate(
(self.realign_idx, np.arange(start=8, stop=8 + 16 + 6)))
elif self.realign_method == "joints_only":
self.realign_idx = np.concatenate(
(np.arange(start=0, stop=8), self.realign_idx + 8,
np.arange(start=8 + 16, stop=8 + 16 + 6)))
elif self.realign_method == "feetcontact_only":
self.realign_idx = np.concatenate((
np.arange(start=0, stop=8 + 16),
self.realign_idx + 8 + 16,
))
elif self.realign_method == "general_joints":
self.realign_idx = np.concatenate((
self.realign_idx,
np.arange(start=8 + 16, stop=8 + 16 + 6),
))
elif self.realign_method == "general_feetcontact":
self.realign_idx = np.concatenate(
(self.realign_idx[:8], np.arange(start=8, stop=8 + 16),
self.realign_idx[8:]))
elif self.realign_method == "joints_feetcontact":
self.realign_idx = np.concatenate(
(np.arange(start=0, stop=8), self.realign_idx))
elif self.realign_method == "general_joints_feetcontact":
self.realign_idx = self.realign_idx
else:
raise NotImplementedError
print(self.realign_idx)
# Maximum joint number is 10.
# a0[0,1,2,3,4,5,6,7,8,9]
# a1[0,1,2,3,4,5,6,7,8,9]
# ...
# a15[0,1,2,3,4,5,6,7,8,9]
# Total possible combinations = (Pr10)^(16-1) = (10!)^15 (A huge number.)
# So we randomly shuffle them, and if repeats, re-shuffle to get a new one.
import numpy as np
from common import seeds
with seeds.temp_seed(0):
for j in range(2):
joint_orders = []
for i in range(16):
joint_order = np.arange(10)
np.random.shuffle(joint_order)
joint_orders.append(joint_order)
joint_orders = np.array(joint_orders)
print(joint_orders)
xml += f"</body>"
max_depth += 1
return xml
def reset_global_variables():
global max_depth, joint_number, max_limb, lowest_point
joint_number = 0
max_depth = 3
max_limb = 10
lowest_point = 0
for i in range(100):
reset_global_variables()
with seeds.temp_seed(i):
xml_worldbody = to_xml(worldbody, 0)
print(f"lowest_point is {lowest_point}")
adjusted_worldbody = worldbody.copy()
above_ground = 0.1
adjusted_worldbody["pos"][
2] = -lowest_point + adjusted_worldbody["length"][2] / 2 + above_ground
print(f"\nadjusted to {adjusted_worldbody['pos'][2]}.\n")
reset_global_variables()
with seeds.temp_seed(i):
xml_worldbody = to_xml(adjusted_worldbody,
adjusted_worldbody["pos"][2])
xml_template = f"""
<!--
joint_number: {joint_number}
