def __init__(self, env): super().__init__(env) self.obs_header_length = self.obs_length - self.max_num_joint * 2 - self.max_num_feet assert common.args.preserve_header or not common.args.preserve_feet_contact, "preserve_feet_contact should only be used when preserve_header is used." if common.args.preserve_header: if common.args.preserve_feet_contact: self.align_idx = list(range(self.max_num_joint * 2)) else: self.align_idx = list( range(self.max_num_joint * 2 + self.max_num_feet)) else: self.align_idx = list(range(self.obs_length)) with temp_seed(common.seed): np.random.shuffle(self.align_idx) with temp_seed(self.robot.robot_id): np.random.shuffle(self.align_idx) if common.args.random_even_same_body: with temp_seed(env.rank): np.random.shuffle(self.align_idx) print(f"Random Wrapper: {self.align_idx}")
def one_exp(num_bodies=2, pick_bodies_from=[], method="aligned", seed=0, repeat=10): global g_all_jobs, g_ge max_num_joints = gym_interface.get_max_num_joints(pick_bodies_from[:1]) with seeds.temp_seed(seed): run_seeds = np.random.randint(low=0, high=10000, size=[repeat]) for run_seed in run_seeds: with seeds.temp_seed(run_seed): train_on_bodies = np.random.choice(pick_bodies_from, size=[num_bodies], replace=False) print("\n# train on bodies: ", train_on_bodies) str_body_selected = ','.join([str(x) for x in train_on_bodies]) custom_alignment = "" # default is aligned if method=="randomized": custom_alignment = g_ge.construct_random_alignment(num_bodies=num_bodies, max_joints=max_num_joints, seed=run_seed) assert g_ge.check_not_in_alignment_history(custom_alignment), "Not lucky! Not a new alignment!" print(f"\n# ==> Alignment : ", custom_alignment) str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest() print(f"# {str_md5}") print(f"sbatch -J $exp_name $submit_to python 1.train.py --seed={run_seed} -f=$exp_name/{num_bodies}_{method}/ --train_bodies={str_body_selected} --topology_wrapper=CustomAlignWrapper --custom_alignment={custom_alignment} --custom_align_max_joints={max_num_joints}") job = { "train_on_bodies": train_on_bodies, "seed": seed, "custom_alignment": custom_alignment, "str_md5": str_md5, "run_seed": run_seed, } g_all_jobs.append(job)
def one_exp(): global g_current_exp_id, g_all_jobs str_body_selected = ','.join([str(x) for x in vanilla_bodies]) assert check_not_in_body_history( str_body_selected), "Not lucky! Not a new set of bodies!" print("\n# train on bodies: ", vanilla_bodies) for seed in range(40): if seed == 0: # I have a good candidate from exp_019 custom_alignment = "1,5,0,4,6,7,3,2::3,7,0,6,2,4,5,1::5,4,7,0,1,6,2,3::3,7,5,4,0,6,2,1" else: custom_alignment = construct_custom_alignment() assert check_not_in_alignment_history( custom_alignment), "Not lucky! Not a new alignment!" print(f"\n# ==> Alignment {seed} : ", custom_alignment) str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest() print(f"# {str_md5}") with seeds.temp_seed(seed): run_seeds = np.random.randint(low=0, high=10000, size=[5]) for run_seed in run_seeds: seed = g_current_exp_id print( f"sbatch -J vanilla4 submit.sh python 1.train.py --train_steps=5e6 --seed={run_seed} --train_bodies={str_body_selected} --test_bodies={str_body_selected} --topology_wrapper=CustomAlignWrapper --custom_alignment={custom_alignment} --tensorboard=tensorboard_vanilla4 --custom_align_max_joints=8" ) g_all_jobs[g_current_exp_id] = { "vanilla_bodies": vanilla_bodies, "custom_alignment": custom_alignment, "seed": seed, "str_md5": str_md5, "run_seed": run_seed, } g_current_exp_id += 1
def construct_random_alignment(self, num_bodies=4, max_joints=8, seed=0): orders = [] with seeds.temp_seed(seed): order = np.arange(max_joints) for i in range(num_bodies): np.random.shuffle(order) orders.append(",".join([str(x) for x in order])) ret = "::".join(orders) return ret
def construct_custom_alignment(): global g_seed_construct_custom_alignment g_seed_construct_custom_alignment += 1 orders = [] with seeds.temp_seed(g_seed_construct_custom_alignment): order = np.arange(8) for i in range(4): np.random.shuffle(order) orders.append(",".join([str(x) for x in order])) ret = "::".join(orders) return ret
def make_body(body, mutate_seed=0): with open(f"{body_template_folder}/{body}.xml", "r") as f: _content = f.read() tempalte_content = string.Template(_content) data = variables[f"body-{body}"].copy() # body specific calculation if body==400: data["torso_posterior"] = - data["torso_length"] / 2 data["torso_anterior"] = data["torso_length"] / 2 old_data = data.copy() for key in old_data: if "_angle" in key: part_name = key[:-len("_angle")] data[f"{part_name}_x"] = data[f"{part_name}_length"] * math.cos(data[f"{part_name}_angle"]/180*math.pi) data[f"{part_name}_z"] = data[f"{part_name}_length"] * math.sin(data[f"{part_name}_angle"]/180*math.pi) data["neck_x"] = data["torso_anterior"] + data["head_x"] elif body==600: data["foot_posterior"] = - data["foot_length"] / 3 data["foot_anterior"] = data["foot_posterior"] + data["foot_length"] elif body==800: data["torso_pos_arm"] = data["torso_length"] / 2 # mutate if mutate_seed>0: largest_noise = 1.4 # Most likely the mutation is within this range (99.7% three deviation) with seeds.temp_seed(body+mutate_seed): mutate_noise = np.random.normal(loc=0, scale=1/3, size=len(data)) mutate_noise = largest_noise**mutate_noise for idx,key in enumerate(data.keys()): data[key] *= mutate_noise[idx] # print(f"{key}: {data[key]} *= {mutate_noise[idx]}") # calculate height fixed_height = 0.1 if body in [300,600,700,800]: data["height"] = data["torso_length"] + data["thigh_length"] + data["leg_length"] + data["foot_thickness"] + fixed_height elif body==400: data["height"] = max(-data["bthigh_z"] - data["bshin_z"] - data["bfoot_z"] + data["bfoot_thickness"], -data["fthigh_z"] - data["fshin_z"] - data["ffoot_z"] + data["ffoot_thickness"]) + fixed_height elif body==500: data["height"] = data["foot_length"] + fixed_height + 0.2 # keep two decimals cleaned_data = {} for key in data: cleaned_data[key] = int(data[key] * 100)/100 _content = tempalte_content.safe_substitute(cleaned_data) assert "$" not in _content, "some variable not substituted." with open(f"{body_folder}/{body+mutate_seed}.xml", "w") as f: print(_content, file=f)
def ga_select_parent(self, population, k=5): # Select two parents # Tournament Selection: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_parent_selection.htm # k-Way tournament # small k means less selection pressure and more randomness with seeds.temp_seed(int(time.time())): selected = np.random.randint(low=0, high=len(population), size=[k]) fitnesses = [] for i in range(k): fitnesses.append(population[selected[i]]["fitness"]) best = np.argsort(fitnesses)[::-1] self.log.record(f"best: {selected[best]}") return population[selected[best[0]]]
def construct_custom_alignment(): global g_seed_construct_custom_alignment g_seed_construct_custom_alignment += 1 orders = [] with seeds.temp_seed(g_seed_construct_custom_alignment): order = np.arange(custom_align_max_joints) for i in range(8): np.random.shuffle(order) orders.append(",".join([str(x) for x in order])) assert 16 % len(orders) == 0 orders = orders * (16 // len(orders)) ret = "::".join(orders) return ret
def construct_custom_alignment(seed, num_joints, num_bodies, num_envs): assert num_envs%num_bodies==0 repeat_times = num_envs//num_bodies orders = [] with seeds.temp_seed(seed): order = np.arange(num_joints) for i in range(num_bodies): np.random.shuffle(order) orders.append(",".join([str(x) for x in order])) unique_alignment = "::".join(orders) repeated_alignments = [] for i in range(repeat_times): repeated_alignments.append(unique_alignment) ret = "::".join(repeated_alignments) return ret
def ga_mutate(alignments, n=2, seed=0): # Mutate individual # Swap Mutation: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_mutation.htm # small n means less mutation and less randomness offspring = copy.deepcopy(alignments) with seeds.temp_seed(seed): target_sequence_ids = np.random.randint(low=0, high=len(offspring), size=[n]) target_positions = [] for i in range(n): target_pos = np.random.choice(np.arange(start=0, stop=len(offspring[0])), size=2) target_positions.append(target_pos) for i in range(n): offspring[target_sequence_ids[i]][target_positions[i][0]], offspring[target_sequence_ids[i]][target_positions[i][1]] = \ offspring[target_sequence_ids[i]][target_positions[i][1]], offspring[target_sequence_ids[i]][target_positions[i][0]] return offspring
def evoke_master(self, individual): # create lock while os.path.exists(f"{self.database_path}.lock"): with seeds.temp_seed(int(time.time())): l = np.random.randint(low=1, high=30) self.log.record(f"sleep for {l} sec...") time.sleep(l) open(f"{self.database_path}.lock", "w").close() # !! entering critical section self.is_in_critical_section = True data = self._evoke_master(individual) self.is_in_critical_section = True # !! leaving critical section os.remove(f"{self.database_path}.lock") return data
def generate_jobs(mutants, label=""): global g_current_exp_id, g_all_jobs for offspring_alignment in mutants: # print(offspring_alignment,run_seed) str_md5 = hashlib.md5(offspring_alignment.encode()).hexdigest() print(f"\n# {str_md5}") print(f"# {offspring_alignment}") with seeds.temp_seed(g_current_exp_id): run_seeds = np.random.randint(low=0, high=1000000, size=[5]) for run_seed in run_seeds: cmd = f"sbatch -J vanilla4_manual_ga submit.sh python 1.train.py --custom_alignment={offspring_alignment} --train_steps=5e6 --seed={run_seed} --train_bodies=399,499,599,699 --test_bodies=399,499,599,699 --topology_wrapper=CustomAlignWrapper --tensorboard=tensorboard_{exp_name} --custom_align_max_joints=8" print(cmd) g_all_jobs[g_current_exp_id] = { # "vanilla_bodies": vanilla_bodies, "custom_alignment": offspring_alignment, "seed": g_current_exp_id, "str_md5": str_md5, "run_seed": run_seed, "label": label, } g_current_exp_id += 1
def ga_mutate(self, parent, n=2): # Mutate individual # Swap Mutation: https://www.tutorialspoint.com/genetic_algorithms/genetic_algorithms_mutation.htm # small n means less mutation and less randomness offspring = copy.deepcopy(parent) offspring["parent"] = parent["id"] with seeds.temp_seed(int(time.time())): target_sequence_ids = np.random.randint(low=0, high=len( offspring["geno"]), size=[n]) target_positions = [] for i in range(n): target_pos = np.random.choice(np.arange( start=0, stop=len(offspring["geno"][0])), size=2) target_positions.append(target_pos) for i in range(n): offspring["geno"][target_sequence_ids[i]][target_positions[i][0]], offspring["geno"][target_sequence_ids[i]][target_positions[i][1]] = \ offspring["geno"][target_sequence_ids[i]][target_positions[i][1]], offspring["geno"][target_sequence_ids[i]][target_positions[i][0]] self.log.record(f"parent geno {parent['geno']}") self.log.record(f"offspring geno {offspring['geno']}") return offspring
def __init__(self, env): # assert not isinstance(env, WalkerWrapper), "Don't padding zero before realign" super().__init__(env) self.realign_method = common.args.realign_method self.num_feet = len(env.robot.foot_list) self.num_joints = env.action_space.shape[0] self.length_general = 8 self.length_joints = self.num_joints * 2 self.length_feet = self.num_feet # print(self.num_feet, self.num_joints) print(self.realign_method) if self.realign_method == "aligned": self.realign_length = 0 elif self.realign_method == "general_only": self.realign_length = self.length_general elif self.realign_method == "joints_only": self.realign_length = self.length_joints elif self.realign_method == "feetcontact_only": self.realign_length = self.length_feet elif self.realign_method == "general_joints": self.realign_length = self.length_general+self.length_joints elif self.realign_method == "general_feetcontact": self.realign_length = self.length_general+self.length_feet elif self.realign_method == "joints_feetcontact": self.realign_length = self.length_joints+self.length_feet elif self.realign_method == "general_joints_feetcontact": self.realign_length = self.length_general+self.length_joints+self.length_feet else: raise NotImplementedError self.realign_idx = np.arange(self.realign_length) with temp_seed(common.seed): np.random.shuffle(self.realign_idx) with temp_seed(self.robot.robot_id): np.random.shuffle(self.realign_idx) if common.args.random_even_same_body: with temp_seed(env.rank): np.random.shuffle(self.realign_idx) if self.realign_method == "aligned": self.realign_idx = np.arange(start=0,stop=self.length_general+self.length_joints+self.length_feet) elif self.realign_method == "general_only": self.realign_idx = np.concatenate(( self.realign_idx, np.arange(start=self.length_general, stop=self.length_general+self.length_joints+self.length_feet) )) elif self.realign_method == "joints_only": self.realign_idx = np.concatenate(( np.arange(start=0, stop=self.length_general), self.realign_idx + self.length_general, np.arange(start=self.length_general+self.length_joints, stop=self.length_general+self.length_joints+self.length_feet) )) elif self.realign_method == "feetcontact_only": self.realign_idx = np.concatenate(( np.arange(start=0, stop=self.length_general+self.length_joints), self.realign_idx + self.length_general + self.length_joints, )) elif self.realign_method == "general_joints": self.realign_idx = np.concatenate(( self.realign_idx, np.arange(start=self.length_general+self.length_joints, stop=self.length_general+self.length_joints+self.length_feet), )) elif self.realign_method == "general_feetcontact": for i, v in enumerate(self.realign_idx): if v>=self.length_general: self.realign_idx[i] = v+self.length_joints self.realign_idx = np.concatenate(( self.realign_idx[:self.length_general], np.arange(start=self.length_general, stop=self.length_general+self.length_joints), self.realign_idx[self.length_general:] )) elif self.realign_method == "joints_feetcontact": self.realign_idx = np.concatenate(( np.arange(start=0, stop=self.length_general), self.realign_idx + self.length_general )) elif self.realign_method == "general_joints_feetcontact": self.realign_idx = self.realign_idx else: raise NotImplementedError print(self.realign_idx)
print(f"# default alignment for 1xx: {str_meaninful_alignment}") str_md5 = hashlib.md5(str_meaninful_alignment.encode()).hexdigest() print(f"# {str_md5}") str_md5 = hashlib.md5("::".join([str_meaninful_alignment] * 2).encode()).hexdigest() print(f"# {str_md5}") num_jobs = 0 all_jobs = [] for num_mutate in [128]: for seed in range(40): new_alignments = ga_mutate(alignments, n=num_mutate, seed=seed) custom_alignment = get_str_alignments(new_alignments) print(f"\n# mutate {num_mutate} alignment for 1xx: {custom_alignment}") str_md5 = hashlib.md5(custom_alignment.encode()).hexdigest() print(f"# {str_md5}") with seeds.temp_seed(seed): run_seeds = np.random.randint(low=0, high=10000, size=[5]) for run_seed in run_seeds: cmd = f"sbatch -J re_search_1xx submit.sh python 1.train.py --seed={run_seed} --custom_alignment={custom_alignment} --tensorboard=tensorboard/1xx_mutate_{num_mutate}_rerun --topology_wrapper=CustomAlignWrapper --custom_align_max_joints=8 --train_bodies=100,101,102,103,104,105,106,107 --test_bodies=100,101,102,103,104,105,106,107 --train_steps=1e7" print(cmd) job = { "id": num_jobs, "num_mutate": num_mutate, "body_seed": seed, "str_md5": str_md5, "custom_alignment": custom_alignment, "run_seed": run_seed, } all_jobs.append(job) num_jobs += 1 print(f"# Total jobs: {num_jobs}.")
def __init__(self, env): super().__init__(env) self.align_idx = list(range(self.obs_length)) with temp_seed(self.robot.robot_id): np.random.shuffle(self.align_idx) print(f"Random Wrapper: {self.align_idx}")
def __init__(self, env: WalkerWrapper): """Before using this wrapper, first wrap with WalkerWrapper""" super().__init__(env) self.realign_method = common.args.realign_method print(self.realign_method) if self.realign_method == "general_only": self.realign_length = 8 elif self.realign_method == "joints_only": self.realign_length = 16 elif self.realign_method == "feetcontact_only": self.realign_length = 6 elif self.realign_method == "general_joints": self.realign_length = 8 + 16 elif self.realign_method == "general_feetcontact": self.realign_length = 8 + 6 elif self.realign_method == "joints_feetcontact": self.realign_length = 16 + 6 elif self.realign_method == "general_joints_feetcontact": self.realign_length = 8 + 16 + 6 else: raise NotImplementedError self.realign_idx = np.arange(self.realign_length) with temp_seed(common.seed): np.random.shuffle(self.realign_idx) with temp_seed(self.robot.robot_id): np.random.shuffle(self.realign_idx) if common.args.random_even_same_body: with temp_seed(env.rank): np.random.shuffle(self.realign_idx) if self.realign_method == "general_only": self.realign_idx = np.concatenate( (self.realign_idx, np.arange(start=8, stop=8 + 16 + 6))) elif self.realign_method == "joints_only": self.realign_idx = np.concatenate( (np.arange(start=0, stop=8), self.realign_idx + 8, np.arange(start=8 + 16, stop=8 + 16 + 6))) elif self.realign_method == "feetcontact_only": self.realign_idx = np.concatenate(( np.arange(start=0, stop=8 + 16), self.realign_idx + 8 + 16, )) elif self.realign_method == "general_joints": self.realign_idx = np.concatenate(( self.realign_idx, np.arange(start=8 + 16, stop=8 + 16 + 6), )) elif self.realign_method == "general_feetcontact": self.realign_idx = np.concatenate( (self.realign_idx[:8], np.arange(start=8, stop=8 + 16), self.realign_idx[8:])) elif self.realign_method == "joints_feetcontact": self.realign_idx = np.concatenate( (np.arange(start=0, stop=8), self.realign_idx)) elif self.realign_method == "general_joints_feetcontact": self.realign_idx = self.realign_idx else: raise NotImplementedError print(self.realign_idx)
# Maximum joint number is 10. # a0[0,1,2,3,4,5,6,7,8,9] # a1[0,1,2,3,4,5,6,7,8,9] # ... # a15[0,1,2,3,4,5,6,7,8,9] # Total possible combinations = (Pr10)^(16-1) = (10!)^15 (A huge number.) # So we randomly shuffle them, and if repeats, re-shuffle to get a new one. import numpy as np from common import seeds with seeds.temp_seed(0): for j in range(2): joint_orders = [] for i in range(16): joint_order = np.arange(10) np.random.shuffle(joint_order) joint_orders.append(joint_order) joint_orders = np.array(joint_orders) print(joint_orders)
xml += f"</body>" max_depth += 1 return xml def reset_global_variables(): global max_depth, joint_number, max_limb, lowest_point joint_number = 0 max_depth = 3 max_limb = 10 lowest_point = 0 for i in range(100): reset_global_variables() with seeds.temp_seed(i): xml_worldbody = to_xml(worldbody, 0) print(f"lowest_point is {lowest_point}") adjusted_worldbody = worldbody.copy() above_ground = 0.1 adjusted_worldbody["pos"][ 2] = -lowest_point + adjusted_worldbody["length"][2] / 2 + above_ground print(f"\nadjusted to {adjusted_worldbody['pos'][2]}.\n") reset_global_variables() with seeds.temp_seed(i): xml_worldbody = to_xml(adjusted_worldbody, adjusted_worldbody["pos"][2]) xml_template = f""" <!-- joint_number: {joint_number}