def get_train_test( eval_setup_name: str, fold_id: int, use_test_split: bool) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: train, dev, test = phyre.get_fold(eval_setup_name, fold_id) if use_test_split: return train + dev, test else: return train, dev
def __init__(self, data_root, split, image_ext='.jpg'): self.data_root = data_root self.split = split self.image_ext = image_ext self.input_size = C.RPIN.INPUT_SIZE # number of input images self.pred_size = eval( f'C.RPIN.PRED_SIZE_{"TRAIN" if split == "train" else "TEST"}') self.seq_size = self.input_size + self.pred_size self.input_height, self.input_width = C.RPIN.INPUT_HEIGHT, C.RPIN.INPUT_WIDTH protocal = C.PHYRE_PROTOCAL fold = C.PHYRE_FOLD num_pos = 400 if split == 'train' else 100 num_neg = 1600 if split == 'train' else 400 eval_setup = f'ball_{protocal}_template' train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold) tasks = train_tasks + dev_tasks if split == 'train' else test_tasks action_tier = phyre.eval_setup_to_action_tier(eval_setup) # all the actions cache = phyre.get_default_100k_cache('ball') training_data = cache.get_sample(tasks, None) # (100000 x 3) actions = training_data['actions'] # (num_tasks x 100000) sim_statuses = training_data['simulation_statuses'] self.simulator = phyre.initialize_simulator(tasks, action_tier) self.video_info = np.zeros((0, 4)) for t_id, t in enumerate(tqdm(tasks)): sim_status = sim_statuses[t_id] pos_acts = actions[sim_status == 1].copy() neg_acts = actions[sim_status == -1].copy() np.random.shuffle(pos_acts) np.random.shuffle(neg_acts) pos_acts = pos_acts[:num_pos] neg_acts = neg_acts[:num_neg] acts = np.concatenate([pos_acts, neg_acts]) video_info = np.zeros((acts.shape[0], 4)) video_info[:, 0] = t_id video_info[:, 1:] = acts self.video_info = np.concatenate([self.video_info, video_info])
def simulate_result(chosen_action, chosen_score, model_number, generation_number): eval_setup = 'ball_cross_template' fold_id = 0 # For simplicity, we will just use one fold for evaluation. train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, 0) action_tier = phyre.eval_setup_to_action_tier(eval_setup) tasks = dev_tasks[0:1] simulator = phyre.initialize_simulator(tasks, action_tier) evaluator = phyre.Evaluator(tasks) # Simulate the given action and add the status from taking the action to the evaluator. simulation_result = simulator.simulate_action(0, chosen_action, need_images=True, need_featurized_objects=True) simulation_score = sf.ScoreFunctionValue(simulation_result) pair = np.array([chosen_action, simulation_score]) timestr = time.strftime("%Y%m%d-%H%M%S") score_pair = [ chosen_score, simulation_score, model_number, generation_number ] score_string = "ScoreLog" + timestr path = "/home/kyra/Desktop/phyre/agents/Scores" np.save(os.path.join(path, score_string), score_pair) return pair, simulation_result
def test(self, start_id=0, end_id=25): random.seed(0) np.random.seed(0) protocal, fold_id = C.PHYRE_PROTOCAL, C.PHYRE_FOLD self.score_model.eval() print(f'testing using protocal {protocal} and fold {fold_id}') # setup the PHYRE evaluation split eval_setup = f'ball_{protocal}_template' action_tier = phyre.eval_setup_to_action_tier(eval_setup) _, _, test_tasks = phyre.get_fold(eval_setup, fold_id) # PHYRE setup candidate_list = [f'{i:05d}' for i in range(start_id, end_id)] # filter tasks test_list = [ task for task in test_tasks if task.split(':')[0] in candidate_list ] simulator = phyre.initialize_simulator(test_list, action_tier) # the action candidates are provided by the author of PHYRE benchmark num_actions = 10000 cache = phyre.get_default_100k_cache('ball') acts = cache.action_array[:num_actions] training_data = cache.get_sample(test_list, None) # some statistics variable when doing the evaluation auccess = np.zeros((len(test_list), 100)) batched_pred = C.SOLVER.BATCH_SIZE objs_color = None all_data, all_acts, all_rois, all_image = [], [], [], [] # cache the initial bounding boxes from the simulator os.makedirs('cache', exist_ok=True) t_list = tqdm(test_list, 'Task') for task_id, task in enumerate(t_list): sim_statuses = training_data['simulation_statuses'][task_id] confs, successes = [], [] boxes_cache_name = f'cache/{task.replace(":", "_")}.hkl' use_cache = os.path.exists(boxes_cache_name) all_boxes = hickle.load(boxes_cache_name) if use_cache else [] valid_act_id = 0 for act_id, act in enumerate( tqdm(acts, 'Candidate Action', leave=False)): sim = simulator.simulate_action(task_id, act, stride=60, need_images=True, need_featurized_objects=True) assert sim.status == sim_statuses[ act_id], 'sanity check not passed' if sim.status == phyre.SimulationStatus.INVALID_INPUT: if act_id == len(acts) - 1 and len( all_data) > 0: # final action is invalid conf_t = self.batch_score(all_data, all_rois, all_image, objs_color) confs = confs + conf_t all_data, all_acts, all_rois, all_image = [], [], [], [] continue successes.append(sim.status == phyre.SimulationStatus.SOLVED) # parse object, prepare input for network, the logic is the same as tools/gen_phyre.py image = cv2.resize(sim.images[0], (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST) all_image.append(image[::-1]) image = phyre.observations_to_float_rgb(image) objs_color = sim.featurized_objects.colors objs_valid = [('BLACK' not in obj_color) and ('PURPLE' not in obj_color) for obj_color in objs_color] objs = sim.featurized_objects.features[:, objs_valid, :] objs_color = np.array(objs_color)[objs_valid] num_objs = objs.shape[1] if use_cache: boxes = all_boxes[valid_act_id] valid_act_id += 1 else: boxes = np.zeros((1, num_objs, 5)) for o_id in range(num_objs): mask = phyre.objects_util.featurized_objects_vector_to_raster( objs[0][[o_id]]) mask_im = phyre.observations_to_float_rgb(mask) mask_im[mask_im == 1] = 0 mask_im = mask_im.sum(-1) > 0 [h, w] = np.where(mask_im) x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max() x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) y1 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) y2 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) boxes[0, o_id] = [o_id, x1, y1, x2, y2] all_boxes.append(boxes) data = image.transpose((2, 0, 1))[None, None, :] data = torch.from_numpy(data.astype(np.float32)) rois = torch.from_numpy(boxes[..., 1:].astype(np.float32))[None, :] all_data.append(data) all_rois.append(rois) if len(all_data) % batched_pred == 0 or act_id == len( acts) - 1: conf_t = self.batch_score(all_data, all_rois, all_image, objs_color) confs = confs + conf_t all_data, all_rois, all_image = [], [], [] if not use_cache: all_boxes = np.stack(all_boxes) hickle.dump(all_boxes, boxes_cache_name, mode='w', compression='gzip') info = f'current AUCESS: ' top_acc = np.array(successes)[np.argsort(confs)[::-1]] for i in range(100): auccess[task_id, i] = int(np.sum(top_acc[:i + 1]) > 0) w = np.array([np.log(k + 1) - np.log(k) for k in range(1, 101)]) s = auccess[:task_id + 1].sum(0) / auccess[:task_id + 1].shape[0] info += f'{np.sum(w * s) / np.sum(w) * 100:.2f}' t_list.set_description(info)
def gen_proposal(self, start_id=0, end_id=25): random.seed(0) np.random.seed(0) protocal = C.PHYRE_PROTOCAL fold_id = C.PHYRE_FOLD print(f'generate proposal for {protocal} fold {fold_id}') max_p_acts, max_n_acts, max_acts = 200, 800, 100000 self.proposal_dir = f'{self.output_dir.split("/")[-1]}_' \ f'p{max_p_acts}n{max_n_acts}a{max_acts // 1000}' eval_setup = f'ball_{protocal}_template' action_tier = phyre.eval_setup_to_action_tier(eval_setup) train_tasks, dev_tasks, test_tasks = phyre.get_fold( eval_setup, fold_id) # filter task train_tasks = train_tasks + dev_tasks candidate_list = [f'{i:05d}' for i in range(start_id, end_id)] for split in ['train', 'test']: train_list = [ task for task in train_tasks if task.split(':')[0] in candidate_list ] test_list = [ task for task in test_tasks if task.split(':')[0] in candidate_list ] if len(eval(f'{split}_list')) == 0: return simulator = phyre.initialize_simulator(eval(f'{split}_list'), action_tier) cache = phyre.get_default_100k_cache('ball') training_data = cache.get_sample(eval(f'{split}_list'), None) actions = cache.action_array[:max_acts] final_list = eval(f'{split}_list') t_list = tqdm(final_list, 'Task') for task_id, task in enumerate(t_list): box_cache_name = f'data/PHYRE_proposal/cache/{task.replace(":", "_")}_box.hkl' act_cache_name = f'data/PHYRE_proposal/cache/{task.replace(":", "_")}_act.hkl' use_cache = os.path.exists(box_cache_name) and os.path.exists( act_cache_name) if use_cache: acts = hickle.load(act_cache_name) all_boxes = hickle.load(box_cache_name) else: sim_statuses = training_data['simulation_statuses'][ task_id] pos_acts = actions[sim_statuses == 1] neg_acts = actions[sim_statuses == -1] np.random.shuffle(pos_acts) np.random.shuffle(neg_acts) pos_acts = pos_acts[:max_p_acts] neg_acts = neg_acts[:max_n_acts] acts = np.concatenate([pos_acts, neg_acts]) hickle.dump(acts, act_cache_name, mode='w', compression='gzip') all_boxes = [] valid_act_id = 0 for act_id, act in enumerate( tqdm(acts, 'Candidate Action', leave=False)): sim = simulator.simulate_action( task_id, act, stride=60, need_images=True, need_featurized_objects=True) if not use_cache: if act_id < len(pos_acts): assert sim.status == phyre.SimulationStatus.SOLVED else: assert sim.status == phyre.SimulationStatus.NOT_SOLVED assert sim.status != phyre.SimulationStatus.INVALID_INPUT raw_images = sim.images rst_images = np.stack([ np.ascontiguousarray( cv2.resize(rst_image, (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST)[::-1]) for rst_image in raw_images ]) # prepare input for network: image = cv2.resize(raw_images[0], (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST) image = phyre.observations_to_float_rgb(image) # parse object objs_color = sim.featurized_objects.colors objs_valid = [('BLACK' not in obj_color) and ('PURPLE' not in obj_color) for obj_color in objs_color] objs = sim.featurized_objects.features[:, objs_valid, :] objs_color = np.array(objs_color)[objs_valid] num_objs = objs.shape[1] if use_cache: boxes = all_boxes[valid_act_id] valid_act_id += 1 else: boxes = np.zeros((1, num_objs, 5)) for o_id in range(num_objs): mask = phyre.objects_util.featurized_objects_vector_to_raster( objs[0][[o_id]]) mask_im = phyre.observations_to_float_rgb(mask) mask_im[mask_im == 1] = 0 mask_im = mask_im.sum(-1) > 0 [h, w] = np.where(mask_im) x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max() x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) y1 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) y2 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) boxes[0, o_id] = [o_id, x1, y1, x2, y2] all_boxes.append(boxes) data = image.transpose((2, 0, 1))[None, None, :] data = torch.from_numpy(data.astype(np.float32)) rois = torch.from_numpy(boxes[..., 1:].astype( np.float32))[None, :] bg_image = rst_images[0].copy() for fg_id in [1, 2, 3, 5]: bg_image[bg_image == fg_id] = 0 boxes, masks = self.generate_trajs(data, rois) rst_masks = np.stack([ self.render_mask_to_image(boxes[0, i], masks[0, i], images=bg_image.copy(), color=objs_color).astype( np.uint8) for i in range(self.pred_rollout) ]) output_dir = f'data/PHYRE_proposal/{self.proposal_dir}/{split}/' output_dir = output_dir + 'pos/' if sim.status == phyre.SimulationStatus.SOLVED else output_dir + 'neg/' output_dir = output_dir + f'{task.replace(":", "_")}/' os.makedirs(output_dir, exist_ok=True) rst_dict = {'gt_im': rst_images, 'pred_im': rst_masks} hickle.dump(rst_dict, f'{output_dir}/{act_id}.hkl', mode='w', compression='gzip') if not use_cache: all_boxes = np.stack(all_boxes) hickle.dump(all_boxes, box_cache_name, mode='w', compression='gzip')
dtrain = '-dtrain' in sys.argv singleviz = '-singleviz' in sys.argv auccess = [] auc_dict = dict() #for eval_setup in ['ball_cross_template', 'ball_within_template']: for eval_setup in ['ball_within_template', 'ball_cross_template']: auccess.append(eval_setup) print(foldstart, folds) for fold_id in range(foldstart, foldstart+folds): solver = FlownetSolver(model_path, type, width, fold=fold_id, setup=eval_setup, bs=batchsize, uniform=uniform, radmode=radmode, scheduled= scheduled, lr=lr, smart=smart, run=run, num_seeds=seeds, epochstart=epochstart, device=device, hidfac=hidfac, dijkstra=dijkstra, dropout=dropout, deepex=deepex, neck=neckfak, altconv=altconv, train_mode=train_mode, puretrain=puretrain) train_ids, dev_ids, test_ids = phyre.get_fold(eval_setup, fold_id) train_ids = train_ids + dev_ids dev_ids = tuple() if "-inspect" in sys.argv: inspect_ids = [] tmp_ids = list(dev_ids)+list(test_ids) count = dict(zip([i[:5] for i in tmp_ids],[0 for _ in tmp_ids])) num_per = 4 if eval_setup == 'ball_within_template' else 10 for tid in tmp_ids: if count[tid[:5]] < num_per: count[tid[:5]] += 1 inspect_ids.append(tid) #L.info("inpsect ids:", inspect_ids) if "-get-all-data" in sys.argv:
import random import numpy as np import phyre from tqdm import tqdm_notebook import animations random.seed(0) # Evaluation Setup eval_setup = 'ball_cross_template' fold_id = 0 # For simplicity, we will just use one fold for evaluation. train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, 0) action_tier = phyre.eval_setup_to_action_tier(eval_setup) tasks = dev_tasks[0:1] print((tasks)) simulator = phyre.initialize_simulator(tasks, action_tier) actions = simulator.build_discrete_action_space(max_actions=1000) def evaluate_random_agent(tasks, tier): # Create a simulator for the task and tier. simulator = phyre.initialize_simulator(tasks, tier) evaluator = phyre.Evaluator(tasks) assert tuple(tasks) == simulator.task_ids images = [] actions = [] for task_index in tqdm_notebook(range(len(tasks)), desc='Evaluate tasks'): while evaluator.get_attempts_for_task( task_index) < phyre.MAX_TEST_ATTEMPTS:
def worker(fold_id, eval_setup): __, __, test_tasks = phyre.get_fold(eval_setup, fold_id) action_tier = phyre.eval_setup_to_action_tier(eval_setup) tasks = test_tasks evaluator = evaluate_simple_agent(tasks, action_tier) return evaluator.get_aucess()
def solve(model, model2, save_images=False): tasks = [ '00000:001', '00000:002', '00000:003', '00000:004', '00000:005', '00001:001', '00001:002', '00001:003', '00001:004', '00001:005', '00002:007', '00002:011', '00002:015', '00002:017', '00002:023', '00003:000', '00003:001', '00003:002', '00003:003', '00003:004', '00004:063', '00004:071', '00004:092', '00004:094', '00004:095' ] tasks = json.load(open("most_tasks.txt", 'r')) eval_setup = 'ball_within_template' fold_id = 0 # For simplicity, we will just use one fold for evaluation. train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold_id) print('Size of resulting splits:\n train:', len(train_tasks), '\n dev:', len(dev_tasks), '\n test:', len(test_tasks)) tasks = train_tasks[:] print("tasks:\n", tasks) sim = phyre.initialize_simulator(tasks, 'ball') init_scenes = sim.initial_scenes X = T.tensor(format(init_scenes)).float() print("Init Scenes Shape:\n", X.shape) base_path = [] action_path = [] for i, t in enumerate(tasks): while True: action = sim.sample(i) action[2] = 0.01 res = sim.simulate_action(i, action, stride=20) if type(res.images) != type(None): base_path.append(rollouts_to_channel([res.images], 2)) action_path.append(rollouts_to_channel([res.images], 1)) break base_path = T.tensor(np.concatenate(base_path)).float() action_path = T.tensor(np.concatenate(base_path)).float() with T.no_grad(): Z = model(X) A = model2(T.cat((X[:, 1:], base_path[:, None], Z), dim=1)) #B = model3(T.cat((X[:,1:], Y[:,None,2], Z, A), dim=1)) #B = extract_action(A, inspect=-2 if save_images else -1) B = extract_action(action_path[:, None], inspect=-2 if save_images else -1) # Saving Images: if save_images: for inspect in range(len(X)): plt.imsave( f"result/flownet/{inspect}_init.png", T.cat(tuple( T.cat((sub, T.ones(32, 1) * 0.5), dim=1) for sub in X[inspect]), dim=1)) plt.imsave(f"result/flownet/{inspect}_base.png", base_path[inspect]) plt.imsave(f"result/flownet/{inspect}_target.png", Z[inspect, 0]) #plt.imsave(f"result/flownet/{inspect}_init_scene.png", np.flip(batch[inspect][0], axis=0)) plt.imsave(f"result/flownet/{inspect}_action.png", A[inspect, 0]) plt.imsave(f"result/flownet/{inspect}_selection.png", B[inspect, 0]) gen_actions = [] for b in B[:, 0]: gen_actions.append(pic_to_values(b)) print(gen_actions) # Feed actions into simulator eva = phyre.Evaluator(tasks) solved, valid, comb = dict(), dict(), dict() for i, t in enumerate(tasks): if not (t[:5] in comb): comb[t[:5]] = 0 valid[t[:5]] = 0 solved[t[:5]] = 0 base_action = gen_actions[i] # Random Agent Intercept: #action = sim.sample() res = sim.simulate_action(i, base_action) tries = 0 alpha = 1 # 100 Tries Max: while eva.get_attempts_for_task(i) < 100: if not res.status.is_solved(): action = np.array(base_action) + np.random.randn(3) * np.array( [0.1, 0.1, 0.1]) * alpha res = sim.simulate_action(i, action) subtries = 0 while subtries < 100 and res.status.is_invalid(): subtries += 1 action_var = np.array(action) + np.random.randn( 3) * np.array([0.05, 0.05, 0.05]) * alpha res = sim.simulate_action(i, action_var) eva.maybe_log_attempt(i, res.status) alpha *= 1.01 else: eva.maybe_log_attempt(i, res.status) tries += 1 if save_images: try: for k, img in enumerate(res.images): plt.imsave(f"result/flownet/{i}_{k}.png", np.flip(img, axis=0)) pass except Exception: pass #print(i, t, res.status.is_solved(), not res.status.is_invalid()) comb[t[:5]] = comb[t[:5]] + 1 if not res.status.is_invalid(): valid[t[:5]] = valid[t[:5]] + 1 if res.status.is_solved(): solved[t[:5]] = solved[t[:5]] + 1 # Prepare Plotting print(eva.compute_all_metrics()) print(eva.get_auccess()) spacing = [1, 2, 3, 4] fig, ax = plt.subplots(5, 5, sharey=True, sharex=True) for i, t in enumerate(comb): ax[i // 5, i % 5].bar(spacing, [ solved[t[:5]] / (valid[t[:5]] if valid[t[:5]] else 1), solved[t[:5]] / comb[t[:5]], valid[t[:5]] / comb[t[:5]], comb[t[:5]] / 100 ]) ax[i // 5, i % 5].set_xlabel(t[:5]) plt.show()
def worker(fold_id, eval_setup): train, dev, test = phyre.get_fold(eval_setup, fold_id) action_tier = phyre.eval_setup_to_action_tier(eval_setup) solved_actions_pdf = train_kde(train, action_tier) return evaluate_agent(test, action_tier, solved_actions_pdf)
import numpy as np from scipy.stats import skewnorm from scipy.stats import gaussian_kde import phyre import ImgToObj tier = 'ball' eval_setup = 'ball_cross_template' fold_id = 2 random.seed(0) train, dev, test = phyre.get_fold(eval_setup, fold_id) cache = phyre.get_default_100k_cache(tier) all_solved_sizes = {} all_solved_loc = {} for index, task_id in enumerate(train): task_type = task_id.split(":")[0] statuses = cache.load_simulation_states(task_id) solved_actions = cache.action_array[statuses == phyre.simulation_cache.SOLVED, :] solved_sizes = solved_actions[:, 2] solved_loc = solved_actions[:, 0:2] if task_type not in all_solved_loc: all_solved_loc[task_type] = solved_loc
def worker(fold_id, eval_setup): __, __, test_tasks = phyre.get_fold(eval_setup, fold_id) action_tier = phyre.eval_setup_to_action_tier(eval_setup) tasks = test_tasks return evaluate_agent(tasks, action_tier)
def generate_data(args): eval_setup = 'ball_cross_template' fold_id = 0 train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold_id) action_tier = phyre.eval_setup_to_action_tier(eval_setup) tasks_2 = [x for x in train_tasks if x.startswith(args.template)==True] tasks = tasks_2 simulator = phyre.initialize_simulator(tasks, action_tier) actions = simulator.build_discrete_action_space(max_actions=100000) try: os.mkdir('./data-generation/numpys') except: pass try: os.mkdir('./data-generation/episodes') except: pass for task_index in tqdm_notebook(range(1, len(tasks_2))): solution_found_counter = 0 # 10 rollouts for each for k in range(10): if solution_found_counter < 5: action = random.choice(actions) simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True, stride=4) num_trys = 0 while (simulation.status != phyre.simulation_cache.SOLVED and num_trys < 10000): num_trys += 1 action = random.choice(actions) simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True, stride=4) if str(simulation.status) == "SimulationStatus.INVALID_INPUT": num_trys -= 1 else: action = random.choice(actions) simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True, stride=4) while (simulation.status != phyre.simulation_cache.NOT_SOLVED): action = random.choice(actions) simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True, stride=4) print(simulation.status) filename = 'data-generation/numpys/task-00023:' + str(task_index + 1) + '_' + str(k) + '.npy' print(filename) np.save(filename, simulation.images) if str(simulation.status) == "SimulationStatus.SOLVED": solution_found_counter += 1 dataset_path = './data-generation/numpys' IMAGE_WIDTH = 256 IMAGE_HEIGHT = 256 max_episode_len = 120 for filename in os.listdir(dataset_path): if filename.startswith('Task'): continue counter = 0 print("Laded file: ", filename) data = os.path.join(dataset_path, filename) try: data = np.load(data) except: continue print(data.shape) if (len(data) >= max_episode_len): images = np.zeros((len(data), 64, 64, 3), dtype=np.uint8) rewards = np.ones((len(data)), dtype=np.float16) actions = np.ones((len(data), 6), dtype=np.float16) orientations = np.ones((len(data), 14), dtype=np.float16) velocity = np.ones((len(data), 9), dtype=np.float16) height = np.ones((len(data)), dtype=np.float16) discount = np.ones((len(data)), dtype=np.float16) for k, scene in enumerate(data): current_image = np.zeros((256, 256, 3), dtype=np.uint8) channel_0 = np.copy(np.flipud((scene))) channel_1 = np.copy(np.flipud((scene))) channel_0[channel_0 == 6] = 0 channel_0[channel_0 > 0] = 255 channel_1[channel_1 != 6] = 0 channel_1[channel_1 == 6] = 255 current_image[:, :, 0] = np.copy(channel_0) current_image[:, :, 1] = np.copy(channel_1) scaled_down_image = np.copy(current_image[::4,::4,:]) images[k] = scaled_down_image numpy_dict = {"image": images[:max_episode_len], "action": actions[:max_episode_len], "reward": rewards[:max_episode_len], "orientations": orientations[:max_episode_len], "velocity": velocity[:max_episode_len], "dsicount": discount[:max_episode_len], "height": height[:max_episode_len]} timestamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') identifier = str(uuid.uuid4().hex) length = len(numpy_dict['reward']) directory = pathlib.Path('data-generation/episodes') filename = pathlib.Path(f'{timestamp}-{identifier}-{length}.npz') filename = directory / filename with io.BytesIO() as f1: np.savez_compressed(f1, **numpy_dict) f1.seek(0) with filename.open('wb') as f2: f2.write(f1.read())
def get_train_test(eval_setup_name, fold_id, use_test_split): train, dev, test = phyre.get_fold(eval_setup_name, fold_id) if use_test_split: return train + dev, test else: return train, dev
def test(self, start_id=0, end_id=25, fold_id=0, protocal='within'): random.seed(0) print(f'testing {protocal} fold {fold_id}') eval_setup = f'ball_{protocal}_template' action_tier = phyre.eval_setup_to_action_tier(eval_setup) _, _, test_tasks = phyre.get_fold(eval_setup, fold_id) # PHYRE setup candidate_list = [f'{i:05d}' for i in range(start_id, end_id)] # filter tasks test_list = [task for task in test_tasks if task.split(':')[0] in candidate_list] simulator = phyre.initialize_simulator(test_list, action_tier) # PHYRE evaluation num_all_actions = [1000, 2000, 5000, 8000, 10000] auccess = np.zeros((len(num_all_actions), len(test_list), 100)) batched_pred = C.SOLVER.BATCH_SIZE # DATA for network: all_data, all_acts, all_rois, all_image = [], [], [], [] cache = phyre.get_default_100k_cache('ball') acts = cache.action_array[:10000] # actions = cache.action_array[:100000] # training_data = cache.get_sample(test_list, None) pos_all, neg_all, pos_correct, neg_correct = 0, 0, 0, 0 objs_color = None for task_id, task in enumerate(test_list): confs, successes, num_valid_act_idx = [], [], [] boxes_cache_name = f'cache/{task.replace(":", "_")}.hkl' use_cache = os.path.exists(boxes_cache_name) all_boxes = hickle.load(boxes_cache_name) if use_cache else [] valid_act_cnt = 0 # sim_statuses = training_data['simulation_statuses'][task_id] # pos_acts = actions[sim_statuses == 1] # neg_acts = actions[sim_statuses == -1] # np.random.shuffle(pos_acts) # np.random.shuffle(neg_acts) # pos_acts = pos_acts[:50] # neg_acts = neg_acts[:200] # acts = np.concatenate([pos_acts, neg_acts]) for act_id, act in enumerate(acts): if act_id == 0: pprint(f'{task}: {task_id} / {len(test_list)}') sim = simulator.simulate_action(task_id, act, stride=60, need_images=True, need_featurized_objects=True) if sim.status == phyre.SimulationStatus.INVALID_INPUT: num_valid_act_idx.append(0) if act_id == len(acts) - 1 and len(all_data) > 0: # final action is invalid conf_t = self.batch_score(all_data, all_acts, all_rois, all_image, objs_color, task) confs = confs + conf_t all_data, all_acts, all_rois, all_image = [], [], [], [] continue num_valid_act_idx.append(1) successes.append(sim.status == phyre.SimulationStatus.SOLVED) if self.score_with_heuristic or self.score_with_mask: # parse object, prepare input for network: image = cv2.resize(sim.images[0], (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST) all_image.append(image[::-1]) # for heuristic method to detect goal location, need to flip image = phyre.observations_to_float_rgb(image) objs_color = sim.featurized_objects.colors objs_valid = [('BLACK' not in obj_color) and ('PURPLE' not in obj_color) for obj_color in objs_color] objs = sim.featurized_objects.features[:, objs_valid, :] objs_color = np.array(objs_color)[objs_valid] num_objs = objs.shape[1] if use_cache: boxes = all_boxes[valid_act_cnt] valid_act_cnt += 1 else: boxes = np.zeros((1, num_objs, 5)) for o_id in range(num_objs): mask = phyre.objects_util.featurized_objects_vector_to_raster(objs[0][[o_id]]) mask_im = phyre.observations_to_float_rgb(mask) mask_im[mask_im == 1] = 0 mask_im = mask_im.sum(-1) > 0 [h, w] = np.where(mask_im) x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max() x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1) y1 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) y2 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1) boxes[0, o_id] = [o_id, x1, y1, x2, y2] all_boxes.append(boxes) data = image.transpose((2, 0, 1))[None, None, :] data = torch.from_numpy(data.astype(np.float32)) rois = torch.from_numpy(boxes[..., 1:].astype(np.float32))[None, :] all_data.append(data) all_rois.append(rois) elif self.score_with_act: init = np.ascontiguousarray(simulator.initial_scenes[task_id][::-1]) init128 = cv2.resize(init, (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST) all_data.append(torch.from_numpy(init128)) all_acts.append(torch.from_numpy(act[None, :])) elif self.score_with_vid_cls: rst_images = np.stack([np.ascontiguousarray( cv2.resize(rst_image, (self.input_width, self.input_height), interpolation=cv2.INTER_NEAREST)[::-1] ) for rst_image in sim.images]) all_data.append(torch.from_numpy(rst_images)) else: raise NotImplementedError if len(all_data) % batched_pred == 0 or act_id == len(acts) - 1: conf_t = self.batch_score(all_data, all_acts, all_rois, all_image, objs_color, task) confs = confs + conf_t all_data, all_acts, all_rois, all_image = [], [], [], [] if self.score_with_heuristic or self.score_with_mask: if not use_cache: all_boxes = np.stack(all_boxes) hickle.dump(all_boxes, boxes_cache_name, mode='w', compression='gzip') else: assert valid_act_cnt == len(all_boxes) pred = np.array(confs) >= 0.5 labels = np.array(successes) pos_all += (labels == 1).sum() neg_all += (labels == 0).sum() pos_correct += (pred == labels)[labels == 1].sum() neg_correct += (pred == labels)[labels == 0].sum() pos_acc = (pred == labels)[labels == 1].sum() / (labels == 1).sum() neg_acc = (pred == labels)[labels == 0].sum() / (labels == 0).sum() info = f'{pos_acc * 100:.1f} / {neg_acc * 100:.1f} ' # info = f'{task}: ' for j, num_acts in enumerate(num_all_actions): num_valid = np.sum(num_valid_act_idx[:num_acts]) top_acc = np.array(successes[:num_valid])[np.argsort(confs[:num_valid])[::-1]] for i in range(100): auccess[j, task_id, i] = int(np.sum(top_acc[:i + 1]) > 0) w = np.array([np.log(k + 1) - np.log(k) for k in range(1, 101)]) s = auccess[j, :task_id + 1].sum(0) / auccess[j, :task_id + 1].shape[0] info += f'{np.sum(w * s) / np.sum(w) * 100:.2f} {np.sum(successes[:num_valid])}/{num_acts // 1000}k | ' pprint(info) pprint(pos_correct, pos_all, pos_correct / pos_all) pprint(neg_correct, neg_all, neg_correct / neg_all) cache_output_dir = f'{self.output_dir.replace("figures/", "")}/' \ f'{self.proposal_setting}_{self.method}_{protocal}_fold_{fold_id}/' os.makedirs(cache_output_dir, exist_ok=True) print(cache_output_dir) stats = { 'auccess': auccess, 'p_c': pos_correct, 'p_a': pos_all, 'n_c': neg_correct, 'n_a': neg_all, } with open(f'{cache_output_dir}/{start_id}_{end_id}.pkl', 'wb') as f: pickle.dump(stats, f, pickle.HIGHEST_PROTOCOL)