def analyze_instruction_set(name, iset, corpus, merge_len): token_lengths = [] demo_lengths = [] token2word, word2token = get_word_to_token_map(corpus) for e, instr_sets in iset.items(): segs = instr_sets[0]["instructions"] if len(segs) > 0: full_path = load_path(e) for seg in segs: if seg["merge_len"] != merge_len: continue tok_i = tokenize_instruction(seg['instruction'], word2token) start_idx = seg["start_idx"] end_idx = seg["end_idx"] seg_path = full_path[start_idx:end_idx] demo_len = path_length(seg_path) demo_lengths.append(demo_len) token_lengths.append(len(tok_i)) avg_tok_len = sum(token_lengths) / len(token_lengths) avg_pth_len = sum(demo_lengths) * 4.7 / (len(demo_lengths) * 1000) print("Dataset: ", name) print(" {} & {} & {:.2f} & {:.2f}".format(len(iset), len(token_lengths), avg_tok_len, avg_pth_len))
def __init__(self, env_list=None, instr_negatives=False, instr_negatives_similar_only=False, seg_level=False, img_scale=1, yaw_rand_range=0, pos_rand_range=0 ): # If data is already loaded in memory, use it self.env_list = env_list self.train_instr, self.dev_instr, self.test_instr, corpus = get_all_instructions() self.all_instr = {**self.train_instr, **self.dev_instr, **self.test_instr} self.token2term, self.word2token = get_word_to_token_map(corpus) self.thesaurus = load_thesaurus() self.include_instr_negatives = instr_negatives if instr_negatives: self.similar_instruction_map = load_similar_instruction_map() self.instr_negatives_similar_only = instr_negatives_similar_only self.img_scale = img_scale self.yaw_rand_range = yaw_rand_range self.pos_rand_range = pos_rand_range self.pos_rand_image = 0 # If the data is supposed to be at seg level (not nested envs + segs), then we can support batching # but we need to correctly infer the dataset size self.seg_level = seg_level if seg_level: self.seg_list = [] for env in self.env_list: for set_idx, set in enumerate(self.all_instr[env]): for seg_idx, seg in enumerate(set["instructions"]): self.seg_list.append([env, set_idx, seg_idx])
def __init__(self): super(AuxLabelsNL, self).__init__() self.thesaurus = load_landmark_alignments() train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions() self.all_instructions = {**train_instructions, **dev_instructions, **test_instructions} self.corpus = corpus self.word2token, self.token2term = get_word_to_token_map(corpus)
def __init__(self, env_list=None, instr_negatives=False, instr_negatives_similar_only=False, seg_level=False, yaw_rand_range=0, img_w=512, img_h=512, map_w=None, map_h=None, incl_path=True, incl_endpoint=False, use_semantic_maps=False): # If data is already loaded in memory, use it self.cuda = False self.env_list = env_list self.train_instr, self.dev_instr, self.test_instr, corpus = get_all_instructions() self.all_instr = {**self.train_instr, **self.dev_instr, **self.test_instr} self.token2term, self.word2token = get_word_to_token_map(corpus) self.thesaurus = load_thesaurus() self.include_instr_negatives = instr_negatives #if instr_negatives: # self.similar_instruction_map = load_similar_instruction_map() self.instr_negatives_similar_only = instr_negatives_similar_only self.use_semantic_maps = use_semantic_maps self.img_w = img_w self.img_h = img_h if map_w is None: self.map_w = self.img_w self.map_h = self.img_h else: self.map_w = map_w self.map_h = map_h self.yaw_rand_range = yaw_rand_range self.latest_img_dbg = None self.latest_rot_img_dbg = None self.incl_endpoint = incl_endpoint self.incl_path = incl_path # If the data is supposed to be at seg level (not nested envs + segs), then we can support batching # but we need to correctly infer the dataset size self.seg_level = seg_level if seg_level: self.seg_list = [] for env in self.env_list: for set_idx, set in enumerate(self.all_instr[env]): for seg_idx, seg in enumerate(set["instructions"]): self.seg_list.append([env, set_idx, seg_idx]) print("Initialzied dataset!") print(" yaw range : " + str(self.yaw_rand_range)) print(" map size: ", self.map_w, self.map_h) print(" img size: ", self.img_w, self.img_h)
def generate_thesaurus(): train_instr, dev_instr, test_instr, corpus = get_all_instructions() _, word2token = get_word_to_token_map(corpus, use_thesaurus=False) term2word, word2term = get_identity_term_mapping(corpus) term2landmark = get_template_term_groundings(corpus, word2term) thesaurus = { "term2word": term2word, "word2term": word2term, "term_groundings": term2landmark, "rejected_words": [] } save_thesaurus(thesaurus)
def __init__( self, model, state=None, epoch=0, name="", run_name="", ): _, _, _, corpus = get_all_instructions() self.token2word, self.word2token = get_word_to_token_map(corpus) self.params = get_current_parameters()["Training"] self.batch_size = self.params['batch_size'] self.weight_decay = self.params['weight_decay'] self.optimizer = self.params['optimizer'] self.num_loaders = self.params['num_loaders'] self.lr = self.params['lr'] self.name = name self.dataset_names = None n_params = get_n_params(model) n_params_tr = get_n_trainable_params(model) print("Training Model:") print("Number of model parameters: " + str(n_params)) print("Trainable model parameters: " + str(n_params_tr)) self.model = model self.run_name = run_name if self.optimizer == "adam": self.optim = optim.Adam(self.get_model_parameters(self.model), self.lr, weight_decay=self.weight_decay) elif self.optimizer == "sgd": self.optim = optim.SGD(self.get_model_parameters(self.model), self.lr, weight_decay=self.weight_decay, momentum=0.9) self.train_epoch_num = epoch self.train_segment = 0 self.test_epoch_num = epoch self.test_segment = 0 self.set_state(state) self.batch_num = 0
def generate_thesaurus(): train_instr, dev_instr, test_instr, corpus = get_all_instructions() _, word2token = get_word_to_token_map(corpus, use_thesaurus=False) terms, rejected_words = cluster_corpus(corpus, train_instr) landmark_names = get_landmark_names() term_groundings, word2term = ground_terms(word2token, terms, landmark_names, train_instr) thesaurus = { "term2word": terms, "word2term": word2term, "term_groundings": term_groundings, "rejected_words": rejected_words } save_thesaurus(thesaurus)
def interactive_demo(): P.initialize_experiment() InteractAPI.launch_ui() rate = Rate(0.1) env = PomdpInterface( is_real=get_current_parameters()["Setup"]["real_drone"]) train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions( ) all_instr = { **train_instructions, **dev_instructions, **train_instructions } token2term, word2token = get_word_to_token_map(corpus) # Run on dev set interact_instructions = dev_instructions env_range_start = get_current_parameters()["Setup"].get( "env_range_start", 0) env_range_end = get_current_parameters()["Setup"].get( "env_range_end", 10e10) interact_instructions = { k: v for k, v in interact_instructions.items() if env_range_start < k < env_range_end } count = 0 stuck_count = 0 model, _ = load_model(get_current_parameters()["Setup"]["model"]) InteractAPI.write_empty_instruction() InteractAPI.write_real_instruction("None") instruction_str = InteractAPI.read_instruction_file() print("Initial instruction: ", instruction_str) for instruction_sets in interact_instructions.values(): for set_idx, instruction_set in enumerate(instruction_sets): env_id = instruction_set['env'] env.set_environment(env_id, instruction_set["instructions"]) presenter = Presenter() cumulative_reward = 0 for seg_idx in range(len(instruction_set["instructions"])): print(f"RUNNING ENV {env_id} SEG {seg_idx}") real_instruction_str = instruction_set["instructions"][ seg_idx]["instruction"] InteractAPI.write_real_instruction(real_instruction_str) valid_segment = env.set_current_segment(seg_idx) if not valid_segment: continue state = env.reset(seg_idx) keep_going = True while keep_going: InteractAPI.write_real_instruction(real_instruction_str) while True: cv2.waitKey(200) instruction = InteractAPI.read_instruction_file() if instruction == "CMD: Next": print("Advancing") keep_going = False InteractAPI.write_empty_instruction() break elif instruction == "CMD: Reset": print("Resetting") env.reset(seg_idx) InteractAPI.write_empty_instruction() elif len(instruction.split(" ")) > 1: instruction_str = instruction break if not keep_going: continue env.override_instruction(instruction_str) tok_instruction = tokenize_instruction( instruction_str, word2token) state = env.reset(seg_idx) print("Executing: f{instruction_str}") while True: rate.sleep() action, internals = model.get_action( state, tok_instruction) state, reward, done, expired, oob = env.step(action) cumulative_reward += reward presenter.show_sample(state, action, reward, cumulative_reward, instruction_str) #show_depth(state.image) if done: break InteractAPI.write_empty_instruction() print("Segment finished!") print("Env finished!")
def train_top_down_pred(): P.initialize_experiment() setup = P.get_current_parameters()["Setup"] launch_ui() env = PomdpInterface() print("model_name:", setup["top_down_model"]) print("model_file:", setup["top_down_model_file"]) model, model_loaded = load_model( model_name_override=setup["top_down_model"], model_file_override=setup["top_down_model_file"]) exec_model, wrapper_model_loaded = load_model( model_name_override=setup["wrapper_model"], model_file_override=setup["wrapper_model_file"]) affine2d = Affine2D() if model.is_cuda: affine2d.cuda() eval_envs = get_correct_eval_env_id_list() print("eval_envs:", eval_envs) train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions( max_size=setup["max_envs"]) all_instr = { **train_instructions, **dev_instructions, **train_instructions } token2term, word2token = get_word_to_token_map(corpus) dataset = model.get_dataset(envs=eval_envs, dataset_name="supervised", eval=True, seg_level=False) dataloader = DataLoader(dataset, collate_fn=dataset.collate_fn, batch_size=1, shuffle=False, num_workers=1, pin_memory=True) for b, batch in list(enumerate(dataloader)): print("batch:", batch) images = batch["images"] instructions = batch["instr"] label_masks = batch["traj_labels"] affines = batch["affines_g_to_s"] env_ids = batch["env_id"] set_idxs = batch["set_idx"] seg_idxs = batch["seg_idx"] env_id = env_ids[0][0] set_idx = set_idxs[0][0] print("env_id of this batch:", env_id) env.set_environment( env_id, instruction_set=all_instr[env_id][set_idx]["instructions"]) env.reset(0) num_segments = len(instructions[0]) print("num_segments in this batch:", num_segments) write_instruction("") write_real_instruction("None") instruction_str = read_instruction_file() print("Initial instruction: ", instruction_str) # TODO: Reset model state here if we keep any temporal memory etc for s in range(num_segments): start_state = env.reset(s) keep_going = True real_instruction = cuda_var(instructions[0][s], setup["cuda"], 0) tmp = list(real_instruction.data.cpu()[0].numpy()) real_instruction_str = debug_untokenize_instruction(tmp) write_real_instruction(real_instruction_str) #write_instruction(real_instruction_str) #instruction_str = real_instruction_str image = cuda_var(images[0][s], setup["cuda"], 0) label_mask = cuda_var(label_masks[0][s], setup["cuda"], 0) affine_g_to_s = affines[0][s] print("Your current environment:") with open( "/storage/dxsun/unreal_config_nl/configs/configs/random_config_" + str(env_id) + ".json") as fp: config = json.load(fp) print(config) while keep_going: write_real_instruction(real_instruction_str) while True: cv2.waitKey(200) instruction = read_instruction_file() if instruction == "CMD: Next": print("Advancing") keep_going = False write_empty_instruction() break elif instruction == "CMD: Reset": print("Resetting") env.reset(s) write_empty_instruction() elif len(instruction.split(" ")) > 1: instruction_str = instruction print("Executing: ", instruction_str) break if not keep_going: continue #instruction_str = read_instruction_file() # TODO: Load instruction from file tok_instruction = tokenize_instruction(instruction_str, word2token) instruction_t = torch.LongTensor(tok_instruction).unsqueeze(0) instruction_v = cuda_var(instruction_t, setup["cuda"], 0) instruction_mask = torch.ones_like(instruction_v) tmp = list(instruction_t[0].numpy()) instruction_dbg_str = debug_untokenize_instruction( tmp, token2term) # import matplotlib.pyplot as plt #plt.plot(image.squeeze(0).permute(1,2,0).cpu().numpy()) #plt.show() res = model(image, instruction_v, instruction_mask) mask_pred = res[0] shp = mask_pred.shape mask_pred = F.softmax(mask_pred.view([2, -1]), 1).view(shp) #mask_pred = softmax2d(mask_pred) # TODO: Rotate the mask_pred to the global frame affine_s_to_g = np.linalg.inv(affine_g_to_s) S = 8.0 affine_scale_up = np.asarray([[S, 0, 0], [0, S, 0], [0, 0, 1]]) affine_scale_down = np.linalg.inv(affine_scale_up) affine_pred_to_g = np.dot( affine_scale_down, np.dot(affine_s_to_g, affine_scale_up)) #affine_pred_to_g_t = torch.from_numpy(affine_pred_to_g).float() mask_pred_np = mask_pred.data.cpu().numpy()[0].transpose( 1, 2, 0) mask_pred_g_np = apply_affine(mask_pred_np, affine_pred_to_g, 32, 32) print("Sum of global mask: ", mask_pred_g_np.sum()) mask_pred_g = torch.from_numpy( mask_pred_g_np.transpose(2, 0, 1)).float()[np.newaxis, :, :, :] exec_model.set_ground_truth_visitation_d(mask_pred_g) # Create a batch axis for pytorch #mask_pred_g = affine2d(mask_pred, affine_pred_to_g_t[np.newaxis, :, :]) mask_pred_np[:, :, 0] -= mask_pred_np[:, :, 0].min() mask_pred_np[:, :, 0] /= (mask_pred_np[:, :, 0].max() + 1e-9) mask_pred_np[:, :, 0] *= 2.0 mask_pred_np[:, :, 1] -= mask_pred_np[:, :, 1].min() mask_pred_np[:, :, 1] /= (mask_pred_np[:, :, 1].max() + 1e-9) presenter = Presenter() presenter.show_image(mask_pred_g_np, "mask_pred_g", torch=False, waitkey=1, scale=4) #import matplotlib.pyplot as plt #print("image.data shape:", image.data.cpu().numpy().shape) #plt.imshow(image.data.squeeze().permute(1,2,0).cpu().numpy()) #plt.show() # presenter.show_image(image.data, "mask_pred_g", torch=False, waitkey=1, scale=4) #import pdb; pdb.set_trace() pred_viz_np = presenter.overlaid_image(image.data, mask_pred_np, channel=0) # TODO: Don't show labels # TODO: OpenCV colours #label_mask_np = p.data.cpu().numpy()[0].transpose(1,2,0) labl_viz_np = presenter.overlaid_image(image.data, label_mask.data, channel=0) viz_img_np = np.concatenate((pred_viz_np, labl_viz_np), axis=1) viz_img_np = pred_viz_np viz_img = presenter.overlay_text(viz_img_np, instruction_dbg_str) cv2.imshow("interactive viz", viz_img) cv2.waitKey(100) rollout_model(exec_model, env, env_ids[0][s], set_idxs[0][s], seg_idxs[0][s], tok_instruction) write_instruction("")
def load_all_envs(self): train_i, dev_i, test_i, corpus = get_all_instructions() all_instructions = merge_instruction_sets(train_i, dev_i, test_i) token2term, word2token = get_word_to_token_map(corpus) env_ids = list(all_instructions.keys()) return env_ids, all_instructions, corpus, token2term, word2token
def __init__(self, model_real, model_sim, model_critic, model_oracle_critic=None, state=None, epoch=0): _, _, _, corpus = get_all_instructions() self.token2word, self.word2token = get_word_to_token_map(corpus) self.params = get_current_parameters()["Training"] self.run_name = get_current_parameters()["Setup"]["run_name"] self.batch_size = self.params['batch_size'] self.iterations_per_epoch = self.params.get("iterations_per_epoch", None) self.weight_decay = self.params['weight_decay'] self.optimizer = self.params['optimizer'] self.critic_loaders = self.params['critic_loaders'] self.model_common_loaders = self.params['model_common_loaders'] self.model_sim_loaders = self.params['model_sim_loaders'] self.lr = self.params['lr'] self.critic_steps = self.params['critic_steps'] self.model_steps = self.params['model_steps'] self.critic_batch_size = self.params["critic_batch_size"] self.model_batch_size = self.params["model_batch_size"] self.disable_wloss = self.params["disable_wloss"] self.sim_steps_per_real_step = self.params.get( "sim_steps_per_real_step", 1) self.real_dataset_names = self.params.get("real_dataset_names") self.sim_dataset_names = self.params.get("sim_dataset_names") self.bidata = self.params.get("bidata", False) self.sim_steps_per_common_step = self.params.get( "sim_steps_per_common_step", 1) n_params_real = get_n_params(model_real) n_params_real_tr = get_n_trainable_params(model_real) n_params_sim = get_n_params(model_sim) n_params_sim_tr = get_n_trainable_params(model_sim) n_params_c = get_n_params(model_critic) n_params_c_tr = get_n_params(model_critic) print("Training Model:") print("Real # model parameters: " + str(n_params_real)) print("Real # trainable parameters: " + str(n_params_real_tr)) print("Sim # model parameters: " + str(n_params_sim)) print("Sim # trainable parameters: " + str(n_params_sim_tr)) print("Critic # model parameters: " + str(n_params_c)) print("Critic # trainable parameters: " + str(n_params_c_tr)) # Share those modules that are to be shared between real and sim models if not self.params.get("disable_domain_weight_sharing"): print("Sharing weights between sim and real modules") model_real.steal_cross_domain_modules(model_sim) else: print("NOT Sharing weights between sim and real modules") self.model_real = model_real self.model_sim = model_sim self.model_critic = model_critic self.model_oracle_critic = model_oracle_critic if self.model_oracle_critic: print("Using oracle critic") if self.optimizer == "adam": Optim = optim.Adam elif self.optimizer == "sgd": Optim = optim.SGD else: raise ValueError(f"Unsuppored optimizer {self.optimizer}") self.optim_models = Optim(self.model_real.both_domain_parameters( self.model_sim), self.lr, weight_decay=self.weight_decay) self.optim_critic = Optim(self.critic_parameters(), self.lr, weight_decay=self.weight_decay) self.train_epoch_num = epoch self.train_segment = 0 self.test_epoch_num = epoch self.test_segment = 0 self.set_state(state)
def automatic_demo(): P.initialize_experiment() instruction_display = InstructionDisplay() rate = Rate(0.1) env = PomdpInterface( is_real=get_current_parameters()["Setup"]["real_drone"]) train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions( ) all_instr = { **train_instructions, **dev_instructions, **train_instructions } token2term, word2token = get_word_to_token_map(corpus) # Run on dev set interact_instructions = dev_instructions env_range_start = get_current_parameters()["Setup"].get( "env_range_start", 0) env_range_end = get_current_parameters()["Setup"].get( "env_range_end", 10e10) interact_instructions = { k: v for k, v in interact_instructions.items() if env_range_start < k < env_range_end } model, _ = load_model(get_current_parameters()["Setup"]["model"]) # Loop over the select few examples while True: for instruction_sets in interact_instructions.values(): for set_idx, instruction_set in enumerate(instruction_sets): env_id = instruction_set['env'] found_example = None for example in examples: if example[0] == env_id: found_example = example if found_example is None: continue env.set_environment(env_id, instruction_set["instructions"]) presenter = Presenter() cumulative_reward = 0 for seg_idx in range(len(instruction_set["instructions"])): if seg_idx != found_example[2]: continue print(f"RUNNING ENV {env_id} SEG {seg_idx}") real_instruction_str = instruction_set["instructions"][ seg_idx]["instruction"] instruction_display.show_instruction(real_instruction_str) valid_segment = env.set_current_segment(seg_idx) if not valid_segment: continue state = env.reset(seg_idx) for i in range(START_PAUSE): instruction_display.tick() time.sleep(1) tok_instruction = tokenize_instruction( real_instruction_str, word2token) state = env.reset(seg_idx) print("Executing: f{instruction_str}") while True: instruction_display.tick() rate.sleep() action, internals = model.get_action( state, tok_instruction) state, reward, done, expired, oob = env.step(action) cumulative_reward += reward #presenter.show_sample(state, action, reward, cumulative_reward, real_instruction_str) #show_depth(state.image) if done: break for i in range(END_PAUSE): instruction_display.tick() time.sleep(1) print("Segment finished!") instruction_display.show_instruction("...") print("Env finished!")
def __init__(self, data=None, env_list=None, dataset_names=["simulator"], dataset_prefix="supervised", domain="sim", max_traj_length=None, aux_provider_names=[], segment_level=False, cache=False): """ Dataset for the replay memory :param data: if data is pre-loaded in memory, this is the training data :param env_list: if data is to be loaded by the dataset, this is the list of environments for which to include data :param dataset_names: list of datasets from which to load data :param dataset_prefix: name of the dataset. Default: supervised will use data collected with collect_supervised_data :param max_traj_length: truncate trajectories to this long :param cuda: :param aux_provider_names: """ # If data is already loaded in memory, use it self.data = data self.prof = SimpleProfiler(torch_sync=False, print=PROFILE) self.min_seg_len = P.get_current_parameters()["Data"].get("min_seg_len", 3) self.do_cache = P.get_current_parameters()["Data"].get("cache", False) self.dataset_prefix = dataset_prefix self.dataset_names = dataset_names self.domain = domain self.env_restrictions = P.get_current_parameters()["Data"].get("dataset_env_restrictions") if self.env_restrictions: self.dataset_restricted_envs = {dname:P.get_current_parameters()["Data"]["EnvRestrictionGroups"][self.env_restrictions[dname]] for dname in dataset_names if dname in self.env_restrictions} print(f"Using restricted envs: {list(self.dataset_restricted_envs.keys())}") else: self.dataset_restricted_envs = {} self.max_traj_length = max_traj_length train_instr, dev_instr, test_instr, corpus = get_all_instructions() # TODO: This shouldn't have access to all instructions. We should really make distinct train, dev, test modes self.all_instr = {**train_instr, **dev_instr, **test_instr} train_instr_full, dev_instr_full, test_instr_full, corpus = get_all_instructions(full=True) self.all_instr_full = {**train_instr_full, **dev_instr_full, **test_instr_full} self.segment_level = segment_level self.sample_ids = [] if self.data is None: assert env_list is not None for i, dataset_name in enumerate(self.dataset_names): dataset_env_list = filter_env_list_has_data(dataset_name, env_list, dataset_prefix) if self.segment_level: dataset_env_list, dataset_seg_list = self.split_into_segments(dataset_env_list, dataset_name) else: dataset_seg_list = [0 for _ in dataset_env_list] for env, seg in zip(dataset_env_list, dataset_seg_list): self.sample_ids.append((dataset_name, env, seg)) self.token2word, self.word2token = get_word_to_token_map(corpus) self.aux_provider_names = aux_provider_names self.aux_label_names = get_aux_label_names(aux_provider_names) self.stackable_names = get_stackable_label_names(aux_provider_names) self.data_cache = {dataset_name:{} for dataset_name in dataset_names} self.traj_len = P.get_current_parameters()["Setup"]["trajectory_length"]