def load_supervised_data(max_envs=None, split_segments=False):
train_i, dev_i, test_i, _ = get_all_instructions(max_envs)
train_env_ids = get_env_ids(train_i)
dev_env_ids = get_env_ids(dev_i)
pool = Pool(processes=20)
# Load dataset for each env in parallel
train_env_data = pool.map(load_single_env_supervised_data, train_env_ids)
dev_env_data = pool.map(load_single_env_supervised_data, dev_env_ids)
pool.close()
#dev_data = []
#train_data = []
# Combine into a single dataset
#for data in train_env_data:
# train_data += data
#for data in dev_env_data:
# dev_data += data
train_data = sanitize_data(train_env_data)
dev_data = sanitize_data(dev_env_data)
if split_segments:
train_data = split_in_segments(train_data)
dev_data = split_in_segments(dev_data)
return train_data, dev_data
def __init__(self):
super(AuxLabelsNL, self).__init__()
self.thesaurus = load_landmark_alignments()
train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions()
self.all_instructions = {**train_instructions, **dev_instructions, **test_instructions}
self.corpus = corpus
self.word2token, self.token2term = get_word_to_token_map(corpus)
def __init__(self,
env_list=None,
instr_negatives=False,
instr_negatives_similar_only=False,
seg_level=False,
img_scale=1,
yaw_rand_range=0,
pos_rand_range=0
):
# If data is already loaded in memory, use it
self.env_list = env_list
self.train_instr, self.dev_instr, self.test_instr, corpus = get_all_instructions()
self.all_instr = {**self.train_instr, **self.dev_instr, **self.test_instr}
self.token2term, self.word2token = get_word_to_token_map(corpus)
self.thesaurus = load_thesaurus()
self.include_instr_negatives = instr_negatives
if instr_negatives:
self.similar_instruction_map = load_similar_instruction_map()
self.instr_negatives_similar_only = instr_negatives_similar_only
self.img_scale = img_scale
self.yaw_rand_range = yaw_rand_range
self.pos_rand_range = pos_rand_range
self.pos_rand_image = 0
# If the data is supposed to be at seg level (not nested envs + segs), then we can support batching
# but we need to correctly infer the dataset size
self.seg_level = seg_level
if seg_level:
self.seg_list = []
for env in self.env_list:
for set_idx, set in enumerate(self.all_instr[env]):
for seg_idx, seg in enumerate(set["instructions"]):
self.seg_list.append([env, set_idx, seg_idx])
def __init__(self,
env_list=None,
instr_negatives=False,
instr_negatives_similar_only=False,
seg_level=False,
yaw_rand_range=0,
img_w=512,
img_h=512,
map_w=None,
map_h=None,
incl_path=True,
incl_endpoint=False,
use_semantic_maps=False):
# If data is already loaded in memory, use it
self.cuda = False
self.env_list = env_list
self.train_instr, self.dev_instr, self.test_instr, corpus = get_all_instructions()
self.all_instr = {**self.train_instr, **self.dev_instr, **self.test_instr}
self.token2term, self.word2token = get_word_to_token_map(corpus)
self.thesaurus = load_thesaurus()
self.include_instr_negatives = instr_negatives
#if instr_negatives:
# self.similar_instruction_map = load_similar_instruction_map()
self.instr_negatives_similar_only = instr_negatives_similar_only
self.use_semantic_maps = use_semantic_maps
self.img_w = img_w
self.img_h = img_h
if map_w is None:
self.map_w = self.img_w
self.map_h = self.img_h
else:
self.map_w = map_w
self.map_h = map_h
self.yaw_rand_range = yaw_rand_range
self.latest_img_dbg = None
self.latest_rot_img_dbg = None
self.incl_endpoint = incl_endpoint
self.incl_path = incl_path
# If the data is supposed to be at seg level (not nested envs + segs), then we can support batching
# but we need to correctly infer the dataset size
self.seg_level = seg_level
if seg_level:
self.seg_list = []
for env in self.env_list:
for set_idx, set in enumerate(self.all_instr[env]):
for seg_idx, seg in enumerate(set["instructions"]):
self.seg_list.append([env, set_idx, seg_idx])
print("Initialzied dataset!")
print(" yaw range : " + str(self.yaw_rand_range))
print(" map size: ", self.map_w, self.map_h)
print(" img size: ", self.img_w, self.img_h)
def __init__(self, run_name="", save_images=True):
super(EvaluateBase, self).__init__()
self.train_i, self.test_i, self.dev_i, corpus = get_all_instructions()
self.passing_distance = LANDMARK_REGION_RADIUS
self.results = ResultsLandmarkSide()
self.presenter = Presenter()
self.run_name = run_name
self.save_images = save_images
def take_pics():
P.initialize_experiment()
train_i, dev_i, test_i, _ = get_all_instructions()
all_instructions = {**train_i, **dev_i, **test_i}
save_dir = paths.get_env_image_path(0)
os.makedirs(os.path.dirname(save_dir), exist_ok=True)
keylist = list(all_instructions.keys())
envs = [PomdpInterface(instance_id=i) for i in range(0, NUM_WORKERS)]
env_id_splits = [[] for _ in range(NUM_WORKERS)]
keylist = [6825]
for i, key in enumerate(keylist):
env_id_splits[i % NUM_WORKERS].append(key)
time.sleep(1.0)
for i in range(len(keylist)):
d = False
# For each worker, start the correct env
for w in range(NUM_WORKERS):
if i >= len(env_id_splits[w]):
continue
env_id = env_id_splits[w][i]
# FIXME: :This assumes that there is only 1 instruction set per env!
fname = paths.get_env_image_path(env_id)
if os.path.isfile(fname):
print("Img exists: " + fname)
continue
d = True
instruction_set = all_instructions[env_id][0]
envs[w].set_environment(env_id, instruction_set["instructions"], fast=True)
print("setting env on worker " + str(w) + " iter " + str(i) + " env_id: " + str(env_id))
# Then for each worker, take a picture and save it
if d:
time.sleep(0.1)
for w in range(NUM_WORKERS):
if i >= len(env_id_splits[w]):
continue
env_id = env_id_splits[w][i]
fname = paths.get_env_image_path(env_id)
if os.path.isfile(fname):
print("Img exists: " + fname)
continue
envs[w].snap_birdseye(fast=True, small_env=SMALL_ENV)
image = envs[w].snap_birdseye(fast=True, small_env=SMALL_ENV)
image = np.flip(image, 0)
imsave(fname, image)
print("saving pic on worker " + str(w) + " iter " + str(i) + " env_id: " + str(env_id))
def __init__(self, run_name="", save_images=True, entire_trajectory=True, custom_instr=None):
super(EvaluateBase, self).__init__()
self.train_i, self.test_i, self.dev_i, corpus = get_all_instructions()
self.all_i = {**self.train_i, **self.test_i, **self.dev_i}
self.passing_distance = DEFAULT_PASSING_DISTANCE
self.results = ResultsLandmarkSide()
self.presenter = Presenter()
self.run_name = run_name
self.save_images = save_images
self.entire_trajectory = entire_trajectory
self.custom_instr = custom_instr
def generate_thesaurus():
train_instr, dev_instr, test_instr, corpus = get_all_instructions()
_, word2token = get_word_to_token_map(corpus, use_thesaurus=False)
term2word, word2term = get_identity_term_mapping(corpus)
term2landmark = get_template_term_groundings(corpus, word2term)
thesaurus = {
"term2word": term2word,
"word2term": word2term,
"term_groundings": term2landmark,
"rejected_words": []
}
save_thesaurus(thesaurus)
def __init__(
self,
model,
state=None,
epoch=0,
name="",
run_name="",
):
_, _, _, corpus = get_all_instructions()
self.token2word, self.word2token = get_word_to_token_map(corpus)
self.params = get_current_parameters()["Training"]
self.batch_size = self.params['batch_size']
self.weight_decay = self.params['weight_decay']
self.optimizer = self.params['optimizer']
self.num_loaders = self.params['num_loaders']
self.lr = self.params['lr']
self.name = name
self.dataset_names = None
n_params = get_n_params(model)
n_params_tr = get_n_trainable_params(model)
print("Training Model:")
print("Number of model parameters: " + str(n_params))
print("Trainable model parameters: " + str(n_params_tr))
self.model = model
self.run_name = run_name
if self.optimizer == "adam":
self.optim = optim.Adam(self.get_model_parameters(self.model),
self.lr,
weight_decay=self.weight_decay)
elif self.optimizer == "sgd":
self.optim = optim.SGD(self.get_model_parameters(self.model),
self.lr,
weight_decay=self.weight_decay,
momentum=0.9)
self.train_epoch_num = epoch
self.train_segment = 0
self.test_epoch_num = epoch
self.test_segment = 0
self.set_state(state)
self.batch_num = 0
def analyze_lani():
P.initialize_experiment()
train_i, dev_i, text_i, corpus = get_all_instructions()
train_i_lani = {k:v for k,v in train_i.items() if int(k) < 6000}
dev_i_lani = {k:v for k,v in dev_i.items() if int(k) < 6000}
test_i_lani = {k:v for k,v in text_i.items() if int(k) < 6000}
train_i_real = {k:v for k,v in train_i.items() if int(k) >= 6000}
dev_i_real = {k:v for k,v in dev_i.items() if int(k) >= 6000}
test_i_real = {k:v for k,v in text_i.items() if int(k) >= 6000}
test_i_small = {k:v for k,v in test_i_real.items() if int(k) in test_small_envs}
dev_i_small = {k:v for k,v in dev_i_real.items() if int(k) in dev_small_envs}
all_i_real = {**train_i_real, **dev_i_real, **test_i_real}
analyze_instruction_set("Lani Train 1Seg", train_i_lani, corpus, merge_len=1)
analyze_instruction_set("Lani Dev 1Seg", dev_i_lani, corpus, merge_len=1)
analyze_instruction_set("Lani Test 1Seg", test_i_lani, corpus, merge_len=1)
analyze_instruction_set("Lani Train 2Seg", train_i_lani, corpus, merge_len=2)
analyze_instruction_set("Lani Dev 2Seg", dev_i_lani, corpus, merge_len=2)
analyze_instruction_set("Lani Test 2Seg", test_i_lani, corpus, merge_len=2)
analyze_instruction_set("Real Train 1Seg", train_i_real, corpus, merge_len=1)
analyze_instruction_set("Real Dev 1Seg", dev_i_real, corpus, merge_len=1)
analyze_instruction_set("Real Test 1Seg", test_i_real, corpus, merge_len=1)
analyze_instruction_set("Real Train 2Seg", train_i_real, corpus, merge_len=2)
analyze_instruction_set("Real Dev 2Seg", dev_i_real, corpus, merge_len=2)
analyze_instruction_set("Real Test 2Seg", test_i_real, corpus, merge_len=2)
#analyze_instruction_set("Real Test 1Seg", test_i_real, corpus, merge_len=1)
#analyze_instruction_set("Real Test 2Seg", test_i_real, corpus, merge_len=2)
#analyze_instruction_set("Small Test 1Seg", test_i_small, corpus, merge_len=1)
#analyze_instruction_set("Small Test 2Seg", test_i_small, corpus, merge_len=2)
analyze_instruction_set("Small Dev 1Seg", dev_i_small, corpus, merge_len=1)
analyze_instruction_set("Small Dev 2Seg", dev_i_small, corpus, merge_len=2)
analyze_instruction_set("Small Dev 1Seg", dev_i_small, corpus, merge_len=1)
analyze_instruction_set("Small Dev 2Seg", dev_i_small, corpus, merge_len=2)
print("ding")
def generate_thesaurus():
train_instr, dev_instr, test_instr, corpus = get_all_instructions()
_, word2token = get_word_to_token_map(corpus, use_thesaurus=False)
terms, rejected_words = cluster_corpus(corpus, train_instr)
landmark_names = get_landmark_names()
term_groundings, word2term = ground_terms(word2token, terms,
landmark_names, train_instr)
thesaurus = {
"term2word": terms,
"word2term": word2term,
"term_groundings": term_groundings,
"rejected_words": rejected_words
}
save_thesaurus(thesaurus)
def dyn_gt_test():
presenter = Presenter()
train_instr, dev_instr, test_instr, corpus = get_all_instructions()
all_instr = {**train_instr, **dev_instr, **test_instr}
for i in range(10):
path = load_path(i)
segments = all_instr[i][0]["instructions"]
for seg in segments:
start_idx = seg["start_idx"]
end_idx = seg["end_idx"]
randInt = random.randint(10, 100)
start_pose = Pose(path[start_idx] - randInt, 0)
if end_idx - start_idx > 0:
randInt = random.randint(10, 100)
new_path = get_dynamic_ground_truth(
path[start_idx:end_idx], (path[start_idx] - randInt))
new_path1 = get_dynamic_ground_truth_smooth(
path[start_idx:end_idx], (path[start_idx] - randInt))
presenter.plot_path(
i, [path[start_idx:end_idx], new_path, new_path1])
def __init__(self,
run_name="",
save_images=True,
entire_trajectory=True,
custom_instr=None,
aug_len=None):
super(EvaluateBase, self).__init__()
self.train_i, self.test_i, self.dev_i, corpus = get_all_instructions()
self.all_i = {**self.train_i, **self.test_i, **self.dev_i}
self.passing_distance = P.get_current_parameters(
)["Units"]["passing_distance"]
self.results = ResultsLandmarkSide()
self.presenter = Presenter()
self.run_name = run_name
self.save_images = save_images
self.entire_trajectory = entire_trajectory
self.custom_instr = custom_instr
self.aug_len = aug_len
self.visible_map = {}
self.hfov = P.get_current_parameters(
)["ModelPVN"]["Stage1"]["cam_h_fov"]
def load_all_envs(self):
train_i, dev_i, test_i, corpus = get_all_instructions()
all_instructions = merge_instruction_sets(train_i, dev_i, test_i)
token2term, word2token = get_word_to_token_map(corpus)
env_ids = list(all_instructions.keys())
return env_ids, all_instructions, corpus, token2term, word2token
def __init__(self,
data=None,
env_list=None,
dataset_names=["simulator"],
dataset_prefix="supervised",
domain="sim",
max_traj_length=None,
aux_provider_names=[],
segment_level=False,
cache=False):
"""
Dataset for the replay memory
:param data: if data is pre-loaded in memory, this is the training data
:param env_list: if data is to be loaded by the dataset, this is the list of environments for which to include data
:param dataset_names: list of datasets from which to load data
:param dataset_prefix: name of the dataset. Default: supervised will use data collected with collect_supervised_data
:param max_traj_length: truncate trajectories to this long
:param cuda:
:param aux_provider_names:
"""
# If data is already loaded in memory, use it
self.data = data
self.prof = SimpleProfiler(torch_sync=False, print=PROFILE)
self.min_seg_len = P.get_current_parameters()["Data"].get("min_seg_len", 3)
self.do_cache = P.get_current_parameters()["Data"].get("cache", False)
self.dataset_prefix = dataset_prefix
self.dataset_names = dataset_names
self.domain = domain
self.env_restrictions = P.get_current_parameters()["Data"].get("dataset_env_restrictions")
if self.env_restrictions:
self.dataset_restricted_envs = {dname:P.get_current_parameters()["Data"]["EnvRestrictionGroups"][self.env_restrictions[dname]] for dname in dataset_names if dname in self.env_restrictions}
print(f"Using restricted envs: {list(self.dataset_restricted_envs.keys())}")
else:
self.dataset_restricted_envs = {}
self.max_traj_length = max_traj_length
train_instr, dev_instr, test_instr, corpus = get_all_instructions()
# TODO: This shouldn't have access to all instructions. We should really make distinct train, dev, test modes
self.all_instr = {**train_instr, **dev_instr, **test_instr}
train_instr_full, dev_instr_full, test_instr_full, corpus = get_all_instructions(full=True)
self.all_instr_full = {**train_instr_full, **dev_instr_full, **test_instr_full}
self.segment_level = segment_level
self.sample_ids = []
if self.data is None:
assert env_list is not None
for i, dataset_name in enumerate(self.dataset_names):
dataset_env_list = filter_env_list_has_data(dataset_name, env_list, dataset_prefix)
if self.segment_level:
dataset_env_list, dataset_seg_list = self.split_into_segments(dataset_env_list, dataset_name)
else:
dataset_seg_list = [0 for _ in dataset_env_list]
for env, seg in zip(dataset_env_list, dataset_seg_list):
self.sample_ids.append((dataset_name, env, seg))
self.token2word, self.word2token = get_word_to_token_map(corpus)
self.aux_provider_names = aux_provider_names
self.aux_label_names = get_aux_label_names(aux_provider_names)
self.stackable_names = get_stackable_label_names(aux_provider_names)
self.data_cache = {dataset_name:{} for dataset_name in dataset_names}
self.traj_len = P.get_current_parameters()["Setup"]["trajectory_length"]
def run(self):
self.mon.run()
def get_command(self):
return self.mon.current_vel
initialize_experiment("nl_datacollect_cage")
teleoper = KeyTeleop()
rate = Rate(0.1)
env = PomdpInterface()
train_instructions, dev_instructions, test_instructions, _ = get_all_instructions(
)
count = 0
stuck_count = 0
def show_depth(image):
grayscale = np.mean(image[:, :, 0:3], axis=2)
depth = image[:, :, 3]
comb = np.stack([grayscale, grayscale, depth], axis=2)
comb -= comb.min()
comb /= (comb.max() + 1e-9)
Presenter().show_image(comb,
"depth_alignment",
torch=False,
waitkey=1,
def interactive_demo():
P.initialize_experiment()
InteractAPI.launch_ui()
rate = Rate(0.1)
env = PomdpInterface(
is_real=get_current_parameters()["Setup"]["real_drone"])
train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions(
)
all_instr = {
**train_instructions,
**dev_instructions,
**train_instructions
}
token2term, word2token = get_word_to_token_map(corpus)
# Run on dev set
interact_instructions = dev_instructions
env_range_start = get_current_parameters()["Setup"].get(
"env_range_start", 0)
env_range_end = get_current_parameters()["Setup"].get(
"env_range_end", 10e10)
interact_instructions = {
k: v
for k, v in interact_instructions.items()
if env_range_start < k < env_range_end
}
count = 0
stuck_count = 0
model, _ = load_model(get_current_parameters()["Setup"]["model"])
InteractAPI.write_empty_instruction()
InteractAPI.write_real_instruction("None")
instruction_str = InteractAPI.read_instruction_file()
print("Initial instruction: ", instruction_str)
for instruction_sets in interact_instructions.values():
for set_idx, instruction_set in enumerate(instruction_sets):
env_id = instruction_set['env']
env.set_environment(env_id, instruction_set["instructions"])
presenter = Presenter()
cumulative_reward = 0
for seg_idx in range(len(instruction_set["instructions"])):
print(f"RUNNING ENV {env_id} SEG {seg_idx}")
real_instruction_str = instruction_set["instructions"][
seg_idx]["instruction"]
InteractAPI.write_real_instruction(real_instruction_str)
valid_segment = env.set_current_segment(seg_idx)
if not valid_segment:
continue
state = env.reset(seg_idx)
keep_going = True
while keep_going:
InteractAPI.write_real_instruction(real_instruction_str)
while True:
cv2.waitKey(200)
instruction = InteractAPI.read_instruction_file()
if instruction == "CMD: Next":
print("Advancing")
keep_going = False
InteractAPI.write_empty_instruction()
break
elif instruction == "CMD: Reset":
print("Resetting")
env.reset(seg_idx)
InteractAPI.write_empty_instruction()
elif len(instruction.split(" ")) > 1:
instruction_str = instruction
break
if not keep_going:
continue
env.override_instruction(instruction_str)
tok_instruction = tokenize_instruction(
instruction_str, word2token)
state = env.reset(seg_idx)
print("Executing: f{instruction_str}")
while True:
rate.sleep()
action, internals = model.get_action(
state, tok_instruction)
state, reward, done, expired, oob = env.step(action)
cumulative_reward += reward
presenter.show_sample(state, action, reward,
cumulative_reward,
instruction_str)
#show_depth(state.image)
if done:
break
InteractAPI.write_empty_instruction()
print("Segment finished!")
print("Env finished!")
def train_top_down_pred():
P.initialize_experiment()
setup = P.get_current_parameters()["Setup"]
launch_ui()
env = PomdpInterface()
print("model_name:", setup["top_down_model"])
print("model_file:", setup["top_down_model_file"])
model, model_loaded = load_model(
model_name_override=setup["top_down_model"],
model_file_override=setup["top_down_model_file"])
exec_model, wrapper_model_loaded = load_model(
model_name_override=setup["wrapper_model"],
model_file_override=setup["wrapper_model_file"])
affine2d = Affine2D()
if model.is_cuda:
affine2d.cuda()
eval_envs = get_correct_eval_env_id_list()
print("eval_envs:", eval_envs)
train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions(
max_size=setup["max_envs"])
all_instr = {
**train_instructions,
**dev_instructions,
**train_instructions
}
token2term, word2token = get_word_to_token_map(corpus)
dataset = model.get_dataset(envs=eval_envs,
dataset_name="supervised",
eval=True,
seg_level=False)
dataloader = DataLoader(dataset,
collate_fn=dataset.collate_fn,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
for b, batch in list(enumerate(dataloader)):
print("batch:", batch)
images = batch["images"]
instructions = batch["instr"]
label_masks = batch["traj_labels"]
affines = batch["affines_g_to_s"]
env_ids = batch["env_id"]
set_idxs = batch["set_idx"]
seg_idxs = batch["seg_idx"]
env_id = env_ids[0][0]
set_idx = set_idxs[0][0]
print("env_id of this batch:", env_id)
env.set_environment(
env_id, instruction_set=all_instr[env_id][set_idx]["instructions"])
env.reset(0)
num_segments = len(instructions[0])
print("num_segments in this batch:", num_segments)
write_instruction("")
write_real_instruction("None")
instruction_str = read_instruction_file()
print("Initial instruction: ", instruction_str)
# TODO: Reset model state here if we keep any temporal memory etc
for s in range(num_segments):
start_state = env.reset(s)
keep_going = True
real_instruction = cuda_var(instructions[0][s], setup["cuda"], 0)
tmp = list(real_instruction.data.cpu()[0].numpy())
real_instruction_str = debug_untokenize_instruction(tmp)
write_real_instruction(real_instruction_str)
#write_instruction(real_instruction_str)
#instruction_str = real_instruction_str
image = cuda_var(images[0][s], setup["cuda"], 0)
label_mask = cuda_var(label_masks[0][s], setup["cuda"], 0)
affine_g_to_s = affines[0][s]
print("Your current environment:")
with open(
"/storage/dxsun/unreal_config_nl/configs/configs/random_config_"
+ str(env_id) + ".json") as fp:
config = json.load(fp)
print(config)
while keep_going:
write_real_instruction(real_instruction_str)
while True:
cv2.waitKey(200)
instruction = read_instruction_file()
if instruction == "CMD: Next":
print("Advancing")
keep_going = False
write_empty_instruction()
break
elif instruction == "CMD: Reset":
print("Resetting")
env.reset(s)
write_empty_instruction()
elif len(instruction.split(" ")) > 1:
instruction_str = instruction
print("Executing: ", instruction_str)
break
if not keep_going:
continue
#instruction_str = read_instruction_file()
# TODO: Load instruction from file
tok_instruction = tokenize_instruction(instruction_str,
word2token)
instruction_t = torch.LongTensor(tok_instruction).unsqueeze(0)
instruction_v = cuda_var(instruction_t, setup["cuda"], 0)
instruction_mask = torch.ones_like(instruction_v)
tmp = list(instruction_t[0].numpy())
instruction_dbg_str = debug_untokenize_instruction(
tmp, token2term)
# import matplotlib.pyplot as plt
#plt.plot(image.squeeze(0).permute(1,2,0).cpu().numpy())
#plt.show()
res = model(image, instruction_v, instruction_mask)
mask_pred = res[0]
shp = mask_pred.shape
mask_pred = F.softmax(mask_pred.view([2, -1]), 1).view(shp)
#mask_pred = softmax2d(mask_pred)
# TODO: Rotate the mask_pred to the global frame
affine_s_to_g = np.linalg.inv(affine_g_to_s)
S = 8.0
affine_scale_up = np.asarray([[S, 0, 0], [0, S, 0], [0, 0, 1]])
affine_scale_down = np.linalg.inv(affine_scale_up)
affine_pred_to_g = np.dot(
affine_scale_down, np.dot(affine_s_to_g, affine_scale_up))
#affine_pred_to_g_t = torch.from_numpy(affine_pred_to_g).float()
mask_pred_np = mask_pred.data.cpu().numpy()[0].transpose(
1, 2, 0)
mask_pred_g_np = apply_affine(mask_pred_np, affine_pred_to_g,
32, 32)
print("Sum of global mask: ", mask_pred_g_np.sum())
mask_pred_g = torch.from_numpy(
mask_pred_g_np.transpose(2, 0,
1)).float()[np.newaxis, :, :, :]
exec_model.set_ground_truth_visitation_d(mask_pred_g)
# Create a batch axis for pytorch
#mask_pred_g = affine2d(mask_pred, affine_pred_to_g_t[np.newaxis, :, :])
mask_pred_np[:, :, 0] -= mask_pred_np[:, :, 0].min()
mask_pred_np[:, :, 0] /= (mask_pred_np[:, :, 0].max() + 1e-9)
mask_pred_np[:, :, 0] *= 2.0
mask_pred_np[:, :, 1] -= mask_pred_np[:, :, 1].min()
mask_pred_np[:, :, 1] /= (mask_pred_np[:, :, 1].max() + 1e-9)
presenter = Presenter()
presenter.show_image(mask_pred_g_np,
"mask_pred_g",
torch=False,
waitkey=1,
scale=4)
#import matplotlib.pyplot as plt
#print("image.data shape:", image.data.cpu().numpy().shape)
#plt.imshow(image.data.squeeze().permute(1,2,0).cpu().numpy())
#plt.show()
# presenter.show_image(image.data, "mask_pred_g", torch=False, waitkey=1, scale=4)
#import pdb; pdb.set_trace()
pred_viz_np = presenter.overlaid_image(image.data,
mask_pred_np,
channel=0)
# TODO: Don't show labels
# TODO: OpenCV colours
#label_mask_np = p.data.cpu().numpy()[0].transpose(1,2,0)
labl_viz_np = presenter.overlaid_image(image.data,
label_mask.data,
channel=0)
viz_img_np = np.concatenate((pred_viz_np, labl_viz_np), axis=1)
viz_img_np = pred_viz_np
viz_img = presenter.overlay_text(viz_img_np,
instruction_dbg_str)
cv2.imshow("interactive viz", viz_img)
cv2.waitKey(100)
rollout_model(exec_model, env, env_ids[0][s], set_idxs[0][s],
seg_idxs[0][s], tok_instruction)
write_instruction("")
from data_io.instructions import get_restricted_env_id_lists, get_all_instructions
import parameters.parameter_server as P
if __name__ == "__main__":
P.initialize_experiment()
train_envs, dev_envs, test_envs = get_restricted_env_id_lists()
train_i, dev_i, test_i, corpus = get_all_instructions()
train_i_envs = set([int(i) for i in train_i.keys()])
for test_env in test_envs:
assert test_env not in train_i_envs, "FAIL"
for dev_env in test_envs:
assert dev_env not in train_i_envs, "FAIL"
print("OK")
def does_seglen_match(env_id, seg_idx, filter_merge_len):
_, _, test_i, _ = get_all_instructions()
seg_merge_len = test_i[int(env_id)][0]["instructions"][int(
seg_idx)]["merge_len"]
return seg_merge_len == filter_merge_len
def __init__(self,
model_real,
model_sim,
model_critic,
model_oracle_critic=None,
state=None,
epoch=0):
_, _, _, corpus = get_all_instructions()
self.token2word, self.word2token = get_word_to_token_map(corpus)
self.params = get_current_parameters()["Training"]
self.run_name = get_current_parameters()["Setup"]["run_name"]
self.batch_size = self.params['batch_size']
self.iterations_per_epoch = self.params.get("iterations_per_epoch",
None)
self.weight_decay = self.params['weight_decay']
self.optimizer = self.params['optimizer']
self.critic_loaders = self.params['critic_loaders']
self.model_common_loaders = self.params['model_common_loaders']
self.model_sim_loaders = self.params['model_sim_loaders']
self.lr = self.params['lr']
self.critic_steps = self.params['critic_steps']
self.model_steps = self.params['model_steps']
self.critic_batch_size = self.params["critic_batch_size"]
self.model_batch_size = self.params["model_batch_size"]
self.disable_wloss = self.params["disable_wloss"]
self.sim_steps_per_real_step = self.params.get(
"sim_steps_per_real_step", 1)
self.real_dataset_names = self.params.get("real_dataset_names")
self.sim_dataset_names = self.params.get("sim_dataset_names")
self.bidata = self.params.get("bidata", False)
self.sim_steps_per_common_step = self.params.get(
"sim_steps_per_common_step", 1)
n_params_real = get_n_params(model_real)
n_params_real_tr = get_n_trainable_params(model_real)
n_params_sim = get_n_params(model_sim)
n_params_sim_tr = get_n_trainable_params(model_sim)
n_params_c = get_n_params(model_critic)
n_params_c_tr = get_n_params(model_critic)
print("Training Model:")
print("Real # model parameters: " + str(n_params_real))
print("Real # trainable parameters: " + str(n_params_real_tr))
print("Sim # model parameters: " + str(n_params_sim))
print("Sim # trainable parameters: " + str(n_params_sim_tr))
print("Critic # model parameters: " + str(n_params_c))
print("Critic # trainable parameters: " + str(n_params_c_tr))
# Share those modules that are to be shared between real and sim models
if not self.params.get("disable_domain_weight_sharing"):
print("Sharing weights between sim and real modules")
model_real.steal_cross_domain_modules(model_sim)
else:
print("NOT Sharing weights between sim and real modules")
self.model_real = model_real
self.model_sim = model_sim
self.model_critic = model_critic
self.model_oracle_critic = model_oracle_critic
if self.model_oracle_critic:
print("Using oracle critic")
if self.optimizer == "adam":
Optim = optim.Adam
elif self.optimizer == "sgd":
Optim = optim.SGD
else:
raise ValueError(f"Unsuppored optimizer {self.optimizer}")
self.optim_models = Optim(self.model_real.both_domain_parameters(
self.model_sim),
self.lr,
weight_decay=self.weight_decay)
self.optim_critic = Optim(self.critic_parameters(),
self.lr,
weight_decay=self.weight_decay)
self.train_epoch_num = epoch
self.train_segment = 0
self.test_epoch_num = epoch
self.test_segment = 0
self.set_state(state)
def automatic_demo():
P.initialize_experiment()
instruction_display = InstructionDisplay()
rate = Rate(0.1)
env = PomdpInterface(
is_real=get_current_parameters()["Setup"]["real_drone"])
train_instructions, dev_instructions, test_instructions, corpus = get_all_instructions(
)
all_instr = {
**train_instructions,
**dev_instructions,
**train_instructions
}
token2term, word2token = get_word_to_token_map(corpus)
# Run on dev set
interact_instructions = dev_instructions
env_range_start = get_current_parameters()["Setup"].get(
"env_range_start", 0)
env_range_end = get_current_parameters()["Setup"].get(
"env_range_end", 10e10)
interact_instructions = {
k: v
for k, v in interact_instructions.items()
if env_range_start < k < env_range_end
}
model, _ = load_model(get_current_parameters()["Setup"]["model"])
# Loop over the select few examples
while True:
for instruction_sets in interact_instructions.values():
for set_idx, instruction_set in enumerate(instruction_sets):
env_id = instruction_set['env']
found_example = None
for example in examples:
if example[0] == env_id:
found_example = example
if found_example is None:
continue
env.set_environment(env_id, instruction_set["instructions"])
presenter = Presenter()
cumulative_reward = 0
for seg_idx in range(len(instruction_set["instructions"])):
if seg_idx != found_example[2]:
continue
print(f"RUNNING ENV {env_id} SEG {seg_idx}")
real_instruction_str = instruction_set["instructions"][
seg_idx]["instruction"]
instruction_display.show_instruction(real_instruction_str)
valid_segment = env.set_current_segment(seg_idx)
if not valid_segment:
continue
state = env.reset(seg_idx)
for i in range(START_PAUSE):
instruction_display.tick()
time.sleep(1)
tok_instruction = tokenize_instruction(
real_instruction_str, word2token)
state = env.reset(seg_idx)
print("Executing: f{instruction_str}")
while True:
instruction_display.tick()
rate.sleep()
action, internals = model.get_action(
state, tok_instruction)
state, reward, done, expired, oob = env.step(action)
cumulative_reward += reward
#presenter.show_sample(state, action, reward, cumulative_reward, real_instruction_str)
#show_depth(state.image)
if done:
break
for i in range(END_PAUSE):
instruction_display.tick()
time.sleep(1)
print("Segment finished!")
instruction_display.show_instruction("...")
print("Env finished!")
