def get_train_test(
eval_setup_name: str, fold_id: int,
use_test_split: bool) -> Tuple[Tuple[str, ...], Tuple[str, ...]]:
train, dev, test = phyre.get_fold(eval_setup_name, fold_id)
if use_test_split:
return train + dev, test
else:
return train, dev
def __init__(self, data_root, split, image_ext='.jpg'):
self.data_root = data_root
self.split = split
self.image_ext = image_ext
self.input_size = C.RPIN.INPUT_SIZE # number of input images
self.pred_size = eval(
f'C.RPIN.PRED_SIZE_{"TRAIN" if split == "train" else "TEST"}')
self.seq_size = self.input_size + self.pred_size
self.input_height, self.input_width = C.RPIN.INPUT_HEIGHT, C.RPIN.INPUT_WIDTH
protocal = C.PHYRE_PROTOCAL
fold = C.PHYRE_FOLD
num_pos = 400 if split == 'train' else 100
num_neg = 1600 if split == 'train' else 400
eval_setup = f'ball_{protocal}_template'
train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold)
tasks = train_tasks + dev_tasks if split == 'train' else test_tasks
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
# all the actions
cache = phyre.get_default_100k_cache('ball')
training_data = cache.get_sample(tasks, None)
# (100000 x 3)
actions = training_data['actions']
# (num_tasks x 100000)
sim_statuses = training_data['simulation_statuses']
self.simulator = phyre.initialize_simulator(tasks, action_tier)
self.video_info = np.zeros((0, 4))
for t_id, t in enumerate(tqdm(tasks)):
sim_status = sim_statuses[t_id]
pos_acts = actions[sim_status == 1].copy()
neg_acts = actions[sim_status == -1].copy()
np.random.shuffle(pos_acts)
np.random.shuffle(neg_acts)
pos_acts = pos_acts[:num_pos]
neg_acts = neg_acts[:num_neg]
acts = np.concatenate([pos_acts, neg_acts])
video_info = np.zeros((acts.shape[0], 4))
video_info[:, 0] = t_id
video_info[:, 1:] = acts
self.video_info = np.concatenate([self.video_info, video_info])
def simulate_result(chosen_action, chosen_score, model_number,
generation_number):
eval_setup = 'ball_cross_template'
fold_id = 0 # For simplicity, we will just use one fold for evaluation.
train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, 0)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
tasks = dev_tasks[0:1]
simulator = phyre.initialize_simulator(tasks, action_tier)
evaluator = phyre.Evaluator(tasks)
# Simulate the given action and add the status from taking the action to the evaluator.
simulation_result = simulator.simulate_action(0,
chosen_action,
need_images=True,
need_featurized_objects=True)
simulation_score = sf.ScoreFunctionValue(simulation_result)
pair = np.array([chosen_action, simulation_score])
timestr = time.strftime("%Y%m%d-%H%M%S")
score_pair = [
chosen_score, simulation_score, model_number, generation_number
]
score_string = "ScoreLog" + timestr
path = "/home/kyra/Desktop/phyre/agents/Scores"
np.save(os.path.join(path, score_string), score_pair)
return pair, simulation_result
def test(self, start_id=0, end_id=25):
random.seed(0)
np.random.seed(0)
protocal, fold_id = C.PHYRE_PROTOCAL, C.PHYRE_FOLD
self.score_model.eval()
print(f'testing using protocal {protocal} and fold {fold_id}')
# setup the PHYRE evaluation split
eval_setup = f'ball_{protocal}_template'
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
_, _, test_tasks = phyre.get_fold(eval_setup, fold_id) # PHYRE setup
candidate_list = [f'{i:05d}'
for i in range(start_id, end_id)] # filter tasks
test_list = [
task for task in test_tasks if task.split(':')[0] in candidate_list
]
simulator = phyre.initialize_simulator(test_list, action_tier)
# the action candidates are provided by the author of PHYRE benchmark
num_actions = 10000
cache = phyre.get_default_100k_cache('ball')
acts = cache.action_array[:num_actions]
training_data = cache.get_sample(test_list, None)
# some statistics variable when doing the evaluation
auccess = np.zeros((len(test_list), 100))
batched_pred = C.SOLVER.BATCH_SIZE
objs_color = None
all_data, all_acts, all_rois, all_image = [], [], [], []
# cache the initial bounding boxes from the simulator
os.makedirs('cache', exist_ok=True)
t_list = tqdm(test_list, 'Task')
for task_id, task in enumerate(t_list):
sim_statuses = training_data['simulation_statuses'][task_id]
confs, successes = [], []
boxes_cache_name = f'cache/{task.replace(":", "_")}.hkl'
use_cache = os.path.exists(boxes_cache_name)
all_boxes = hickle.load(boxes_cache_name) if use_cache else []
valid_act_id = 0
for act_id, act in enumerate(
tqdm(acts, 'Candidate Action', leave=False)):
sim = simulator.simulate_action(task_id,
act,
stride=60,
need_images=True,
need_featurized_objects=True)
assert sim.status == sim_statuses[
act_id], 'sanity check not passed'
if sim.status == phyre.SimulationStatus.INVALID_INPUT:
if act_id == len(acts) - 1 and len(
all_data) > 0: # final action is invalid
conf_t = self.batch_score(all_data, all_rois,
all_image, objs_color)
confs = confs + conf_t
all_data, all_acts, all_rois, all_image = [], [], [], []
continue
successes.append(sim.status == phyre.SimulationStatus.SOLVED)
# parse object, prepare input for network, the logic is the same as tools/gen_phyre.py
image = cv2.resize(sim.images[0],
(self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)
all_image.append(image[::-1])
image = phyre.observations_to_float_rgb(image)
objs_color = sim.featurized_objects.colors
objs_valid = [('BLACK' not in obj_color)
and ('PURPLE' not in obj_color)
for obj_color in objs_color]
objs = sim.featurized_objects.features[:, objs_valid, :]
objs_color = np.array(objs_color)[objs_valid]
num_objs = objs.shape[1]
if use_cache:
boxes = all_boxes[valid_act_id]
valid_act_id += 1
else:
boxes = np.zeros((1, num_objs, 5))
for o_id in range(num_objs):
mask = phyre.objects_util.featurized_objects_vector_to_raster(
objs[0][[o_id]])
mask_im = phyre.observations_to_float_rgb(mask)
mask_im[mask_im == 1] = 0
mask_im = mask_im.sum(-1) > 0
[h, w] = np.where(mask_im)
x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max()
x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1)
x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1)
y1 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT -
1)
y2 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT -
1)
boxes[0, o_id] = [o_id, x1, y1, x2, y2]
all_boxes.append(boxes)
data = image.transpose((2, 0, 1))[None, None, :]
data = torch.from_numpy(data.astype(np.float32))
rois = torch.from_numpy(boxes[...,
1:].astype(np.float32))[None, :]
all_data.append(data)
all_rois.append(rois)
if len(all_data) % batched_pred == 0 or act_id == len(
acts) - 1:
conf_t = self.batch_score(all_data, all_rois, all_image,
objs_color)
confs = confs + conf_t
all_data, all_rois, all_image = [], [], []
if not use_cache:
all_boxes = np.stack(all_boxes)
hickle.dump(all_boxes,
boxes_cache_name,
mode='w',
compression='gzip')
info = f'current AUCESS: '
top_acc = np.array(successes)[np.argsort(confs)[::-1]]
for i in range(100):
auccess[task_id, i] = int(np.sum(top_acc[:i + 1]) > 0)
w = np.array([np.log(k + 1) - np.log(k) for k in range(1, 101)])
s = auccess[:task_id + 1].sum(0) / auccess[:task_id + 1].shape[0]
info += f'{np.sum(w * s) / np.sum(w) * 100:.2f}'
t_list.set_description(info)
def gen_proposal(self, start_id=0, end_id=25):
random.seed(0)
np.random.seed(0)
protocal = C.PHYRE_PROTOCAL
fold_id = C.PHYRE_FOLD
print(f'generate proposal for {protocal} fold {fold_id}')
max_p_acts, max_n_acts, max_acts = 200, 800, 100000
self.proposal_dir = f'{self.output_dir.split("/")[-1]}_' \
f'p{max_p_acts}n{max_n_acts}a{max_acts // 1000}'
eval_setup = f'ball_{protocal}_template'
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
train_tasks, dev_tasks, test_tasks = phyre.get_fold(
eval_setup, fold_id)
# filter task
train_tasks = train_tasks + dev_tasks
candidate_list = [f'{i:05d}' for i in range(start_id, end_id)]
for split in ['train', 'test']:
train_list = [
task for task in train_tasks
if task.split(':')[0] in candidate_list
]
test_list = [
task for task in test_tasks
if task.split(':')[0] in candidate_list
]
if len(eval(f'{split}_list')) == 0:
return
simulator = phyre.initialize_simulator(eval(f'{split}_list'),
action_tier)
cache = phyre.get_default_100k_cache('ball')
training_data = cache.get_sample(eval(f'{split}_list'), None)
actions = cache.action_array[:max_acts]
final_list = eval(f'{split}_list')
t_list = tqdm(final_list, 'Task')
for task_id, task in enumerate(t_list):
box_cache_name = f'data/PHYRE_proposal/cache/{task.replace(":", "_")}_box.hkl'
act_cache_name = f'data/PHYRE_proposal/cache/{task.replace(":", "_")}_act.hkl'
use_cache = os.path.exists(box_cache_name) and os.path.exists(
act_cache_name)
if use_cache:
acts = hickle.load(act_cache_name)
all_boxes = hickle.load(box_cache_name)
else:
sim_statuses = training_data['simulation_statuses'][
task_id]
pos_acts = actions[sim_statuses == 1]
neg_acts = actions[sim_statuses == -1]
np.random.shuffle(pos_acts)
np.random.shuffle(neg_acts)
pos_acts = pos_acts[:max_p_acts]
neg_acts = neg_acts[:max_n_acts]
acts = np.concatenate([pos_acts, neg_acts])
hickle.dump(acts,
act_cache_name,
mode='w',
compression='gzip')
all_boxes = []
valid_act_id = 0
for act_id, act in enumerate(
tqdm(acts, 'Candidate Action', leave=False)):
sim = simulator.simulate_action(
task_id,
act,
stride=60,
need_images=True,
need_featurized_objects=True)
if not use_cache:
if act_id < len(pos_acts):
assert sim.status == phyre.SimulationStatus.SOLVED
else:
assert sim.status == phyre.SimulationStatus.NOT_SOLVED
assert sim.status != phyre.SimulationStatus.INVALID_INPUT
raw_images = sim.images
rst_images = np.stack([
np.ascontiguousarray(
cv2.resize(rst_image,
(self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)[::-1])
for rst_image in raw_images
])
# prepare input for network:
image = cv2.resize(raw_images[0],
(self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)
image = phyre.observations_to_float_rgb(image)
# parse object
objs_color = sim.featurized_objects.colors
objs_valid = [('BLACK' not in obj_color)
and ('PURPLE' not in obj_color)
for obj_color in objs_color]
objs = sim.featurized_objects.features[:, objs_valid, :]
objs_color = np.array(objs_color)[objs_valid]
num_objs = objs.shape[1]
if use_cache:
boxes = all_boxes[valid_act_id]
valid_act_id += 1
else:
boxes = np.zeros((1, num_objs, 5))
for o_id in range(num_objs):
mask = phyre.objects_util.featurized_objects_vector_to_raster(
objs[0][[o_id]])
mask_im = phyre.observations_to_float_rgb(mask)
mask_im[mask_im == 1] = 0
mask_im = mask_im.sum(-1) > 0
[h, w] = np.where(mask_im)
x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max()
x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH -
1)
x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH -
1)
y1 *= (self.input_height -
1) / (phyre.SCENE_HEIGHT - 1)
y2 *= (self.input_height -
1) / (phyre.SCENE_HEIGHT - 1)
boxes[0, o_id] = [o_id, x1, y1, x2, y2]
all_boxes.append(boxes)
data = image.transpose((2, 0, 1))[None, None, :]
data = torch.from_numpy(data.astype(np.float32))
rois = torch.from_numpy(boxes[..., 1:].astype(
np.float32))[None, :]
bg_image = rst_images[0].copy()
for fg_id in [1, 2, 3, 5]:
bg_image[bg_image == fg_id] = 0
boxes, masks = self.generate_trajs(data, rois)
rst_masks = np.stack([
self.render_mask_to_image(boxes[0, i],
masks[0, i],
images=bg_image.copy(),
color=objs_color).astype(
np.uint8)
for i in range(self.pred_rollout)
])
output_dir = f'data/PHYRE_proposal/{self.proposal_dir}/{split}/'
output_dir = output_dir + 'pos/' if sim.status == phyre.SimulationStatus.SOLVED else output_dir + 'neg/'
output_dir = output_dir + f'{task.replace(":", "_")}/'
os.makedirs(output_dir, exist_ok=True)
rst_dict = {'gt_im': rst_images, 'pred_im': rst_masks}
hickle.dump(rst_dict,
f'{output_dir}/{act_id}.hkl',
mode='w',
compression='gzip')
if not use_cache:
all_boxes = np.stack(all_boxes)
hickle.dump(all_boxes,
box_cache_name,
mode='w',
compression='gzip')
dtrain = '-dtrain' in sys.argv
singleviz = '-singleviz' in sys.argv
auccess = []
auc_dict = dict()
#for eval_setup in ['ball_cross_template', 'ball_within_template']:
for eval_setup in ['ball_within_template', 'ball_cross_template']:
auccess.append(eval_setup)
print(foldstart, folds)
for fold_id in range(foldstart, foldstart+folds):
solver = FlownetSolver(model_path, type, width, fold=fold_id, setup=eval_setup, bs=batchsize, uniform=uniform, radmode=radmode,
scheduled= scheduled, lr=lr, smart=smart, run=run, num_seeds=seeds, epochstart=epochstart,
device=device, hidfac=hidfac, dijkstra=dijkstra, dropout=dropout, deepex=deepex,
neck=neckfak, altconv=altconv, train_mode=train_mode, puretrain=puretrain)
train_ids, dev_ids, test_ids = phyre.get_fold(eval_setup, fold_id)
train_ids = train_ids + dev_ids
dev_ids = tuple()
if "-inspect" in sys.argv:
inspect_ids = []
tmp_ids = list(dev_ids)+list(test_ids)
count = dict(zip([i[:5] for i in tmp_ids],[0 for _ in tmp_ids]))
num_per = 4 if eval_setup == 'ball_within_template' else 10
for tid in tmp_ids:
if count[tid[:5]] < num_per:
count[tid[:5]] += 1
inspect_ids.append(tid)
#L.info("inpsect ids:", inspect_ids)
if "-get-all-data" in sys.argv:
import random
import numpy as np
import phyre
from tqdm import tqdm_notebook
import animations
random.seed(0)
# Evaluation Setup
eval_setup = 'ball_cross_template'
fold_id = 0 # For simplicity, we will just use one fold for evaluation.
train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, 0)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
tasks = dev_tasks[0:1]
print((tasks))
simulator = phyre.initialize_simulator(tasks, action_tier)
actions = simulator.build_discrete_action_space(max_actions=1000)
def evaluate_random_agent(tasks, tier):
# Create a simulator for the task and tier.
simulator = phyre.initialize_simulator(tasks, tier)
evaluator = phyre.Evaluator(tasks)
assert tuple(tasks) == simulator.task_ids
images = []
actions = []
for task_index in tqdm_notebook(range(len(tasks)), desc='Evaluate tasks'):
while evaluator.get_attempts_for_task(
task_index) < phyre.MAX_TEST_ATTEMPTS:
def worker(fold_id, eval_setup):
__, __, test_tasks = phyre.get_fold(eval_setup, fold_id)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
tasks = test_tasks
evaluator = evaluate_simple_agent(tasks, action_tier)
return evaluator.get_aucess()
def solve(model, model2, save_images=False):
tasks = [
'00000:001', '00000:002', '00000:003', '00000:004', '00000:005',
'00001:001', '00001:002', '00001:003', '00001:004', '00001:005',
'00002:007', '00002:011', '00002:015', '00002:017', '00002:023',
'00003:000', '00003:001', '00003:002', '00003:003', '00003:004',
'00004:063', '00004:071', '00004:092', '00004:094', '00004:095'
]
tasks = json.load(open("most_tasks.txt", 'r'))
eval_setup = 'ball_within_template'
fold_id = 0 # For simplicity, we will just use one fold for evaluation.
train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold_id)
print('Size of resulting splits:\n train:', len(train_tasks), '\n dev:',
len(dev_tasks), '\n test:', len(test_tasks))
tasks = train_tasks[:]
print("tasks:\n", tasks)
sim = phyre.initialize_simulator(tasks, 'ball')
init_scenes = sim.initial_scenes
X = T.tensor(format(init_scenes)).float()
print("Init Scenes Shape:\n", X.shape)
base_path = []
action_path = []
for i, t in enumerate(tasks):
while True:
action = sim.sample(i)
action[2] = 0.01
res = sim.simulate_action(i, action, stride=20)
if type(res.images) != type(None):
base_path.append(rollouts_to_channel([res.images], 2))
action_path.append(rollouts_to_channel([res.images], 1))
break
base_path = T.tensor(np.concatenate(base_path)).float()
action_path = T.tensor(np.concatenate(base_path)).float()
with T.no_grad():
Z = model(X)
A = model2(T.cat((X[:, 1:], base_path[:, None], Z), dim=1))
#B = model3(T.cat((X[:,1:], Y[:,None,2], Z, A), dim=1))
#B = extract_action(A, inspect=-2 if save_images else -1)
B = extract_action(action_path[:, None], inspect=-2 if save_images else -1)
# Saving Images:
if save_images:
for inspect in range(len(X)):
plt.imsave(
f"result/flownet/{inspect}_init.png",
T.cat(tuple(
T.cat((sub, T.ones(32, 1) * 0.5), dim=1)
for sub in X[inspect]),
dim=1))
plt.imsave(f"result/flownet/{inspect}_base.png",
base_path[inspect])
plt.imsave(f"result/flownet/{inspect}_target.png", Z[inspect, 0])
#plt.imsave(f"result/flownet/{inspect}_init_scene.png", np.flip(batch[inspect][0], axis=0))
plt.imsave(f"result/flownet/{inspect}_action.png", A[inspect, 0])
plt.imsave(f"result/flownet/{inspect}_selection.png", B[inspect,
0])
gen_actions = []
for b in B[:, 0]:
gen_actions.append(pic_to_values(b))
print(gen_actions)
# Feed actions into simulator
eva = phyre.Evaluator(tasks)
solved, valid, comb = dict(), dict(), dict()
for i, t in enumerate(tasks):
if not (t[:5] in comb):
comb[t[:5]] = 0
valid[t[:5]] = 0
solved[t[:5]] = 0
base_action = gen_actions[i]
# Random Agent Intercept:
#action = sim.sample()
res = sim.simulate_action(i, base_action)
tries = 0
alpha = 1
# 100 Tries Max:
while eva.get_attempts_for_task(i) < 100:
if not res.status.is_solved():
action = np.array(base_action) + np.random.randn(3) * np.array(
[0.1, 0.1, 0.1]) * alpha
res = sim.simulate_action(i, action)
subtries = 0
while subtries < 100 and res.status.is_invalid():
subtries += 1
action_var = np.array(action) + np.random.randn(
3) * np.array([0.05, 0.05, 0.05]) * alpha
res = sim.simulate_action(i, action_var)
eva.maybe_log_attempt(i, res.status)
alpha *= 1.01
else:
eva.maybe_log_attempt(i, res.status)
tries += 1
if save_images:
try:
for k, img in enumerate(res.images):
plt.imsave(f"result/flownet/{i}_{k}.png",
np.flip(img, axis=0))
pass
except Exception:
pass
#print(i, t, res.status.is_solved(), not res.status.is_invalid())
comb[t[:5]] = comb[t[:5]] + 1
if not res.status.is_invalid():
valid[t[:5]] = valid[t[:5]] + 1
if res.status.is_solved():
solved[t[:5]] = solved[t[:5]] + 1
# Prepare Plotting
print(eva.compute_all_metrics())
print(eva.get_auccess())
spacing = [1, 2, 3, 4]
fig, ax = plt.subplots(5, 5, sharey=True, sharex=True)
for i, t in enumerate(comb):
ax[i // 5, i % 5].bar(spacing, [
solved[t[:5]] /
(valid[t[:5]] if valid[t[:5]] else 1), solved[t[:5]] / comb[t[:5]],
valid[t[:5]] / comb[t[:5]], comb[t[:5]] / 100
])
ax[i // 5, i % 5].set_xlabel(t[:5])
plt.show()
def worker(fold_id, eval_setup):
train, dev, test = phyre.get_fold(eval_setup, fold_id)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
solved_actions_pdf = train_kde(train, action_tier)
return evaluate_agent(test, action_tier, solved_actions_pdf)
import numpy as np
from scipy.stats import skewnorm
from scipy.stats import gaussian_kde
import phyre
import ImgToObj
tier = 'ball'
eval_setup = 'ball_cross_template'
fold_id = 2
random.seed(0)
train, dev, test = phyre.get_fold(eval_setup, fold_id)
cache = phyre.get_default_100k_cache(tier)
all_solved_sizes = {}
all_solved_loc = {}
for index, task_id in enumerate(train):
task_type = task_id.split(":")[0]
statuses = cache.load_simulation_states(task_id)
solved_actions = cache.action_array[statuses ==
phyre.simulation_cache.SOLVED, :]
solved_sizes = solved_actions[:, 2]
solved_loc = solved_actions[:, 0:2]
if task_type not in all_solved_loc:
all_solved_loc[task_type] = solved_loc
def worker(fold_id, eval_setup):
__, __, test_tasks = phyre.get_fold(eval_setup, fold_id)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
tasks = test_tasks
return evaluate_agent(tasks, action_tier)
def generate_data(args):
eval_setup = 'ball_cross_template'
fold_id = 0
train_tasks, dev_tasks, test_tasks = phyre.get_fold(eval_setup, fold_id)
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
tasks_2 = [x for x in train_tasks if x.startswith(args.template)==True]
tasks = tasks_2
simulator = phyre.initialize_simulator(tasks, action_tier)
actions = simulator.build_discrete_action_space(max_actions=100000)
try:
os.mkdir('./data-generation/numpys')
except:
pass
try:
os.mkdir('./data-generation/episodes')
except:
pass
for task_index in tqdm_notebook(range(1, len(tasks_2))):
solution_found_counter = 0
# 10 rollouts for each
for k in range(10):
if solution_found_counter < 5:
action = random.choice(actions)
simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True,
stride=4)
num_trys = 0
while (simulation.status != phyre.simulation_cache.SOLVED and num_trys < 10000):
num_trys += 1
action = random.choice(actions)
simulation = simulator.simulate_action(task_index, action, need_images=True,
need_featurized_objects=True, stride=4)
if str(simulation.status) == "SimulationStatus.INVALID_INPUT":
num_trys -= 1
else:
action = random.choice(actions)
simulation = simulator.simulate_action(task_index, action, need_images=True, need_featurized_objects=True,
stride=4)
while (simulation.status != phyre.simulation_cache.NOT_SOLVED):
action = random.choice(actions)
simulation = simulator.simulate_action(task_index, action, need_images=True,
need_featurized_objects=True, stride=4)
print(simulation.status)
filename = 'data-generation/numpys/task-00023:' + str(task_index + 1) + '_' + str(k) + '.npy'
print(filename)
np.save(filename, simulation.images)
if str(simulation.status) == "SimulationStatus.SOLVED":
solution_found_counter += 1
dataset_path = './data-generation/numpys'
IMAGE_WIDTH = 256
IMAGE_HEIGHT = 256
max_episode_len = 120
for filename in os.listdir(dataset_path):
if filename.startswith('Task'):
continue
counter = 0
print("Laded file: ", filename)
data = os.path.join(dataset_path, filename)
try:
data = np.load(data)
except:
continue
print(data.shape)
if (len(data) >= max_episode_len):
images = np.zeros((len(data), 64, 64, 3), dtype=np.uint8)
rewards = np.ones((len(data)), dtype=np.float16)
actions = np.ones((len(data), 6), dtype=np.float16)
orientations = np.ones((len(data), 14), dtype=np.float16)
velocity = np.ones((len(data), 9), dtype=np.float16)
height = np.ones((len(data)), dtype=np.float16)
discount = np.ones((len(data)), dtype=np.float16)
for k, scene in enumerate(data):
current_image = np.zeros((256, 256, 3), dtype=np.uint8)
channel_0 = np.copy(np.flipud((scene)))
channel_1 = np.copy(np.flipud((scene)))
channel_0[channel_0 == 6] = 0
channel_0[channel_0 > 0] = 255
channel_1[channel_1 != 6] = 0
channel_1[channel_1 == 6] = 255
current_image[:, :, 0] = np.copy(channel_0)
current_image[:, :, 1] = np.copy(channel_1)
scaled_down_image = np.copy(current_image[::4,::4,:])
images[k] = scaled_down_image
numpy_dict = {"image": images[:max_episode_len], "action": actions[:max_episode_len], "reward": rewards[:max_episode_len], "orientations": orientations[:max_episode_len], "velocity": velocity[:max_episode_len], "dsicount": discount[:max_episode_len], "height": height[:max_episode_len]}
timestamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
identifier = str(uuid.uuid4().hex)
length = len(numpy_dict['reward'])
directory = pathlib.Path('data-generation/episodes')
filename = pathlib.Path(f'{timestamp}-{identifier}-{length}.npz')
filename = directory / filename
with io.BytesIO() as f1:
np.savez_compressed(f1, **numpy_dict)
f1.seek(0)
with filename.open('wb') as f2:
f2.write(f1.read())
def get_train_test(eval_setup_name, fold_id, use_test_split):
train, dev, test = phyre.get_fold(eval_setup_name, fold_id)
if use_test_split:
return train + dev, test
else:
return train, dev
def test(self, start_id=0, end_id=25, fold_id=0, protocal='within'):
random.seed(0)
print(f'testing {protocal} fold {fold_id}')
eval_setup = f'ball_{protocal}_template'
action_tier = phyre.eval_setup_to_action_tier(eval_setup)
_, _, test_tasks = phyre.get_fold(eval_setup, fold_id) # PHYRE setup
candidate_list = [f'{i:05d}' for i in range(start_id, end_id)] # filter tasks
test_list = [task for task in test_tasks if task.split(':')[0] in candidate_list]
simulator = phyre.initialize_simulator(test_list, action_tier)
# PHYRE evaluation
num_all_actions = [1000, 2000, 5000, 8000, 10000]
auccess = np.zeros((len(num_all_actions), len(test_list), 100))
batched_pred = C.SOLVER.BATCH_SIZE
# DATA for network:
all_data, all_acts, all_rois, all_image = [], [], [], []
cache = phyre.get_default_100k_cache('ball')
acts = cache.action_array[:10000]
# actions = cache.action_array[:100000]
# training_data = cache.get_sample(test_list, None)
pos_all, neg_all, pos_correct, neg_correct = 0, 0, 0, 0
objs_color = None
for task_id, task in enumerate(test_list):
confs, successes, num_valid_act_idx = [], [], []
boxes_cache_name = f'cache/{task.replace(":", "_")}.hkl'
use_cache = os.path.exists(boxes_cache_name)
all_boxes = hickle.load(boxes_cache_name) if use_cache else []
valid_act_cnt = 0
# sim_statuses = training_data['simulation_statuses'][task_id]
# pos_acts = actions[sim_statuses == 1]
# neg_acts = actions[sim_statuses == -1]
# np.random.shuffle(pos_acts)
# np.random.shuffle(neg_acts)
# pos_acts = pos_acts[:50]
# neg_acts = neg_acts[:200]
# acts = np.concatenate([pos_acts, neg_acts])
for act_id, act in enumerate(acts):
if act_id == 0:
pprint(f'{task}: {task_id} / {len(test_list)}')
sim = simulator.simulate_action(task_id, act, stride=60, need_images=True, need_featurized_objects=True)
if sim.status == phyre.SimulationStatus.INVALID_INPUT:
num_valid_act_idx.append(0)
if act_id == len(acts) - 1 and len(all_data) > 0: # final action is invalid
conf_t = self.batch_score(all_data, all_acts, all_rois, all_image, objs_color, task)
confs = confs + conf_t
all_data, all_acts, all_rois, all_image = [], [], [], []
continue
num_valid_act_idx.append(1)
successes.append(sim.status == phyre.SimulationStatus.SOLVED)
if self.score_with_heuristic or self.score_with_mask:
# parse object, prepare input for network:
image = cv2.resize(sim.images[0], (self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)
all_image.append(image[::-1]) # for heuristic method to detect goal location, need to flip
image = phyre.observations_to_float_rgb(image)
objs_color = sim.featurized_objects.colors
objs_valid = [('BLACK' not in obj_color) and ('PURPLE' not in obj_color) for obj_color in objs_color]
objs = sim.featurized_objects.features[:, objs_valid, :]
objs_color = np.array(objs_color)[objs_valid]
num_objs = objs.shape[1]
if use_cache:
boxes = all_boxes[valid_act_cnt]
valid_act_cnt += 1
else:
boxes = np.zeros((1, num_objs, 5))
for o_id in range(num_objs):
mask = phyre.objects_util.featurized_objects_vector_to_raster(objs[0][[o_id]])
mask_im = phyre.observations_to_float_rgb(mask)
mask_im[mask_im == 1] = 0
mask_im = mask_im.sum(-1) > 0
[h, w] = np.where(mask_im)
x1, x2, y1, y2 = w.min(), w.max(), h.min(), h.max()
x1 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1)
x2 *= (self.input_width - 1) / (phyre.SCENE_WIDTH - 1)
y1 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1)
y2 *= (self.input_height - 1) / (phyre.SCENE_HEIGHT - 1)
boxes[0, o_id] = [o_id, x1, y1, x2, y2]
all_boxes.append(boxes)
data = image.transpose((2, 0, 1))[None, None, :]
data = torch.from_numpy(data.astype(np.float32))
rois = torch.from_numpy(boxes[..., 1:].astype(np.float32))[None, :]
all_data.append(data)
all_rois.append(rois)
elif self.score_with_act:
init = np.ascontiguousarray(simulator.initial_scenes[task_id][::-1])
init128 = cv2.resize(init, (self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)
all_data.append(torch.from_numpy(init128))
all_acts.append(torch.from_numpy(act[None, :]))
elif self.score_with_vid_cls:
rst_images = np.stack([np.ascontiguousarray(
cv2.resize(rst_image, (self.input_width, self.input_height),
interpolation=cv2.INTER_NEAREST)[::-1]
) for rst_image in sim.images])
all_data.append(torch.from_numpy(rst_images))
else:
raise NotImplementedError
if len(all_data) % batched_pred == 0 or act_id == len(acts) - 1:
conf_t = self.batch_score(all_data, all_acts, all_rois, all_image, objs_color, task)
confs = confs + conf_t
all_data, all_acts, all_rois, all_image = [], [], [], []
if self.score_with_heuristic or self.score_with_mask:
if not use_cache:
all_boxes = np.stack(all_boxes)
hickle.dump(all_boxes, boxes_cache_name, mode='w', compression='gzip')
else:
assert valid_act_cnt == len(all_boxes)
pred = np.array(confs) >= 0.5
labels = np.array(successes)
pos_all += (labels == 1).sum()
neg_all += (labels == 0).sum()
pos_correct += (pred == labels)[labels == 1].sum()
neg_correct += (pred == labels)[labels == 0].sum()
pos_acc = (pred == labels)[labels == 1].sum() / (labels == 1).sum()
neg_acc = (pred == labels)[labels == 0].sum() / (labels == 0).sum()
info = f'{pos_acc * 100:.1f} / {neg_acc * 100:.1f} '
# info = f'{task}: '
for j, num_acts in enumerate(num_all_actions):
num_valid = np.sum(num_valid_act_idx[:num_acts])
top_acc = np.array(successes[:num_valid])[np.argsort(confs[:num_valid])[::-1]]
for i in range(100):
auccess[j, task_id, i] = int(np.sum(top_acc[:i + 1]) > 0)
w = np.array([np.log(k + 1) - np.log(k) for k in range(1, 101)])
s = auccess[j, :task_id + 1].sum(0) / auccess[j, :task_id + 1].shape[0]
info += f'{np.sum(w * s) / np.sum(w) * 100:.2f} {np.sum(successes[:num_valid])}/{num_acts // 1000}k | '
pprint(info)
pprint(pos_correct, pos_all, pos_correct / pos_all)
pprint(neg_correct, neg_all, neg_correct / neg_all)
cache_output_dir = f'{self.output_dir.replace("figures/", "")}/' \
f'{self.proposal_setting}_{self.method}_{protocal}_fold_{fold_id}/'
os.makedirs(cache_output_dir, exist_ok=True)
print(cache_output_dir)
stats = {
'auccess': auccess,
'p_c': pos_correct,
'p_a': pos_all,
'n_c': neg_correct,
'n_a': neg_all,
}
with open(f'{cache_output_dir}/{start_id}_{end_id}.pkl', 'wb') as f:
pickle.dump(stats, f, pickle.HIGHEST_PROTOCOL)
