def main_watch_video(domain_path, instance_path, session, ontology_path): #input grounded role labeling dataset session here
domain_path_inserted_predicates = str(domain_path).replace(".pddl", "-inserted-predicates.pddl")
instance_path_inserted_predicates = str(instance_path).replace(".pddl", "-parsed-objects.pddl")
my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(domain_path_inserted_predicates, \
instance_path_inserted_predicates, ontology_path)
# dict from bounding boxes to parsed_problem_objects_dict.keys()
bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'], 'g_drawer': ['g_drawer1'],
'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'], 'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'], 'knife': ['knife1'], 'plate': ['plate1'],
'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'], 'cucumber': ['cucumber1'], 'sink': ['sink1'],
'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'], 'spice_holder': ['spice_holder1'],
'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'], 'l_hand': ['l_hand'],
'g_drawer1': ['g_drawer11']}
###ATTENTION: Where the videos are stored won't be automated, as this is different for each dataset
video_annotation = video_annotation_mod.VideoAnnotation(
settings.VIDEO_DIR_PATH, \
settings.ANNOT_DIR_PATH, \
bb_to_pddl_obj_dict)
# ['s13-d25', 's28-d25', 's37-d25', 's21-d21', 's31-d25', 's23-d21', 's13-d21', 's27-d21', 's37-d21', 's22-d25']
#for current_session in video_annotation.session_names:
for current_session in [session]:
#for current_session in ["s37-d21"]:
touch_events = [] #all binary starts and ends of overlaps between two objects (frame_no, [all touches])
possible_actions_session = [] #all actions from manipulator_events and non_manipulator_events where the object predicates match to an action (frame_no, [all_actions])
print(current_session)
video_annotation.create_single_session_dicts(current_session)
show_annotation(my_parsed_problem, video_annotation, bb_to_pddl_obj_dict, touch_events, possible_actions_session)
#show_annotation(random.choice(session_names), parsed_problem)
print(current_session, "done")
pickle.dump(touch_events, \
open(str(Path(settings.ROOT_DIR) / Path("data/overlap_detections/touch_events_"+str(current_session)+".p")), "wb"))
pickle.dump(possible_actions_session, \
open(str(Path(settings.ROOT_DIR) / Path("data/possible_actions/possible_actions_session_"+str(current_session)+".p")), "wb"))
def __init__(self):
self.t_start = time.time()
# compile re for later use
self.re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")
self.my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(
"/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl", \
"/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects.pddl")
# dict from bounding boxes to parsed_problem_objects_dict.keys()
self.bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'],
'g_drawer': ['g_drawer1'],
'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'],
'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'],
'knife': ['knife1'], 'plate': ['plate1'],
'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'],
'cucumber': ['cucumber1'], 'sink': ['sink1'],
'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'],
'spice_holder': ['spice_holder1'],
'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'],
'l_hand': ['l_hand'],
'g_drawer1': ['g_drawer11']}
self.video_annotation = video_annotation_mod.VideoAnnotation(
'/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/Videos/', \
'/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/', \
self.bb_to_pddl_obj_dict)
self.session_name = "s13-d25"
print(self.session_name)
self.touch_events = pickle.load(open(
"/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/overlap_detections/touch_events_" + self.session_name + ".p",
"rb"))
self.possible_actions_session = pickle.load(open(
"/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/possible_actions/possible_actions_session_" + self.session_name + ".p",
"rb"))
# lama2011: /home/mk/Planning/fastdownwardplanner/fast-downward.py --alias seq-sat-lama-2011 /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl
# cmd = '/home/mk/Planning/fastdownwardplanner/fast-downward.py --validate /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl --search "astar(lmcut())" > fast_downward_out.txt'
# get current state from parsed problem
self.current_state_FD = self.my_parsed_problem.parsed_problem_FD.init
self.current_state_set = set()
for atom in self.current_state_FD:
if atom.predicate != "=" and self.my_parsed_problem.onto[
atom.predicate] is None: # TODO only add non-static atoms (static atoms defined in domain need to be checked, too?)
s = "(" + str(atom.predicate) + " " + " ".join(atom.args) + ")"
self.current_state_set.add(s)
def __init__(self, domain_inserted_predicates_path, instance_parsed_objects_path, session_name, ontology_path, \
save_after_X_iterations, experiment_name, current_results_dir, goal_path, possible_actions_percentage):
self.t_start = time.time()
self.n_iter = 0
self.save_after_X_iterations = save_after_X_iterations
self.experiment_name = experiment_name
self.current_results_dir = current_results_dir
# compile re for later use
self.re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")
self.domain_inserted_predicates_path = domain_inserted_predicates_path
self.instance_parsed_objects_path = instance_parsed_objects_path
self.ontology_path = ontology_path
self.possible_actions_percentage = possible_actions_percentage
self.goal_path = goal_path
self.no_goals = 0
self.goals = []
self.goal_sets = []
with open(self.goal_path) as f:
for i, l in enumerate(f):
#goal string
self.goals.append(l)
#goal set
found_atoms_list = Node.re_compiled.findall(l)
g_s = set()
for a in found_atoms_list:
g_s.add(a)
self.goal_sets.append(g_s)
self.no_goals = i + 1
self.my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(domain_inserted_predicates_path,
instance_parsed_objects_path, ontology_path)
# dict from bounding boxes to parsed_problem_objects_dict.keys()
self.bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'],
'g_drawer': ['g_drawer1'],
'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'],
'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'],
'knife': ['knife1'], 'plate': ['plate1'],
'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'],
'cucumber': ['cucumber1'], 'sink': ['sink1'],
'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'],
'spice_holder': ['spice_holder1'],
'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'],
'l_hand': ['l_hand'],
'g_drawer1': ['g_drawer11']}
self.video_annotation = video_annotation_mod.VideoAnnotation(
settings.VIDEO_DIR_PATH, \
settings.ANNOT_DIR_PATH, \
self.bb_to_pddl_obj_dict)
self.session_name = session_name
print(self.session_name)
self.touch_events = pickle.load(open(Path(ROOT_DIR) /
Path("data/overlap_detections/touch_events_" + self.session_name + ".p"),
"rb"))
self.possible_actions_session_full_length = pickle.load(open(Path(ROOT_DIR) /
Path("data/possible_actions/possible_actions_session_" + self.session_name + ".p"),
"rb"))
self.possible_actions_session = self.possible_actions_session_full_length[0:int(len(self.possible_actions_session_full_length)*self.possible_actions_percentage + 0.5)]
# get current state from parsed problem
self.current_state_FD = self.my_parsed_problem.parsed_problem_FD.init
self.current_state_set = set()
for atom in self.current_state_FD:
if atom.predicate != "=" and self.my_parsed_problem.onto[
atom.predicate] is None:
s = "(" + str(atom.predicate) + " " + " ".join(atom.args) + ")"
self.current_state_set.add(s)
def build_tree():
t0 = time.time()
t_start = time.time()
#compile re for later use
re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")
my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl", \
"/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects.pddl")
# dict from bounding boxes to parsed_problem_objects_dict.keys()
bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'], 'g_drawer': ['g_drawer1'],
'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'], 'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'], 'knife': ['knife1'], 'plate': ['plate1'],
'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'], 'cucumber': ['cucumber1'], 'sink': ['sink1'],
'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'], 'spice_holder': ['spice_holder1'],
'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'], 'l_hand': ['l_hand'],
'g_drawer1': ['g_drawer11']}
video_annotation = video_annotation_mod.VideoAnnotation(
'/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/Videos/', \
'/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/', \
bb_to_pddl_obj_dict)
session_name = "s13-d25"
print(session_name)
touch_events = pickle.load( open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/overlap_detections/touch_events_"+session_name+".p", "rb" ) )
possible_actions_session = pickle.load( open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/possible_actions/possible_actions_session_"+session_name+".p", "rb" ) )
#lama2011: /home/mk/Planning/fastdownwardplanner/fast-downward.py --alias seq-sat-lama-2011 /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl
# cmd = '/home/mk/Planning/fastdownwardplanner/fast-downward.py --validate /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl --search "astar(lmcut())" > fast_downward_out.txt'
#get current state from parsed problem
current_state_FD = my_parsed_problem.parsed_problem_FD.init
current_state_set = set()
for atom in current_state_FD:
if atom.predicate != "=" and my_parsed_problem.onto[atom.predicate] is None: #TODO only add non-static atoms (static atoms defined in domain need to be checked, too?)
s = "("+ str(atom.predicate) + " " + " ".join(atom.args) + ")"
current_state_set.add(s)
#print(s)
current_state_string = " ".join(sorted(current_state_set))
#put init state at root of tree
#https://treelib.readthedocs.io/en/latest/
tree = Tree()
tree.create_node(str(current_state_string), 0, data=[current_state_set, ""]) #in the root there was no previous action
open_state_set = [0] #put the initial state as first element into the open state set
state_string_to_node_id_dict = dict()
state_string_to_node_id_dict[str(current_state_string)] = 0
###TODO the paragraph below is probably duplicate. it is done in the loop anyway
template_instance = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects-insert-init.pddl", "r")
template_instance_string = "".join(template_instance.readlines())
parsed_template_instance_string = template_instance_string.replace("<insert_init>", current_state_string)
f = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl", "w")
f.write(parsed_template_instance_string)
f.close()
tot_os_cmd_time = 0
tree_id_counter = 1
tree.show()
for timestep in possible_actions_session:
frame_no = timestep[0]
open_state_set_for_timestep = []
for s in open_state_set:
current_plan_path_string = "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/plans/current_sas_plan_try_"
try_action_strings = []
for i, try_action in enumerate(timestep[1]):
try_action_string = "(" + " ".join(try_action) + ")" #create the plan for the action to try
try_action_strings.append(try_action_string)
f = open(current_plan_path_string+str(i), "w")
f.write(try_action_string + "\n;cost = 1 (unit cost)")
f.close()
current_state_set = tree[s].data[0]
current_state_string = " ".join(current_state_set)
template_instance = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects-insert-init.pddl", \
"r")
template_instance_string = "".join(template_instance.readlines())
parsed_template_instance_string = template_instance_string.replace("<insert_init>", current_state_string)
f = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl", "w")
f.write(parsed_template_instance_string)
f.close()
cmd = '/home/mk/Planning/VAL/validate -v /home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl \
/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl ' \
+ " ".join([current_plan_path_string+str(i) for i in range(len(timestep[1]))]) + \
' > /home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt'
#cmd = '/home/mk/Planning/VAL/validate -v /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/current-state-take-put-instance-no-handempty.pddl /home/mk/PycharmProjects/pic-to-plan/val_exp/open_sas_plan > plan_val_output.txt' #check for unsatisfied precondition --> action not applicable in current state
t_os_start = time.time()
os.system(cmd)
t_os_end = time.time()
tot_os_cmd_time += t_os_end - t_os_start
plan_val_output = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt", "r")
plan_val_output_joined = "".join(plan_val_output.readlines())
#print(plan_val_output_joined)
new_state_sets = []
plan_val_output = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt", "r") #TODO superfluous open, but readlines changes the iterator...
for line in plan_val_output:
if "Checking plan" in line:
new_atom_added = False
new_state_sets.append(copy.deepcopy(current_state_set))
elif "Deleting" in line:
new_atom_added = True
m = re_compiled.search(line)
atom_to_delete = m.group(0)
#print("DEL", atom_to_delete)
if atom_to_delete in new_state_sets[-1]:
new_state_sets[-1].remove(atom_to_delete)
elif "Adding" in line:
new_atom_added = True
m = re_compiled.search(line)
atom_to_add = m.group(0)
#print("ADD", atom_to_add)
if atom_to_add not in new_state_sets[-1]:
new_state_sets[-1].add(atom_to_add)
sorted_new_state_strings = [" ".join(sorted(new_s)) for new_s in new_state_sets]
for i, sorted_new_state_string in enumerate(sorted_new_state_strings):
if new_atom_added and sorted_new_state_string not in state_string_to_node_id_dict.keys(): #new atom short circuits logical expression
tree.create_node(sorted_new_state_string, tree_id_counter, data=[new_state_sets[i], try_action_strings[i]], parent=tree[s].identifier) #in the root there was no previous action
open_state_set_for_timestep.append(tree_id_counter)
state_string_to_node_id_dict[sorted_new_state_string] = tree_id_counter
tree_id_counter += 1
t1 = time.time()
if t1-t0 > 30:
t0 = t1
tree.show()
print(frame_no, tree_id_counter)
if time.time() - t_start > 10:
break
open_state_set.extend(open_state_set_for_timestep)
tree.show()
t_stop = time.time()
processing_time = t_stop - t_start
print("FINISHED", "Frame no:", frame_no, "Node count:", tree_id_counter, "Tree depth:", tree.depth())
print("Processing time:", processing_time, "Video length:", frame_no/30.0, "Realtime factor:", frame_no/(30.0 * processing_time))
print("Time spent in os.system", tot_os_cmd_time)