def main_watch_video(domain_path, instance_path, session, ontology_path): #input grounded role labeling dataset session here
    domain_path_inserted_predicates = str(domain_path).replace(".pddl", "-inserted-predicates.pddl")
    instance_path_inserted_predicates = str(instance_path).replace(".pddl", "-parsed-objects.pddl")
    my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(domain_path_inserted_predicates, \
                                                            instance_path_inserted_predicates, ontology_path)
    # dict from bounding boxes to parsed_problem_objects_dict.keys()
    bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'], 'g_drawer': ['g_drawer1'],
     'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'], 'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
     'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'], 'knife': ['knife1'], 'plate': ['plate1'],
     'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'], 'cucumber': ['cucumber1'], 'sink': ['sink1'],
     'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'], 'spice_holder': ['spice_holder1'],
     'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'], 'l_hand': ['l_hand'],
     'g_drawer1': ['g_drawer11']}

    ###ATTENTION: Where the videos are stored won't be automated, as this is different for each dataset
    video_annotation = video_annotation_mod.VideoAnnotation(
        settings.VIDEO_DIR_PATH, \
        settings.ANNOT_DIR_PATH, \
        bb_to_pddl_obj_dict)

    # ['s13-d25', 's28-d25', 's37-d25', 's21-d21', 's31-d25', 's23-d21', 's13-d21', 's27-d21', 's37-d21', 's22-d25']
    #for current_session in video_annotation.session_names:
    for current_session in [session]:
    #for current_session in ["s37-d21"]:
        touch_events = []                       #all binary starts and ends of overlaps between two objects (frame_no, [all touches])
        possible_actions_session = []          #all actions from manipulator_events and non_manipulator_events where the object predicates match to an action (frame_no, [all_actions])
        print(current_session)
        video_annotation.create_single_session_dicts(current_session)
        show_annotation(my_parsed_problem, video_annotation, bb_to_pddl_obj_dict, touch_events, possible_actions_session)
        #show_annotation(random.choice(session_names), parsed_problem)
        print(current_session, "done")
        pickle.dump(touch_events, \
                    open(str(Path(settings.ROOT_DIR) / Path("data/overlap_detections/touch_events_"+str(current_session)+".p")), "wb"))
        pickle.dump(possible_actions_session, \
                    open(str(Path(settings.ROOT_DIR) / Path("data/possible_actions/possible_actions_session_"+str(current_session)+".p")), "wb"))
示例#2
0
    def __init__(self):
        self.t_start = time.time()

        # compile re for later use
        self.re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")

        self.my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(
            "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl", \
            "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects.pddl")
        # dict from bounding boxes to parsed_problem_objects_dict.keys()
        self.bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'],
                               'g_drawer': ['g_drawer1'],
                               'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'],
                               'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
                               'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'],
                               'knife': ['knife1'], 'plate': ['plate1'],
                               'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'],
                               'cucumber': ['cucumber1'], 'sink': ['sink1'],
                               'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'],
                               'spice_holder': ['spice_holder1'],
                               'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'],
                               'l_hand': ['l_hand'],
                               'g_drawer1': ['g_drawer11']}

        self.video_annotation = video_annotation_mod.VideoAnnotation(
            '/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/Videos/', \
            '/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/', \
            self.bb_to_pddl_obj_dict)

        self.session_name = "s13-d25"
        print(self.session_name)
        self.touch_events = pickle.load(open(
            "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/overlap_detections/touch_events_" + self.session_name + ".p",
            "rb"))
        self.possible_actions_session = pickle.load(open(
            "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/possible_actions/possible_actions_session_" + self.session_name + ".p",
            "rb"))

        # lama2011: /home/mk/Planning/fastdownwardplanner/fast-downward.py  --alias seq-sat-lama-2011 /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl
        # cmd = '/home/mk/Planning/fastdownwardplanner/fast-downward.py --validate /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl --search "astar(lmcut())" > fast_downward_out.txt'

        # get current state from parsed problem
        self.current_state_FD = self.my_parsed_problem.parsed_problem_FD.init
        self.current_state_set = set()
        for atom in self.current_state_FD:
            if atom.predicate != "=" and self.my_parsed_problem.onto[
                atom.predicate] is None:  # TODO only add non-static atoms (static atoms defined in domain need to be checked, too?)
                s = "(" + str(atom.predicate) + " " + " ".join(atom.args) + ")"
                self.current_state_set.add(s)
    def __init__(self, domain_inserted_predicates_path, instance_parsed_objects_path, session_name, ontology_path, \
                 save_after_X_iterations, experiment_name, current_results_dir, goal_path, possible_actions_percentage):
        self.t_start = time.time()
        self.n_iter = 0
        self.save_after_X_iterations = save_after_X_iterations
        self.experiment_name = experiment_name
        self.current_results_dir = current_results_dir
        # compile re for later use
        self.re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")
        self.domain_inserted_predicates_path = domain_inserted_predicates_path
        self.instance_parsed_objects_path = instance_parsed_objects_path
        self.ontology_path = ontology_path
        self.possible_actions_percentage = possible_actions_percentage
        self.goal_path = goal_path
        self.no_goals = 0
        self.goals = []
        self.goal_sets = []
        with open(self.goal_path) as f:
            for i, l in enumerate(f):
                #goal string
                self.goals.append(l)
                #goal set
                found_atoms_list = Node.re_compiled.findall(l)
                g_s = set()
                for a in found_atoms_list:
                    g_s.add(a)
                self.goal_sets.append(g_s)
            self.no_goals = i + 1

        self.my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem(domain_inserted_predicates_path,
                                                                      instance_parsed_objects_path, ontology_path)

        # dict from bounding boxes to parsed_problem_objects_dict.keys()
        self.bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'],
                                    'g_drawer': ['g_drawer1'],
                                    'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'],
                                    'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
                                    'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'],
                                    'knife': ['knife1'], 'plate': ['plate1'],
                                    'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'],
                                    'cucumber': ['cucumber1'], 'sink': ['sink1'],
                                    'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'],
                                    'spice_holder': ['spice_holder1'],
                                    'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'],
                                    'l_hand': ['l_hand'],
                                    'g_drawer1': ['g_drawer11']}

        self.video_annotation = video_annotation_mod.VideoAnnotation(
            settings.VIDEO_DIR_PATH, \
            settings.ANNOT_DIR_PATH, \
            self.bb_to_pddl_obj_dict)

        self.session_name = session_name
        print(self.session_name)
        self.touch_events = pickle.load(open(Path(ROOT_DIR) /
                                             Path("data/overlap_detections/touch_events_" + self.session_name + ".p"),
                                             "rb"))
        self.possible_actions_session_full_length = pickle.load(open(Path(ROOT_DIR) /
                                                                     Path("data/possible_actions/possible_actions_session_" + self.session_name + ".p"),
                                                                     "rb"))
        self.possible_actions_session = self.possible_actions_session_full_length[0:int(len(self.possible_actions_session_full_length)*self.possible_actions_percentage + 0.5)]

        # get current state from parsed problem
        self.current_state_FD = self.my_parsed_problem.parsed_problem_FD.init
        self.current_state_set = set()
        for atom in self.current_state_FD:
            if atom.predicate != "=" and self.my_parsed_problem.onto[
                atom.predicate] is None:
                s = "(" + str(atom.predicate) + " " + " ".join(atom.args) + ")"
                self.current_state_set.add(s)
def build_tree():
    t0 = time.time()
    t_start = time.time()

    #compile re for later use
    re_compiled = re.compile(r"\([A-Za-z0-9_ ]+\)")

    my_parsed_problem = parsed_problem_mod.ParsedPDDLProblem("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl", \
                                                                "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects.pddl")
    # dict from bounding boxes to parsed_problem_objects_dict.keys()
    bb_to_pddl_obj_dict = {'plastic_bag': ['plastic_bag1'], 'plastic_paper_bag': ['plastic_paper_bag1'], 'g_drawer': ['g_drawer1'],
         'sponge': ['sponge1'], 'drawer': ['drawer1', 'drawer2'], 'cuttingboard': ['cuttingboard1'], 'end': ['end1'],
         'bowl': ['bowl1'], 'r_hand': ['r_hand'], 'bread': ['bread1', 'bread2', 'bread3'], 'knife': ['knife1'], 'plate': ['plate1'],
         'cupboard': ['cupboard1'], 'peel': ['peel1'], 'fridge': ['fridge1'], 'cucumber': ['cucumber1'], 'sink': ['sink1'],
         'peeler': ['peeler1'], 'spice': ['spice1'], 'faucet': ['faucet1'], 'spice_holder': ['spice_holder1'],
         'towel': ['towel1'], 'counter': ['counter1'], 'spice_shaker': ['spice_shaker1'], 'l_hand': ['l_hand'],
         'g_drawer1': ['g_drawer11']}

    video_annotation = video_annotation_mod.VideoAnnotation(
            '/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/Videos/', \
            '/media/hdd1/Datasets/GroundingSemanticRoleLabelingForCookingDataset/Video_annotation/Video_annotation/', \
            bb_to_pddl_obj_dict)

    session_name = "s13-d25"
    print(session_name)
    touch_events = pickle.load( open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/overlap_detections/touch_events_"+session_name+".p", "rb" ) )
    possible_actions_session = pickle.load( open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/data/possible_actions/possible_actions_session_"+session_name+".p", "rb" ) )

    #lama2011: /home/mk/Planning/fastdownwardplanner/fast-downward.py  --alias seq-sat-lama-2011 /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl
    # cmd = '/home/mk/Planning/fastdownwardplanner/fast-downward.py --validate /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/take-put-instance.pddl --search "astar(lmcut())" > fast_downward_out.txt'

    #get current state from parsed problem
    current_state_FD = my_parsed_problem.parsed_problem_FD.init
    current_state_set = set()
    for atom in current_state_FD:
        if atom.predicate != "=" and my_parsed_problem.onto[atom.predicate] is None: #TODO only add non-static atoms (static atoms defined in domain need to be checked, too?)
            s = "("+ str(atom.predicate) + " " + " ".join(atom.args) + ")"
            current_state_set.add(s)
            #print(s)
    current_state_string = " ".join(sorted(current_state_set))

    #put init state at root of tree
    #https://treelib.readthedocs.io/en/latest/
    tree = Tree()
    tree.create_node(str(current_state_string), 0, data=[current_state_set, ""]) #in the root there was no previous action
    open_state_set = [0]  #put the initial state as first element into the open state set
    state_string_to_node_id_dict = dict()
    state_string_to_node_id_dict[str(current_state_string)] = 0

    ###TODO the paragraph below is probably duplicate. it is done in the loop anyway
    template_instance = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects-insert-init.pddl", "r")
    template_instance_string = "".join(template_instance.readlines())
    parsed_template_instance_string = template_instance_string.replace("<insert_init>", current_state_string)
    f = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl", "w")
    f.write(parsed_template_instance_string)
    f.close()

    tot_os_cmd_time = 0

    tree_id_counter = 1
    tree.show()
    for timestep in possible_actions_session:
        frame_no = timestep[0]
        open_state_set_for_timestep = []
        for s in open_state_set:
            current_plan_path_string = "/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/plans/current_sas_plan_try_"
            try_action_strings = []
            for i, try_action in enumerate(timestep[1]):
                try_action_string = "(" + " ".join(try_action) + ")"    #create the plan for the action to try
                try_action_strings.append(try_action_string)
                f = open(current_plan_path_string+str(i), "w")
                f.write(try_action_string + "\n;cost = 1 (unit cost)")
                f.close()
            current_state_set = tree[s].data[0]
            current_state_string = " ".join(current_state_set)

            template_instance = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/template-instance-parsed-objects-insert-init.pddl", \
                    "r")
            template_instance_string = "".join(template_instance.readlines())
            parsed_template_instance_string = template_instance_string.replace("<insert_init>", current_state_string)
            f = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl", "w")
            f.write(parsed_template_instance_string)
            f.close()

            cmd = '/home/mk/Planning/VAL/validate -v /home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/domains/template-domain-inserted-predicates.pddl \
                    /home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/instances/current-state-instance.pddl ' \
                    + " ".join([current_plan_path_string+str(i) for i in range(len(timestep[1]))]) + \
                    ' > /home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt'
            #cmd = '/home/mk/Planning/VAL/validate -v  /home/mk/PycharmProjects/pic-to-plan/take-put-domain.pddl /home/mk/PycharmProjects/pic-to-plan/current-state-take-put-instance-no-handempty.pddl /home/mk/PycharmProjects/pic-to-plan/val_exp/open_sas_plan > plan_val_output.txt' #check for unsatisfied precondition --> action not applicable in current state
            t_os_start = time.time()
            os.system(cmd)
            t_os_end = time.time()
            tot_os_cmd_time += t_os_end - t_os_start

            plan_val_output = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt", "r")
            plan_val_output_joined = "".join(plan_val_output.readlines())
            #print(plan_val_output_joined)
            new_state_sets = []
            plan_val_output = open("/home/mk/PycharmProjects/pic-to-plan-v2-git/pic_to_plan_v2/pddl/val_output/plan_val_output.txt", "r") #TODO superfluous open, but readlines changes the iterator...
            for line in plan_val_output:
                if "Checking plan" in line:
                    new_atom_added = False
                    new_state_sets.append(copy.deepcopy(current_state_set))
                elif "Deleting" in line:
                    new_atom_added = True
                    m = re_compiled.search(line)
                    atom_to_delete = m.group(0)
                    #print("DEL", atom_to_delete)
                    if atom_to_delete in new_state_sets[-1]:
                        new_state_sets[-1].remove(atom_to_delete)
                elif "Adding" in line:
                    new_atom_added = True
                    m = re_compiled.search(line)
                    atom_to_add = m.group(0)
                    #print("ADD", atom_to_add)
                    if atom_to_add not in new_state_sets[-1]:
                        new_state_sets[-1].add(atom_to_add)

            sorted_new_state_strings = [" ".join(sorted(new_s)) for new_s in new_state_sets]
            for i, sorted_new_state_string in enumerate(sorted_new_state_strings):
                if new_atom_added and sorted_new_state_string not in state_string_to_node_id_dict.keys(): #new atom short circuits logical expression
                    tree.create_node(sorted_new_state_string, tree_id_counter, data=[new_state_sets[i], try_action_strings[i]], parent=tree[s].identifier) #in the root there was no previous action
                    open_state_set_for_timestep.append(tree_id_counter)
                    state_string_to_node_id_dict[sorted_new_state_string] = tree_id_counter
                    tree_id_counter += 1

            t1 = time.time()

            if t1-t0 > 30:
                t0 = t1
                tree.show()
                print(frame_no, tree_id_counter)
            if time.time() - t_start > 10:
                break

        open_state_set.extend(open_state_set_for_timestep)

    tree.show()
    t_stop = time.time()
    processing_time = t_stop - t_start
    print("FINISHED", "Frame no:", frame_no, "Node count:", tree_id_counter, "Tree depth:", tree.depth())
    print("Processing time:", processing_time, "Video length:", frame_no/30.0, "Realtime factor:", frame_no/(30.0 * processing_time))
    print("Time spent in os.system", tot_os_cmd_time)