def task1(self, trajectories_json_file): q_values = {} # Your code here print(self.helper.get_all_actions()) #print(self.helper.get_all_actions(2)) api.execute_action('moveF', {}, 'robot1') #self.helper.execute_action('moveF', {}, 'robot2') return q_values
def raw_execute_action(self, list_of_actions_to_execute, bot_location): for action in list_of_actions_to_execute: actions = {} actions["action_name"] = action actions["action_params"] = {} if (action == "CLEAN"): actions["action_name"] = "clean" actions["action_params"]["dirt_id"] = str(self.dirt_location_to_id_mapping[bot_location]) temp, next_state = api.execute_action(actions["action_name"], actions["action_params"]) return next_state
def random_walk(self): print("Robot can perform following actions: {}".format( self.action_list)) while True: if api.is_terminal_state(self.current_state): print "Goal Reached" break possible_actions = api.get_possible_actions(self.current_state) print("Possible actions in current state: {}".format( possible_actions)) for action in possible_actions: print "Action {}".format(action) if action == "pick": # try to pick book 1 action_params = {"book_name": "book_1"} elif action == "place": action_params = { "book_name": "book_1", "bin_name": "trolly_2" } else: action_params = {} states = api.get_possible_states(self.current_state, action, action_params) print "Possible states are:" for state in states: next_state = states[state][0] probability = states[state][1] print state print "State: ", next_state print "Probability: ", probability print "Reward: ", api.get_reward(self.current_state, action, next_state) print "" idx = random.randint(0, len(possible_actions) - 1) chosen_action = possible_actions[idx] if chosen_action == "pick": # try to pick book 1 action_params = {"book_name": "book_1"} elif chosen_action == "place": action_params = {"book_name": "book_1", "bin_name": "trolly_2"} else: action_params = {} print "Executing action: {} with params: {}".format( chosen_action, action_params) success, next_state = api.execute_action(chosen_action, action_params) if success == 1: print "Successfully executed" else: print "Action failed" self.current_state = next_state print "updated current state:" print self.current_state raw_input("\nPress Enter to continue execution...")
if __name__ == "__main__": args = parser.parse_args() if args.command == 'get_current_state': print(api.get_current_state()) elif args.command == 'is_terminal_state': current_state = api.get_current_state() print(api.is_terminal_state(current_state)) elif args.command == 'reset_world': print(api.reset_world()) elif args.command == 'get_all_actions': print(api.get_all_actions()) elif args.command == 'get_possible_actions': current_state = api.get_current_state() print(api.get_possible_actions(current_state)) elif args.command == 'get_possible_states': current_state = api.get_current_state() for robot in current_state['robots'].keys(): for action in api.get_possible_actions(current_state, robot): print( api.get_possible_states(current_state, action, {'robot': robot})) elif args.command == 'execute_action': current_state = api.get_current_state() success, next_state = api.execute_action(args.action, {}) print(success) print(api.get_reward(current_state, args.action, next_state)) elif args.command == 'get_path': print(api.get_path('robot0', (0.0, -2.0)))
action = line[line.index(':') + 1:].lower().split() if (action[0] == 'move'): robot = action[3] if 'house' in action[2]: target = action[2].split('_')[1] target = current_state['houses'][target]['deliver_loc'] elif 'box' in action[2]: target = action[2][:-5] target = current_state['packages'][target]['load_loc'][0] else: assert False actions = api.get_path(robot, target) for act in actions: success, current_state = api.execute_action(act, []) if success != 1: print(act + ' failed!') break elif (action[0] == 'load'): robot = action[4] success, current_state = api.execute_action( robot + '-load', []) if success != 1: print('load failed!') break elif (action[0] == 'deliver'): robot = action[5] success, current_state = api.execute_action(
def path_generation(self): print("Robot can perform following actions: {}".format( self.action_list)) #f=open(plan_file, "r") #contents =f.readlines() with open(ROOT_PATH + "/objects.json", 'r') as json_file: try: objects = json.load(json_file) except (ValueError, KeyError, TypeError): print "JSON error" #print objects cans_cups = objects["cans"] cans_cups.update(objects["cups"]) from_location = State(0.0, 0.0, 'EAST') f = open(ROOT_PATH + '/state_action.txt', 'w+') init_state = (api.get_current_state()['robot']['x'], api.get_current_state()['robot']['y'], api.get_current_state()['robot']['orientation']) f.write("%s\t" % str(init_state)) while cans_cups: choice = random.choice(cans_cups.values()) print choice["load_loc"] key = cans_cups.keys()[cans_cups.values().index(choice)] actions, dummy_from_location, result = self.get_path( from_location, choice["load_loc"], choice["loc"]) print actions, dummy_from_location, result for action in actions: action_params = {} #print "Executing actions:",action success, next_state = api.execute_action(action, action_params) if not success: break f.write("%s \n" % action) init_state = (api.get_current_state()['robot']['x'], api.get_current_state()['robot']['y'], api.get_current_state()['robot']['orientation']) f.write("%s \t" % str(init_state)) print "Successfully reached object:", key if result: del cans_cups[key] if "can" in key: can_num = key.split("_")[1] obj_name = "coke@can{}".format(can_num) else: cup_num = key.split("_")[1] obj_name = "plastic@cup{}".format(cup_num) success, next_state = api.execute_action( 'sense', {"obj_name": obj_name}) if next_state["sees_can"]: print("I saw a can!!!") success, next_state = api.execute_action( 'pick', {"can_name": key}) f.write("pick_%s \t" % str(key)) print "I have executed pick action:", next_state actions, dummy_from_location, result = self.get_path( dummy_from_location, objects["bins"]["bin"]["load_loc"], objects["bins"]["bin"]["loc"]) print actions, dummy_from_location, result for action in actions: action_params = {} #print "Executing actions:",action success, next_state = api.execute_action( action, action_params) if not success: break f.write("%s \n" % action) init_state = ( api.get_current_state()['robot']['x'], api.get_current_state()['robot']['y'], api.get_current_state()['robot']['orientation']) f.write("%s \t" % str(init_state)) print "Successfully reached bin" success, next_state = api.execute_action( 'place', { "can_name": key, "bin_name": "bin" }) print "I have executed place action:", next_state f.write("place_%s \t" % str(key)) else: print("I saw a cup!!!") from_location = dummy_from_location f.close() print "Successfully Completed"
def task5(self, episodes): '''for running the simulation after training''' q_values = {} # Your code here actions_json_file = '/action_config.json' with open(self.root_path + actions_json_file) as json_file: try: self.action_reference = json.load(json_file, parse_float=float) except (ValueError, KeyError, TypeError): print "JSON error" self.book_penalty = -100 self.bump_penalty = -100 # ============================================================================= # q tables initialized to zero # ============================================================================= #(x,y, orientation, c1,c2,tbot_near,action) #c1,c2 will be zero if available, and one if picked up q1 = np.load('q_table_r1_6.npy') q2 = np.load('q_table_r2_6.npy') # ============================================================================= # Create Agents # ============================================================================= agent1_books = [1, 2, 3] agent2_books = [4, 5, 6] agent1 = Agent('robot1', q1, agent1_books, more_books=True) agent2 = Agent('robot2', q2, agent2_books, more_books=True) tbot_list = [agent1, agent2] R_cumulative = {agent1.name: [], agent2.name: []} # ============================================================================= # acting # ============================================================================= tbot_active = agent1 tbot_passive = agent2 epsilon = 0 # a single episode api.reset_world() R_cumulative_active = 0 R_cumulative_passive = 0 initial_state = api.get_current_state() current_state = initial_state state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple #pdb.set_trace() while not api.is_terminal_state(current_state): through_api = True # flag for going through API for an action, if False, then reward is given manually # ============================================================================= # active tbot acts and learns # ============================================================================= #pick action for tbot_active #choose either random or exploit, according to epsilon=epsilon_calc(epsilon_initial, epsilon_decay, i) # ============================================================================= # if state_active[0]>=5 or state_passive[0]>=5: # pdb.set_trace() # ============================================================================= action_A, action_items_A, action_params_A, action_string_A = self.choose_action( tbot_active, epsilon, state_active) #selects action through_api, next_state, reward = self.reward_prune( current_state, state_active, state_passive, action_A, action_items_A, tbot_active, tbot_passive ) #prunes by checking for invalid actions, in which case we don't run through environment_api #pdb.set_trace() if through_api: success, next_state = api.execute_action( action_A, action_params_A, tbot_active.name) reward = api.get_reward(current_state, action_A, next_state) R_cumulative_active += reward next_state_active = tbot_active.dict_to_np_state( next_state, tbot_passive) #update q_values of tbot_active ONLY state_action_idx = tuple(state_active) + tuple( [tbot_active.idx_to_action.index(action_string_A)]) current_state = next_state # udpate current state so the other tbot knows the updated state before choosing an action etc. state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple # ============================================================================= # passive tbot acts and does NOT learn # ============================================================================= action_P, action_items_P, action_params_P, action_string_P = self.choose_action( tbot_passive, epsilon, state_passive) through_api, next_state, reward = self.reward_prune( current_state, state_passive, state_active, action_P, action_items_P, tbot_passive, tbot_active) #reward won't be used if through_api: success, next_state = api.execute_action( action_P, action_params_P, tbot_passive.name) reward = api.get_reward(current_state, action_P, next_state) R_cumulative_passive += reward current_state = next_state # udpate current state for active tbot state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple R_cumulative[tbot_active.name].append(R_cumulative_active) R_cumulative[tbot_passive.name].append(R_cumulative_passive) print(R_cumulative)
def task4(self, episodes): ''' train with 3 books for each tbot''' #pdb.set_trace() q_values = {} actions_json_file = '/action_config.json' with open(self.root_path + actions_json_file) as json_file: try: self.action_reference = json.load(json_file, parse_float=float) except (ValueError, KeyError, TypeError): print "JSON error" # ============================================================================= # episode parameters # ============================================================================= #there are actually 2*episodes episodes, since there are two tbots episode_update = 2 #the amount of episodes a tbot will train while the other tbots policy remains constant. must be a divisor of episodes episode_blocks = int( episodes / episode_update ) #number of episode blocks, each episode block one tbot is updating their q table while the other only acts # ============================================================================= # epsilon parameters & set penalty values # ============================================================================= epsilon_initial = .95 epsilon_decay = .002 epsilon_calc = lambda epsilon_initial, epsilon_decay, i: max( 0.05, epsilon_initial - epsilon_decay * i) self.book_penalty = -100 self.bump_penalty = -100 # ============================================================================= # q tables initialized to zero # ============================================================================= q1 = np.zeros( (7, 7, 4, 2, 2, 2, 2, 2, 2, 5, 9) ) #(x,y, orientation, c1,c2,c3,c4,c5,c6,tbot_near,action) #c1,c2 will be zero if available, and one if picked up q2 = np.zeros((7, 7, 4, 2, 2, 2, 2, 2, 2, 5, 9)) # ============================================================================= # Create Agents # ============================================================================= agent1_books = [1, 2, 3] agent2_books = [4, 5, 6] agent1 = Agent('robot1', q1, agent1_books, more_books=True) agent2 = Agent('robot2', q2, agent2_books, more_books=True) tbot_list = [agent1, agent2] R_cumulative = {agent1.name: [], agent2.name: []} # ============================================================================= # acting and training # ============================================================================= for i in range(episode_blocks): epsilon = epsilon_calc( epsilon_initial, epsilon_decay, i) #epsilon can only changes every episode block for tbot in tbot_list: #determines which tbot is learning, active updates table, passive does not tbot_active = tbot tbot_passive_set = set(tbot_list) - set([tbot]) tbot_passive = tbot_passive_set.pop() for e in range( episode_update ): #cycle through the episodes inside an episode block epsilon = epsilon_calc(epsilon_initial, epsilon_decay, i * episode_update + e) # a single episode api.reset_world() R_cumulative_active = 0 R_cumulative_passive = 0 initial_state = api.get_current_state() current_state = initial_state state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple #pdb.set_trace() print('episode_block {0} episode {1} for tbot {2}'.format( i, e, tbot_active.name)) while not api.is_terminal_state(current_state): through_api = True # flag for going through API for an action, if False, then reward is given manually # ============================================================================= # active tbot acts and learns # ============================================================================= #pick action for tbot_active #choose either random or exploit, according to epsilon=epsilon_calc(epsilon_initial, epsilon_decay, i) # ============================================================================= # if state_active[0]>=5 or state_passive[0]>=5: # pdb.set_trace() # ============================================================================= action_A, action_items_A, action_params_A, action_string_A = self.choose_action( tbot_active, epsilon, state_active) #selects action through_api, next_state, reward = self.reward_prune( current_state, state_active, state_passive, action_A, action_items_A, tbot_active, tbot_passive ) #prunes by checking for invalid actions, in which case we don't run through environment_api #pdb.set_trace() if through_api: success, next_state = api.execute_action( action_A, action_params_A, tbot_active.name) reward = api.get_reward(current_state, action_A, next_state) R_cumulative_active += reward next_state_active = tbot_active.dict_to_np_state( next_state, tbot_passive) #update q_values of tbot_active ONLY state_action_idx = tuple(state_active) + tuple( [tbot_active.idx_to_action.index(action_string_A)]) tbot_active.q[state_action_idx] = ( 1 - self.alpha ) * tbot_active.q[state_action_idx] + self.alpha * ( reward + self.gamma * max(tbot_active.q[next_state_active])) current_state = next_state # udpate current state so the other tbot knows the updated state before choosing an action etc. state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple # ============================================================================= # passive tbot acts and does NOT learn # ============================================================================= action_P, action_items_P, action_params_P, action_string_P = self.choose_action( tbot_passive, epsilon, state_passive) through_api, next_state, reward = self.reward_prune( current_state, state_passive, state_active, action_P, action_items_P, tbot_passive, tbot_active) #reward won't be used if through_api: success, next_state = api.execute_action( action_P, action_params_P, tbot_passive.name) reward = api.get_reward(current_state, action_P, next_state) R_cumulative_passive += reward current_state = next_state # udpate current state for active tbot state_active = tbot_active.dict_to_np_state( current_state, tbot_passive) #active bots state tuple state_passive = tbot_passive.dict_to_np_state( current_state, tbot_active) #passive bots state tuple R_cumulative[tbot_active.name].append(R_cumulative_active) R_cumulative[tbot_passive.name].append( R_cumulative_passive) print(R_cumulative) print('on the {0}th episode is {1}'.format( i * episode_update + e, epsilon)) np.save('q_table_r1_6.npy', agent1.q) np.save('q_table_r2_6.npy', agent2.q) import pickle with open("robot1_rewards_6.txt", "wb") as f: pickle.dump(R_cumulative['robot1'], f) with open("robot2_rewards_6.txt", "wb") as f: pickle.dump(R_cumulative['robot2'], f)