def __init__(self, task): self.env = gym.make(task) LearningAgent.__init__(self, self.env) self.nrFeatures = getNrFeaturesForTask(task) self.nrActions = self.env.action_space.n self.tF = np.zeros((self.nrActions, self.nrFeatures), 'float32')
def __init__(self, task, gamma): env = gym.make(task) LearningAgent.__init__(self, env) self.discretizer = Discretizer(task) self.numStates, self.numActions = self.discretizer.getStateActionInfo() self.gamma = gamma self.Q = np.zeros((self.numStates, self.numActions))
def __init__(self, task): self.task = task self.env = gym.make(task) LearningAgent.__init__(self, self.env) self.nrFeatures = getNrFeaturesForTask(task) self.nrActions = getActionsForTask(task) self.tF = np.random.randn(self.nrActions, self.nrFeatures)
def __init__(self, task): self.env = gym.make(task) LearningAgent.__init__(self, self.env) self.nrFeatures = getNrFeaturesForTask(task) self.nrActions = getActionsForTask(task) self.tF = np.zeros((self.nrActions, self.nrFeatures),'float32') self.population = np.random.rand(self.pop,self.nrActions, self.nrFeatures)
def ask_player_mode(number, chips, bet_value, model): print "Player" + str(number) + " Mode : " print " 1 : Manual player (You play)" print " 2 : Random Agent" print " 3 : All-In Agent" print " 4 : Bucket_Agent" print " 5 : Bayesian Agent ( with pre-computed model )" print " 6 : Learning Agent " print " 7 : Regret-Minimization Agent" x = raw_input(" What mode ? [1-7] : ") gamer = None if x == "1": gamer = player("player" + str(number), chips) elif x == "2": gamer = random_agent("player" + str(number), chips) elif x == "3": gamer = All_In_Agent("player" + str(number), chips) elif x == "4": gamer = Agent_Bucket("player" + str(number), chips) elif x == "5": gamer = bayesian("player" + str(number), chips, list(model)) elif x == "6": gamer = LearningAgent("player" + str(number), chips, 2) elif x == "7": gamer = Regret_Agent("player" + str(number), chips, bet_value) else: return False return gamer
def run_epoch(envt, oracle, central_agent, value_function, DAY, is_training, agents_predefined=None, TRAINING_FREQUENCY: int = 1): # INITIALISATIONS Experience.envt = envt # Initialising agents if agents_predefined is not None: agents = deepcopy(agents_predefined) else: initial_states = envt.get_initial_states(envt.NUM_AGENTS, is_training) agents = [ LearningAgent(agent_idx, initial_state) for agent_idx, initial_state in enumerate(initial_states) ] # ITERATING OVER TIMESTEPS print("DAY: {}".format(DAY)) down_sample = 1 if Settings.has_value("down_sample") and Settings.get_value("down_sample"): down_sample = Settings.get_value("down_sample") request_generator = envt.get_request_batch(DAY, downsample=down_sample) total_value_generated = 0 num_total_requests = 0 ret_dictionary = { 'epoch_requests_completed': [], 'epoch_requests_accepted': [], 'epoch_dropoff_delay': [], 'epoch_requests_seen': [], 'epoch_requests_accepted_profit': [], 'epoch_each_agent_profit': [], 'epoch_locations_all': [], 'epoch_locations_accepted': [], } while True: # Get new requests try: current_requests = next(request_generator) if Settings.has_value("print_verbose") and Settings.get_value( "print_verbose"): print("Current time: {} or {} on DAY {}".format( envt.current_time, datetime.timedelta(seconds=envt.current_time), DAY)) print("Number of new requests: {}".format( len(current_requests))) except StopIteration: break ret_dictionary['epoch_locations_all'].append( [i.pickup for i in current_requests]) for i in current_requests: envt.requests_region[envt.labels[i.pickup]] += 1 # Get feasible actions feasible_actions_all_agents = oracle.get_feasible_actions( agents, current_requests) # Score feasible actions experience = Experience(deepcopy(agents), feasible_actions_all_agents, envt.current_time, len(current_requests)) scored_actions_all_agents = value_function.get_value([experience]) # Choose actions for each agent scored_final_actions = central_agent.choose_actions( scored_actions_all_agents, is_training=is_training, epoch_num=envt.num_days_trained) # Assign final actions to agents for agent_idx, (action, _) in enumerate(scored_final_actions): agents[agent_idx].path = deepcopy(action.new_path) position = experience.agents[agent_idx].position.next_location time_driven = 0 for request in action.requests: time_driven += envt.get_travel_time(request.pickup, request.dropoff) time_to_request = sum([ envt.get_travel_time(position, request.pickup) for request in action.requests ]) envt.driver_utilities[agent_idx] += max( (time_driven - time_to_request), 0) # Calculate reward for selected actions rewards = [] locations_served = [] for action, _ in scored_final_actions: reward = len(action.requests) locations_served += [request.pickup for request in action.requests] for request in action.requests: envt.success_region[envt.labels[request.pickup]] += 1 rewards.append(reward) total_value_generated += reward if Settings.has_value("print_verbose") and Settings.get_value( "print_verbose"): print("Reward for epoch: {}".format(sum(rewards))) profits, agent_profits = get_profit_distribution(scored_final_actions) for i, j in agent_profits: envt.driver_profits[i] += j # Update if (is_training): # Update replay buffer value_function.remember(experience) # Update value function every TRAINING_FREQUENCY timesteps if ((int(envt.current_time) / int(envt.EPOCH_LENGTH)) % TRAINING_FREQUENCY == TRAINING_FREQUENCY - 1): value_function.update(central_agent) # Sanity check for agent in agents: assert envt.has_valid_path(agent) # Writing statistics to logs value_function.add_to_logs( 'rewards_day_{}'.format(envt.num_days_trained), sum(rewards), envt.current_time) avg_capacity = sum([agent.path.current_capacity for agent in agents]) / envt.NUM_AGENTS value_function.add_to_logs( 'avg_capacity_day_{}'.format(envt.num_days_trained), avg_capacity, envt.current_time) epoch_dictionary = {} for agent in agents: agent_dictionary = get_statistics_next_epoch(agent, envt) if epoch_dictionary == {}: epoch_dictionary = agent_dictionary else: for key in agent_dictionary: epoch_dictionary[key] += agent_dictionary[key] # Simulate the passing of time envt.simulate_motion(agents, current_requests) num_total_requests += len(current_requests) ret_dictionary['epoch_requests_completed'].append( epoch_dictionary['requests_served']) ret_dictionary['epoch_dropoff_delay'].append( epoch_dictionary['total_delivery_delay']) ret_dictionary['epoch_requests_accepted'].append(sum(rewards)) ret_dictionary['epoch_requests_seen'].append(len(current_requests)) ret_dictionary['epoch_requests_accepted_profit'].append(sum(profits)) ret_dictionary['epoch_each_agent_profit'].append(agent_profits) ret_dictionary['epoch_locations_accepted'].append(locations_served) if Settings.has_value("print_verbose") and Settings.get_value( "print_verbose"): print("Requests served {}".format( np.sum(ret_dictionary["epoch_requests_completed"]))) print("Requests accepted {}".format(sum(rewards))) # Printing statistics for current epoch print('Number of requests accepted: {}'.format(total_value_generated)) print('Number of requests seen: {}'.format(num_total_requests)) ret_dictionary['total_requests_accepted'] = total_value_generated return ret_dictionary
max_test_score = test_score if test_score > max_test_score else max_test_score envt.num_days_trained += 1 if value_num in NEURAL_VALUE_FUNCTIONS: value_function.model.save('../models/{}_{}.h5'.format( num_agents, envt.num_days_trained)) # Reset the driver utilities envt.reset() central_agent.reset() for day in TEST_DAYS: initial_states = envt.get_initial_states(envt.NUM_AGENTS, is_training=False) agents = [ LearningAgent(agent_idx, initial_state) for agent_idx, initial_state in enumerate(initial_states) ] epoch_data = run_epoch(envt, oracle, central_agent, value_function, day, is_training=False, agents_predefined=agents) total_requests_served = epoch_data['total_requests_accepted'] print("(TEST) DAY: {}, Requests: {}\n\n".format( day, total_requests_served)) # Write our pickled resutls
from Agent import * from SensorAgent import SensorAgent from KinematicsAgent import KinematicsAgent from ControllingAgent import ControllingAgent from SupervisorAgent import SupervisorAgent from LearningAgent import LearningAgent #Agent initialization sensorAgent = SensorAgent(AgentType.SensorAgent) kinematicsAgent = KinematicsAgent(AgentType.KinematicsAgent) controllingAgent = ControllingAgent(AgentType.ControllingAgent) supervisorAgent = SupervisorAgent(AgentType.SupervisorAgent) learningAgent = LearningAgent(AgentType.LearningAgent) #Adding peer agents #Sensor sensorAgent.AddPeerAgent(learningAgent) #Kinematics kinematicsAgent.AddPeerAgent(learningAgent) kinematicsAgent.AddPeerAgent(controllingAgent) #Controlling controllingAgent.AddPeerAgent(kinematicsAgent) #Supervisor supervisorAgent.AddPeerAgent(learningAgent) #Learning learningAgent.AddPeerAgent(sensorAgent) learningAgent.AddPeerAgent(kinematicsAgent)
def run_epoch(envt, oracle, central_agent, value_function, DAY, is_training, agents_predefined=None, TRAINING_FREQUENCY: int=1): # INITIALISATIONS Experience.envt = envt # Initialising agents if agents_predefined is not None: agents = deepcopy(agents_predefined) else: initial_states = envt.get_initial_states(envt.NUM_AGENTS, is_training) agents = [LearningAgent(agent_idx, initial_state) for agent_idx, initial_state in enumerate(initial_states)] # ITERATING OVER TIMESTEPS print("DAY: {}".format(DAY)) request_generator = envt.get_request_batch(DAY) total_value_generated = 0 num_total_requests = 0 while True: # Get new requests try: current_requests = next(request_generator) print("Current time: {}".format(envt.current_time)) print("Number of new requests: {}".format(len(current_requests))) except StopIteration: break # Get feasible actions feasible_actions_all_agents = oracle.get_feasible_actions(agents, current_requests) # Score feasible actions experience = Experience(deepcopy(agents), feasible_actions_all_agents, envt.current_time, len(current_requests)) scored_actions_all_agents = value_function.get_value([experience]) # Choose actions for each agent scored_final_actions = central_agent.choose_actions(scored_actions_all_agents, is_training=is_training, epoch_num=envt.num_days_trained) # Assign final actions to agents for agent_idx, (action, _) in enumerate(scored_final_actions): agents[agent_idx].path = deepcopy(action.new_path) # Calculate reward for selected actions rewards = [] for action, _ in scored_final_actions: reward = envt.get_reward(action) rewards.append(reward) total_value_generated += reward print("Reward for epoch: {}".format(sum(rewards))) # Update if (is_training): # Update replay buffer value_function.remember(experience) # Update value function every TRAINING_FREQUENCY timesteps if ((int(envt.current_time) / int(envt.EPOCH_LENGTH)) % TRAINING_FREQUENCY == TRAINING_FREQUENCY - 1): value_function.update(central_agent) # Diagnostics for action, score in scored_actions_all_agents[0]: print("{}: {}, {}, {}".format(score, action.requests, action.new_path, action.new_path.total_delay)) print() for idx, (action, score) in enumerate(scored_final_actions[:10]): print("{}: {}, {}, {}".format(score, action.requests, action.new_path, action.new_path.total_delay)) # Sanity check for agent in agents: assert envt.has_valid_path(agent) # Writing statistics to logs value_function.add_to_logs('rewards_day_{}'.format(envt.num_days_trained), sum(rewards), envt.current_time) avg_capacity = sum([agent.path.current_capacity for agent in agents]) / envt.NUM_AGENTS value_function.add_to_logs('avg_capacity_day_{}'.format(envt.num_days_trained), avg_capacity, envt.current_time) # Simulate the passing of time envt.simulate_motion(agents, current_requests) num_total_requests += len(current_requests) # Printing statistics for current epoch print('Number of requests accepted: {}'.format(total_value_generated)) print('Number of requests seen: {}'.format(num_total_requests)) return total_value_generated
total_requests_served = run_epoch(envt, oracle, central_agent, value_function, day, is_training=False) print("\n(TEST) DAY: {}, Requests: {}\n\n".format(day, total_requests_served)) test_score += total_requests_served value_function.add_to_logs('validation_score', test_score, envt.num_days_trained) # TODO: Save results better if (isinstance(value_function, NeuralNetworkBased)): if (test_score > max_test_score or (envt.num_days_trained % SAVE_FREQ) == (SAVE_FREQ - 1)): value_function.model.save('../models/{}_{}agent_{}capacity_{}delay_{}interval_{}_{}.h5'.format(type(value_function).__name__, args.numagents, args.capacity, args.pickupdelay, args.decisioninterval, envt.num_days_trained, test_score)) max_test_score = test_score if test_score > max_test_score else max_test_score envt.num_days_trained += 1 # CHECK TEST SCORE # value_function_baseline = RewardPlusDelay(DELAY_COEFFICIENT=1e-7, log_dir=LOG_DIR) for day in TEST_DAYS: # Initialising agents initial_states = envt.get_initial_states(envt.NUM_AGENTS, is_training=False) agents = [LearningAgent(agent_idx, initial_state) for agent_idx, initial_state in enumerate(initial_states)] total_requests_served = run_epoch(envt, oracle, central_agent, value_function, day, is_training=False, agents_predefined=agents) print("\n(TEST) DAY: {}, Requests: {}\n\n".format(day, total_requests_served)) value_function.add_to_logs('test_requests_served', total_requests_served, envt.num_days_trained) # total_requests_served = run_epoch(envt, oracle, central_agent, value_function_baseline, day, is_training=False, agents_predefined=agents) # print("\n(TEST) DAY: {}, Requests: {}\n\n".format(day, total_requests_served)) # value_function_baseline.add_to_logs('test_requests_served', total_requests_served, envt.num_days_trained) envt.num_days_trained += 1
from LearningAgent import LearningAgent from importSVM import import_model #Get the data and the cards, then generate the proper files to send to the Preference Learning Toolbox la = LearningAgent() la.process_and_save_data('all_data.csv','processed_deck_features.json','test.cdata','test.pdatas') la.select_card('Ba3awX1yiTkSIlV7ESWD6Tw', 'WalAkS9-bTt-x2d7Y6RfDVA', 'WkTQ-d3JfSdeAMp2vQMwK6g') #Get the RankSVM model from the toolbox by choosing the best parameters #TODO: Add those parameters later #Create a new Learning Agent based on the model generated using the Preference Learning Toolbox ##bot.learn_from_model('RankSVM.model') #import_model('RankSVM.model', 'test.cdata')
def __init__(self, Qs, env): self.agents = [LearningAgent(Q, env.action_space[0].n) for Q in Qs] self.env = env self.current_episode = 0 self.episodes_amount = 100