def singleEffect(filename, parameter): seed = rnd.randint(0, Global.BIG_NUM) playAllAgainst(filename, Agents.DepMCTS(), str(parameter), seed) playAllAgainst(filename, Agents.DepUnDepMCTS(0.5), str(parameter), seed) playAllAgainst(filename, Agents.SimulatedMCTS(0.97), str(parameter), seed) playAllAgainst(filename, Agents.FlippingMCTS(), str(parameter), seed) playAllAgainst(filename, Agents.NormalMCTS(), str(parameter), seed)
def test_mountain_car(n_states,qhat_file='Data/mc_qhat.pkl',max_eps = 10000): """ Compares aggregated mountain car with n_states with a simple discretised version of the same. Supplying a precalculated qhat file will drastically speed computation """ divs = int(np.floor(np.sqrt(n_states))) if qhat_file: with open(qhat_file,'rb') as f: qhat = pickle.load(f) else: qhat = evaluate_MC_qhat() agg = Aggregation.generateAggregation(qhat,target_divisions=n_states) mc_d = Problems.MountainCar(representation='disc',divisions = divs) mc_a = Problems.MountainCar(representation='aggr',aggregation=agg,divisions=100) ag_d = Agents.QAgent(mc_d,alpha=1e-3) ag_a = Agents.QAgent(mc_a,alpha=1e-3) d_eps = ag_d.run_n_episodes(max_eps) a_eps = ag_a.run_n_episodes(max_eps) n_eps = np.array([2**i for i in range(1,int(np.log2(max_eps))+1)]) data = pd.DataFrame() data['n_eps'] = n_eps data['disc'] = d_eps data['aggr'] = a_eps return data
def testMaze(n_train, n_nav): ValueLearning.DBG_LVL = 1 move_distance = 0.99 # Experiment parameters nx = 6 ny = 6 n_fields = round(1.0 * (nx + 3) * (ny + 3)) Hippocampus.N_CELLS_PER_FIELD = 1 n_cells = n_fields * Hippocampus.N_CELLS_PER_FIELD # Maze creation maze = Environment.RandomGoalOpenField(nx, ny, move_distance) # Generate place fields and place cells place_fields = Hippocampus.setupPlaceFields(maze, n_fields) place_cells = Hippocampus.assignPlaceCells(n_cells, place_fields) # Create Actor and Critic actor = Agents.RandomAgent(maze.getActions(), n_cells) critic = Agents.Critic(n_cells) ValueLearning.learnValueFunction(n_train, maze, place_cells, actor, critic, max_steps=1000)
def updateGame(debug): global base while True: Pygame.update() # Checks job list for available jobs and assigns them to workers. if Agents.jobList: for agent in agents: if agent.getRole( ) == "worker" and agent.getState() != "upgrading": agent.setJob(Agents.jobList[0]) Agents.removeFromJobList() agent.setState(StateManager.upgrading()) if not Agents.jobList: break # Executes the agents current state for agent in agents: agent.state.execute(agent) FogOfWar.updateFogOfWar(agents) Pygame.drawAgents(agents) # Checks if victory requirements have been met. if agents[0].base.getCoal() >= 200 and agents[0].base.getIron() >= 20: print("Victory!") return True if debug: infoPrints()
def playAllAgainst(filename, agent, parameter): agentName = type(agent).__name__ result = playGame([Agents.NormalMCTS(), agent]).split("-") writeFile( filename, "NormalMCTS" + ", " + agentName + ", " + str(parameter) + ", " + result[0] + ", " + result[1]) result = playGame([agent, Agents.NormalMCTS()]).split("-") writeFile( filename, agentName + ", " + "NormalMCTS" + ", " + str(parameter) + ", " + result[0] + ", " + result[1])
def make_ag(): return Agents.SmallFC_FW( in_d=in_dims, out_d=out_dims, num_hidden=num_hidden, hidden_dim=hidden_dim, output_normalisation=normalise_output_with)
def __set_agent_values__(self): int_rate = self.model_params["int_rate"] min_holding = self.model_params['min_holding'] init_cash = self.model_params['init_cash'] position = self.model_params['init_holding'] min_cash = self.model_params['min_cash'] tolerance = self.model_params['tolerance'] mistake_threshold = self.model_params['mistake_threshold'] make_mistakes = self.model_params['make_mistakes'] for i in range(int(self.model_params['num_agents'])): agent = Agents.Agent(id=i, name='Agent ' + str(i), int_rate=int_rate, min_holding=min_holding, init_cash=init_cash, position=position, forecast_params=self.forecast_params, dividend=self.init_dividend, price=self.init_asset_price, conditions=self.conditions, min_cash=min_cash, risk_aversion=self.__gen_agent_risk__, tolerance=tolerance, mistake_threshold=mistake_threshold, make_mistakes=make_mistakes) agent.__set_holdings__() self.population.append(agent)
def initialize_agents(grid, num_agents=NUM_AGENTS): agents = [] for i in range(num_agents): agent = Agents.BasicAgent(pygame, grid, i) agents.append(agent) grid.add_agent(agent) return agents
def setupStartPos(r): global base map = Map.map R = copy.deepcopy(r) redo = True # Finds suitable spawning area while redo: startPos = (random.randrange(1, 99), random.randrange(1, 99)) for neighbour in R: if map[startPos[0] + neighbour[0]][startPos[1] + neighbour[1]] in ("B", "V", "T", "I"): redo = True break else: redo = False map[startPos[0]][startPos[1]] = "S" base = BaseManager.base(startPos) # Spawn workers around spawning area for i in range(StatParser.statDict["workers"]): for next in R: if next[2] == 7: continue agents.append( Agents.agent((startPos[0] + next[0], startPos[1] + next[1]), base, i + 1)) next[2] += 1 break return startPos
def test_Q(self): p_raw, p_agg = setupESA() ql_raw = Agents.QAgent(p_raw, 1) ql_agg = Agents.QAgent(p_agg, 1) ql_raw.episode(timeout=1000) ql_agg.episode(timeout=1000) delta_r = sum(abs(ql_raw.qValues[0] - p_raw.qValues[0])) / 4 delta_a = sum(abs(ql_agg.qValues[0] - p_agg.qValues[0])) / 2 print("\nQ learning raw delta = {}, agg delta = {}".format( delta_r, delta_a)) self.assertTrue(delta_r < 1e-1) self.assertTrue(delta_a < 1e-1)
def test_UT(self): p_raw, p_agg = setupESA() ut_raw = Agents.UTreeAgent(p_raw) ut_raw.episode(timeout=1000) ut_raw.utree.print_tree()
def test_SL(self): p_raw, p_agg = setupESA() sl_raw = Agents.SarsaLambda(p_raw, 1) sl_agg = Agents.SarsaLambda(p_agg, 1) sl_raw.episode(timeout=1000) sl_agg.episode(timeout=1000) delta_r = sum(sl_raw.qValues[0] - p_raw.qValues[0]) / 4 delta_a = sum(sl_agg.qValues[0] - p_agg.qValues[0]) / 2 print("\nSarsa(l) raw delta = {}, agg delta = {}".format( delta_r, delta_a)) self.assertTrue(delta_r < 1e-1) self.assertTrue(delta_a < 1e-1)
def compare_raw_agg_ql(param_tuples,timeout=1000,gamma=0.5,aggtype='q',log_prob=False,rep=1,alpha=0.005,decayAlpha=False): """ Generates random problems from parameter lists, runs qlearning and records value function deviations for each problem after timeout steps log optionally records problems so that they can be retrieved later """ dmat = np.zeros((len(param_tuples)*rep,9)) if log_prob: problem_dict = {} d = datetime.today().strftime('%d-%m-%Y--%H_%M_%S') filename = 'problems' + d + '.pkl' for i, (n,n_agg,b,acts,e_noise) in enumerate(param_tuples): for j in range(rep): p_r, p_a, _ = Problems.genRandomProblems(n,n_agg,acts,b,gamma=gamma,e_noise=e_noise) pid = hash(str(p_r.transitions)) if log_prob: problem_dict[pid] = {'raw':p_r,'agg':p_a} agent_r = Agents.QAgent(p_r,alpha=alpha) agent_a = Agents.QAgent(p_a,alpha=alpha) agent_r.episode(timeout=timeout,decayAlpha=decayAlpha) agent_a.episode(timeout=timeout,decayAlpha=decayAlpha) delta_r = Evaluation.getDeltas(agent_r,p_r) delta_a = Evaluation.getDeltas(agent_a,p_a,agg=p_a.aggregation) dtilde = Evaluation.nonMarkovianity(p_a.transitions[0], p_a.aggregation) dmat[i*rep+j] = (pid,n,n_agg,b,acts,e_noise,dtilde,np.average(delta_r),np.average(delta_a)) if log_prob: with open(path+filename,'wb') as f: pickle.dump(problem_dict,f) data = pd.DataFrame(data=dmat,columns=['pid','n','n_agg','b','acts','e_noise','nonmarkovianity','d_r','d_a']) return data
def test_VI(self): p_raw, _ = setupESA() vi_raw = Agents.VIAgent(p_raw, 1) vi_raw.VISweep() delta_r = sum(vi_raw.qValues[0] - p_raw.qValues[0]) / 4 print("\nValue Iteration raw delta = {}".format(delta_r)) self.assertTrue(delta_r < 1e-4)
def __init__(self, rnn_size=256, embed_dim=256, learning_rate=0.001, vocab_size=8638, batch_size=512, dropout=0.75): self.rnn_size = rnn_size self.embed_dim = embed_dim self.learning_rate = learning_rate self.vocab_size = vocab_size + 1 self.batch_size = batch_size self.drop_out = dropout # Structures define self.img_dim = 2048 self.FE_m = Agents.FeatEmbed(self.rnn_size, self.img_dim, name='FE_m') self.FE_o = Agents.FeatEmbed(self.rnn_size, self.img_dim, name='FE_o') self.FE_w = Agents.FeatEmbed(self.rnn_size, self.img_dim, name='FE_w') self.attention = Agents.Att(self.rnn_size, self.img_dim, name='Att') self.initialize_model()
def test_convergence_ql(param_tuple,timeout=100,interval=100,gamma=0.9,aggtype='q'): """ Generates a random problem from (single) parameter tuple, runs qlearning and records value function deviation at regular intervals. """ n, n_agg, b, acts, e_noise = param_tuple p_r, p_a, _ = problems.genRandomProblems(n,n_agg,acts,b,gamma=gamma,e_noise=e_noise) agent_r = Agents.QAgent(p_r,alpha=1e-1) agent_a = Agents.QAgent(p_a,alpha=1e-1) dmat = np.zeros((intervals,3)) for i in range(intervals): agent_r.episode(timeout=timeout) agent_a.episode(timeout=timeout) delta_a = Evaluation.getDeltas(agent_a,p_a) delta_r = Evaluation.getDeltas(agent_r,p_r) dmat[i] = (i*interval,delta_a,delta_r) data = pd.DataFrame(data=dmat,columns=['n','d_a','d_r']) return data
def main(): global base r = ([0, 0, 0], [1, 1, 0], [1, 0, 0], [0, 1, 0], [-1, 1, 0], [1, -1, 0], [-1, 0, 0], [0, -1, 0], [-1, -1, 0]) StatParser.readStats() Map.makeMap() startPos = setupStartPos(r) FogOfWar.createFogOfWar(startPos) FogOfWar.updateFogOfWar(agents) Resources.findMaterials() Pygame.init() # Adds jobs to the job list for workers to be assigned to. for j in range(StatParser.statDict["workers"]): if j < 30: Agents.addToJobList("woodCutter") elif j >= 30 and j < 40: Agents.addToJobList("miner") elif j >= 40 and j < 46: Agents.addToJobList("explorer") else: Agents.addToJobList("builder") # Adds some buildings to the build list so the builders know what to build. for k in range(8): if k < 6: base.addToBuildList(["coalFurnace", 0]) else: base.addToBuildList(["smeltery", 0]) # Allows user to speed up the application (can be laggy). TimeMultiplier.setTimeMultiplier(int(input("Set a time multiplier: "))) # Allows the user to choose if they want debug info to be displayed or not debug = input("Debug info y/n?: ") if debug == "y": debug = True else: debug = False # Runs and times update to see how long it took for the AI to complete the wanted task gameStart = time.time() if updateGame(debug): print("Time to finish: " + str((time.time() - gameStart) * TimeMultiplier.timeMultiplier)) return True
def playAllAgainst(filename, agent, parameter, seed): agentName = agent.__class__.__name__ Agents.fixSeed(seed) result = playGame([Agents.NormalMCTS(), agent]).split("-") writeFile( filename, "NormalMCTS" + ", " + agentName + ", " + str(parameter) + ", " + result[0] + ", " + result[1]) Agents.fixSeed(seed) result = playGame([agent, Agents.NormalMCTS()]).split("-") writeFile( filename, agentName + ", " + "NormalMCTS" + ", " + str(parameter) + ", " + result[0] + ", " + result[1])
def createWidgets(self): """ Create qt widgets QTabWidget / QWidget \/ QWidget \_________________ | | | ______________________________________| """ self.steps = Steps.StepsQWidget(self) self.descrs = Descriptions.DescriptionsQWidget(self) self.parameters = Parameters.ParametersQWidget(self) self.parametersOutput = Parameters.ParametersQWidget(self, forParamsOutput=True) self.probes = Probes.ProbesQWidget(self) self.agents = Agents.AgentsQWidget(self) self.adapters = Adapters.AdaptersQWidget(self, testParams=self, testDescrs=self.descrs) self.libraries = Libraries.LibrariesQWidget(self, testParams=self, testDescrs=self.descrs) self.parametersTab = QTabWidget() self.parametersTab.setTabPosition(QTabWidget.North) self.parametersTab.setStyleSheet("QTabWidget { border: 0px; }") # remove 3D border self.parametersTab.addTab(self.descrs, QIcon(":/test-config.png"), "Description") self.parametersTab.addTab(self.steps, QIcon(":/run-state.png"), "Steps") self.paramsTab = QTabWidget() self.paramsTab.setStyleSheet("QTabWidget { border: 0px; }") # remove 3D border self.paramsTab.setTabPosition(QTabWidget.North) self.paramsTab.addTab(self.parameters, QIcon(":/test-input.png"), "Inputs") self.paramsTab.addTab(self.parametersOutput, QIcon(":/test-output.png"), "Outputs") self.paramsTab.addTab(self.adapters, QIcon(":/adapters.png"), "Adapters") self.paramsTab.addTab(self.libraries, QIcon(":/libraries.png"), "Libraries") self.miscsTab = QTabWidget() self.miscsTab.setStyleSheet("QTabWidget { border: 0px; }") # remove 3D border self.miscsTab.setTabPosition(QTabWidget.North) self.miscsTab.addTab(self.agents, QIcon(":/agent.png"), "Agents") self.miscsTab.addTab(self.probes, QIcon(":/probe.png"), "Probes") self.title = QLabel("Test Properties") font = QFont() font.setBold(True) self.title.setFont(font) self.labelHelp = QLabel("Prepare the test.") font = QFont() font.setItalic(True) self.labelHelp.setFont(font) self.mainTab = QTabWidget() self.mainTab.setTabPosition(QTabWidget.North) self.mainTab.addTab(self.parametersTab, QIcon(":/test-description.png"), "Test Design") self.mainTab.addTab(self.paramsTab, QIcon(":/repository.png"), "Test Data") self.mainTab.addTab(self.miscsTab, QIcon(":/server-config.png"), "Miscellaneous") if Settings.instance().readValue( key = 'TestProperties/inputs-default-tab' ) == "True": self.mainTab.setCurrentIndex(1) self.paramsTab.setCurrentIndex(TAB_INPUTS) layout = QVBoxLayout() layout.addWidget( self.title ) layout.addWidget( self.labelHelp ) layout.addWidget(self.mainTab) layout.setContentsMargins(0,0,0,0) self.setLayout(layout)
def execute(self, agent): readyToBuild = False # Where to build if agent.base.getBuildList() and not agent.getLocked(): pos = agent.getPos() # find unused buildable tile for next in Map.r: if Map.map[pos[0] + next[0]][pos[1] + next[1]] in ( "M", "G") and FogOfWar.fogOfWar[pos[0] + next[0]][pos[1] + next[1]]: # What to build for building in agent.base.getBuildList(): if building[0] == "coalFurnace": if building[1] == 0 or building[1] == agent.getId( ): readyToBuild = agent.base.buildCoalFurnace( agent, (pos[0] + next[0], pos[1] + next[1])) if readyToBuild == True: agent.setPos( (pos[0] + next[0], pos[1] + next[1])) building[1] = agent.getId() agent.setLocked(True) break elif building[0] == "smeltery": if building[1] == 0 or building[1] == agent.getId( ): readyToBuild = agent.base.buildSmeltery( agent, (pos[0] + next[0], pos[1] + next[1])) if readyToBuild == True: agent.setPos( (pos[0] + next[0], pos[1] + next[1])) building[1] = agent.getId() agent.setLocked(True) break elif building[0] == "blacksmith": if building[1] == 0 or building[1] == agent.getId( ): readyToBuild = agent.base.buildBlacksmith( agent, (pos[0] + next[0], pos[1] + next[1])) if readyToBuild: agent.setPos( (pos[0] + next[0], pos[1] + next[1])) building[1] = agent.getId() agent.setLocked(True) break elif building[0] == "trainingCamp": if building[1] == 0 or building[1] == agent.getId( ): agent.base.buildTrainingcamp( agent, (pos[0] + next[0], pos[1] + next[1])) building[1] = agent.getId() if readyToBuild: agent.setPos( (pos[0] + next[0], pos[1] + next[1])) agent.setLocked(True) break break elif agent.getPos() == agent.getJob()[1]: diff = (time.time() - agent.getTimer()) * TimeMultiplier.timeMultiplier if agent.getJob()[0] == "coalFurnace*": if diff >= StatParser.statDict["cfBuildTime"]: agent.base.addBuilding( BaseManager.coalFurnace(agent.getPos())) agent.setJob("builder") agent.setLocked(False) agent.setState(returnHome()) elif agent.getJob()[0] == "smeltery*": if diff >= StatParser.statDict["smelteryBuildTime"]: agent.base.addBuilding(BaseManager.smeltery( agent.getPos())) agent.setJob("builder") agent.setLocked(False) agent.setState(returnHome()) elif agent.getJob()[0] == "blacksmith*": if diff >= StatParser.statDict["bsBuildTime"]: agent.base.addBuilding( BaseManager.blacksmith(agent.getPos())) agent.setJob("builder") agent.setLocked(False) agent.setState(returnHome()) elif agent.getJob()[0] == "trainingCamp": if diff >= StatParser.statDict["tcBuildTime"]: agent.base.addBuilding( BaseManager.trainingCamp(agent.getPos())) agent.setJob("builder") agent.setLocked(False) agent.setState(returnHome()) # When to hire elif agent.getJob()[0] == "coalFurnace": if StatParser.statDict["cfBuildTime"] - ( time.time() - agent.getTimer() ) <= StatParser.statDict["artisanUpgradeTime"]: Agents.addToJobList("coalWorker") agent.setJob(("coalFurnace*", agent.getJob()[1])) elif agent.getJob()[0] == "smeltery": if StatParser.statDict["smelteryBuildTime"] - ( time.time() - agent.getTimer() ) <= StatParser.statDict["artisanUpgradeTime"]: Agents.addToJobList("smelteryWorker") agent.setJob(("smeltery*", agent.getJob()[1])) elif agent.getJob()[0] == "blacksmith": if StatParser.statDict["bsBuildTime"] - ( time.time() - agent.getTimer() ) <= StatParser.statDict["artisanUpgradeTime"]: Agents.addToJobList("weaponSmith") agent.setJob(("blacksmith*", agent.getJob()[1])) elif agent.base.getBuildList() == []: # When finished go idle until build list has content agent.setState(returnHome())
def make_ag(): return Agents.Agent1d(min(problem.env.phi_vals), max(problem.env.phi_vals))
import pyparsing as pp import Agents import argparse from settings import * import re global args agent = Agents.Agent(args.groups) #=========================== def printenv(e): # print("") # print("========ENVIRONMENT=======") # for k in e.keys(): # print("\t"+k) # print("\t\t"+str(e[k])) # print("--------------------------") # print("") pass def dprintenv(e): print("") print("========ENVIRONMENT=======") for k in e.keys(): print("\t" + k.replace("_", " ")) print("\t\t" + str(e[k])) print("--------------------------") print("")
cost_function = 'exp' # data set parameter DATA_PATH = r'D:\workspaces\datasets\USCensus1990\USCensus1990.data.txt' predictor = ['iMobillim', 'iWork89'] target = 'iMilitary' with_data_set = False # agent hyperparameter rho = 0.5 v1 = 1 # agents # agent1 = Agents.HierarchicalOptimisticOptimization([0, 1], v1=v1, rho=rho, setup=setup) # agent2 = Agents.HierarchicalOptimisticOptimization([0, 1], v1=v1, rho=rho, setup=setup) agent1 = Agents.Zooming([0, 1]) agent2 = Agents.Zooming([0, 1]) # agent1 = Agents.Random([0, 1]) # agent2 = Agents.Random([0, 1]) discrete_actions = [[0.01, 0.07, 0.13, 0.19, 0.25, 0.31, 0.37, 0.43, 0.49], [0.01, 0.045, 0.08, 0.115, 0.15, 0.185, 0.22, 0.255, 0.29]] # agent1 = Agents.EpsilonGreedy(discrete_actions[0], eps_greedy=0.999, eps_decay=0.9996) # agent2 = Agents.EpsilonGreedy(discrete_actions[1], eps_greedy=0.999, eps_decay=0.9996) # DPG parameter logistic_growth = 0.2 betas = [5, -3] sigmas_j_square = [.5, .3] # 'loo' or 'shapley' pricing_mechanism = 'loo' r_max = 15
def __init__(self, ng=c.numOfGoods, nr=c.numofRounds, mem=c.memory, al=c.alpha, maxCost=c.max_fixedCost): # This list will be used to keep track of goods that agents recieve... # ..though it is not their consumption goods, i.e. goods agents... # choose to carry for purpose of indirect trade c.numOfBaseAgents = ng * (ng - 1) c.numOfGoods = ng c.numofRounds = nr c.memory = mem c.alpha = al c.max_fixedCost = maxCost self.listofMoney = [0] * (c.numOfGoods) # agent self.types self.types = [] # array of all trades self.allTrades = [] # array of goods traded self.goodsTraded = [] # cost list self.costList_unchanging = [] # create a dictionary to store the cost of trading goods, the structure will be of the following kind # {(0,1):[0.1,0.02,0.5],(0,2):[0.1,0.6,0.02],(1,0):[0.5,0.3]} # (0,1) records the cost of the agent who recieved 0 # we initialize an empty dictionary with the intention of filling it up with the above self.tradeCosts = dict() # create two list of number of goods self.goods_listOne = [i for i in range(c.numOfGoods)] self.goods_listTwo = [i for i in range(c.numOfGoods)] # create a list of combinations of goods, it will look like this: [(0,1),(0,2),(1,0) and so on] self.goodsCombinations = [] for i in self.goods_listOne: for j in self.goods_listTwo: self.goodsCombinations.append((i, j)) # attaches an empty list to a dictionary where elements shows combinations of goods, so we get {(0,1):[],(0,2):[],(1,0):[]} for x in self.goodsCombinations: self.tradeCosts[x] = list() # print "fixed costs", c.max_fixedCost # list of costs assigned to each agent for i in range(0, c.numOfGoods): self.costList_unchanging.append(random.uniform(0, c.max_fixedCost)) self.costList = copy.deepcopy(self.costList_unchanging) # We generate a list of agents self.agentList = [ag.simpleAgents() for count in xrange(c.numOfBaseAgents)] self.listofMoney = [0] * (c.numOfGoods) # initialize agent self.types for i in range(0, c.numOfGoods): for j in range(0, c.numOfGoods): if i != j: nlist = [str(i), str(j), str(j)] self.types.append(nlist) # And we give each agents agent1 list of consumption good,.. # ...production good and carry good. Initially the production... # ...good and carry good are the same for i in range(0, len(self.types)): self.agentList[i].goods = self.types[i] for i in self.agentList: i.cost = self.costList # print "costs", self.costList # register callback funciton if necessary self.callback_function = None
def return_if_have_shot(self, start_node, target): '''Uses linear equation to determine if, starting from a particular tile, if it has a clean shot at the target''' #First, define the starting and ending positions start_cell = Terrain.terrain_obj.terrain_dict[start_node] target_coord = Terrain.terrain_obj.return_cell_index( target.x, target.y) target_cell = Terrain.terrain_obj.terrain_dict[target_coord] #Linear equation parameters t = 10 x = start_cell.sprite.x y = start_cell.sprite.y bottom_fact = math.sqrt(((target_cell.sprite.x - start_cell.sprite.x) * (target_cell.sprite.x - start_cell.sprite.x)) + \ ((target_cell.sprite.y - start_cell.sprite.y) * (target_cell.sprite.y - start_cell.sprite.y))) x_fact = (target_cell.sprite.x - start_cell.sprite.x) / bottom_fact y_fact = (target_cell.sprite.y - start_cell.sprite.y) / bottom_fact #List of cells that pass between two points available_coords = [] #While loop until reached target cell test_limit = 1000 iter = 0 while True: x += t * x_fact y += t * y_fact new_coord = Terrain.terrain_obj.return_sprite_index(x, y) #print(x,y) #print(new_coord) if new_coord not in available_coords: available_coords.append(new_coord) #If searched has reached its end, then find neighbors and return false if mountains, else true if target_coord in available_coords: test_lazer = Agents.Laser(x=None, y=None, rotation=0, target=None) for coord in available_coords: test_lazer.sprite.x = Terrain.terrain_obj.terrain_dict[ coord].sprite.x test_lazer.sprite.y = Terrain.terrain_obj.terrain_dict[ coord].sprite.y test_lazer.rotation = -math.degrees( Functions.angle( point_1=(test_lazer.sprite.x, test_lazer.sprite.y), point_2=(target.sprite.x, target.sprite.y))) test_lazer.sprite.rotation = test_lazer.rotation #print('laser position',test_lazer.sprite.x,test_lazer.sprite.y) for mountain in [obj for obj in Terrain.terrain_obj.terrain_dict if \ Terrain.terrain_obj.terrain_dict[obj].terrain_mov_mod == math.inf]: mountain_cell = Terrain.terrain_obj.terrain_dict[ mountain] #print('mountain cell position',mountain_cell.sprite.x,mountain_cell.sprite.y) if test_lazer.return_if_x_y_in_sprite_loc( mountain_cell.sprite.x, mountain_cell.sprite.y): #print('mountain cell position',mountain_cell.sprite.x,mountain_cell.sprite.y) return False ''' for coord in available_coords: if Terrain.terrain_obj.terrain_dict[coord].terrain_mov_mod == math.inf: #print('mountain the way') return False ''' #Terrain.terrain_obj.color_path(available_coords) #print('has a shot') return True iter += 1 if iter > test_limit: print(available_coords) #Terrain.terrain_obj.color_path(available_coords) print('got to iter end') return False
#%% # import necessary packages import numpy as np # for numeric calculations import pandas as pd import func as f import Agents as ag agent_list = [ag.RLWM_noneFree(),ag.RLWM_allFree(),ag.RLWM_noise(),ag.RLWM_modulation(), ag.RLWM_actionSoftmax()] numMod, numBlocks = len(agent_list), len(agent_list[0].Sub) ModOrder = iter(range(numMod)) AICs, BICs, Mods = np.zeros((numBlocks, numMod)), np.zeros((numBlocks, numMod)), np.empty(numMod, dtype="object") AICs_te, BICs_te = np.zeros((numBlocks, numMod)), np.zeros((numBlocks, numMod)) #%% for i in agent_list: Order = next(ModOrder) agent = agent_list[Order] ModFit = np.load('../ModelFit/ModFit'+agent.name+'.npz',allow_pickle=True) # loads your saved array into variable a. te_ModFit = np.load('../Testing_ModelFit/Testing_ModelFit' + agent.name + '.npz', allow_pickle=True) f.ParamBar(agent.name,agent.pname, ModFit['Params'], ModFit['Ks']) AICs[:, Order], BICs[:, Order], Mods[Order] = ModFit['AICs'], ModFit['BICs'], ModFit['agtName'] AICs_te[:, Order], BICs_te[:, Order] = te_ModFit['AICs'], te_ModFit['BICs'] #%% Mods = ['noneFree', 'allFree', 'noise', 'modulation', 'action confusion'] f.ICBar(AICs, BICs, Mods, degree = 10)
def learnValueFunction(n_trials, environment, place_cells, actor=None, critic=None, max_steps=np.Inf): """ Main function responsible for learning value function for a given environment INPUTS: ------- n_trials: (INTEGER) Number of trials allowed on the task environment: (Maze) Physical space in which the task has to be learnt place_cells: (PlaceCell) Entity that encodes a particular location as a population <OPTIONAL INPUTS> actor: Pre-trained actor critic: Pre-trained critic OUTPUTS: -------- actor: (Actor Class) Entity that learns actions for a given state critic: (Critic Class) Entity that evaluates the value for a particular state. These values are used for taking actions. """ # Visualize place fields for a few cells and then the aggregate activity # Set up the actor and critic based on the place fields if critic is None: critic = Agents.Critic(len(place_cells)) else: assert (critic.getNFields() == len(place_cells)) if actor is None: actor = Agents.Actor(environment.getActions(), len(place_cells)) # actor = Agents.RandomAgent(environment.getActions(), len(place_cells)) # actor = Agents.IdealActor(environment, critic, place_cells) else: assert (actor.getNFields() == len(place_cells)) n_steps = np.zeros(n_trials, dtype=float) for trial in range(n_trials): # Path is visualized using a graphics object canvas = Graphics.WallMazeCanvas(environment) if DBG_LVL > 2: n_cells_to_visualize = 4 for _ in range(n_cells_to_visualize): sample_cell = random.randint(0, len(place_cells)) canvas.visualizePlaceField(place_cells[sample_cell]) canvas.visualizeAggregatePlaceFields(place_cells) # Initialize a new location and adjust for the optimal number of steps # needed to get to the goal. environment.redrawInitLocation() optimal_steps_to_goal = environment.getOptimalDistanceToGoal() n_steps[trial] = -optimal_steps_to_goal initial_state = environment.getCurrentState() canvas.update(initial_state) terminate_trial = False while not terminate_trial: terminate_trial = environment.reachedGoalState() if (n_steps[trial] > max_steps * environment.MOVE_DISTANCE): break n_steps[trial] += environment.MOVE_DISTANCE current_state = environment.getCurrentState() if DBG_LVL > 1: print('On state: (%.2f, %.2f)' % (current_state[0], current_state[1])) # Get the place field activity based on the current location pf_activity = [pf.getActivity(current_state) for pf in place_cells] # Get an action based on the place field activity next_action = actor.getAction(pf_activity) if DBG_LVL > 1: print('Selected Action: %s' % next_action) # Apply this action onto the environment reward = environment.move(next_action) # canvas.update(environment.getCurrentState()) # Use the obtained reward to update the value new_environment_state = environment.getCurrentState() canvas.update(new_environment_state) new_pf_activity = [ pf.getActivity(new_environment_state) for pf in place_cells ] prediction_error = critic.updateValue(pf_activity, new_pf_activity, reward) actor.updateWeights(pf_activity, prediction_error) if (DBG_LVL > 0): print('Ended trial %d moving %.1f.' % (trial, n_steps[trial])) # At debug level 1, only the first and the last trajectories, and # corresponding value functions are shown. At higher debug levels, # the entire trajectory is shown for every iteration if (DBG_LVL > 1) or (trial == 1) or (trial == n_trials - 1): # Plot the trajectory taken for this trial canvas.plotTrajectory() # This takes extremely long when using a population of neurons canvas.plotValueFunction(place_cells, critic, limits=False, continuous=True) # Plot a histogram of the weightS """ critic_weights = np.reshape(critic.getWeights(), -1) Graphics.histogram(critic_weights) """ if (DBG_LVL > 0): Graphics.plot(n_steps) else: print('Step Statistics - Mean (%.2f), STD (%.2f)' % (np.mean(n_steps), np.std(n_steps))) return (actor, critic, n_steps)
import numpy as np # for numeric calculations import time #calculate runtime import Agents as ag import func as f #%% agent_list = [ ag.RLWM_noneFree(), ag.RLWM_allFree(), ag.RLWM_noise(), ag.RLWM_modulation(), ag.RLWM_actionSoftmax() ] start_time = time.time() AICs, BICs = f.modRec(agent_list, initSample=22, endSample=33) print("--- %s seconds ---" % (time.time() - start_time)) np.savez('Model_Recovery', AICs=AICs, BICs=BICs) # save the file as "outfile_name.npy"
#%% # import necessary packages import pandas as pd # for python data frame import time #calculate runtime import numpy as np # for numeric calculations import Agents as ag import func as f #%% agent_list = [ ag.RLWM_noneFree(), ag.RLWM_allFree(), ag.RLWM_noise(), ag.RLWM_modulation() ] #%% for agent in agent_list: agent.te_IC()
def __init__(self, ng=c.numOfGoods, nr=c.numofRounds, mem=c.memory, al=c.alpha, maxCost=c.max_fixedCost): #This list will be used to keep track of goods that agents recieve... #..though it is not their consumption goods, i.e. goods agents... #choose to carry for purpose of indirect trade c.numOfBaseAgents = ng * (ng - 1) c.numOfGoods = ng c.numofRounds = nr c.memory = mem c.alpha = al c.max_fixedCost = maxCost self.listofMoney = [0] * (c.numOfGoods) #agent self.types self.types = [] #array of all trades self.allTrades = [] #array of goods traded self.goodsTraded = [] #cost list self.costList_unchanging = [] # create a dictionary to store the cost of trading goods, the structure will be of the following kind # {(0,1):[0.1,0.02,0.5],(0,2):[0.1,0.6,0.02],(1,0):[0.5,0.3]} # (0,1) records the cost of the agent who recieved 0 # we initialize an empty dictionary with the intention of filling it up with the above self.tradeCosts = dict() # create two list of number of goods self.goods_listOne = [i for i in range(c.numOfGoods)] self.goods_listTwo = [i for i in range(c.numOfGoods)] # create a list of combinations of goods, it will look like this: [(0,1),(0,2),(1,0) and so on] self.goodsCombinations = [] for i in self.goods_listOne: for j in self.goods_listTwo: self.goodsCombinations.append((i, j)) # attaches an empty list to a dictionary where elements shows combinations of goods, so we get {(0,1):[],(0,2):[],(1,0):[]} for x in self.goodsCombinations: self.tradeCosts[x] = list() #print "fixed costs", c.max_fixedCost #list of costs assigned to each agent for i in range(0, c.numOfGoods): self.costList_unchanging.append(random.uniform(0, c.max_fixedCost)) self.costList = copy.deepcopy(self.costList_unchanging) # We generate a list of agents self.agentList = [ ag.simpleAgents() for count in xrange(c.numOfBaseAgents) ] self.listofMoney = [0] * (c.numOfGoods) #initialize agent self.types for i in range(0, c.numOfGoods): for j in range(0, c.numOfGoods): if i != j: nlist = [str(i), str(j), str(j)] self.types.append(nlist) # And we give each agents agent1 list of consumption good,.. # ...production good and carry good. Initially the production... # ...good and carry good are the same for i in range(0, len(self.types)): self.agentList[i].goods = self.types[i] for i in self.agentList: i.cost = self.costList #print "costs", self.costList #register callback funciton if necessary self.callback_function = None
def testMaze(): """ No comments here. Look at single_maze_learning_agent.py for more details! """ ValueLearning.DBG_LVL = 0 nx = 6 ny = 6 # Set the number of cells to be used per "place field" - Same for all the environments Hippocampus.N_CELLS_PER_FIELD = 1 n_fields = round(1.0 * (nx + 3) * (ny + 3)) n_cells = Hippocampus.N_CELLS_PER_FIELD * n_fields move_distance = 0.99 n_training_trials = 100 n_single_env_episodes = 2 n_alternations = 1 max_train_steps = 1000 # First Environment: Has its own place cells and place fields env_E1 = Environment.RandomGoalOpenField(nx, ny, move_distance) canvas_E1 = Graphics.WallMazeCanvas(env_E1) place_fields_E1 = Hippocampus.setupPlaceFields(env_E1, n_fields) place_cells_E1 = Hippocampus.assignPlaceCells(n_cells, place_fields_E1) # Train a critic on the first environment print('Training Critic solely on Env A') critic_E1 = None weights_E1 = np.empty((n_cells, n_single_env_episodes), dtype=float) for episode in range(n_single_env_episodes): (_, critic_E1, _) = ValueLearning.learnValueFunction(n_training_trials, env_E1, place_cells_E1, critic=critic_E1, max_steps=max_train_steps) weights_E1[:, episode] = critic_E1.getWeights() # Get a trajectory in the environment and plot the value function canvas_E1.plotValueFunction(place_cells_E1, critic_E1, continuous=True) input('Press return to run next environment...') components_E1 = Graphics.showDecomposition(weights_E1, title='Environment 01') # Create empty actors and critics actor = Agents.RandomAgent(env_E1.getActions(), n_cells) critic = Agents.Critic(n_cells) # Second Environment: This has a different set (but the same number) of # place fields and place cells (also has a bunch of walls) nx = 6 ny = 6 lp_wall = Environment.Wall((0, 3), (3, 3)) rp_wall = Environment.Wall((4, 3), (6, 3)) env_E2 = Environment.MazeWithWalls(nx, ny, [lp_wall, rp_wall], move_distance=move_distance) canvas_E2 = Graphics.WallMazeCanvas(env_E2) place_fields_E2 = Hippocampus.setupPlaceFields(env_E2, n_fields) place_cells_E2 = Hippocampus.assignPlaceCells(n_cells, place_fields_E2) # Train another critic on the second environment print() print('Training Critic solely on Env B') critic_E2 = None weights_E2 = np.empty((n_cells, n_single_env_episodes), dtype=float) for episode in range(n_single_env_episodes): (_, critic_E2, _) = ValueLearning.learnValueFunction(n_training_trials, env_E2, place_cells_E2, critic=critic_E2, max_steps=max_train_steps) weights_E2[:, episode] = critic_E2.getWeights() components_E2 = Graphics.showDecomposition(weights_E2, title='Environment 02') canvas_E2.plotValueFunction(place_cells_E2, critic_E2, continuous=True) # Look at the projection of one environment's weights on the other's principal components Graphics.showDecomposition(weights_E1, components=components_E2, title='E2 on E1') Graphics.showDecomposition(weights_E2, components=components_E1, title='E1 on E2') input('Press any key to start Alternation.') # This can be used to just reinforce the fact that the agent is indeed # random! The steps taken to goal would not change over time because of the # way the agent behaves. learning_steps_E1 = np.zeros((n_alternations, 1), dtype=float) learning_steps_E2 = np.zeros((n_alternations, 1), dtype=float) # keep track of weights for PCA weights = np.empty((n_cells, n_alternations * 2), dtype=float) for alt in range(n_alternations): n_alternation_trials = n_single_env_episodes * n_training_trials # n_alternation_trials = n_training_trials print('Alternation: %d' % alt) # First look at the performance of the agent in the task before it is # allowed to learn anything. Then allow learning print('Learning Environment A') (actor, critic, steps_E1) = ValueLearning.learnValueFunction( n_alternation_trials, env_E1, place_cells_E1, actor, critic, max_train_steps) learning_steps_E1[alt] = np.mean(steps_E1) weights[:, 2 * alt] = critic.getWeights() # Repeat for environment 1 print('Learning Environment B') (actor, critic, steps_E2) = ValueLearning.learnValueFunction( n_alternation_trials, env_E2, place_cells_E2, actor, critic, max_train_steps) learning_steps_E2[alt] = np.mean(steps_E2) weights[:, 2 * alt + 1] = critic.getWeights() # Show the alternation weights in the two basis Graphics.showDecomposition(weights, components=components_E1, title='Alternation weights in E1') Graphics.showDecomposition(weights, components=components_E2, title='Alternation weights in E2') # Show the value functions for both the environments input('Press return for Value Function of E1') canvas_E1.plotValueFunction(place_cells_E1, critic, continuous=True) canvas_E1.plotValueFunction(place_cells_E1, critic_E1, continuous=True) canvas_E1.plotValueFunction(place_cells_E1, critic_E2, continuous=True) # Plot the ideal value function ideal_critic = Agents.IdealValueAgent(env_E1, place_cells_E1) optimal_value_function = ideal_critic.getValueFunction() scaling_factor = 1.0 / (1 - critic_E1.getDiscountFactor()) # Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), \ range=(env_E1.NON_GOAL_STATE_REWARD, scaling_factor * env_E1.GOAL_STATE_REWARD)) input('Press return for Value Function of E2') canvas_E2.plotValueFunction(place_cells_E2, critic, continuous=True) canvas_E2.plotValueFunction(place_cells_E2, critic_E2, continuous=True) canvas_E2.plotValueFunction(place_cells_E2, critic_E1, continuous=True) # Plot the ideal value function ideal_critic = Agents.IdealValueAgent(env_E2, place_cells_E2) optimal_value_function = ideal_critic.getValueFunction() scaling_factor = 1.0 / (1 - critic_E2.getDiscountFactor()) # Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), \ range=(env_E2.NON_GOAL_STATE_REWARD, scaling_factor * env_E2.GOAL_STATE_REWARD)) input('Press any key to exit!')