class Foraging(Model):
number_of_bean = 0
number_of_corn = 0
number_of_soy = 0
def __init__(self, width=50, height=50, torus=True, num_bug=50, seed=42, strategy=None):
super().__init__(seed=seed)
self.number_of_bug = num_bug
if not(strategy in ["stick", "switch"]):
raise TypeError("'strategy' must be one of {stick, switch}")
self.strategy = strategy
self.grid = SingleGrid(width, height, torus)
self.schedule = RandomActivation(self)
data = {"Bean": lambda m: m.number_of_bean,
"Corn": lambda m: m.number_of_corn,
"Soy": lambda m: m.number_of_soy,
"Bug": lambda m: m.number_of_bug,
}
self.datacollector = DataCollector(data)
# create foods
self._populate(Bean)
self._populate(Corn)
self._populate(Soy)
# create bugs
for i in range(self.number_of_bug):
pos = self.grid.find_empty()
bug = Bug(i, self)
bug.strategy = self.strategy
self.grid.place_agent(bug, pos)
self.schedule.add(bug)
def step(self):
self.schedule.step()
self.datacollector.collect(self)
if not(self.grid.exists_empty_cells()):
self.running = False
def _populate(self, food_type):
prefix = "number_of_{}"
counter = 0
while counter < food_type.density * (self.grid.width * self.grid.height):
pos = self.grid.find_empty()
food = food_type(counter, self)
self.grid.place_agent(food, pos)
self.schedule.add(food)
food_name = food_type.__name__.lower()
attr_name = prefix.format(food_name)
val = getattr(self, attr_name)
val += 1
setattr(self, attr_name, val)
counter += 1
class SchellingModel(Model):
'''
Model class for the Schelling segregation model.
'''
def __init__(self, height, width, density, type_pcs=[.2, .2, .2, .2, .2]):
'''
'''
self.height = height
self.width = width
self.density = density
self.type_pcs = type_pcs
self.schedule = RandomActivation(self)
self.grid = SingleGrid(height, width, torus=False)
self.happy = 0
self.datacollector = DataCollector(
{"happy": lambda m: m.happy}, # Model-level count of happy agents
# For testing purposes, agent's individual x and y
{
"x": lambda a: a.pos[0],
"y": lambda a: a.pos[1]
})
self.running = True
# Set up agents
# We use a grid iterator that returns
# the coordinates of a cell as well as
# its contents. (coord_iter)
total_agents = self.height * self.width * self.density
agents_by_type = [total_agents * val for val in self.type_pcs]
for loc, types in enumerate(agents_by_type):
for i in range(int(types)):
pos = self.grid.find_empty()
agent = SchellingAgent(pos, self, loc)
self.grid.position_agent(agent, pos)
self.schedule.add(agent)
def step(self):
'''
Run one step of the model. If All agents are happy, halt the model.
'''
self.happy = 0 # Reset counter of happy agents
self.schedule.step()
self.datacollector.collect(self)
if self.happy == self.schedule.get_agent_count():
self.running = False
class TestSingleGrid(unittest.TestCase):
def setUp(self):
self.space = SingleGrid(50, 50, False)
self.agents = []
for i, pos in enumerate(TEST_AGENTS_GRID):
a = MockAgent(i, None)
self.agents.append(a)
self.space.place_agent(a, pos)
def test_agent_positions(self):
"""
Ensure that the agents are all placed properly.
"""
for i, pos in enumerate(TEST_AGENTS_GRID):
a = self.agents[i]
assert a.pos == pos
def test_remove_agent(self):
for i, pos in enumerate(TEST_AGENTS_GRID):
a = self.agents[i]
assert a.pos == pos
assert self.space.grid[pos[0]][pos[1]] == a
self.space.remove_agent(a)
assert a.pos is None
assert self.space.grid[pos[0]][pos[1]] is None
def test_empty_cells(self):
if self.space.exists_empty_cells():
pytest.deprecated_call(self.space.find_empty)
for i, pos in enumerate(list(self.space.empties)):
a = MockAgent(-i, pos)
self.space.position_agent(a, x=pos[0], y=pos[1])
assert self.space.find_empty() is None
with self.assertRaises(Exception):
self.space.move_to_empty(a)
def move_agent(self):
agent_number = 0
initial_pos = TEST_AGENTS_GRID[agent_number]
final_pos = (7, 7)
_agent = self.agents[agent_number]
assert _agent.pos == initial_pos
assert self.space.grid[initial_pos[0]][initial_pos[1]] == _agent
assert self.space.grid[final_pos[0]][final_pos[1]] is None
self.space.move_agent(_agent, final_pos)
assert _agent.pos == final_pos
assert self.space.grid[initial_pos[0]][initial_pos[1]] is None
assert self.space.grid[final_pos[0]][final_pos[1]] == _agent
class SchellingModel(Model):
'''
Model class for the Schelling segregation model.
'''
def __init__(self, height, width, density, type_pcs=[.2, .2, .2, .2, .2]):
'''
'''
self.height = height
self.width = width
self.density = density
self.type_pcs = type_pcs
self.schedule = RandomActivation(self)
self.grid = SingleGrid(height, width, torus=False)
self.happy = 0
self.datacollector = DataCollector(
{"happy": lambda m: m.happy}, # Model-level count of happy agents
# For testing purposes, agent's individual x and y
{"x": lambda a: a.pos[0], "y": lambda a: a.pos[1]})
self.running = True
# Set up agents
# We use a grid iterator that returns
# the coordinates of a cell as well as
# its contents. (coord_iter)
total_agents = self.height * self.width * self.density
agents_by_type = [total_agents*val for val in self.type_pcs]
for loc, types in enumerate(agents_by_type):
for i in range(int(types)):
pos = self.grid.find_empty()
agent = SchellingAgent(pos, self, loc)
self.grid.position_agent(agent, pos)
self.schedule.add(agent)
def step(self):
'''
Run one step of the model. If All agents are happy, halt the model.
'''
self.happy = 0 # Reset counter of happy agents
self.schedule.step()
self.datacollector.collect(self)
if self.happy == self.schedule.get_agent_count():
self.running = False
class ConwayGameOfLifeModel(Model):
"""
Class describing Conway's game of life using the mesa agent based model framowork
The model contains a grid. In each cell of the grid there is an agent. The agent can be dead or
alive. At each step of the simulation, the state of the agent can change.
The model is responsible of creating the grid and handling the iteration of the simulation
"""
def __init__(self, grid_height, grid_width, percentage_of_cell_alive):
"""
Constructor
"""
self.grid = SingleGrid(grid_width, grid_height, False)
self.scheduler = SimultaneousActivation(self)
self.number_of_agent = grid_width * grid_height
# Creation of all agent
for i in range(self.number_of_agent):
# Randomly chooses the initial state of the agent (0 is alive and 1 is dead)
# We use choices from the random module because it allows us to specify a distribution
# (ie. a list of probability for each state). Choices will return a list with ne element
# which is our state
probability_alive = percentage_of_cell_alive / 100
probability_dead = 1 - probability_alive
state = choices([0, 1], [probability_dead, probability_alive])[0]
# Creating the agent and adding it to the scheduler
agent = CellAgent(i, state, self)
self.scheduler.add(agent)
# Adding the new agent to the grid
agent_coordinates = self.grid.find_empty()
self.grid.place_agent(agent, agent_coordinates)
# Define if the simulation is running or not
self.running = True
def step(self):
"""
Method to advance the model by one step. It will call the step method of each agent.
We use a simultaneous scheduler which means we we'll be iterating through all the agent at
once to determine their next state and then apply the new state
"""
self.scheduler.step()
class Foraging(Model):
number_of_bean = 0
number_of_corn = 0
number_of_soy = 0
def __init__(self,
width=50,
height=50,
torus=True,
num_bug=50,
seed=42,
strategy=None):
super().__init__(seed=seed)
self.number_of_bug = num_bug
if not (strategy in ["stick", "switch"]):
raise TypeError("'strategy' must be one of {stick, switch}")
self.strategy = strategy
self.grid = SingleGrid(width, height, torus)
self.schedule = RandomActivation(self)
data = {
"Bean": lambda m: m.number_of_bean,
"Corn": lambda m: m.number_of_corn,
"Soy": lambda m: m.number_of_soy,
"Bug": lambda m: m.number_of_bug,
}
self.datacollector = DataCollector(data)
# create foods
self._populate(Bean)
self._populate(Corn)
self._populate(Soy)
# create bugs
for i in range(self.number_of_bug):
pos = self.grid.find_empty()
bug = Bug(i, self)
bug.strategy = self.strategy
self.grid.place_agent(bug, pos)
self.schedule.add(bug)
def step(self):
self.schedule.step()
self.datacollector.collect(self)
if not (self.grid.exists_empty_cells()):
self.running = False
def _populate(self, food_type):
prefix = "number_of_{}"
counter = 0
while counter < food_type.density * (self.grid.width *
self.grid.height):
pos = self.grid.find_empty()
food = food_type(counter, self)
self.grid.place_agent(food, pos)
self.schedule.add(food)
food_name = food_type.__name__.lower()
attr_name = prefix.format(food_name)
val = getattr(self, attr_name)
val += 1
setattr(self, attr_name, val)
counter += 1
class Factory(Model):
"""The Factory model that maintains the state of the whole factory."""
def __init__(self, grid_w, grid_h, n_robots):
"""Initialize factory."""
# Initialize.
self.orders = 0
self.n_robots = n_robots
self.scheduler = RandomActivation(self)
self.grid = SingleGrid(grid_w, grid_h, torus=False)
self.init_astar()
# Initialize departments.
self.machine = Machine("machine", self, self.grid.find_empty())
self.store = Store("store", self, self.grid.find_empty())
self.packaging = Packaging("packaging", self, self.grid.find_empty())
self.dept_positions = [self.machine.pos, self.store.pos, self.packaging.pos]
# Initialize robots.
for i in range(self.n_robots):
# Create robot.
r = Robot(i, self)
# Initialize random location.
pos = self.grid.find_empty()
self.grid.place_agent(r, pos)
# Register with scheduler.
self.scheduler.add(r)
# Initialize visualization.
plt.ion()
def add_order(self):
"""Increment the number of orders to the factory."""
self.orders += 1
def step(self):
"""Advance the factory by one step."""
# Step through factory. Check for orders.
if self.orders > 0:
self.store.orders += 1
self.orders -= 1
# Step through departments.
self.store.step()
self.machine.step()
self.packaging.step()
# Step through robots.
self.scheduler.step()
# Visualize.
self.visualize()
def init_astar(self):
"""Initialize a-star resources so that it doesn't have to calculated for each robot.
Initialized in such a way that:
* A diagonal paths are allowed.
* The path calculated takes into account all obstacles in the grid.
"""
def get_empty_neighborhood(pos):
"""A sub function to calculate empty neighbors of a point for a-star."""
neighbors = self.grid.get_neighborhood(pos=pos, moore=True)
return [n for n in neighbors if self.grid.is_cell_empty(n)]
# Initialize a path finder object once for the entire factory.
self.path_finder = astar.pathfinder(neighbors=get_empty_neighborhood,
distance=astar.absolute_distance,
cost=astar.fixed_cost(1))
def find_nearest_aimless_robot(self, pos):
"""Find the nearest aimless robot to a given position in the factory."""
def is_aimless(robot, pos):
"""Check if the robot satisfied aimless condition."""
if robot.destination is None:
return True
else:
return False
aimless_robots = [robot for robot in self.scheduler.agents if is_aimless(robot, pos)]
if len(aimless_robots) != 0:
robot_distances = [astar.absolute_distance(pos, robot.pos) for robot in aimless_robots]
nearest_index = np.argmin(robot_distances)
return aimless_robots[nearest_index]
else:
return None
def find_robot_at_position(self, pos):
"""Find robot that is at a given location in the factory that is not busy."""
for robot in self.scheduler.agents:
if robot.pos == pos:
return robot
return None
def find_next_position_towards_destination(self, curr_pos, dest_pos):
"""Find the next empty position to move in the direction of the destination."""
n_steps, path = self.path_finder(curr_pos, dest_pos) # Handles non-empty locations.
# NOTE: We cannot find a valid path to the destination when:
# 1) The destination has an another robot located inside it, which also occurs when curr_pos and
# dest_pos are the same.
# 2) The path is entirely blocked.
# In these cases we return the next position to be the curr_pos, in order to wait until things
# clear up.
if n_steps is None or n_steps <= 0: # No valid path to destination
next_pos = curr_pos
print("[MOVE] Warning: No path to destination from {} --> {}".format(curr_pos, dest_pos))
# This mean there's a valid path to destination.
else:
# index 0, is the curr_pos, index 1 is the next position.
next_pos = path[1]
return next_pos
def find_next_position_for_random_walk(self, curr_pos):
"""Find a valid location for a robot to just randomly walk into."""
def is_pos_empty(pos):
"""A sub function if a cell is empty for random walking."""
if self.grid.is_cell_empty(pos) and pos not in self.dept_positions:
return True
else:
return False
neighborhood = self.grid.get_neighborhood(curr_pos, moore=True)
empty_neighborhood = [n for n in neighborhood if is_pos_empty(n)]
if len(empty_neighborhood) > 0:
next_index = np.random.randint(len(empty_neighborhood))
next_pos = empty_neighborhood[next_index]
else:
next_pos = curr_pos
return next_pos
def visualize(self):
"""A chess board type visualization."""
def heatmap(a):
cMap = ListedColormap(['grey', 'black', 'green', 'orange', 'red', 'blue'])
sns.heatmap(a, vmin=0, vmax=6, cmap=cMap, linewidths=1)
plt.pause(0.15)
plt.clf()
g = np.zeros((self.grid.height, self.grid.width), dtype=int)
g[self.store.pos] = 3
g[self.machine.pos] = 4
g[self.packaging.pos] = 5
for robot in self.scheduler.agents:
if robot.destination is None:
g[robot.pos] = 1
else:
g[robot.pos] = 2
heatmap(g)
class CHModel(Model):
"""A model with some number of agents."""
def __init__(self, width, height, random_n = 0, cow_n = 0, plan_n = 0, mc_n = 0, td_n = 0, episode_number = 0, t_mc_n = 0, old_Q_values = None):
self.running = True
#self.num_agents = N
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
self.id_count = 0 #to assign each agent a unique ID
#self.max_timesteps = 500 #max timesteps for each episode
# To keep score
self.total_cow_count = 0.0
self.current_cow_count = 0.0
self.score = 0.0
self.previous_cow_count = 0.0
# Save model for agent use
self.wallLocations = [(1,5), (1,6), (1,7), (2,7), (3,7), (4,7), (5,7), (6,7), (6,6), (6,5)]
self.goalState = [(2,5), (3,5), (4,5), (5,5), (2,6), (3,6), (4,6), (5,6)]
self.goalTarget = (3,5) #corral "entrance" that plan agents herd towards
self.state = None # encode state at each timestep
self.number_random_agents = random_n
self.number_cow_agents = cow_n
self.number_plan_agents = plan_n
self.number_monte_carlo_agents = mc_n
self.number_td_agents = td_n
self.number_trained_mc_agents = t_mc_n
# load pre-trained data to add to or make new Q tables for MC Agents
# set to false to make a new Q table
#loadpretrained = True
#if (loadpretrained and (not old_Q_values)):
# print("loading pkl file")
# with open('mc_q_save.pkl', 'rb') as file:
# self.Q_values = dill.load(file)
# Monte Carlo Agent model save
self.Q_table_sharing = True ## If true, agents share a Q table
self.vision_range = 2 # How far the MC agents can see
if old_Q_values: #load previous Q tables if they exist
self.Q_values = old_Q_values
else:
self.Q_values = [] #no previous Q tables, so make new ones
if (self.Q_table_sharing):
# Just one Q table
self.Q_values.append(defaultdict(lambda: np.zeros(len(rl_methods.action_space))))
else:
#every agent gets it's own Q table
for agent in range(self.number_monte_carlo_agents):
self.Q_values.append(defaultdict(lambda: np.zeros(len(rl_methods.action_space))))
self.mc_agents = []
self.episode = episode_number
#calculate episilon based on episode
#epsilon = 1 / i_episode
####### tweak episilon to get better results #######
self.epsilon = 1.0/((episode_number/800) + 1)
#self.epsilon = 1.0/((episode_number/8000)+1)
# Place wall agents
for i in range(len(self.wallLocations)):
a = WallAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(a)
#print("placing ", a, " at ", self.corralLocations[i])
self.grid.place_agent(a, self.wallLocations[i])
# Place random agents
for i in range(self.number_random_agents):
a = RandomAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(a)
cell_location = self.grid.find_empty()
self.grid.place_agent(a, cell_location)
# Place cow agents
for i in range(self.number_cow_agents):
c = CowAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(c)
#self.cow_agent_list.append(c) #make a list of cows
cell_location = self.grid.find_empty()
self.grid.place_agent(c, cell_location)
# Place plan agents
for i in range(self.number_plan_agents):
p = PlanAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(p)
cell_location = self.grid.find_empty()
self.grid.place_agent(p, cell_location)
# Place monte carlo agents
for i in range(self.number_monte_carlo_agents):
Q_table_to_use = None
if (self.Q_table_sharing): # If sharing Q tables, everyone gets a copy of the same Q table
Q_table_to_use = self.Q_values[0]
else:
Q_table_to_use = self.Q_values[i] # If not sharing, everyone gets a different Q table
m = MonteCarloAgent(self.id_count, self, Q_table_to_use, self.epsilon, vision = self.vision_range) # init MC agents with previous Q tables
self.mc_agents.append(m) # save MC agents to retrieve Q values
self.id_count += 1
self.schedule.add(m)
cell_location = self.grid.find_empty()
self.grid.place_agent(m, cell_location)
# Place trained monte carlo agents
# open/load trained Q table
if (self.number_trained_mc_agents > 0):
loaded_Q = None
with open('mc_q_save.pkl', 'rb') as file:
loaded_Q = dill.load(file)
if loaded_Q:
for i in range(self.number_trained_mc_agents):
tm = TrainedMonteCarloAgent(self.id_count, self, loaded_Q, vision = self.vision_range)
self.id_count += 1
self.schedule.add(tm)
cell_location = self.grid.find_empty()
self.grid.place_agent(tm, cell_location)
else:
print("Can't load Q table for trained MC Agents")
# Place TD agents
for i in range(self.number_td_agents):
t = TDAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(t)
cell_location = self.grid.find_empty()
self.grid.place_agent(t, cell_location)
def step(self):
self.state = rl_methods.encode_state(self.grid)
self.schedule.step()
self.update_score()
#print(np.matrix(self.state))
#print("the current score is ", self.score)
# Update rewards of Monte Carlo agents
rewards_type = 3
# if rewards_type is
###1 use the actual current score
###2 use number of cows in goal
###3 cows in goal with penalty if cow leaves goal
# how penalized do you want the agents to be for letting cow escape?
penalty_modifier = 0.0
# how much of a bonus for getting cows to go in the goal?
bonus_modifier = 100.0
# bonus for keeping cows in goal
bonus_cows = 5.0
for mcagent in self.mc_agents:
if (rewards_type == 1):
mcagent.update_rewards(self.score)
elif (rewards_type == 2):
mcagent.update_rewards(self.current_cow_count)
elif (rewards_type == 3):
penalty = 0.0
bonus = 0.0
no_cow_penalty = -1.0
if (self.current_cow_count < self.previous_cow_count):
print("calculating penalty ESCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPE")
cows_escaped = (float(self.previous_cow_count) - float(self.current_cow_count))
#print("this many escaped: ", cows_escaped, ", modifier: ", penalty_modifier)
penalty = penalty_modifier * cows_escaped
#print("prev cows ", self.previous_cow_count, ", cows ", self.current_cow_count, ", penalty ", penalty)
if (self.current_cow_count > self.previous_cow_count):
print("calculating penalty COWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW")
cows_gained = (float(self.current_cow_count) - float(self.previous_cow_count))
#print("this many escaped: ", cows_escaped, ", modifier: ", penalty_modifier)
bonus = bonus_modifier * cows_gained
if (self.current_cow_count < self.number_cow_agents):
penalty = penalty - (no_cow_penalty * (float(self.number_cow_agents) - float(self.current_cow_count)))
mcagent.update_rewards((self.current_cow_count * bonus_cows) - penalty + bonus)
print("current cow count: ", self.current_cow_count, ", penalty: ", penalty, ", bonus: ", bonus, ", no cow ")
print("total reward: ", (self.current_cow_count * bonus_cows) - penalty + bonus)
else:
printing("using default reward")
mcagent.update_rewards(self.score)
def update_score(self):
self.previous_cow_count = self.current_cow_count
self.current_cow_count = cow_methods.cows_in_goal(self, self.goalState)
self.total_cow_count += self.current_cow_count
print(self.total_cow_count, self.current_cow_count, self.schedule.time, " Episode: ", self.episode)
self.score = self.total_cow_count / self.schedule.time
def get_new_Q_values(self):
""" Update model Q values at the end of the episode, called by run after each episode """
new_Q = []
if(self.Q_table_sharing): #If all agents are sharing Q table data
updated_Q = None
for agent in self.mc_agents:
# Update the Q table then pass it on to the next agent on the team to update
updated_Q = agent.Q_table_update(shared_Q_table = updated_Q)
new_Q.append(copy.deepcopy(updated_Q))
else:
# If all agents have their own Q tables, update and save for next episode
for agent in self.mc_agents:
updated_Q = agent.Q_table_update()
new_Q.append(copy.deepcopy(updated_Q))
return new_Q
class ReactionDiffusionModel(Model):
"""A model with some number of agents."""
#Initialize a model that includes a grid of side-length N and one agent for each grid
def __init__(self, N):
#Number of agents
self.num_agents = N * N
#The two grids can have just one agent per cell, it is dimensions NxN, and it is toroidal
self.oldActivatorGrid = SingleGrid(N, N, True)
self.oldInhibitorGrid = SingleGrid(N, N, True)
self.currentActivatorGrid = SingleGrid(N, N, True)
self.currentInhibitorGrid = SingleGrid(N, N, True)
#Determine how our model will pick agent to interact with
self.schedule = RandomActivation(self)
# Create agents
for i in range(self.num_agents):
#Initialize a cell with uniqueID = i
a = Cell(i, self)
#Add our agent to our scheduler
self.schedule.add(a)
#Choose a random, unoccupied cell in our grid and add our agent to it
#position_agent stores the x and y value for each of our agents
locationTuple = self.oldActivatorGrid.find_empty()
if (locationTuple) == (N / 2, N / 2):
a.act = 2 * a.act
self.oldActivatorGrid.place_agent(a, locationTuple)
self.oldInhibitorGrid.place_agent(a, locationTuple)
self.currentActivatorGrid.place_agent(a, locationTuple)
self.currentInhibitorGrid.place_agent(a, locationTuple)
#Method to get activator values in our current activator grid
def getActivatorGrid(self):
activator_Grid = np.zeros((self.currentActivatorGrid.width,
self.currentActivatorGrid.height))
for cell in model.currentActivatorGrid.coord_iter():
cell_content, x, y = cell
activator_Grid[x][y] = cell_content.act
return activator_Grid
#Method to get inhibitor values in our current inhibitor grid
def getInhibitorGrid(self):
inhibitor_Grid = np.zeros((self.currentInhibitorGrid.width,
self.currentInhibitorGrid.height))
for cell in model.currentInhibitorGrid.coord_iter():
cell_content, x, y = cell
inhibitor_Grid[x][y] = cell_content.inh
return inhibitor_Grid
def step(self):
#Determine what the original activator and inhibitor distributions are
oldActivatorGrid = self.currentActivatorGrid
oldInhibitorGrid = self.currentInhibitorGrid
#Perform a step of the model, where we calculate all of the new concentrations
self.schedule.step()
#Determine the new activator and inhibitor distributions
currentActivatorGrid = self.getActivatorGrid()
currentInhibitorGrid = self.getInhibitorGrid()
class DiseaseModel(Model):
"""
A model with some number of agents.
highS: Number of agents with high sociability.
middleS: Number of agents with middle sociability.
lowS: Number of agents with low sociability.
width: Width of the grid.
height: Height of the grid.
edu_setting: If true, agents will follow a schedule and sit in classrooms,
else they will move freely through an open grid.
cureProb: Probability of agent getting better.
cureProbFac: Factor of cureProb getting higher.
mutateProb: Probability of a disease mutating.
diseaseRate: Rate at which the disease spreads.
"""
def __init__(self, highS, middleS, lowS, width, height, edu_setting=True,
cureProb=0.1, cureProbFac=2/1440, mutateProb=0.0050,
diseaseRate=0.38):
super().__init__()
self.num_agents = highS + middleS + lowS
self.lowS = lowS
self.middleS = middleS
self.highS = highS
self.initialCureProb = cureProb
self.cureProbFac = cureProbFac
self.mutateProb = mutateProb
self.diseaseRate = diseaseRate
self.edu_setting = edu_setting
self.maxDisease = 0 # amount of mutations
self.counter = 540 # keeps track of timesteps
self.removed = []
self.exit = (width - 1, floor(height / 2))
# Check if agents fit within grid
if self.num_agents > width * height:
raise ValueError("Number of agents exceeds grid capacity.")
# Create grid with random activation
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
if edu_setting:
# Create walls
numberRooms = 3
self.add_walls(numberRooms, width, height)
self.midWidthRoom = floor(width / numberRooms / 2)
self.midHeightRoom = floor(height / numberRooms / 2)
self.widthRoom = floor(width / numberRooms)
self.heightRoom = floor(height / numberRooms)
numberRows = floor((self.heightRoom) / 2)
widthRows = self.widthRoom - 4
location = [[] for _ in range(numberRooms * 2)]
for i in range(numberRooms):
for j in range(0, numberRows, 2):
startWidth = 2 + (i % 3) * self.widthRoom
for currentWidth in range(widthRows):
location[i] += [(startWidth + currentWidth, j)]
for i in range(3, numberRooms * 2):
for j in range(0, numberRows, 2):
startWidth = 2 + (i % 3) * self.widthRoom
for currentWidth in range(widthRows):
location[i] += [(startWidth + currentWidth,
height - 1 - j)]
# Set 3 goals per roster
self.roster = [[location[0], location[3], location[1]],
[location[5], location[2], location[0]],
[location[4], location[1], location[5]]]
# Create agents
self.addAgents(lowS, 0, 0)
self.addAgents(middleS, lowS, 1)
self.addAgents(highS, lowS + highS, 2)
# set up data collecter
self.datacollector = DataCollector(
model_reporters={"diseasepercentage": disease_collector},
agent_reporters={"disease": "disease"})
def heuristic(self, start, goal):
"""
Returns manhattan distance.
start: current location (x,y)
goal: goal location (x,y)
"""
dx = abs(start[0] - goal[0])
dy = abs(start[1] - goal[1])
return dx + dy
def get_vertex_neighbors(self, pos):
"""
Returns all neighbors.
pos: current position
"""
n = self.grid.get_neighborhood(pos, moore=False)
neighbors = []
for item in n:
if not abs(item[0] - pos[0]) > 1 and not abs(item[1] - pos[1]) > 1:
neighbors += [item]
return neighbors
def move_cost(self, location):
"""
Return the cost of a location.
"""
if self.grid.is_cell_empty(location):
return 1 # Normal movement cost
else:
return 100 # Very difficult to go through walls
def add_walls(self, n, widthGrid, heightGrid):
"""
Add walls in grid.
n: number of rooms horizontally
widthGrid: width of the grid
heightGrid: height of the grid
"""
widthRooms = floor(widthGrid / n)
heightRooms = floor(heightGrid / n)
heightHall = heightGrid - 2 * heightRooms
# Add horizontal walls
for i in range(n - 1):
for y in range(heightRooms):
brick = wall(self.num_agents, self)
self.grid.place_agent(brick, ((i + 1) * widthRooms, y))
self.grid.place_agent(brick, ((i + 1) * widthRooms, y +
heightRooms + heightHall))
doorWidth = 2
# Add vertical walls
for x in range(widthGrid):
if (x % widthRooms) < (widthRooms - doorWidth):
brick = wall(self.num_agents, self)
self.grid.place_agent(brick, (x, heightRooms))
self.grid.place_agent(brick, (x, heightRooms + heightHall - 1))
def addAgents(self, n, startID, sociability):
"""
Add agents with a sociability.
n: number of agents
startID: ID of the first added agent
sociability: sociability of the agents
"""
disease_list = np.random.randint(0, 2, n)
for i in range(n):
# Set schedule for every agent if educational setting
if self.edu_setting:
a_roster = []
rosterNumber = self.random.randrange(len(self.roster))
rooms = self.roster[rosterNumber]
for roomNumber in range(len(rooms)):
loc = self.random.choice(rooms[roomNumber])
a_roster += [loc]
(self.roster[rosterNumber][roomNumber]).remove(loc)
else:
a_roster = []
a = DiseaseAgent(i + startID, sociability, self, disease_list[i],
a_roster)
self.schedule.add(a)
# Set agent outside grid, ready to enter, if edu setting
# else randomly place on empty spot on grid
if self.edu_setting:
self.removed += [a]
a.pos = None
else:
self.grid.place_agent(a, self.grid.find_empty())
def step(self):
"""
Continue one step in simulation.
"""
self.counter += 1
self.datacollector.collect(self)
self.schedule.step()
class DiseaseModel(Model):
"""
A model with some number of agents.
highS: Number of agents with high sociability.
middleS: Number of agents with middle sociability.
lowS: Number of agents with low sociability.
width: Width of the grid.
height: Height of the grid.
edu_setting: Classrooms and set schedule if true, else random free movement.
cureProb: Probability of agent getting better.
cureProbFac: Factor of cureProb getting higher.
mutateProb: Probability of a disease mutating.
diseaseRate: Rate at which the disease spreads.
"""
def __init__(self, highS, middleS, lowS, width, height, edu_setting=True, cureProb=0.1, cureProbFac=2/1440, mutateProb=0.0050, diseaseRate=0.38):
super().__init__()
self.num_agents = highS + middleS + lowS
self.lowS = lowS
self.middleS = middleS
self.highS = highS
self.initialCureProb = cureProb
self.cureProbFac = cureProbFac
self.mutateProb = mutateProb
self.diseaseRate = diseaseRate
self.edu_setting = edu_setting
self.maxDisease = 0# amount of mutations
self.counter = 540 # keeps track of timesteps
self.removed = []
self.exit = (width-1,floor(height/2))
# Check if agents fit within grid
if self.num_agents > width * height:
raise ValueError("Number of agents exceeds grid capacity.")
# Create grid with random activation
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
if edu_setting:
# Create walls
numberRooms = 3
self.add_walls(numberRooms, width, height)
self.midWidthRoom = floor(width / numberRooms / 2)
self.midHeightRoom = floor(height / numberRooms / 2)
# Calculate the centers of the 6 rooms
roomLeftDown = (5 * self.midWidthRoom, self.midHeightRoom)
roomLeftMid = (3 * self.midWidthRoom, self.midHeightRoom)
roomLeftUp = (self.midWidthRoom, self.midHeightRoom)
roomRightDown = (5 * self.midWidthRoom, 5 * self.midHeightRoom, )
roomRightMid = (3 * self.midWidthRoom, 5 * self.midHeightRoom)
roomRightUp = (self.midWidthRoom, 5 * self.midHeightRoom)
# Set 3 goals per roster
self.roster = [[roomLeftDown, roomLeftUp, roomRightMid], [roomRightMid, roomLeftDown, roomRightDown],
[roomRightUp, roomRightDown, roomLeftUp]]
# Create agents
self.addAgents(lowS, 0, 0)
self.addAgents(middleS, lowS, 1)
self.addAgents(highS, lowS + highS, 2)
self.datacollector = DataCollector(
model_reporters={"diseasepercentage": disease_collector},
agent_reporters={"disease": "disease"})
def heuristic(self, start, goal):
"""
Returns manhattan distance.
start: current location (x,y)
goal: goal location (x,y)
"""
dx = abs(start[0] - goal[0])
dy = abs(start[1] - goal[1])
return dx + dy
def get_vertex_neighbors(self, pos):
"""
Returns all neighbors.
pos: current position
"""
n = self.grid.get_neighborhood(pos, moore=False)
neighbors = []
for item in n:
if not abs(item[0]-pos[0]) > 1 and not abs(item[1]-pos[1]) > 1:
neighbors += [item]
return neighbors
def move_cost(self, location):
"""
Return the cost of a location.
"""
if self.grid.is_cell_empty(location):
return 1 # Normal movement cost
else:
return 100
def add_walls(self, n, widthGrid, heightGrid):
"""
Add walls in grid.
n: number of rooms horizontally
widthGrid: width of the grid
heightGrid: height of the grid
"""
widthRooms = floor(widthGrid/n)
heightRooms = floor(heightGrid/n)
widthHall = widthGrid - 2 * widthRooms
heightHall = heightGrid - 2 * heightRooms
# Add horizontal walls
for i in range(n - 1):
for y in range(heightRooms):
brick = wall(self.num_agents, self)
self.grid.place_agent(brick, ((i + 1) * widthRooms, y))
self.grid.place_agent(brick, ((i + 1) * widthRooms, y + heightRooms + heightHall))
doorWidth = 2
# Add vertical walls
for x in range(widthGrid):
if (x % widthRooms) < (widthRooms - doorWidth):
brick = wall(self.num_agents, self)
self.grid.place_agent(brick, (x, heightRooms))
self.grid.place_agent(brick, (x, heightRooms + heightHall - 1))
def addAgents(self, n, startID, sociability):
"""
Add agents with a sociability.
n: number of agents
startID: ID of the first added agent
sociability: sociability of the agents
"""
disease_list = np.random.randint(0,2,n)
for i in range(n):
a = DiseaseAgent(i + startID, sociability,self,disease_list[i])
self.schedule.add(a)
# Add the agent to a random grid cell
location = self.grid.find_empty()
self.grid.place_agent(a, location)
def step(self):
"""
Continue one step in simulation.
"""
self.counter += 1
self.datacollector.collect(self)
self.schedule.step()
class MondoModel(Model):
"""Questo è il mondo fatto a griglia"""
def __init__(self, popolazione, width, height):
super().__init__()
self.popolazione = popolazione
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
self.points = []
# Create agents
for i in range(self.popolazione):
a = PersonAgent(i, self)
self.schedule.add(a)
emptyspace = self.grid.find_empty()
if emptyspace is not None:
self.grid.place_agent(a, emptyspace)
paziente_zero = self.schedule.agents[0]
paziente_zero.virus = Virus(mortalita=20,
tempo_incubazione=3,
infettivita=70)
paziente_zero.ttl = paziente_zero.virus.tempo_incubazione
def step(self):
'''Advance the model by one step.'''
self.schedule.step()
suscettibili = 0
infetti = 0
morti = 0
immuni = 0
for persona in self.schedule.agents:
if persona.isAlive is False:
morti += 1
elif persona.isImmune is True:
immuni += 1
elif persona.virus is not None:
infetti += 1
else:
suscettibili += 1
self.points.append([suscettibili, infetti, morti, immuni])
def crea_grafico(self):
global_health_status = np.zeros((self.grid.width, self.grid.height))
for persona, x, y in self.grid.coord_iter(
): # ctrl+click per spiegare meglio
if persona is None:
global_health_status[x][y] = StatoCella.vuoto
elif persona.isAlive is False:
global_health_status[x][y] = StatoCella.morto
elif persona.isImmune is True:
global_health_status[x][y] = StatoCella.guarito
elif persona.virus is not None:
global_health_status[x][y] = StatoCella.infetto
else:
global_health_status[x][y] = StatoCella.suscettibile
cmap = matplotlib.colors.ListedColormap([(0, 0, 0), (1, 0, 0),
(1, 1, 0), (0, 0, 1),
(0, 1, 0)])
img = plt.imshow(global_health_status,
interpolation='nearest',
vmin=0,
vmax=4,
cmap=cmap)
plt.colorbar(img, ticks=[0, 1, 2, 3, 4])
plt.show()
def crea_grafico_2(self):
matplotlib.pyplot.plot(self.points)
matplotlib.pyplot.show()
class VariableSchelModel(Model):
def __init__(self,
density,
width,
height,
satisfaction_ratio=[0.5, 0.5],
group_count=2,
group_pct=[0.5]):
"""
Schelling's Model with a variable number of 'groups'
:param density: density of the total number of agents, expressed as a percentage of size of model
:param width: width of the model
:param height: height of the model
:param group_count: number of 'groups'
:param satisfaction_ratio: satisfaction ratios for the different groups, takes an array of length equivalent
to the number of 'groups'
:param group_pct: group population size as a percentage of total number of agents, takes an array of length one less
than the number of groups. Last group percentage is calculated automatically
"""
if isinstance(density, str):
print('string')
if group_count < 2:
raise ValueError('Group Count cannot be less than 2!')
if group_count - 1 != len(group_pct):
raise ValueError(
'Group Percentages and Group Count mismatch! '
'Group percentages should be 1 less than the number of groups')
if len(satisfaction_ratio) != group_count:
raise ValueError(
'Satisfaction ratio should be a list with length equivalent to the group count'
)
self.num_agents = int(density * (width * height))
self.running = True
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
self.satisfaction_ratio = satisfaction_ratio
self.reached_equilibrium = False
self.agents = []
self.ratio_sum = 0
self.lowest_ratio = 1
self.mean_ratio = 0
id_offset = 0
for group in range(group_count):
if group == group_count - 1:
# Final count (Percentage isn't explicit, have to calculate,i.e., group_pct length doesnt match ratio)
pct = 1 - sum(group_pct)
for i in range(id_offset,
id_offset + int(pct * self.num_agents)):
a = SchelAgent(i, self, group)
self.agents.append(a)
id_offset += 1
self.schedule.add(a)
self.grid.place_agent(a, self.grid.find_empty())
else:
for i in range(
id_offset,
id_offset + int(group_pct[group] * self.num_agents)):
a = SchelAgent(i, self, group)
self.agents.append(a)
id_offset += 1
self.schedule.add(a)
self.grid.place_agent(a, self.grid.find_empty())
self.data_collector = DataCollector(model_reporters={
"mean ratio value":
lambda model: model.mean_ratio,
"lowest ratio value":
lambda model: model.lowest_ratio
},
agent_reporters={
"coordinates":
lambda agent: agent.pos,
"group":
lambda agent: agent.group,
"satisfaction":
lambda agent: agent.satisfied,
"agent ratio value":
lambda agent: agent.ratio
})
self.data_collector.collect(self)
def step(self):
self.ratio_sum = 0
self.lowest_ratio = 1.0
self.reached_equilibrium = True
self.schedule.step()
self.mean_ratio = self.ratio_sum / self.num_agents
self.data_collector.collect(self)
# Debugging for server charts
# print(self.mean_ratio)
# print(self.lowest_ratio)
# Stops model if model reached equilibrium
self.running = False if self.reached_equilibrium is True else True
class CHModel(Model):
"""A model with some number of agents."""
def __init__(self,
width,
height,
random_n=0,
cow_n=0,
plan_n=0,
mc_n=0,
td_n=0,
episode_number=0,
t_mc_n=0,
old_Q_values=None):
self.running = True
#self.num_agents = N
self.grid = SingleGrid(width, height, True)
self.schedule = RandomActivation(self)
self.id_count = 0 #to assign each agent a unique ID
self.done = False # if all cows are herded, set to true
# To keep score
self.total_cow_count = 0.0
self.current_cow_count = 0.0
self.score = 0.0
self.previous_cow_count = 0.0
# Save model for agent use
self.wallLocations = [(1, 5), (1, 6), (1, 7), (2, 7), (3, 7), (4, 7),
(5, 7), (6, 7), (6, 6), (6, 5)]
self.goalState = [(2, 5), (3, 5), (4, 5), (5, 5), (2, 6), (3, 6),
(4, 6), (5, 6)]
self.back_of_goal = [(2, 6), (3, 6), (4, 6), (5, 6)]
self.front_of_goal = [(2, 5), (3, 5), (4, 5), (5, 5)]
self.goalTarget = (3, 5
) #corral "entrance" that plan agents herd towards
self.state = None # encode state at each timestep
self.number_random_agents = random_n
self.number_cow_agents = cow_n
self.number_plan_agents = plan_n
self.number_monte_carlo_agents = mc_n
self.number_td_agents = td_n
self.number_trained_mc_agents = t_mc_n
# Monte Carlo Agent model save
self.Q_table_sharing = True ## If true, agents share a Q table
self.vision_range = 2 # How far the MC agents can see
if old_Q_values: #load previous Q tables if they exist
self.Q_values = old_Q_values
else:
self.Q_values = [] #no previous Q tables, so make new ones
if (self.Q_table_sharing):
# Just one Q table
self.Q_values.append(
defaultdict(
lambda: np.zeros(len(rl_methods.action_space))))
else:
#every agent gets it's own Q table
for agent in range(self.number_monte_carlo_agents):
self.Q_values.append(
defaultdict(
lambda: np.zeros(len(rl_methods.action_space))))
self.mc_agents = []
self.episode = episode_number
#calculate episilon based on episode
#epsilon = 1 / i_episode
####### tweak episilon to get better results #######
self.epsilon = 1.0 / ((episode_number / 800) + 1)
#self.epsilon = 1.0/((episode_number/8000)+1)
# Place wall agents
for i in range(len(self.wallLocations)):
a = WallAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(a)
#print("placing ", a, " at ", self.corralLocations[i])
self.grid.place_agent(a, self.wallLocations[i])
# Place random agents
for i in range(self.number_random_agents):
a = RandomAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(a)
cell_location = self.grid.find_empty()
self.grid.place_agent(a, cell_location)
# Place cow agents
for i in range(self.number_cow_agents):
c = CowAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(c)
#self.cow_agent_list.append(c) #make a list of cows
cell_location = self.grid.find_empty()
self.grid.place_agent(c, cell_location)
# Place plan agents
for i in range(self.number_plan_agents):
p = PlanAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(p)
cell_location = self.grid.find_empty()
self.grid.place_agent(p, cell_location)
# Place monte carlo agents
for i in range(self.number_monte_carlo_agents):
Q_table_to_use = None
if (
self.Q_table_sharing
): # If sharing Q tables, everyone gets a copy of the same Q table
Q_table_to_use = self.Q_values[0]
else:
Q_table_to_use = self.Q_values[
i] # If not sharing, everyone gets a different Q table
m = MonteCarloAgent(self.id_count,
self,
Q_table_to_use,
self.epsilon,
vision=self.vision_range
) # init MC agents with previous Q tables
self.mc_agents.append(m) # save MC agents to retrieve Q values
self.id_count += 1
self.schedule.add(m)
cell_location = self.grid.find_empty()
self.grid.place_agent(m, cell_location)
# Place trained monte carlo agents
# open/load trained Q table
if (self.number_trained_mc_agents > 0):
loaded_Q = None
with open('mc_q_save.pkl', 'rb') as file:
loaded_Q = dill.load(file)
if loaded_Q:
for i in range(self.number_trained_mc_agents):
tm = TrainedMonteCarloAgent(self.id_count,
self,
loaded_Q,
vision=self.vision_range)
self.id_count += 1
self.schedule.add(tm)
cell_location = self.grid.find_empty()
self.grid.place_agent(tm, cell_location)
else:
print("Can't load Q table for trained MC Agents")
# Place TD agents
for i in range(self.number_td_agents):
t = TDAgent(self.id_count, self)
self.id_count += 1
self.schedule.add(t)
cell_location = self.grid.find_empty()
self.grid.place_agent(t, cell_location)
def step(self):
self.state = rl_methods.encode_state(self.grid)
self.schedule.step()
self.update_score()
# Update rewards of Monte Carlo agents
reward = 0.0
cows_in_goal = cow_methods.cows_in_goal(self, self.goalState)
# if a cow goes into the goal, give a huge bonus and stop the cow
if (cows_in_goal > self.previous_cow_count):
print("New cow in the goal:", cows_in_goal)
reward = 50
else:
reward = -1.0
print("cows in goal: ", cows_in_goal, ", previous_cow_count: ",
self.previous_cow_count, " reward: ", reward)
for mcagent in self.mc_agents:
mcagent.update_rewards(reward)
if (cows_in_goal == self.number_cow_agents):
print(
"all cows herded in model!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
self.done = True
def update_score(self):
self.previous_cow_count = self.current_cow_count
self.current_cow_count = cow_methods.cows_in_goal(self, self.goalState)
self.total_cow_count += self.current_cow_count
print(self.total_cow_count, self.current_cow_count, self.schedule.time,
" Episode: ", self.episode)
self.score = self.total_cow_count / self.schedule.time
def get_new_Q_values(self):
""" Update model Q values at the end of the episode, called by run after each episode """
new_Q = []
if (self.Q_table_sharing): #If all agents are sharing Q table data
updated_Q = None
for agent in self.mc_agents:
# Update the Q table then pass it on to the next agent on the team to update
updated_Q = agent.Q_table_update(shared_Q_table=updated_Q)
new_Q.append(copy.deepcopy(updated_Q))
else:
# If all agents have their own Q tables, update and save for next episode
for agent in self.mc_agents:
updated_Q = agent.Q_table_update()
new_Q.append(copy.deepcopy(updated_Q))
return new_Q
