def __init__(self, sim, connections, **kwargs): """ Initialize the workload with the simulation (containing both the environment and the topology for work), the set of connections to cause outages for as a group, and any additional arguments. """ self.sim = sim self.connections = connections self.do_outage = Bernoulli( kwargs.pop('outage_prob', settings.simulation.outage_prob)) # NOTE: This will not call any methods on the connections (on purpose) self._state = ONLINE # Distribution of outage duration self.outage_duration = BoundedNormal( kwargs.pop('outage_mean', settings.simulation.outage_mean), kwargs.pop('outage_stddev', settings.simulation.outage_stddev), floor=10.0, ) # Distribution of online duration self.online_duration = BoundedNormal( kwargs.pop('online_mean', settings.simulation.online_mean), kwargs.pop('online_stddev', settings.simulation.online_stddev), floor=10.0, ) # Initialize the Process super(OutageGenerator, self).__init__(sim.env)
def __init__(self, sim, **kwargs): # Distributions to change locations and devices self.do_move = Bernoulli(kwargs.get('move_prob', settings.simulation.move_prob)) self.do_switch = Bernoulli(kwargs.get('switch_prob', settings.simulation.switch_prob)) # Initialize the Process super(MobileWorkload, self).__init__(sim, **kwargs)
class FederatedEventualReplica(EventualReplica): """ Implements eventual consistency while allowing for integration with strongly consistent replicas. """ def __init__(self, simulation, **kwargs): super(FederatedEventualReplica, self).__init__(simulation, **kwargs) # Federated settings self.do_sync = Bernoulli(kwargs.get('sync_prob', SYNC_PROB)) self.do_local = Bernoulli(kwargs.get('local_prob', LOCAL_PROB)) def select_anti_entropy_neighbor(self): """ Selects a neighbor to perform anti-entropy with, prioritizes local neighbors over remote ones and also actively selects the sequential consistency nodes to perform anti-entropy with. """ # Decide if we should sync with the core consensus group if self.do_sync.get(): # Find a strong consensus node that is local neighbors = self.neighbors( consistency=Consistency.STRONG, location=self.location ) # If we have local nodes, choose one of them if neighbors: return random.choice(neighbors) # Otherwise choose any strong node that exists neighbors = self.neighbors(consistency=Consistency.STRONG) if neighbors: return random.choice(neighbors) # Decide if we should do anti-entropy locally or across the wide area. if self.do_local.get(): # Find all local nodes with the same consistency. neighbors = self.neighbors( consistency=[Consistency.EVENTUAL, Consistency.STENTOR], location=self.location ) # If we have local nodes, choose one of them if neighbors: return random.choice(neighbors) return random.choice(self.neighbors()) # At this point return a wide area node that doesn't have strong consistency neighbors = self.neighbors( consistency=Consistency.STRONG, location=self.location, exclude=True ) # If we have wide area nodes, choose one of them if neighbors: return random.choice(neighbors) # Last resort, simply choose any neighbor we possibly can! return random.choice(self.neighbors())
class FederatedEventualReplica(EventualReplica): """ Implements eventual consistency while allowing for integration with strongly consistent replicas. """ def __init__(self, simulation, **kwargs): super(FederatedEventualReplica, self).__init__(simulation, **kwargs) # Federated settings self.do_sync = Bernoulli(kwargs.get('sync_prob', SYNC_PROB)) self.do_local = Bernoulli(kwargs.get('local_prob', LOCAL_PROB)) def select_anti_entropy_neighbor(self): """ Selects a neighbor to perform anti-entropy with, prioritizes local neighbors over remote ones and also actively selects the sequential consistency nodes to perform anti-entropy with. """ # Decide if we should sync with the core consensus group if self.do_sync.get(): # Find a strong consensus node that is local neighbors = self.neighbors(consistency=Consistency.STRONG, location=self.location) # If we have local nodes, choose one of them if neighbors: return random.choice(neighbors) # Otherwise choose any strong node that exists neighbors = self.neighbors(consistency=Consistency.STRONG) if neighbors: return random.choice(neighbors) # Decide if we should do anti-entropy locally or across the wide area. if self.do_local.get(): # Find all local nodes with the same consistency. neighbors = self.neighbors( consistency=[Consistency.EVENTUAL, Consistency.STENTOR], location=self.location) # If we have local nodes, choose one of them if neighbors: return random.choice(neighbors) return random.choice(self.neighbors()) # At this point return a wide area node that doesn't have strong consistency neighbors = self.neighbors(consistency=Consistency.STRONG, location=self.location, exclude=True) # If we have wide area nodes, choose one of them if neighbors: return random.choice(neighbors) # Last resort, simply choose any neighbor we possibly can! return random.choice(self.neighbors())
def __init__(self, sim, devices, **kwargs): """ Instead of specifying a single device for the workload, specify multiple devices that we ping pong betweeen with accesses. """ if len(devices) < 2: raise WorkloadException( "Ping Pong requires at least two devices to play") self.players = devices self.do_move = Bernoulli( kwargs.get('move_prob', settings.simulation.move_prob)) kwargs['device'] = Discrete(devices).get() super(PingPongWorkload, self).__init__(sim, **kwargs)
def __init__(self, sim, n_objects=None, conflict_prob=None, loc_max_users=None, **defaults): """ Initialize the conflict workload allocation with the following params: - n_objects: number of objects per user (constant or range) - conflict_prob: the likelihood of assigning an object to multiple replicas - loc_max_users: the maximum users per location during allocation Workloads are allocated to each location in a round robin fashion up to the maximum number of users or if the location maximum limits are reached (or no devices remain to allocate to). """ # Initialize the topology workload super(ConflictWorkloadAllocation, self).__init__(sim, **defaults) # Initialize parameters or get from settings self.n_objects = n_objects self.loc_max_users = loc_max_users self.do_conflict = Bernoulli(conflict_prob or settings.simulation.conflict_prob) # Reorganize the devices into locations tracking the location index # as well as how many users are assigned to each location via a map. self.locidx = 0 self.locations = {device.location: 0 for device in self.devices} self.devices = { location: [ device for device in self.devices if device.location == location ] for location in self.locations.keys() }
def __init__(self, sim, connections, **kwargs): """ Initialize the workload with the simulation (containing both the environment and the topology for work), the set of connections to cause outages for as a group, and any additional arguments. """ self.sim = sim self.connections = connections self.do_outage = Bernoulli(kwargs.pop('outage_prob', settings.simulation.outage_prob)) # NOTE: This will not call any methods on the connections (on purpose) self._state = ONLINE # Distribution of outage duration self.outage_duration = BoundedNormal( kwargs.pop('outage_mean', settings.simulation.outage_mean), kwargs.pop('outage_stddev', settings.simulation.outage_stddev), floor = 10.0, ) # Distribution of online duration self.online_duration = BoundedNormal( kwargs.pop('online_mean', settings.simulation.online_mean), kwargs.pop('online_stddev', settings.simulation.online_stddev), floor = 10.0, ) # Initialize the Process super(OutageGenerator, self).__init__(sim.env)
class PingPongWorkload(RoutineWorkload): """ The ping pong workload shifts a single user between a set of devices such that multiple devices access the same object space, but that they do not occur at the same time. This is similar to the mobile workload but with a more routine bouncing between the specified replicas. """ def __init__(self, sim, devices, **kwargs): """ Instead of specifying a single device for the workload, specify multiple devices that we ping pong betweeen with accesses. """ if len(devices) < 2: raise WorkloadException( "Ping Pong requires at least two devices to play" ) self.players = devices self.do_move = Bernoulli(kwargs.get('move_prob', settings.simulation.move_prob)) kwargs['device'] = Discrete(devices).get() super(PingPongWorkload, self).__init__(sim, **kwargs) def move(self): """ Moves the workload to a new device """ self.device = Discrete([ player for player in self.players if player != self.device ]).get() def update(self, **kwargs): """ Updates the device with the possibility of switching devices. """ # Update the current object by calling super. super(PingPongWorkload, self).update(**kwargs) if self.do_move.get() or self.device is None: # Execute the move self.move() # Log the move self.sim.logger.info( "{} has moved to {} on {}.".format( self.name, self.location, self.device ) )
def __init__(self, sim, **kwargs): """ Initialize workload probabilities and distributions before passing all optional keyword arguments to the super class. """ # Distribution for whether or not to change objects self.do_object = Bernoulli(kwargs.pop("object_prob", settings.simulation.object_prob)) self.do_read = Bernoulli(kwargs.pop("read_prob", settings.simulation.read_prob)) # Interval distribution for the wait (in ms) to the next access. self.next_access = BoundedNormal( kwargs.pop("access_mean", settings.simulation.access_mean), kwargs.pop("access_stddev", settings.simulation.access_stddev), floor=1.0, ) # Initialize the Workload super(RoutineWorkload, self).__init__(sim, **kwargs) # If current is None, update the state of the workload: if self.current is None: self.update()
def __init__(self, sim, n_objects=None, conflict_prob=None, loc_max_users=None, **defaults): """ Initialize the conflict workload allocation with the following params: - n_objects: number of objects per user (constant or range) - conflict_prob: the likelihood of assigning an object to multiple replicas - loc_max_users: the maximum users per location during allocation Workloads are allocated to each location in a round robin fashion up to the maximum number of users or if the location maximum limits are reached (or no devices remain to allocate to). """ # Initialize the topology workload super(ConflictWorkloadAllocation, self).__init__(sim, **defaults) # Initialize parameters or get from settings self.n_objects = n_objects self.loc_max_users = loc_max_users self.do_conflict = Bernoulli(conflict_prob or settings.simulation.conflict_prob) # Reorganize the devices into locations tracking the location index # as well as how many users are assigned to each location via a map. self.locidx = 0 self.locations = {device.location: 0 for device in self.devices} self.devices = { location: [device for device in self.devices if device.location == location] for location in self.locations.keys() }
def __init__(self, sim, devices, **kwargs): """ Instead of specifying a single device for the workload, specify multiple devices that we ping pong betweeen with accesses. """ if len(devices) < 2: raise WorkloadException( "Ping Pong requires at least two devices to play" ) self.players = devices self.do_move = Bernoulli(kwargs.get('move_prob', settings.simulation.move_prob)) kwargs['device'] = Discrete(devices).get() super(PingPongWorkload, self).__init__(sim, **kwargs)
def __init__(self, sim, **kwargs): """ Initialize workload probabilities and distributions before passing all optional keyword arguments to the super class. """ # Distribution for whether or not to change objects self.do_object = Bernoulli( kwargs.pop('object_prob', settings.simulation.object_prob)) self.do_read = Bernoulli( kwargs.pop('read_prob', settings.simulation.read_prob)) # Interval distribution for the wait (in ms) to the next access. self.next_access = BoundedNormal( kwargs.pop('access_mean', settings.simulation.access_mean), kwargs.pop('access_stddev', settings.simulation.access_stddev), floor=1.0, ) # Initialize the Workload super(RoutineWorkload, self).__init__(sim, **kwargs) # If current is None, update the state of the workload: if self.current is None: self.update()
class PingPongWorkload(RoutineWorkload): """ The ping pong workload shifts a single user between a set of devices such that multiple devices access the same object space, but that they do not occur at the same time. This is similar to the mobile workload but with a more routine bouncing between the specified replicas. """ def __init__(self, sim, devices, **kwargs): """ Instead of specifying a single device for the workload, specify multiple devices that we ping pong betweeen with accesses. """ if len(devices) < 2: raise WorkloadException( "Ping Pong requires at least two devices to play") self.players = devices self.do_move = Bernoulli( kwargs.get('move_prob', settings.simulation.move_prob)) kwargs['device'] = Discrete(devices).get() super(PingPongWorkload, self).__init__(sim, **kwargs) def move(self): """ Moves the workload to a new device """ self.device = Discrete([ player for player in self.players if player != self.device ]).get() def update(self, **kwargs): """ Updates the device with the possibility of switching devices. """ # Update the current object by calling super. super(PingPongWorkload, self).update(**kwargs) if self.do_move.get() or self.device is None: # Execute the move self.move() # Log the move self.sim.logger.info("{} has moved to {} on {}.".format( self.name, self.location, self.device))
class ConflictWorkloadAllocation(TopologyWorkloadAllocation): """ A specialized workload allocation that will be the default in simulations when not using a trace file or other specialized hook. This allocation does not allow users to "move" or "switch" devices as in the original mobile workloads. Instead, each user is assigned to a location using a specific strategy and generates routine workload accesses. Conflict is defined by a likelihood and specifies how objects are assigned to users for routine accesses. A conflict likelihood of 1 means the exact same objects are assigned to all users. A conflict of zero means that no objects will overlap for any of the users. """ workload_class = RoutineWorkload object_factory = CharacterSequence(upper=True) def __init__(self, sim, n_objects=None, conflict_prob=None, loc_max_users=None, **defaults): """ Initialize the conflict workload allocation with the following params: - n_objects: number of objects per user (constant or range) - conflict_prob: the likelihood of assigning an object to multiple replicas - loc_max_users: the maximum users per location during allocation Workloads are allocated to each location in a round robin fashion up to the maximum number of users or if the location maximum limits are reached (or no devices remain to allocate to). """ # Initialize the topology workload super(ConflictWorkloadAllocation, self).__init__(sim, **defaults) # Initialize parameters or get from settings self.n_objects = n_objects self.loc_max_users = loc_max_users self.do_conflict = Bernoulli(conflict_prob or settings.simulation.conflict_prob) # Reorganize the devices into locations tracking the location index # as well as how many users are assigned to each location via a map. self.locidx = 0 self.locations = {device.location: 0 for device in self.devices} self.devices = { location: [ device for device in self.devices if device.location == location ] for location in self.locations.keys() } @setter def n_objects(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_objects_accessed if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the number of objects as a range: (min, max)" ) return Uniform(*value) else: raise ImproperlyConfigured( "Specify the number of objects as a constant " "or uniform random range." ) @setter def loc_max_users(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_users_location if value is None: return None if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the max users per location as a range: (min, max)" ) return Uniform(*value) else: raise ImproperlyConfigured( "Specify the maximum number of users per location as a " "constant or uniform random range (or None)." ) def select(self, attempts=0): """ Make a device selection by assigning the users in a round robin """ # Get the current location and update the location index. location = self.locations.keys()[self.locidx] # Update the location index to go around the back end self.locidx += 1 if self.locidx >= len(self.locations.keys()): self.locidx = 0 # Test to see if we have any locations left if not self.devices[location]: if attempts > len(self.locations.keys()): raise WorkloadException( "Cannot select device for allocation, no devices left!" ) return self.select(attempts + 1) # Test to see if we have reached the location limit if self.loc_max_users is not None: # TODO: Change this so that the users is randomly allocated in # advance rather than on the fly with different selections per # area (e.g. fix the random allocations per location). if self.locations[location] >= self.loc_max_users.get(): if attempts > len(self.locations.keys()): raise WorkloadException( "Cannot allocate any more users, " "max users per location reached!" ) return self.select(attempts + 1) # We will definitely make a selection for this location below here # so increment the location selection count to limit allocation. self.locations[location] += 1 # Round robin device selection from the location if self.selection == ROUNDS_SELECT: return self.devices[location].pop() # Random device selection from the location if self.selection == RANDOM_SELECT: device = Discrete(self.devices[location]).get() self.devices[location].remove(device) return device # How did we end up here?! raise WorkloadException( "Unable to select a device for allocation!" ) def allocate(self, objects=None, current=None, **kwargs): """ Allocate is overriden here to provide a warning to folks who call it directly -- it will simply call super (allocating the next device with the specified object space) and it WILL NOT maintain the conflict object distribution. Instead, it is preferable to call allocate_many with the number of users that you wish to allocate, and in fact - to only do it once! """ warnings.warn(WorkloadWarning( "Conflict space is not allocated correctly! " "This function will allocate a device from the topology with the " "specified objects but will not maintain the conflict likelihood." " Use allocate_many to correctly allocate using this class." )) super(ConflictWorkloadAllocation, self).allocate(objects, current, **kwargs) def allocate_many(self, n_users, **kwargs): """ This is the correct entry point for allocating users with different conflict probability per object across the simulation. It assigns an object space to n_users by allocating every object from the object factory to users in a round robin fashion with the given conflict probabilty. """ # Define the maximum number of objects per user. max_user_objects = { idx: self.n_objects.get() for idx in range(n_users) } # Define each user's object space object_space = [[] for _ in range(n_users)] # Start allocating objects to the users in a round robin fashion. for _ in range(sum(max_user_objects.values())): # Get the next object as a candidate for assignment obj = self.object_factory.next() assigned = False # Go through each user and determine if we should assign. for idx, space in enumerate(object_space): # If this space is already full, carry on. if len(space) >= max_user_objects[idx]: continue # If the object is not assigned, or on conflict probabilty, # Then assign the object to that particular object space. if not assigned or self.do_conflict.get(): space.append(obj) assigned = True # If we've gotten to the end without assignment, we're done! if not assigned: break # Now go through and allocate all the workloads for objects in object_space: device = self.select() current = Discrete(objects).get() extra = self.defaults.copy() extra.update(kwargs) self.workloads.append( self.workload_class( self.sim, device=device, objects=objects, current=current, **extra ) )
def __init__(self, simulation, **kwargs): super(FederatedEventualReplica, self).__init__(simulation, **kwargs) # Federated settings self.do_sync = Bernoulli(kwargs.get('sync_prob', SYNC_PROB)) self.do_local = Bernoulli(kwargs.get('local_prob', LOCAL_PROB))
class OutageGenerator(NamedProcess): """ A process that causes outages to occur across the wide or local area or across both areas, or to occur for leaders only. Outages are generated on a collection of connections, usually collected together based on the connection type. The outages script is run as follows in the simulation: - determine the probability of online vs. outage - use the normal distribution of online/outage to determine duration - cause outage if outage, wait until duration is over. - repeat from the first step. Outages can also do more than cut the connection, they can also vary the latency for that connection by changing the network parameters. """ def __init__(self, sim, connections, **kwargs): """ Initialize the workload with the simulation (containing both the environment and the topology for work), the set of connections to cause outages for as a group, and any additional arguments. """ self.sim = sim self.connections = connections self.do_outage = Bernoulli( kwargs.pop('outage_prob', settings.simulation.outage_prob)) # NOTE: This will not call any methods on the connections (on purpose) self._state = ONLINE # Distribution of outage duration self.outage_duration = BoundedNormal( kwargs.pop('outage_mean', settings.simulation.outage_mean), kwargs.pop('outage_stddev', settings.simulation.outage_stddev), floor=10.0, ) # Distribution of online duration self.online_duration = BoundedNormal( kwargs.pop('online_mean', settings.simulation.online_mean), kwargs.pop('online_stddev', settings.simulation.online_stddev), floor=10.0, ) # Initialize the Process super(OutageGenerator, self).__init__(sim.env) @setter def connections(self, value): """ Allows passing a single connection instance or multiple. """ if not isinstance(value, (tuple, list)): value = (value, ) return tuple(value) @setter def state(self, state): """ When the state is set on the outage generator, update connections. """ if state == ONLINE: self.update_online_state() elif state == OUTAGE: self.update_outage_state() else: raise OutagesException( "Unknown state: '{}' set either {} or {}".format( state, ONLINE, OUTAGE)) return state def update_online_state(self): """ Sets the state of the generator to online. NOTE - should not be called by clients but can be subclassed! """ # If we were previously offline: if self.state == OUTAGE: for conn in self.connections: conn.up() self.sim.logger.debug("{} is now online".format(conn)) def update_outage_state(self): """ Sets the state of the generator to outage. NOTE - should not be called by clients but can be subclassed! """ # If we were previously online: if self.state == ONLINE: for conn in self.connections: conn.down() self.sim.logger.debug("{} is now offline".format(conn)) def duration(self): """ Returns the duration of the current state in milliseconds. """ if self.state == ONLINE: return self.online_duration.get() if self.state == OUTAGE: return self.outage_duration.get() def update(self): """ Updates the state of the connections according to the outage probability. This method should be called routinely according to the outage and online duration distributions. """ if self.do_outage.get(): self.state = OUTAGE else: self.state = ONLINE def run(self): """ The action that generates outages on the passed in set of connections. """ while True: # Get the duration of the current state duration = self.duration() # Log (info) the outage/online state and duration self.sim.logger.info("{} connections {} for {}".format( len(self.connections), self.state, humanizedelta(milliseconds=duration))) # Wait for the duration yield self.env.timeout(duration) # Update the state of the outage self.update() def __str__(self): """ String representation of the outage generator. """ return "{}: {} connections {}".format(self.name, len(self.connections), self.state)
class RoutineWorkload(Workload): """ A routine workload generates accesses according to the following params: - A normal distribution time between accesses - A probability of reads (vs. writes) - A probability of switching objects (vs. continuing with current) This is the simplest workload that is completely implemented. """ def __init__(self, sim, **kwargs): """ Initialize workload probabilities and distributions before passing all optional keyword arguments to the super class. """ # Distribution for whether or not to change objects self.do_object = Bernoulli( kwargs.pop('object_prob', settings.simulation.object_prob)) self.do_read = Bernoulli( kwargs.pop('read_prob', settings.simulation.read_prob)) # Interval distribution for the wait (in ms) to the next access. self.next_access = BoundedNormal( kwargs.pop('access_mean', settings.simulation.access_mean), kwargs.pop('access_stddev', settings.simulation.access_stddev), floor=1.0, ) # Initialize the Workload super(RoutineWorkload, self).__init__(sim, **kwargs) # If current is None, update the state of the workload: if self.current is None: self.update() def update(self, **kwargs): """ Uses the do_object distribution to determine whether or not to change the currently accessed object to a new object. """ # Do we switch the current object? if self.current is None or self.do_object.get(): if len(self.objects) == 1: # There is only one choice, no switching! self.current = self.objects[0] else: # Randomly select an object that is not the current object. self.current = Discrete([ obj for obj in self.objects if obj != self.current ]).get() # Call to the super update method super(RoutineWorkload, self).update(**kwargs) def wait(self): """ Utilizes the bounded normal distribution to return the wait (in milliseconds) until the next access. """ return self.next_access.get() def access(self): """ Utilizes the do_read distribution to determine whether or not to issue a write or a read access, and calls the device method for it. """ # Make sure that there is a device to write to! if not self.device: raise WorkloadException( "No device specified to trigger the access on!") # Make sure that there is a current object to write! if not self.current: raise WorkloadException( "No object specified as currently open on the workload!") # Determine if we are reading or writing. access = READ if self.do_read.get() else WRITE # Log the results on the timeseries for the access. self.sim.results.update( access, (self.device.id, self.location, self.current, self.env.now)) if access == READ: # Read the latest version of the current object return self.device.read(self.current) if access == WRITE: # Write to the current version (e.g. call nextv) return self.device.write(self.current)
class OutageGenerator(NamedProcess): """ A process that causes outages to occur across the wide or local area or across both areas, or to occur for leaders only. Outages are generated on a collection of connections, usually collected together based on the connection type. The outages script is run as follows in the simulation: - determine the probability of online vs. outage - use the normal distribution of online/outage to determine duration - cause outage if outage, wait until duration is over. - repeat from the first step. Outages can also do more than cut the connection, they can also vary the latency for that connection by changing the network parameters. """ def __init__(self, sim, connections, **kwargs): """ Initialize the workload with the simulation (containing both the environment and the topology for work), the set of connections to cause outages for as a group, and any additional arguments. """ self.sim = sim self.connections = connections self.do_outage = Bernoulli(kwargs.pop('outage_prob', settings.simulation.outage_prob)) # NOTE: This will not call any methods on the connections (on purpose) self._state = ONLINE # Distribution of outage duration self.outage_duration = BoundedNormal( kwargs.pop('outage_mean', settings.simulation.outage_mean), kwargs.pop('outage_stddev', settings.simulation.outage_stddev), floor = 10.0, ) # Distribution of online duration self.online_duration = BoundedNormal( kwargs.pop('online_mean', settings.simulation.online_mean), kwargs.pop('online_stddev', settings.simulation.online_stddev), floor = 10.0, ) # Initialize the Process super(OutageGenerator, self).__init__(sim.env) @setter def connections(self, value): """ Allows passing a single connection instance or multiple. """ if not isinstance(value, (tuple, list)): value = (value,) return tuple(value) @setter def state(self, state): """ When the state is set on the outage generator, update connections. """ if state == ONLINE: self.update_online_state() elif state == OUTAGE: self.update_outage_state() else: raise OutagesException( "Unknown state: '{}' set either {} or {}".format( state, ONLINE, OUTAGE ) ) return state def update_online_state(self): """ Sets the state of the generator to online. NOTE - should not be called by clients but can be subclassed! """ # If we were previously offline: if self.state == OUTAGE: for conn in self.connections: conn.up() self.sim.logger.debug( "{} is now online".format(conn) ) def update_outage_state(self): """ Sets the state of the generator to outage. NOTE - should not be called by clients but can be subclassed! """ # If we were previously online: if self.state == ONLINE: for conn in self.connections: conn.down() self.sim.logger.debug( "{} is now offline".format(conn) ) def duration(self): """ Returns the duration of the current state in milliseconds. """ if self.state == ONLINE: return self.online_duration.get() if self.state == OUTAGE: return self.outage_duration.get() def update(self): """ Updates the state of the connections according to the outage probability. This method should be called routinely according to the outage and online duration distributions. """ if self.do_outage.get(): self.state = OUTAGE else: self.state = ONLINE def run(self): """ The action that generates outages on the passed in set of connections. """ while True: # Get the duration of the current state duration = self.duration() # Log (info) the outage/online state and duration self.sim.logger.info( "{} connections {} for {}".format( len(self.connections), self.state, humanizedelta(milliseconds=duration) ) ) # Wait for the duration yield self.env.timeout(duration) # Update the state of the outage self.update() def __str__(self): """ String representation of the outage generator. """ return "{}: {} connections {}".format( self.name, len(self.connections), self.state )
class ConflictWorkloadAllocation(TopologyWorkloadAllocation): """ A specialized workload allocation that will be the default in simulations when not using a trace file or other specialized hook. This allocation does not allow users to "move" or "switch" devices as in the original mobile workloads. Instead, each user is assigned to a location using a specific strategy and generates routine workload accesses. Conflict is defined by a likelihood and specifies how objects are assigned to users for routine accesses. A conflict likelihood of 1 means the exact same objects are assigned to all users. A conflict of zero means that no objects will overlap for any of the users. """ workload_class = RoutineWorkload object_factory = CharacterSequence(upper=True) def __init__(self, sim, n_objects=None, conflict_prob=None, loc_max_users=None, **defaults): """ Initialize the conflict workload allocation with the following params: - n_objects: number of objects per user (constant or range) - conflict_prob: the likelihood of assigning an object to multiple replicas - loc_max_users: the maximum users per location during allocation Workloads are allocated to each location in a round robin fashion up to the maximum number of users or if the location maximum limits are reached (or no devices remain to allocate to). """ # Initialize the topology workload super(ConflictWorkloadAllocation, self).__init__(sim, **defaults) # Initialize parameters or get from settings self.n_objects = n_objects self.loc_max_users = loc_max_users self.do_conflict = Bernoulli(conflict_prob or settings.simulation.conflict_prob) # Reorganize the devices into locations tracking the location index # as well as how many users are assigned to each location via a map. self.locidx = 0 self.locations = {device.location: 0 for device in self.devices} self.devices = { location: [device for device in self.devices if device.location == location] for location in self.locations.keys() } @setter def n_objects(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_objects_accessed if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the number of objects as a range: (min, max)") return Uniform(*value) else: raise ImproperlyConfigured( "Specify the number of objects as a constant " "or uniform random range.") @setter def loc_max_users(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_users_location if value is None: return None if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the max users per location as a range: (min, max)" ) return Uniform(*value) else: raise ImproperlyConfigured( "Specify the maximum number of users per location as a " "constant or uniform random range (or None).") def select(self, attempts=0): """ Make a device selection by assigning the users in a round robin """ # Get the current location and update the location index. location = self.locations.keys()[self.locidx] # Update the location index to go around the back end self.locidx += 1 if self.locidx >= len(self.locations.keys()): self.locidx = 0 # Test to see if we have any locations left if not self.devices[location]: if attempts > len(self.locations.keys()): raise WorkloadException( "Cannot select device for allocation, no devices left!") return self.select(attempts + 1) # Test to see if we have reached the location limit if self.loc_max_users is not None: # TODO: Change this so that the users is randomly allocated in # advance rather than on the fly with different selections per # area (e.g. fix the random allocations per location). if self.locations[location] >= self.loc_max_users.get(): if attempts > len(self.locations.keys()): raise WorkloadException("Cannot allocate any more users, " "max users per location reached!") return self.select(attempts + 1) # We will definitely make a selection for this location below here # so increment the location selection count to limit allocation. self.locations[location] += 1 # Round robin device selection from the location if self.selection == ROUNDS_SELECT: return self.devices[location].pop() # Random device selection from the location if self.selection == RANDOM_SELECT: device = Discrete(self.devices[location]).get() self.devices[location].remove(device) return device # How did we end up here?! raise WorkloadException("Unable to select a device for allocation!") def allocate(self, objects=None, current=None, **kwargs): """ Allocate is overriden here to provide a warning to folks who call it directly -- it will simply call super (allocating the next device with the specified object space) and it WILL NOT maintain the conflict object distribution. Instead, it is preferable to call allocate_many with the number of users that you wish to allocate, and in fact - to only do it once! """ warnings.warn( WorkloadWarning( "Conflict space is not allocated correctly! " "This function will allocate a device from the topology with the " "specified objects but will not maintain the conflict likelihood." " Use allocate_many to correctly allocate using this class.")) super(ConflictWorkloadAllocation, self).allocate(objects, current, **kwargs) def allocate_many(self, n_users, **kwargs): """ This is the correct entry point for allocating users with different conflict probability per object across the simulation. It assigns an object space to n_users by allocating every object from the object factory to users in a round robin fashion with the given conflict probabilty. """ # Define the maximum number of objects per user. max_user_objects = { idx: self.n_objects.get() for idx in range(n_users) } # Define each user's object space object_space = [[] for _ in range(n_users)] # Start allocating objects to the users in a round robin fashion. for _ in range(sum(max_user_objects.values())): # Get the next object as a candidate for assignment obj = self.object_factory.next() assigned = False # Go through each user and determine if we should assign. for idx, space in enumerate(object_space): # If this space is already full, carry on. if len(space) >= max_user_objects[idx]: continue # If the object is not assigned, or on conflict probabilty, # Then assign the object to that particular object space. if not assigned or self.do_conflict.get(): space.append(obj) assigned = True # If we've gotten to the end without assignment, we're done! if not assigned: break # Now go through and allocate all the workloads for objects in object_space: device = self.select() current = Discrete(objects).get() extra = self.defaults.copy() extra.update(kwargs) self.workloads.append( self.workload_class(self.sim, device=device, objects=objects, current=current, **extra))
class RoutineWorkload(Workload): """ A routine workload generates accesses according to the following params: - A normal distribution time between accesses - A probability of reads (vs. writes) - A probability of switching objects (vs. continuing with current) This is the simplest workload that is completely implemented. """ def __init__(self, sim, **kwargs): """ Initialize workload probabilities and distributions before passing all optional keyword arguments to the super class. """ # Distribution for whether or not to change objects self.do_object = Bernoulli(kwargs.pop("object_prob", settings.simulation.object_prob)) self.do_read = Bernoulli(kwargs.pop("read_prob", settings.simulation.read_prob)) # Interval distribution for the wait (in ms) to the next access. self.next_access = BoundedNormal( kwargs.pop("access_mean", settings.simulation.access_mean), kwargs.pop("access_stddev", settings.simulation.access_stddev), floor=1.0, ) # Initialize the Workload super(RoutineWorkload, self).__init__(sim, **kwargs) # If current is None, update the state of the workload: if self.current is None: self.update() def update(self, **kwargs): """ Uses the do_object distribution to determine whether or not to change the currently accessed object to a new object. """ # Do we switch the current object? if self.current is None or self.do_object.get(): if len(self.objects) == 1: # There is only one choice, no switching! self.current = self.objects[0] else: # Randomly select an object that is not the current object. self.current = Discrete([obj for obj in self.objects if obj != self.current]).get() # Call to the super update method super(RoutineWorkload, self).update(**kwargs) def wait(self): """ Utilizes the bounded normal distribution to return the wait (in milliseconds) until the next access. """ return self.next_access.get() def access(self): """ Utilizes the do_read distribution to determine whether or not to issue a write or a read access, and calls the device method for it. """ # Make sure that there is a device to write to! if not self.device: raise WorkloadException("No device specified to trigger the access on!") # Make sure that there is a current object to write! if not self.current: raise WorkloadException("No object specified as currently open on the workload!") # Determine if we are reading or writing. access = READ if self.do_read.get() else WRITE # Log the results on the timeseries for the access. self.sim.results.update(access, (self.device.id, self.location, self.current, self.env.now)) if access == READ: # Read the latest version of the current object return self.device.read(self.current) if access == WRITE: # Write to the current version (e.g. call nextv) return self.device.write(self.current)
class MobileWorkload(RoutineWorkload): """ This workload extends the RoutineWorkload model by allowing the user to switch both locations and devices in a location by specifying two new probabilities: - Probability of moving locations - Probability of switching devices at a location This workload is intended to simulate a user moving between work, home, and mobile devices in a meaningful way. Note that this means that some devices could have "multiple" users generating accesses on them. Note that locations and devices are filtered via the settings. """ # This kills the property from the super class. # TODO: this is a HACK -- remove it! location = None # Specify what locations are valid to move to. invalid_locations = frozenset([ Location.get(loc) for loc in settings.simulation.invalid_locations ]) # Specify what device types are invalid to switch to. invalid_types = frozenset([ Device.get(dev) for dev in settings.simulation.invalid_types ]) def __init__(self, sim, **kwargs): # Distributions to change locations and devices self.do_move = Bernoulli(kwargs.get('move_prob', settings.simulation.move_prob)) self.do_switch = Bernoulli(kwargs.get('switch_prob', settings.simulation.switch_prob)) # Initialize the Process super(MobileWorkload, self).__init__(sim, **kwargs) @memoized def locations(self): """ Gets the unique locations of the replicas. Automatically filters locations that aren't workable or should be ignored. """ # Create a mapping of locations to replica devices locations = defaultdict(list) for replica in self.sim.replicas: # Filter invalid replicas if replica.type in self.invalid_types: continue # Filter invalid locations if replica.location in self.invalid_locations: continue # Associate the location with the replica locations[replica.location].append(replica) # If no locations exist, then raise a workload error if not locations: raise WorkloadException( "No valid locations or replicas associated with workload!" ) return locations def move(self): """ Moves the user to a new location """ if len(self.locations) == 1: # There is only one choice, no switching! self.location = self.locations.keys()[0] self.switch() return False self.location = Discrete([ location for location in self.locations.keys() if location != self.location ]).get() self.switch() return True def switch(self): """ Switches the device the user is currently working on """ if len(self.locations[self.location]) == 1: # There is only one choice, no switching! self.device = self.locations[self.location][0] return False self.device = Discrete([ device for device in self.locations[self.location] if device != self.device ]).get() return True def update(self, **kwargs): """ Updates the device and location to simulate random user movement. """ # Update the current object by calling super. super(MobileWorkload, self).update(**kwargs) if self.do_move.get() or self.location is None: if self.move(): self.sim.logger.info( "{} has moved to {} on {}.".format( self.name, self.location, self.device ) ) return True return False if self.do_switch.get() or self.device is None: if self.switch(): self.sim.logger.debug( "{} has switched devices to {} ({})".format( self.name, self.device, self.location ) ) return True return False return False