def handle(self, args): """ Uses built in workloads and the TracesWriter to generate a trace file. """ # Disable logging during trace generation logger = logging.getLogger('cloudscope.simulation') logger.disabled = True # Update settings arguments settings.simulation.conflict_prob = args.conflict settings.simulation.access_mean = args.access_mean settings.simulation.access_stddev = args.access_stddev # Simulation arguments kwargs = { 'users': args.users, 'objects': args.objects, 'max_sim_time': args.timesteps, 'trace': None, } # Create simulation simulation = ConsistencySimulation.load(args.data[0], **kwargs) simulation.trace = None # Create or select the correct simulation if args.best_case or args.tiered: workload = BestCaseAllocation( simulation, args.objects, selection='random', ) workload.allocate_many(args.users) elif args.ping_pong: factory = CharacterSequence(upper=True) objects = [factory.next() for _ in range(args.objects)] workload = PingPongWorkload( simulation, simulation.replicas[:args.users], objects=objects, ) else: simulation.script() workload = simulation.workload # Create the traces writer and write the traces to disk writer = TracesWriter(workload, args.timesteps) counts = writer.write(args.output) return ( "traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n" "object space contains {objects:,} object names:\n" " {mean_objects_per_device:,} average objects per device | " "{mean_devices_per_object:,} average devices per object\n" " {mean_accesses_per_device:,} average accesses per device | " "{mean_accesses_per_object:,} average accesses per object\n" "wrote the trace file to {0}").format(args.output.name, **counts)
def handle(self, args): """ Uses built in workloads and the TracesWriter to generate a trace file. """ # Disable logging during trace generation logger = logging.getLogger('cloudscope.simulation') logger.disabled = True # Update settings arguments settings.simulation.conflict_prob = args.conflict settings.simulation.access_mean = args.access_mean settings.simulation.access_stddev = args.access_stddev # Simulation arguments kwargs = { 'users': args.users, 'objects': args.objects, 'max_sim_time': args.timesteps, 'trace': None, } # Create simulation simulation = ConsistencySimulation.load(args.data[0], **kwargs) simulation.trace = None # Create or select the correct simulation if args.best_case or args.tiered: workload = BestCaseAllocation( simulation, args.objects, selection='random', ) workload.allocate_many(args.users) elif args.ping_pong: factory = CharacterSequence(upper=True) objects = [factory.next() for _ in range(args.objects)] workload = PingPongWorkload( simulation, simulation.replicas[:args.users], objects=objects, ) else: simulation.script() workload = simulation.workload # Create the traces writer and write the traces to disk writer = TracesWriter(workload, args.timesteps) counts = writer.write(args.output) return ( "traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n" "object space contains {objects:,} object names:\n" " {mean_objects_per_device:,} average objects per device | " "{mean_devices_per_object:,} average devices per object\n" " {mean_accesses_per_device:,} average accesses per device | " "{mean_accesses_per_object:,} average accesses per object\n" "wrote the trace file to {0}" ).format(args.output.name, **counts)
def test_character_sequence(self): """ Ensure that an "infinite" character sequence works as expected """ letters = 'abcdefghijklmnopqrstuvwxyz' sequence = CharacterSequence() self.assertEqual(sequence.next(), 'a') for idx in xrange(1, 1000): val = sequence.next() self.assertEqual(len(val), int(math.log(idx, 26)) + 1) self.assertEqual(val[-1], letters[idx % 26])
def test_character_sequence_upper(self): """ Ensure that uppercase in the character sequence works as expected """ letters = 'ABCDEFGHIJKLMNOPQRSTUVWYZ' sequence = CharacterSequence(upper=True) self.assertEqual(sequence.next(), 'A') for idx in xrange(1, 1000): val = sequence.next() self.assertEqual(len(val), int(math.log(idx, 26)) + 1) self.assertEqual(val[-1], letters[idx % 26])
class BestCaseAllocation(TopologyWorkloadAllocation): """ Allocates each device it's own object space by maintaining a static reference to an object factory, so that no matter what, replica servers get their own object space defined. """ object_factory = CharacterSequence(upper=True) def __init__(self, sim, n_objects=None, **defaults): """ Initialize the best case allocation with the number of objects per replica server such that no device will get the same object space. Allocate can then be called at will with no further parameters. """ super(BestCaseAllocation, self).__init__(sim, **defaults) self.n_objects = n_objects or settings.simulation.max_objects_accessed def allocate(self, **kwargs): """ Allocates the next device with the next object space. """ objects = [self.object_factory.next() for _ in range(self.n_objects)] current = Discrete(objects).get() # Allocate the workload super(BestCaseAllocation, self).allocate(objects, current, **kwargs)
def test_log_index(self): """ Test finding the index of a version in a log """ versions = CharacterSequence(upper=True) log = WriteLog() for term in xrange(5): for _ in xrange(10): log.append(versions.next(), term) versions.reset() for idx in xrange(1, len(log)): version = versions.next() self.assertEqual(log.index(version), idx) self.assertEqual(log[log.index(version)].version, version)
def test_log_remove(self): """ Test the remove of an item from a log """ log = WriteLog() versions = CharacterSequence(upper=True) for term in xrange(5): for _ in xrange(10): log.append(versions.next(), term) loglen = len(log) versions.reset() for idx in xrange(1, loglen): version = versions.next() self.assertEqual(version, log.remove(version), "log must return the removed version") self.assertEqual(len(log), loglen-idx, "log must decrease in size") self.assertNotIn(version, log, "log must not contain version")
def test_reset_character_sequence(self): """ Ensure that a sequence can be reset """ sequence = CharacterSequence() for idx in xrange(27): sequence.next() self.assertEqual(sequence.value, "aa") sequence.reset() self.assertEqual(sequence.next(), "a")
class ObjectFactory(object): """ Creates a new object class with versioning. """ def __init__(self, namespace=None): self.namespace = namespace or Namespace() self.counter = CharacterSequence(upper=True) def __call__(self): return self.namespace(self.counter.next()) def reset(self): """ Resets the namespace """ self.namespace.reset()
def test_log_remove(self): """ Test the remove of an item from a log """ log = WriteLog() versions = CharacterSequence(upper=True) for term in xrange(5): for _ in xrange(10): log.append(versions.next(), term) loglen = len(log) versions.reset() for idx in xrange(1, loglen): version = versions.next() self.assertEqual(version, log.remove(version), "log must return the removed version") self.assertEqual(len(log), loglen - idx, "log must decrease in size") self.assertNotIn(version, log, "log must not contain version")
class ConflictWorkloadAllocation(TopologyWorkloadAllocation): """ A specialized workload allocation that will be the default in simulations when not using a trace file or other specialized hook. This allocation does not allow users to "move" or "switch" devices as in the original mobile workloads. Instead, each user is assigned to a location using a specific strategy and generates routine workload accesses. Conflict is defined by a likelihood and specifies how objects are assigned to users for routine accesses. A conflict likelihood of 1 means the exact same objects are assigned to all users. A conflict of zero means that no objects will overlap for any of the users. """ workload_class = RoutineWorkload object_factory = CharacterSequence(upper=True) def __init__(self, sim, n_objects=None, conflict_prob=None, loc_max_users=None, **defaults): """ Initialize the conflict workload allocation with the following params: - n_objects: number of objects per user (constant or range) - conflict_prob: the likelihood of assigning an object to multiple replicas - loc_max_users: the maximum users per location during allocation Workloads are allocated to each location in a round robin fashion up to the maximum number of users or if the location maximum limits are reached (or no devices remain to allocate to). """ # Initialize the topology workload super(ConflictWorkloadAllocation, self).__init__(sim, **defaults) # Initialize parameters or get from settings self.n_objects = n_objects self.loc_max_users = loc_max_users self.do_conflict = Bernoulli(conflict_prob or settings.simulation.conflict_prob) # Reorganize the devices into locations tracking the location index # as well as how many users are assigned to each location via a map. self.locidx = 0 self.locations = {device.location: 0 for device in self.devices} self.devices = { location: [device for device in self.devices if device.location == location] for location in self.locations.keys() } @setter def n_objects(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_objects_accessed if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the number of objects as a range: (min, max)") return Uniform(*value) else: raise ImproperlyConfigured( "Specify the number of objects as a constant " "or uniform random range.") @setter def loc_max_users(self, value): """ Creates a uniform probability distribution for the given value. If the value is an integer, then it will return that value constantly. If it is a range, it will return the uniform distribution. If the value is None, it will look the value up in the configuration. """ value = value or settings.simulation.max_users_location if value is None: return None if isinstance(value, int): return Uniform(value, value) if isinstance(value, (tuple, list)): if len(value) != 2: raise ImproperlyConfigured( "Specify the max users per location as a range: (min, max)" ) return Uniform(*value) else: raise ImproperlyConfigured( "Specify the maximum number of users per location as a " "constant or uniform random range (or None).") def select(self, attempts=0): """ Make a device selection by assigning the users in a round robin """ # Get the current location and update the location index. location = self.locations.keys()[self.locidx] # Update the location index to go around the back end self.locidx += 1 if self.locidx >= len(self.locations.keys()): self.locidx = 0 # Test to see if we have any locations left if not self.devices[location]: if attempts > len(self.locations.keys()): raise WorkloadException( "Cannot select device for allocation, no devices left!") return self.select(attempts + 1) # Test to see if we have reached the location limit if self.loc_max_users is not None: # TODO: Change this so that the users is randomly allocated in # advance rather than on the fly with different selections per # area (e.g. fix the random allocations per location). if self.locations[location] >= self.loc_max_users.get(): if attempts > len(self.locations.keys()): raise WorkloadException("Cannot allocate any more users, " "max users per location reached!") return self.select(attempts + 1) # We will definitely make a selection for this location below here # so increment the location selection count to limit allocation. self.locations[location] += 1 # Round robin device selection from the location if self.selection == ROUNDS_SELECT: return self.devices[location].pop() # Random device selection from the location if self.selection == RANDOM_SELECT: device = Discrete(self.devices[location]).get() self.devices[location].remove(device) return device # How did we end up here?! raise WorkloadException("Unable to select a device for allocation!") def allocate(self, objects=None, current=None, **kwargs): """ Allocate is overriden here to provide a warning to folks who call it directly -- it will simply call super (allocating the next device with the specified object space) and it WILL NOT maintain the conflict object distribution. Instead, it is preferable to call allocate_many with the number of users that you wish to allocate, and in fact - to only do it once! """ warnings.warn( WorkloadWarning( "Conflict space is not allocated correctly! " "This function will allocate a device from the topology with the " "specified objects but will not maintain the conflict likelihood." " Use allocate_many to correctly allocate using this class.")) super(ConflictWorkloadAllocation, self).allocate(objects, current, **kwargs) def allocate_many(self, n_users, **kwargs): """ This is the correct entry point for allocating users with different conflict probability per object across the simulation. It assigns an object space to n_users by allocating every object from the object factory to users in a round robin fashion with the given conflict probabilty. """ # Define the maximum number of objects per user. max_user_objects = { idx: self.n_objects.get() for idx in range(n_users) } # Define each user's object space object_space = [[] for _ in range(n_users)] # Start allocating objects to the users in a round robin fashion. for _ in range(sum(max_user_objects.values())): # Get the next object as a candidate for assignment obj = self.object_factory.next() assigned = False # Go through each user and determine if we should assign. for idx, space in enumerate(object_space): # If this space is already full, carry on. if len(space) >= max_user_objects[idx]: continue # If the object is not assigned, or on conflict probabilty, # Then assign the object to that particular object space. if not assigned or self.do_conflict.get(): space.append(obj) assigned = True # If we've gotten to the end without assignment, we're done! if not assigned: break # Now go through and allocate all the workloads for objects in object_space: device = self.select() current = Discrete(objects).get() extra = self.defaults.copy() extra.update(kwargs) self.workloads.append( self.workload_class(self.sim, device=device, objects=objects, current=current, **extra))
def __init__(self, namespace=None): self.namespace = namespace or Namespace() self.counter = CharacterSequence(upper=True)