def test_unit_sequence(self): """ Ensure an "infinite" sequence works as expected """ sequence = Sequence() for idx in xrange(1, 100000): self.assertEqual(sequence.next(), idx)
def test_step_sequence(self): """ Ensure that a stepped sequence works as expected """ sequence = Sequence(step=10) for idx in xrange(10, 100000, 10): self.assertEqual(sequence.next(), idx)
def new(klass, name): """ Returns a new subclass of the version for a specific object and resets the global counter on the object, for multi-version systems. """ name = name or "foo" # Handle passing None into the new method. return type(name, (klass, ), {"counter": Sequence()})
class SimulationLogger(WrappedLogger): """ This will correctly log messages from the simulation. """ counter = Sequence() logger = logging.getLogger('cloudscope.simulation') def __init__(self, env, **kwargs): self.env = env self._user = kwargs.pop('user', None) super(SimulationLogger, self).__init__(**kwargs) @property def user(self): if not self._user: self._user = getpass.getuser() return self._user def log(self, level, message, *args, **kwargs): """ Provide current user as extra context to the logger """ extra = kwargs.pop('extra', {}) extra.update({ 'user': self.user, 'msgid': self.counter.next(), 'time': self.env.now, }) kwargs['extra'] = extra super(SimulationLogger, self).log(level, message, *args, **kwargs)
def test_limit_sequence(self): """ Ensure that a sequence can be limited """ with self.assertRaises(StopIteration): sequence = Sequence(limit=1000) for idx in xrange(1, 100000): self.assertEqual(sequence.next(), idx)
def test_reset_sequence(self): """ Ensure that a sequence can be reset """ sequence = Sequence() for idx in xrange(1, 100): sequence.next() self.assertGreater(sequence.value, 1) sequence.reset() self.assertEqual(sequence.next(), 1)
class NamedProcess(Process): """ A process with a sequence counter and self identification. """ counter = Sequence() def __init__(self, env): self._id = self.counter.next() super(NamedProcess, self).__init__(env) @property def name(self): return "{} #{}".format(self.__class__.__name__, self._id)
def new(klass): return klass(Sequence(), Sequence())
class Version(object): """ A representation of a write to an object in the replica; the Version tracks all information associated with that write (e.g. the version that it was written from, when and how long it took to replicate the write). """ @classmethod def new(klass, name): """ Returns a new subclass of the version for a specific object and resets the global counter on the object, for multi-version systems. """ name = name or "foo" # Handle passing None into the new method. return type(name, (klass, ), {"counter": Sequence()}) # Autoincrementing ID counter = Sequence() @classmethod def increment_version(klass, replica): """ Returns the next unique version number in the global sequence. This method takes as input the replica writing the version number so that subclasses can implement replica-specific versioning. """ return klass.counter.next() @classmethod def latest_version(klass): """ Returns the latest global version of this object. """ return klass.counter.value def __init__(self, replica, parent=None, **kwargs): """ Creation of an initial version for the version tree. """ self.writer = replica self.version = self.increment_version(replica) self.parent = parent self.children = [] self.committed = False self.tag = kwargs.get('tag', None) # This seems very tightly coupled, should we do something different? self.replicas = set([replica.id]) # Level is depcrecated and no longer used. # TODO: Remove the level cleanly self.level = kwargs.get('level', replica.consistency) self.created = kwargs.get('created', replica.env.now) self.updated = kwargs.get('updated', replica.env.now) @property def name(self): """ Returns the name if it was created via the new function. (e.g. is not `Version`) """ name = self.__class__.__name__ if name != "Version": return name @property def access(self): """ Can reconstruct an access given the information in the version, or the API allows the use of the setter to assign the Write access that created the version for storage and future use. """ if not hasattr(self, '_access'): self._access = Write(self.name, self.writer, version=self, started=self.created, finished=self.updated) return self._access @access.setter def access(self, access): """ Assign a Write event that created this particular version for use in passing the event along in the distributed system. """ self._access = access def update(self, replica, commit=False, **kwargs): """ Replicas call this to update on remote writes. This method also tracks visibility latency for right now. """ self.updated = replica.env.now if replica.id not in self.replicas: self.replicas.add(replica.id) # Track replication over time visibility = float(len(self.replicas)) / float( len(replica.sim.replicas)) self.writer.sim.results.update( 'visibility', (self.writer.id, str(self), visibility, self.created, self.updated)) # Is this version completely replicated? if len(self.replicas) == len(replica.sim.replicas): # Track the visibility latency self.writer.sim.results.update( 'visibility latency', (self.writer.id, str(self), self.created, self.updated)) if commit and not self.committed: self.committed = True self.writer.sim.results.update( 'commit latency', (self.writer.id, str(self), self.created, self.updated)) def is_committed(self): """ Alias for committed. """ return self.committed def is_visible(self): """ Compares the set of replicas with the global replica set to determine if the version is fully visible on the cluster. (Based on who updates) """ return len(self.replicas) == len(self.writer.sim.replicas) def is_stale(self): """ Compares the version of this object to the global counter to determine if this vesion is the latest or not. """ return self.version < self.latest_version() def is_forked(self): """ Detect if we have multiple children or not. This is a "magic" method of determining if the fork exists or not ... for now. We need to make this "non-magic" soon. """ # TODO: Non-magic version of this # Right now we are computing how many un-dropped children exist dropped = lambda child: not child.access.is_dropped() return len(filter(dropped, self.children)) > 1 def nextv(self, replica, **kwargs): """ Returns a clone of this version, incremented to the next version. """ # TODO: ADD THIS BACK IN! # The problem is that this is global knowledge and cannot be checked. # Do not allow dropped writes to be incremented # if self.access.is_dropped(): # msg = ( # "Cannot write to a dropped version! " # "{} {} attempting to write {}" # ).format(self.writer.__class__.__name__, self.writer, self) # raise WorkloadException(msg) # Detect if we're a stale write (e.g. the parent version is stale). # NOTE: You have to do this before you create the next version # otherwise by definition the parent is stale! if self.is_stale(): # Count the number of stale writes as well as provide a mechanism # for computing time and version staleness for the write. self.writer.sim.results.update('stale writes', ( self.writer.id, self.writer.env.now, self.created, self.latest_version(), self.version, )) # Create the next version at this point. nv = self.__class__(replica, parent=self, level=self.level) # Append the next version to your children self.children.append(nv) # Detect if we've forked the write if self.is_forked(): # Count the number of forked writes self.writer.sim.results.update( 'forked writes', (self.writer.id, self.writer.env.now)) return nv def __str__(self): def mkvers(item): if item.name: return "{}.{}".format(item.name, item.version) return item.version if self.parent: return "{}->{}".format(mkvers(self.parent), mkvers(self)) return "root->{}".format(mkvers(self)) def __repr__(self): return repr(str(self)) ## Version comparison methods def __lt__(self, other): return self.version < other.version def __le__(self, other): return self.version <= other.version def __eq__(self, other): if other is None: return False if self.version == other.version: if self.parent is None: return other.parent is None return self.parent.version == other.parent.version return False def __ne__(self, other): return not self == other def __gt__(self, other): return self.version > other.version def __ge__(self, other): return self.version >= other.version
class Replica(Node): """ A replica is a network node that implements version handling. """ # Autoincrementing ID counter = Sequence() def __init__(self, sim, **kwargs): # Initialze Node super(Replica, self).__init__(sim.env) # Simulation Environment self.sim = sim # Replica Properties self.id = kwargs.get('id', 'r{}'.format(self.counter.next())) self.type = kwargs.get('type', settings.simulation.default_replica) self.label = kwargs.get('label', "{}-{}".format(self.type, self.id)) self.state = kwargs.get('state', State.READY) self.location = kwargs.get('location', 'unknown') self.consistency = Consistency.get( kwargs.get('consistency', settings.simulation.default_consistency)) ###################################################################### ## Properties ###################################################################### @property def state(self): """ Manages the state of the replica when set. """ if not hasattr(self, '_state'): self._state = State.ERRORED return self._state @state.setter def state(self, state): """ When setting the state, calls `on_state_change` so that replicas can modify their state machines accordingly. Note that the state is changed before this call, so replicas should inspect the new state. """ state = State.get(state) self._state = state self.on_state_change() ###################################################################### ## Core Methods (Replica API) ###################################################################### def send(self, target, value): """ Intermediate step towards Node.send (which handles simulation network) - this method logs information about the message, records metrics for results analysis, and does final preperatory work for sent messages. Simply logs that the message has been sent. """ try: # Call the super method to queue the message on the network event = super(Replica, self).send(target, value) except NetworkError: # Drop the message if the network connection is down return self.on_dropped_message(target, value) # Track total number of sent messages and get message type for logging. message = event.value mtype = self.sim.results.messages.update(message, SENT) # Debug logging of the message sent self.sim.logger.debug("message {} sent at {} from {} to {}".format( mtype, self.env.now, message.source, message.target)) # Track time series of sent messages if settings.simulation.trace_messages: self.sim.results.update( SENT, (message.source.id, message.target.id, self.env.now, mtype)) return event def recv(self, event): """ Intermediate step towards Node.recv (which handles simulation network) - this method logs information about the message, records metrics for results analysis, and detects and passes the message to the correct event handler for that RPC type. Subclasses should create methods of the form `on_[type]_rpc` where the type is the lowercase class name of the RPC named tuple. The recv method will route incomming messages to their correct RPC handler or raise an exception if it cannot find the access method. """ try: # Get the unpacked message from the event. message = super(Replica, self).recv(event) except NetworkError: # Drop the message if the network connection is down return self.on_dropped_message(event.value.target, event.value.value) # Track total number of sent messages and get message type for logging. mtype = self.sim.results.messages.update(message, RECV) # Track the message delay statistics of all received messages self.sim.results.latencies.update(message) # Debug logging of the message recv self.sim.logger.debug( "protocol {!r} received by {} from {} ({}ms delayed)".format( mtype, message.target, message.source, message.delay)) # Track time series of recv messages if settings.simulation.trace_messages: self.sim.results.update(RECV, (message.target.id, message.source.id, self.env.now, mtype, message.delay)) # Dispatch the RPC to the correct handler return self.dispatch(message) def read(self, name, **kwargs): """ Exposes the read API for every replica server and is one of the two primary methods of replica interaction. The read method expects the name of the object to read from and creates a Read event with meta information about the read. It is the responsibility of the subclasses to determine if the read is complete or not. Note that name can also be a Read event (in the case of remote reads that want to access identical read functionality on the replica). The read method will not create a new event, but will pass through the passed in Read event. This is in direct contrast to the old read method, where the replica did the work of logging and metrics - now this is all in the read event (when complete is triggered). """ if not name: raise AccessError( "Must supply a name to read from the replica server") return Read.create(name, self, **kwargs) def write(self, name, **kwargs): """ Exposes the write API for every replica server and is the second of the two primary methods of replica interaction. the write method exepcts the name of the object to write to and creates a Write event with meta informationa bout the write. It is the responsibility of sublcasses to perform the actual write on their local stores with replication. Note that name can also be a Write event and this method is in very different than the old write method (see details in read). This method will be adapted in the future to deal with write sizes, blocks, and other write meta information. """ if not name: raise AccessError( "Must supply a name to write to the replica server") return Write.create(name, self, **kwargs) def serialize(self): """ Outputs a simple object representation of the state of the replica. """ return dict([(attr, getattr(self, attr)) for attr in ('id', 'type', 'label', 'location', 'consistency')]) ###################################################################### ## Helper Methods ###################################################################### def dispatch(self, message): """ Dispatches an RPC message to the correct handler. Because RPC message values are expected to be typed namedtuples, the dispatcher looks for a handler method on the replica named similar to: def on_[type]_rpc(self, message): pass Where [type] is the snake_case of the RPC class, for example, the AppendEntries handler would be named on_append_entries_rpc. The dispatch returns the result of the handler. """ name = message.value.__class__.__name__ handler = "on_{}_rpc".format(decamelize(name)) # Check to see if the replica has the handler. if not hasattr(self, handler): NotImplementedError( "Handler for '{}' not implemented, add '{}' to {}".format( name, handler, self.__class__)) # Get the handler, call on message and return. handler = getattr(self, handler) return handler(message) def neighbors(self, consistency=None, location=None, exclude=False): """ Returns all nodes that are connected to the local node, filtering on either consistency of location. If neither consistency nor location are supplied, then this simply returns all of the neighboring nodes. If exclude is False, this returns any node that has the specified consistency or location. If exclude is True it returns any node that doesn't have the specified consistency or location. """ neighbors = self.connections.keys() # Filter based on consistency level if consistency is not None: # Convert a single consistenty level or a string into a collection if isinstance(consistency, (Consistency, basestring)): consistency = [Consistency.get(consistency)] else: consistency = map(Consistency.get, consistency) # Convert the consistencies into a set for lookup consistency = frozenset(consistency) # Filter connections in that consistency level if exclude: is_neighbor = lambda r: r.consistency not in consistency else: is_neighbor = lambda r: r.consistency in consistency neighbors = filter(is_neighbor, neighbors) # Filter based on location if location is not None: # Convert a single location into a collection if isinstance(location, (Location, basestring)): location = [location] # Convert the locations into a set for lookup. location = frozenset(location) # Filter connections in that consistency level if exclude: is_neighbor = lambda r: r.location not in location else: is_neighbor = lambda r: r.location in location neighbors = filter(is_neighbor, neighbors) return neighbors ###################################################################### ## Event Handlers ###################################################################### def on_state_change(self): """ Subclasses can call this to handle instance state. See the `state` property for more detail (this is called on set). """ pass def on_dropped_message(self, target, value): """ Called when there is a network error and a message that is being sent is dropped - subclasses can choose to retry the message or send to someone else. For now, we'll just record and log the drop. """ # Create a dummy message dummy = Message(self, target, value, None) mtype = self.sim.results.messages.update(message, DROP) # Debug logging of the message dropped self.sim.logger.debug("message {} dropped from {} to {} at {}".format( mtype, self, target, self.env.now)) # Track time series of dropped messages if settings.simulation.trace_messages: self.sim.results.update(DROP, (self.id, target.id, self.env.now, mtype)) ###################################################################### ## Object data model ###################################################################### def __str__(self): return "{} ({})".format(self.label, self.id)
try: from unittest import mock except ImportError: import mock from cloudscope.replica import Replica, Location, Consistency, Device from cloudscope.dynamo import Sequence from cloudscope.simulation.base import Simulation from cloudscope.results import Results MockEnvironment = mock.create_autospec(simpy.Environment, autospec=True) MockSimulation = mock.create_autospec(Simulation, autospec=True) MockResults = mock.create_autospec(Results, autospec=True) MockReplica = mock.create_autospec(Replica, autospec=True) sequence = Sequence() def get_mock_simulation(**kwargs): simulation = MockSimulation() simulation.env = MockEnvironment() simulation.results = MockResults() simulation.__name__ = "MockSimulation" # Set specific properties and attributes simulation.env.process = mock.MagicMock() simulation.env.now = kwargs.get('now', 42) simulation.replicas = [ get_mock_replica(simulation) for x in xrange(kwargs.get('replicas', 5)) ]