def _load_persistence_layer(self): """ Loads the persistence layer """ # Setup my persistence layer self.persis = SqlitePersistenceLayer(self.my_name) self.persis.init_persistence()
def setUp(self): self.persis = SqlitePersistenceLayer('test_layer', ':memory:') self.persis.init_persistence()
class TestSqlitePersistenceLayer(TestCase): """ Tests the sqlite persistence layer """ def setUp(self): self.persis = SqlitePersistenceLayer('test_layer', ':memory:') self.persis.init_persistence() def tearDown(self): self.persis.conn.close() def test_simple_put(self): """ Tests a put """ self.persis.put_key('foo', 'this is my data') result = self.persis.conn.execute("SELECT * FROM key_values") rows = [row for row in result] self.assertEquals(len(rows), 1) row = rows[0][0:3] expected_values = (1, 'foo', 'this is my data') self.assertEquals(row, expected_values) def test_simple_get(self): """ Tests a get """ self.persis.put_key('foo', 'this is my data') result = self.persis.get_key('foo') self.assertEquals(len(result), 1) self.assertEquals(result[0][0:2], (1, 'this is my data')) def test_put_multiple_values_per_key(self): """ Ensures that we can put multiple values for the same key in the persistence layer. """ self.persis.put_key('foo', 'this is my data') self.persis.put_key('foo', 'this is my data #2') result = self.persis.conn.execute("SELECT * FROM key_values") rows = [row for row in result] self.assertEquals(len(rows), 2) def test_get_multiple_values_per_key(self): """ Ensures that we can get multiple values for the same key in the persistence layer. """ self.persis.put_key('foo', 'this is my data') self.persis.put_key('foo', 'this is my data #2') result = self.persis.get_key('foo') self.assertEquals(len(result), 2) expected_rows = [(1, 'this is my data'), (2, 'this is my data #2')] for row in result: row = row[0:2] self.assertTrue(row in expected_rows) expected_rows.remove(row)
class StorageNode(object): """ A storage node. """ GET = 'GET' PUT = 'PUT' def __init__(self, servers, port): """ Parameters: servers : list(str) A list of servers. Each server name is in the format {host/ip}:port port : int Port number to start on """ self.port = int(port) self.server = None if servers is None: servers = [] # Add myself to the servers list self.my_name = str(self) servers.append(self.my_name) self.datastore_view = DataStoreView(servers) # Load the persistence layer self._load_persistence_layer() def __del__(self): """ Destructor """ if self.persis: self.persis.close() if self.server: self.server.server_close() def __str__(self): """ Builds a string representation of the storage node :rtype: str :returns: A string representation of the storage node """ if getattr(self, 'port'): return '%s:%s' % (socket.gethostbyname( socket.gethostname()), self.port) else: return '%s' % socket.gethostbyname(socket.gethostname()) # ------------------------------------------------------ # Public methods # ------------------------------------------------------ def run(self): """ Main storage node loop """ self.server = SimpleXMLRPCServer(('', self.port), allow_none=True) self.server.register_function(self.get, "get") self.server.register_function(self.put, "put") self.server.serve_forever() # ------------------------------------------------------ # RPC methods # ------------------------------------------------------ def get(self, key): """ Gets a key :Parameters: key : str The key value """ logging.debug('Getting key=%s' % key) # Make sure I am supposed to have this key respon_node = self.datastore_view.get_node(key) if respon_node != self.my_name: logging.info("I'm not responsible for %s (%s vs %s)" % (key, respon_node, self.my_name)) return None # Read it from the database result = self.persis.get_key(key) # If the contexts don't line up then return the most recent value = None if len(result) == 1: value = result[0][1] else: value = self._reconcile_conflict(result)[0] logging.debug('Returning value=%s' % value) return value def put(self, key, value, context=None): """ Puts a key value in the datastore :Parameters: key : str The key name value : str The value context : str Should be only be None for now. In the future an application will be able to add a custom context string :rtype: str :returns 200 if the operation succeeded, 400 otherwise """ # Make sure I am supposed to have this key if self.datastore_view.get_node(key) != self.my_name: logging.info("I'm not responsible for %s" % key) return None res_code = None try: # Read it from the database result = self.persis.put_key(key, value) res_code = '200' except: logging.error( 'Error putting key=%s value=%s into the persistence layer' % (key, value)) res_code = '400' return res_code # ------------------------------------------------------ # Private methods # ------------------------------------------------------ def _reconcile_conflict(self, result): """ Reconciles the conflict between a number of values. Note that currently this defaults to taking the last written value. In the future this will be expanded to allow application specific conflict resolution :Parameters: result : list(tuples) A list of result tuples from the persistence layer in the form [(id, "value", "date"), ...] :rtype: tuple(int, str) :returns An id, string tuple of the chosen version """ last_result = None last_date = None for res in result: if last_result is None: last_result = res[1] last_date = self._parse_date(res[2]) else: date = self._parse_date(res[2]) if date > last_date: last_date = date last_result = res[1] return (last_result, last_date) def _load_persistence_layer(self): """ Loads the persistence layer """ # Setup my persistence layer self.persis = SqlitePersistenceLayer(self.my_name) self.persis.init_persistence() def _parse_date(self, datestr): """ Parses an iso formatted date :Parameters: datestr : str An iso formatted date :rtype: datetime :returns A date object """ date_str, micros = datestr.split('.') date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") date += timedelta(microseconds=float(micros)) return date
class StorageNode(object): """ A storage node. """ GET = "GET" PUT = "PUT" def __init__(self, servers, port): """ Parameters: servers : list(str) A list of servers. Each server name is in the format {host/ip}:port port : int Port number to start on """ self.port = int(port) self.server = None if servers is None: servers = [] # Add myself to the servers list self.my_name = str(self) servers.append(self.my_name) self.datastore_view = DataStoreView(servers) # Load the persistence layer self._load_persistence_layer() def __del__(self): """ Destructor """ if self.persis: self.persis.close() if self.server: self.server.server_close() def __str__(self): """ Builds a string representation of the storage node :rtype: str :returns: A string representation of the storage node """ if getattr(self, "port"): return "%s:%s" % (socket.gethostbyname(socket.gethostname()), self.port) else: return "%s" % socket.gethostbyname(socket.gethostname()) # ------------------------------------------------------ # Public methods # ------------------------------------------------------ def run(self): """ Main storage node loop """ self.server = SimpleXMLRPCServer(("", self.port), allow_none=True) self.server.register_function(self.get, "get") self.server.register_function(self.put, "put") self.server.serve_forever() # ------------------------------------------------------ # RPC methods # ------------------------------------------------------ def get(self, key): """ Gets a key :Parameters: key : str The key value """ logging.debug("Getting key=%s" % key) # Make sure I am supposed to have this key respon_node = self.datastore_view.get_node(key) if respon_node != self.my_name: logging.info("I'm not responsible for %s (%s vs %s)" % (key, respon_node, self.my_name)) return None # Read it from the database result = self.persis.get_key(key) # If the contexts don't line up then return the most recent value = None if len(result) == 1: value = result[0][1] else: value = self._reconcile_conflict(result)[0] logging.debug("Returning value=%s" % value) return value def put(self, key, value, context=None): """ Puts a key value in the datastore :Parameters: key : str The key name value : str The value context : str Should be only be None for now. In the future an application will be able to add a custom context string :rtype: str :returns 200 if the operation succeeded, 400 otherwise """ # Make sure I am supposed to have this key if self.datastore_view.get_node(key) != self.my_name: logging.info("I'm not responsible for %s" % key) return None res_code = None try: # Read it from the database result = self.persis.put_key(key, value) res_code = "200" except: logging.error("Error putting key=%s value=%s into the persistence layer" % (key, value)) res_code = "400" return res_code # ------------------------------------------------------ # Private methods # ------------------------------------------------------ def _reconcile_conflict(self, result): """ Reconciles the conflict between a number of values. Note that currently this defaults to taking the last written value. In the future this will be expanded to allow application specific conflict resolution :Parameters: result : list(tuples) A list of result tuples from the persistence layer in the form [(id, "value", "date"), ...] :rtype: tuple(int, str) :returns An id, string tuple of the chosen version """ last_result = None last_date = None for res in result: if last_result is None: last_result = res[1] last_date = self._parse_date(res[2]) else: date = self._parse_date(res[2]) if date > last_date: last_date = date last_result = res[1] return (last_result, last_date) def _load_persistence_layer(self): """ Loads the persistence layer """ # Setup my persistence layer self.persis = SqlitePersistenceLayer(self.my_name) self.persis.init_persistence() def _parse_date(self, datestr): """ Parses an iso formatted date :Parameters: datestr : str An iso formatted date :rtype: datetime :returns A date object """ date_str, micros = datestr.split(".") date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") date += timedelta(microseconds=float(micros)) return date