def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def setUp(self):
self.persis = SqlitePersistenceLayer('test_layer', ':memory:')
self.persis.init_persistence()
class TestSqlitePersistenceLayer(TestCase):
"""
Tests the sqlite persistence layer
"""
def setUp(self):
self.persis = SqlitePersistenceLayer('test_layer', ':memory:')
self.persis.init_persistence()
def tearDown(self):
self.persis.conn.close()
def test_simple_put(self):
"""
Tests a put
"""
self.persis.put_key('foo', 'this is my data')
result = self.persis.conn.execute("SELECT * FROM key_values")
rows = [row for row in result]
self.assertEquals(len(rows), 1)
row = rows[0][0:3]
expected_values = (1, 'foo', 'this is my data')
self.assertEquals(row, expected_values)
def test_simple_get(self):
"""
Tests a get
"""
self.persis.put_key('foo', 'this is my data')
result = self.persis.get_key('foo')
self.assertEquals(len(result), 1)
self.assertEquals(result[0][0:2], (1, 'this is my data'))
def test_put_multiple_values_per_key(self):
"""
Ensures that we can put multiple values for the same key in the
persistence layer.
"""
self.persis.put_key('foo', 'this is my data')
self.persis.put_key('foo', 'this is my data #2')
result = self.persis.conn.execute("SELECT * FROM key_values")
rows = [row for row in result]
self.assertEquals(len(rows), 2)
def test_get_multiple_values_per_key(self):
"""
Ensures that we can get multiple values for the same key in the
persistence layer.
"""
self.persis.put_key('foo', 'this is my data')
self.persis.put_key('foo', 'this is my data #2')
result = self.persis.get_key('foo')
self.assertEquals(len(result), 2)
expected_rows = [(1, 'this is my data'),
(2, 'this is my data #2')]
for row in result:
row = row[0:2]
self.assertTrue(row in expected_rows)
expected_rows.remove(row)
def setUp(self):
self.persis = SqlitePersistenceLayer('test_layer', ':memory:')
self.persis.init_persistence()
class TestSqlitePersistenceLayer(TestCase):
"""
Tests the sqlite persistence layer
"""
def setUp(self):
self.persis = SqlitePersistenceLayer('test_layer', ':memory:')
self.persis.init_persistence()
def tearDown(self):
self.persis.conn.close()
def test_simple_put(self):
"""
Tests a put
"""
self.persis.put_key('foo', 'this is my data')
result = self.persis.conn.execute("SELECT * FROM key_values")
rows = [row for row in result]
self.assertEquals(len(rows), 1)
row = rows[0][0:3]
expected_values = (1, 'foo', 'this is my data')
self.assertEquals(row, expected_values)
def test_simple_get(self):
"""
Tests a get
"""
self.persis.put_key('foo', 'this is my data')
result = self.persis.get_key('foo')
self.assertEquals(len(result), 1)
self.assertEquals(result[0][0:2], (1, 'this is my data'))
def test_put_multiple_values_per_key(self):
"""
Ensures that we can put multiple values for the same key in the
persistence layer.
"""
self.persis.put_key('foo', 'this is my data')
self.persis.put_key('foo', 'this is my data #2')
result = self.persis.conn.execute("SELECT * FROM key_values")
rows = [row for row in result]
self.assertEquals(len(rows), 2)
def test_get_multiple_values_per_key(self):
"""
Ensures that we can get multiple values for the same key in the
persistence layer.
"""
self.persis.put_key('foo', 'this is my data')
self.persis.put_key('foo', 'this is my data #2')
result = self.persis.get_key('foo')
self.assertEquals(len(result), 2)
expected_rows = [(1, 'this is my data'), (2, 'this is my data #2')]
for row in result:
row = row[0:2]
self.assertTrue(row in expected_rows)
expected_rows.remove(row)
class StorageNode(object):
"""
A storage node.
"""
GET = 'GET'
PUT = 'PUT'
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
def __del__(self):
"""
Destructor
"""
if self.persis:
self.persis.close()
if self.server:
self.server.server_close()
def __str__(self):
"""
Builds a string representation of the storage node
:rtype: str
:returns: A string representation of the storage node
"""
if getattr(self, 'port'):
return '%s:%s' % (socket.gethostbyname(
socket.gethostname()), self.port)
else:
return '%s' % socket.gethostbyname(socket.gethostname())
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(('', self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key
:Parameters:
key : str
The key value
"""
logging.debug('Getting key=%s' % key)
# Make sure I am supposed to have this key
respon_node = self.datastore_view.get_node(key)
if respon_node != self.my_name:
logging.info("I'm not responsible for %s (%s vs %s)" %
(key, respon_node, self.my_name))
return None
# Read it from the database
result = self.persis.get_key(key)
# If the contexts don't line up then return the most recent
value = None
if len(result) == 1:
value = result[0][1]
else:
value = self._reconcile_conflict(result)[0]
logging.debug('Returning value=%s' % value)
return value
def put(self, key, value, context=None):
"""
Puts a key value in the datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
# Make sure I am supposed to have this key
if self.datastore_view.get_node(key) != self.my_name:
logging.info("I'm not responsible for %s" % key)
return None
res_code = None
try:
# Read it from the database
result = self.persis.put_key(key, value)
res_code = '200'
except:
logging.error(
'Error putting key=%s value=%s into the persistence layer' %
(key, value))
res_code = '400'
return res_code
# ------------------------------------------------------
# Private methods
# ------------------------------------------------------
def _reconcile_conflict(self, result):
"""
Reconciles the conflict between a number of values. Note
that currently this defaults to taking the last written value.
In the future this will be expanded to allow application specific
conflict resolution
:Parameters:
result : list(tuples)
A list of result tuples from the persistence layer in the form
[(id, "value", "date"), ...]
:rtype: tuple(int, str)
:returns An id, string tuple of the chosen version
"""
last_result = None
last_date = None
for res in result:
if last_result is None:
last_result = res[1]
last_date = self._parse_date(res[2])
else:
date = self._parse_date(res[2])
if date > last_date:
last_date = date
last_result = res[1]
return (last_result, last_date)
def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def _parse_date(self, datestr):
"""
Parses an iso formatted date
:Parameters:
datestr : str
An iso formatted date
:rtype: datetime
:returns A date object
"""
date_str, micros = datestr.split('.')
date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
date += timedelta(microseconds=float(micros))
return date
class StorageNode(object):
"""
A storage node.
"""
GET = "GET"
PUT = "PUT"
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
def __del__(self):
"""
Destructor
"""
if self.persis:
self.persis.close()
if self.server:
self.server.server_close()
def __str__(self):
"""
Builds a string representation of the storage node
:rtype: str
:returns: A string representation of the storage node
"""
if getattr(self, "port"):
return "%s:%s" % (socket.gethostbyname(socket.gethostname()), self.port)
else:
return "%s" % socket.gethostbyname(socket.gethostname())
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(("", self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key
:Parameters:
key : str
The key value
"""
logging.debug("Getting key=%s" % key)
# Make sure I am supposed to have this key
respon_node = self.datastore_view.get_node(key)
if respon_node != self.my_name:
logging.info("I'm not responsible for %s (%s vs %s)" % (key, respon_node, self.my_name))
return None
# Read it from the database
result = self.persis.get_key(key)
# If the contexts don't line up then return the most recent
value = None
if len(result) == 1:
value = result[0][1]
else:
value = self._reconcile_conflict(result)[0]
logging.debug("Returning value=%s" % value)
return value
def put(self, key, value, context=None):
"""
Puts a key value in the datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
# Make sure I am supposed to have this key
if self.datastore_view.get_node(key) != self.my_name:
logging.info("I'm not responsible for %s" % key)
return None
res_code = None
try:
# Read it from the database
result = self.persis.put_key(key, value)
res_code = "200"
except:
logging.error("Error putting key=%s value=%s into the persistence layer" % (key, value))
res_code = "400"
return res_code
# ------------------------------------------------------
# Private methods
# ------------------------------------------------------
def _reconcile_conflict(self, result):
"""
Reconciles the conflict between a number of values. Note
that currently this defaults to taking the last written value.
In the future this will be expanded to allow application specific
conflict resolution
:Parameters:
result : list(tuples)
A list of result tuples from the persistence layer in the form
[(id, "value", "date"), ...]
:rtype: tuple(int, str)
:returns An id, string tuple of the chosen version
"""
last_result = None
last_date = None
for res in result:
if last_result is None:
last_result = res[1]
last_date = self._parse_date(res[2])
else:
date = self._parse_date(res[2])
if date > last_date:
last_date = date
last_result = res[1]
return (last_result, last_date)
def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def _parse_date(self, datestr):
"""
Parses an iso formatted date
:Parameters:
datestr : str
An iso formatted date
:rtype: datetime
:returns A date object
"""
date_str, micros = datestr.split(".")
date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
date += timedelta(microseconds=float(micros))
return date
