def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
"""
self.port = int(port)
self.server = None
if not servers:
raise exceptions.ValueError("Cannot have empty server list")
# Create the load balancer's view of the storage node ring
self.datastore_view = DataStoreView(servers)
# Open connections to each server
self.server_conns = {}
for server in servers:
self.server_conns[server] = xmlrpclib.ServerProxy(self.CONN_STR %
server)
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
"""
self.port = int(port)
self.server = None
if not servers:
raise exceptions.ValueError("Cannot have empty server list")
# Create the load balancer's view of the storage node ring
self.datastore_view = DataStoreView(servers)
# Open connections to each server
self.server_conns = {}
for server in servers:
self.server_conns[server] = xmlrpclib.ServerProxy(self.CONN_STR % server)
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
class LoadBalancer(object):
"""
A load balancer that routes requests to the appropriate storage node
"""
CONN_STR = 'http://%s'
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
"""
self.port = int(port)
self.server = None
if not servers:
raise exceptions.ValueError("Cannot have empty server list")
# Create the load balancer's view of the storage node ring
self.datastore_view = DataStoreView(servers)
# Open connections to each server
self.server_conns = {}
for server in servers:
self.server_conns[server] = xmlrpclib.ServerProxy(self.CONN_STR % server)
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(('', self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key from the appropriate storage node
:Parameters:
key : str
The key value
"""
value = None
try:
# Find the responsbile node
respon_node = self.datastore_view.get_node(key)
logging.debug('Getting key=%s from node=%s' % (key, respon_node))
# Get the value from that node
value = self.server_conns[respon_node].get(key)
logging.debug('Value=%s' % value)
except:
logging.error('Error getting the key=%s' % key)
value = None
return value
def put(self, key, value, context=None):
"""
Puts a key in the appropriate datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
respon_code = None
try:
# Find the responsbile node
respon_node = self.datastore_view.get_node(key)
logging.debug('Putting key=%s on node=%s' % (key, respon_node))
# Get the value from that node
respon_code = self.server_conns[respon_node].put(key, value)
except:
logging.error("Error putting key=%s, value=%s" % (key,value))
respon_code = "400"
return respon_code
class StorageNode(object):
"""
A storage node.
"""
GET = 'GET'
PUT = 'PUT'
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
def __del__(self):
"""
Destructor
"""
if self.persis:
self.persis.close()
if self.server:
self.server.server_close()
def __str__(self):
"""
Builds a string representation of the storage node
:rtype: str
:returns: A string representation of the storage node
"""
if getattr(self, 'port'):
return '%s:%s' % (socket.gethostbyname(
socket.gethostname()), self.port)
else:
return '%s' % socket.gethostbyname(socket.gethostname())
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(('', self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key
:Parameters:
key : str
The key value
"""
logging.debug('Getting key=%s' % key)
# Make sure I am supposed to have this key
respon_node = self.datastore_view.get_node(key)
if respon_node != self.my_name:
logging.info("I'm not responsible for %s (%s vs %s)" %
(key, respon_node, self.my_name))
return None
# Read it from the database
result = self.persis.get_key(key)
# If the contexts don't line up then return the most recent
value = None
if len(result) == 1:
value = result[0][1]
else:
value = self._reconcile_conflict(result)[0]
logging.debug('Returning value=%s' % value)
return value
def put(self, key, value, context=None):
"""
Puts a key value in the datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
# Make sure I am supposed to have this key
if self.datastore_view.get_node(key) != self.my_name:
logging.info("I'm not responsible for %s" % key)
return None
res_code = None
try:
# Read it from the database
result = self.persis.put_key(key, value)
res_code = '200'
except:
logging.error(
'Error putting key=%s value=%s into the persistence layer' %
(key, value))
res_code = '400'
return res_code
# ------------------------------------------------------
# Private methods
# ------------------------------------------------------
def _reconcile_conflict(self, result):
"""
Reconciles the conflict between a number of values. Note
that currently this defaults to taking the last written value.
In the future this will be expanded to allow application specific
conflict resolution
:Parameters:
result : list(tuples)
A list of result tuples from the persistence layer in the form
[(id, "value", "date"), ...]
:rtype: tuple(int, str)
:returns An id, string tuple of the chosen version
"""
last_result = None
last_date = None
for res in result:
if last_result is None:
last_result = res[1]
last_date = self._parse_date(res[2])
else:
date = self._parse_date(res[2])
if date > last_date:
last_date = date
last_result = res[1]
return (last_result, last_date)
def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def _parse_date(self, datestr):
"""
Parses an iso formatted date
:Parameters:
datestr : str
An iso formatted date
:rtype: datetime
:returns A date object
"""
date_str, micros = datestr.split('.')
date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
date += timedelta(microseconds=float(micros))
return date
class StorageNode(object):
"""
A storage node.
"""
GET = "GET"
PUT = "PUT"
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
port : int
Port number to start on
"""
self.port = int(port)
self.server = None
if servers is None:
servers = []
# Add myself to the servers list
self.my_name = str(self)
servers.append(self.my_name)
self.datastore_view = DataStoreView(servers)
# Load the persistence layer
self._load_persistence_layer()
def __del__(self):
"""
Destructor
"""
if self.persis:
self.persis.close()
if self.server:
self.server.server_close()
def __str__(self):
"""
Builds a string representation of the storage node
:rtype: str
:returns: A string representation of the storage node
"""
if getattr(self, "port"):
return "%s:%s" % (socket.gethostbyname(socket.gethostname()), self.port)
else:
return "%s" % socket.gethostbyname(socket.gethostname())
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(("", self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key
:Parameters:
key : str
The key value
"""
logging.debug("Getting key=%s" % key)
# Make sure I am supposed to have this key
respon_node = self.datastore_view.get_node(key)
if respon_node != self.my_name:
logging.info("I'm not responsible for %s (%s vs %s)" % (key, respon_node, self.my_name))
return None
# Read it from the database
result = self.persis.get_key(key)
# If the contexts don't line up then return the most recent
value = None
if len(result) == 1:
value = result[0][1]
else:
value = self._reconcile_conflict(result)[0]
logging.debug("Returning value=%s" % value)
return value
def put(self, key, value, context=None):
"""
Puts a key value in the datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
# Make sure I am supposed to have this key
if self.datastore_view.get_node(key) != self.my_name:
logging.info("I'm not responsible for %s" % key)
return None
res_code = None
try:
# Read it from the database
result = self.persis.put_key(key, value)
res_code = "200"
except:
logging.error("Error putting key=%s value=%s into the persistence layer" % (key, value))
res_code = "400"
return res_code
# ------------------------------------------------------
# Private methods
# ------------------------------------------------------
def _reconcile_conflict(self, result):
"""
Reconciles the conflict between a number of values. Note
that currently this defaults to taking the last written value.
In the future this will be expanded to allow application specific
conflict resolution
:Parameters:
result : list(tuples)
A list of result tuples from the persistence layer in the form
[(id, "value", "date"), ...]
:rtype: tuple(int, str)
:returns An id, string tuple of the chosen version
"""
last_result = None
last_date = None
for res in result:
if last_result is None:
last_result = res[1]
last_date = self._parse_date(res[2])
else:
date = self._parse_date(res[2])
if date > last_date:
last_date = date
last_result = res[1]
return (last_result, last_date)
def _load_persistence_layer(self):
"""
Loads the persistence layer
"""
# Setup my persistence layer
self.persis = SqlitePersistenceLayer(self.my_name)
self.persis.init_persistence()
def _parse_date(self, datestr):
"""
Parses an iso formatted date
:Parameters:
datestr : str
An iso formatted date
:rtype: datetime
:returns A date object
"""
date_str, micros = datestr.split(".")
date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
date += timedelta(microseconds=float(micros))
return date
class LoadBalancer(object):
"""
A load balancer that routes requests to the appropriate storage node
"""
CONN_STR = 'http://%s'
def __init__(self, servers, port):
"""
Parameters:
servers : list(str)
A list of servers. Each server name is in the
format {host/ip}:port
"""
self.port = int(port)
self.server = None
if not servers:
raise exceptions.ValueError("Cannot have empty server list")
# Create the load balancer's view of the storage node ring
self.datastore_view = DataStoreView(servers)
# Open connections to each server
self.server_conns = {}
for server in servers:
self.server_conns[server] = xmlrpclib.ServerProxy(self.CONN_STR %
server)
# ------------------------------------------------------
# Public methods
# ------------------------------------------------------
def run(self):
"""
Main storage node loop
"""
self.server = SimpleXMLRPCServer(('', self.port), allow_none=True)
self.server.register_function(self.get, "get")
self.server.register_function(self.put, "put")
self.server.serve_forever()
# ------------------------------------------------------
# RPC methods
# ------------------------------------------------------
def get(self, key):
"""
Gets a key from the appropriate storage node
:Parameters:
key : str
The key value
"""
value = None
try:
# Find the responsbile node
respon_node = self.datastore_view.get_node(key)
logging.debug('Getting key=%s from node=%s' % (key, respon_node))
# Get the value from that node
value = self.server_conns[respon_node].get(key)
logging.debug('Value=%s' % value)
except:
logging.error('Error getting the key=%s' % key)
value = None
return value
def put(self, key, value, context=None):
"""
Puts a key in the appropriate datastore
:Parameters:
key : str
The key name
value : str
The value
context : str
Should be only be None for now. In the future an application will be
able to add a custom context string
:rtype: str
:returns 200 if the operation succeeded, 400 otherwise
"""
respon_code = None
try:
# Find the responsbile node
respon_node = self.datastore_view.get_node(key)
logging.debug('Putting key=%s on node=%s' % (key, respon_node))
# Get the value from that node
respon_code = self.server_conns[respon_node].put(key, value)
except:
logging.error("Error putting key=%s, value=%s" % (key, value))
respon_code = "400"
return respon_code
