def test_chr_add_node_duplicate(self):
hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.nodes, set(hosts))
hashring.add_node(("127.0.0.1", "cache2"))
self.assertEqual(hashring.nodes, set(hosts))
self.assertEqual(hashring.nodes_len, 3)
def test_chr_remove_node_missing(self):
hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.nodes, set(hosts))
hashring.remove_node(("127.0.0.1", "cache4"))
self.assertEqual(hashring.nodes, set(hosts))
self.assertEqual(hashring.nodes_len, 3)
def test_chr_get_node_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"), ("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.get_node('hosts.worker1.cpu'),
('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b'))
self.assertEqual(hashring.get_node('hosts.worker2.cpu'),
('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))
def test_chr_compute_ring_position(self):
hosts = [("127.0.0.1", "cache0"), ("127.0.0.1", "cache1"),
("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'),
64833)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'),
38509)
def test_chr_get_nodes(self):
hosts = [("127.0.0.1", "cache0"), ("127.0.0.1", "cache1"),
("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
node = hashring.get_nodes('hosts.worker1.cpu')
self.assertEqual(node,
[('127.0.0.1', 'cache2'), ('127.0.0.1', 'cache0'),
('127.0.0.1', 'cache1')])
def test_chr_remove_node_missing(self):
hosts = [("127.0.0.1", "cache0"), ("127.0.0.1", "cache1"),
("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.nodes, set(hosts))
hashring.remove_node(("127.0.0.1", "cache4"))
self.assertEqual(hashring.nodes, set(hosts))
self.assertEqual(hashring.nodes_len, 3)
def test_chr_add_node_duplicate(self):
hosts = [("127.0.0.1", "cache0"), ("127.0.0.1", "cache1"),
("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.nodes, set(hosts))
hashring.add_node(("127.0.0.1", "cache2"))
self.assertEqual(hashring.nodes, set(hosts))
self.assertEqual(hashring.nodes_len, 3)
def test_chr_compute_ring_position_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'),
59573)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'),
35749)
def test_chr_get_node_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.get_node('hosts.worker1.cpu'),
('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b'))
self.assertEqual(hashring.get_node('hosts.worker2.cpu'),
('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))
def __init__(self, hosts, timeout):
self.hosts = [(server, instance) for (server, port, instance) in hosts]
self.ports = dict(
((server, instance), port) for (server, port, instance) in hosts)
self.timeout = float(timeout)
self.hashRing = ConsistentHashRing(self.hosts)
self.connections = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(self.hosts)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
self.hash_ring = ConsistentHashRing(self.hosts)
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(self.hosts)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
class CarbonLinkPool:
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(self.hosts)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
key = self.keyfunc(metric)
nodes = []
servers = set()
for node in self.hash_ring.get_nodes(key):
(server, instance) = node
if server in servers:
continue
servers.add(server)
nodes.append(node)
if len(servers) >= settings.REPLICATION_FACTOR:
break
available = [ n for n in nodes if self.is_available(n) ]
return random.choice(available or nodes)
def is_available(self, host):
now = time.time()
last_fail = self.last_failure.get(host, 0)
return (now - last_fail) < settings.CARBONLINK_RETRY_DELAY
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
try:
connection.connect( (server, port) )
except:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt( socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1 )
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache("CarbonLink cache-query request for %s returned %d datapoints" % (metric, len(results['datapoints'])))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" % (metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata', metric=metric, key=key, value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" % (metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
result = {}
result.setdefault('datapoints', [])
if metric.startswith(settings.CARBON_METRIC_PREFIX):
return self.send_request_to_all(request)
host = self.select_host(metric)
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" % (metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception,e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" % (str(host), e))
else:
class CarbonLinkPool:
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
self.hash_ring = ConsistentHashRing(self.hosts)
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
return self.hash_ring.get_node(metric)
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
try:
connection.connect( (server, port) )
except:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt( socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1 )
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache("CarbonLink cache-query request for %s returned %d datapoints" % (metric, len(results)))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" % (metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata', metric=metric, key=key, value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" % (metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
host = self.select_host(metric)
conn = self.get_connection(host)
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except:
self.last_failure[host] = time.time()
raise
else:
self.connections[host].add(conn)
if 'error' in result:
raise CarbonLinkRequestError(result['error'])
else:
return result
def recv_response(self, conn):
len_prefix = recv_exactly(conn, 4)
body_size = struct.unpack("!L", len_prefix)[0]
body = recv_exactly(conn, body_size)
return pickle.loads(body)
def test_chr_compute_ring_position(self):
hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), 64833)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), 38509)
class CarbonLinkPool:
def __init__(self, hosts, timeout):
self.hosts = [(server, instance) for (server, port, instance) in hosts]
self.ports = dict(
((server, instance), port) for (server, port, instance) in hosts)
self.timeout = float(timeout)
self.hashRing = ConsistentHashRing(self.hosts)
self.connections = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def selectHost(self, metric):
"Returns the carbon host that has data for the given metric"
return self.hashRing.get_node(metric)
def getConnection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
connection.connect((server, port))
connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
return connection
def putConnectionInPool(self, host, connection):
connectionPool = self.connections[host]
connectionPool.add(connection)
def removeConnectionFromPool(self, host, connection):
connectionPool = self.connections.get(host, set())
connectionPool.discard(connection)
def sendRequest(self, metric):
"Sends a request and returns a completion callback"
host = self.selectHost(metric)
query = struct.pack(
"!L", len(metric)) + metric # 32-bit length prefix string
connection = None
try:
connection = self.getConnection(host)
connection.sendall(query)
# To keep things asynchronous we return a result callback
def receiveResponse():
try:
buf = ''
remaining = 4
message_size = None
while remaining:
packet = connection.recv(remaining)
assert packet, "CarbonLink lost connection to %s" % str(
host)
buf += packet
if message_size is None:
if len(buf) == 4:
remaining = message_size = struct.unpack(
"!L", buf)[0]
buf = ''
continue
remaining -= len(packet)
# We're done with the connection for this request, put it in the pool
self.putConnectionInPool(host, connection)
# Now parse the response
points = pickle.loads(buf)
log.cache("CarbonLink to %s, retrieved %d points for %s" %
(host, len(points), metric))
for point in points:
yield point
except:
log.exception(
"CarbonLink to %s, exception while getting response" %
str(host))
self.removeConnectionFromPool(host, connection)
return receiveResponse
except:
log.exception("CarbonLink to %s, exception while sending request" %
str(host))
if connection:
self.removeConnectionFromPool(host, connection)
noResults = lambda: []
return noResults
instances = []
unwelcome_instances = []
for arg in sys.argv[1:]:
unwelcome = False
if arg.startswith('-'):
arg = arg[1:]
unwelcome = True
instance = tuple(arg.split(':', 2))
instances.append(instance)
if unwelcome:
unwelcome_instances.append(instance)
if 0 == len(instances):
print('Usage: python whisper-clean.py [-]<address>:<instance>[...]')
sys.exit(1)
ring = ConsistentHashRing(instances)
for dirname, dirnames, filenames in os.walk('/var/lib/graphite/whisper'):
for filename in filenames:
pathname = os.path.join(dirname, filename)
basename, ext = os.path.splitext(filename)
if '.wsp' != ext:
print('skipping %s' %
os.path.relpath(pathname, '/var/lib/graphite/whisper'))
if ring.get_node(
os.path.relpath(os.path.join(dirname, basename),
'/var/lib/graphite/whisper').replace(
'/', '.')) in unwelcome_instances:
print('unlinking %s' % pathname)
os.unlink(pathname)
def test_chr_get_nodes(self):
hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
hashring = ConsistentHashRing(hosts)
node = hashring.get_nodes('hosts.worker1.cpu')
self.assertEqual(node, [('127.0.0.1', 'cache2'), ('127.0.0.1', 'cache0'), ('127.0.0.1', 'cache1')])
class CarbonLinkPool(object):
def __init__(self, hosts, timeout):
self.hosts = [(server, instance) for (server, port, instance) in hosts]
self.ports = {(server, instance): port
for (server, port, instance) in hosts}
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" %
(settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(
self.hosts, hash_type=settings.CARBONLINK_HASHING_TYPE)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
key = self.keyfunc(metric)
nodes = []
servers = set()
for node in self.hash_ring.get_nodes(key):
(server, instance) = node
if server in servers:
continue
servers.add(server)
nodes.append(node)
if len(servers) >= settings.REPLICATION_FACTOR:
break
available = [n for n in nodes if self.is_available(n)]
return random.choice(available or nodes)
def is_available(self, host):
now = time.time()
last_fail = self.last_failure.get(host, 0)
return (now - last_fail) < settings.CARBONLINK_RETRY_DELAY
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
try:
connection = socket.create_connection((server, port), self.timeout)
except socket.error:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache(
"CarbonLink cache-query request for %s returned %d datapoints" %
(metric, len(results['datapoints'])))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" %
(metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata',
metric=metric,
key=key,
value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" %
(metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(
request, protocol=settings.CARBONLINK_PICKLE_PROTOCOL)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
result = {}
result.setdefault('datapoints', [])
if metric.startswith(settings.CARBON_METRIC_PREFIX):
return self.send_request_to_all(request)
if not self.hosts:
log.cache(
"CarbonLink is not connected to any host. Returning empty nodes list"
)
return result
host = self.select_host(metric)
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" %
(metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception as e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" %
(str(host), e))
else:
self.connections[host].add(conn)
if 'error' in result:
log.cache("Error getting data from cache: %s" %
result['error'])
raise CarbonLinkRequestError(result['error'])
log.cache("CarbonLink finished receiving %s from %s" %
(str(metric), str(host)))
return result
def send_request_to_all(self, request):
metric = request['metric']
serialized_request = pickle.dumps(
request, protocol=settings.CARBONLINK_PICKLE_PROTOCOL)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
results = {}
results.setdefault('datapoints', {})
for host in self.hosts:
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" %
(metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception as e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" %
(str(host), e))
else:
self.connections[host].add(conn)
if 'error' in result:
log.cache("Error getting data from cache %s: %s" %
(str(host), result['error']))
else:
if len(result['datapoints']) > 1:
results['datapoints'].update(result['datapoints'])
log.cache("CarbonLink finished receiving %s from %s" %
(str(metric), str(host)))
return results
def recv_response(self, conn):
len_prefix = self.recv_exactly(conn, 4)
body_size = struct.unpack("!L", len_prefix)[0]
body = self.recv_exactly(conn, body_size)
return unpickle.loads(body)
@staticmethod
def recv_exactly(conn, num_bytes):
buf = b''
while len(buf) < num_bytes:
data = conn.recv(num_bytes - len(buf))
if not data:
raise Exception("Connection lost")
buf += data
return buf
class CarbonLinkPool:
def __init__(self, hosts, timeout):
self.hosts = [(server, instance) for (server, port, instance) in hosts]
self.ports = dict(
((server, instance), port) for (server, port, instance) in hosts)
self.timeout = float(timeout)
self.hash_ring = ConsistentHashRing(self.hosts)
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
return self.hash_ring.get_node(metric)
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
try:
connection.connect((server, port))
except:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache(
"CarbonLink cache-query request for %s returned %d datapoints" %
(metric, len(results)))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" %
(metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata',
metric=metric,
key=key,
value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" %
(metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
host = self.select_host(metric)
conn = self.get_connection(host)
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except:
self.last_failure[host] = time.time()
raise
else:
self.connections[host].add(conn)
if 'error' in result:
raise CarbonLinkRequestError(result['error'])
else:
return result
def recv_response(self, conn):
len_prefix = recv_exactly(conn, 4)
body_size = struct.unpack("!L", len_prefix)[0]
body = recv_exactly(conn, body_size)
return pickle.loads(body)
def test_chr_compute_ring_position_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), 59573)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), 35749)
class CarbonLinkPool:
def __init__(self, hosts, timeout):
self.hosts = [(server, instance) for (server, port, instance) in hosts]
self.ports = dict(
((server, instance), port) for (server, port, instance) in hosts)
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" %
(settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(self.hosts)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
key = self.keyfunc(metric)
nodes = []
servers = set()
for node in self.hash_ring.get_nodes(key):
(server, instance) = node
if server in servers:
continue
servers.add(server)
nodes.append(node)
if len(servers) >= settings.REPLICATION_FACTOR:
break
available = [n for n in nodes if self.is_available(n)]
return random.choice(available or nodes)
def is_available(self, host):
now = time.time()
last_fail = self.last_failure.get(host, 0)
return (now - last_fail) < settings.CARBONLINK_RETRY_DELAY
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
try:
connection.connect((server, port))
except:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache(
"CarbonLink cache-query request for %s returned %d datapoints" %
(metric, len(results['datapoints'])))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" %
(metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata',
metric=metric,
key=key,
value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" %
(metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
result = {}
result.setdefault('datapoints', [])
if metric.startswith(settings.CARBON_METRIC_PREFIX):
return self.send_request_to_all(request)
host = self.select_host(metric)
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" %
(metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception, e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" %
(str(host), e))
else:
class CarbonLinkPool(object):
def __init__(self, hosts, timeout):
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict(
((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
servers = set([server for (server, port, instance) in hosts])
if len(servers) < settings.REPLICATION_FACTOR:
raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (
settings.REPLICATION_FACTOR, len(servers)))
self.hash_ring = ConsistentHashRing(
self.hosts, hash_type=settings.CARBONLINK_HASHING_TYPE)
self.keyfunc = load_keyfunc()
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
for host in self.hosts:
self.connections[host] = set()
def select_host(self, metric):
"Returns the carbon host that has data for the given metric"
key = self.keyfunc(metric)
nodes = []
servers = set()
for node in self.hash_ring.get_nodes(key):
(server, instance) = node
if server in servers:
continue
servers.add(server)
nodes.append(node)
if len(servers) >= settings.REPLICATION_FACTOR:
break
available = [ n for n in nodes if self.is_available(n) ]
return random.choice(available or nodes)
def is_available(self, host):
now = time.time()
last_fail = self.last_failure.get(host, 0)
return (now - last_fail) < settings.CARBONLINK_RETRY_DELAY
def get_connection(self, host):
# First try to take one out of the pool for this host
(server, instance) = host
port = self.ports[host]
connectionPool = self.connections[host]
try:
return connectionPool.pop()
except KeyError:
pass #nothing left in the pool, gotta make a new connection
log.cache("CarbonLink creating a new socket for %s" % str(host))
connection = socket.socket()
connection.settimeout(self.timeout)
try:
connection.connect((server, port))
except:
self.last_failure[host] = time.time()
raise
else:
connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
return connection
def query(self, metric):
request = dict(type='cache-query', metric=metric)
results = self.send_request(request)
log.cache("CarbonLink cache-query request for %s returned %d datapoints" % (
metric, len(results['datapoints'])))
return results['datapoints']
def get_metadata(self, metric, key):
request = dict(type='get-metadata', metric=metric, key=key)
results = self.send_request(request)
log.cache("CarbonLink get-metadata request received for %s:%s" % (metric, key))
return results['value']
def set_metadata(self, metric, key, value):
request = dict(type='set-metadata', metric=metric, key=key, value=value)
results = self.send_request(request)
log.cache("CarbonLink set-metadata request received for %s:%s" % (metric, key))
return results
def send_request(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
result = {}
result.setdefault('datapoints', [])
if metric.startswith(settings.CARBON_METRIC_PREFIX):
return self.send_request_to_all(request)
if not self.hosts:
log.cache("CarbonLink is not connected to any host. Returning empty nodes list")
return result
host = self.select_host(metric)
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" % (metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception as e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" % (str(host), e))
else:
self.connections[host].add(conn)
if 'error' in result:
log.cache("Error getting data from cache: %s" % result['error'])
raise CarbonLinkRequestError(result['error'])
log.cache("CarbonLink finished receiving %s from %s" % (str(metric), str(host)))
return result
def send_request_to_all(self, request):
metric = request['metric']
serialized_request = pickle.dumps(request, protocol=-1)
len_prefix = struct.pack("!L", len(serialized_request))
request_packet = len_prefix + serialized_request
results = {}
results.setdefault('datapoints', {})
for host in self.hosts:
conn = self.get_connection(host)
log.cache("CarbonLink sending request for %s to %s" % (metric, str(host)))
try:
conn.sendall(request_packet)
result = self.recv_response(conn)
except Exception as e:
self.last_failure[host] = time.time()
log.cache("Exception getting data from cache %s: %s" % (str(host), e))
else:
self.connections[host].add(conn)
if 'error' in result:
log.cache("Error getting data from cache %s: %s" % (str(host), result['error']))
else:
if len(result['datapoints']) > 1:
results['datapoints'].update(result['datapoints'])
log.cache("CarbonLink finished receiving %s from %s" % (str(metric), str(host)))
return results
def recv_response(self, conn):
len_prefix = self.recv_exactly(conn, 4)
body_size = struct.unpack("!L", len_prefix)[0]
body = self.recv_exactly(conn, body_size)
return unpickle.loads(body)
@staticmethod
def recv_exactly(conn, num_bytes):
buf = b''
while len(buf) < num_bytes:
data = conn.recv(num_bytes - len(buf))
if not data:
raise Exception("Connection lost")
buf += data
return buf