def proxy_client(server, bucket):
#for this bucket on this node what is the proxy ?
rest = RestConnection(server)
bucket_info = rest.get_bucket(bucket)
nodes = bucket_info.nodes
for node in nodes:
if node.ip == server.ip and int(node.port) == int(server.port):
client = MemcachedClient(server.ip, node.moxi)
vBuckets = RestConnection(server).get_vbuckets(bucket)
client.vbucket_count = len(vBuckets)
if bucket_info.authType == "sasl":
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
return client
raise Exception("unable to find {0} in get_nodes()".format(server.ip))
class VBucketAwareCouchbaseClient(object):
#poll server every few seconds to see if the vbucket-map
#has changes
def __init__(self, url, bucket, password="", verbose=False):
self.log = logger.logger("VBucketAwareMemcachedClient")
self.bucket = bucket
self.rest_username = bucket
self.rest_password = password
self._memcacheds = {}
self._vBucketMap = {}
self._vBucketMap_lock = Lock()
self._vBucketMapFastForward = {}
self._vBucketMapFastForward_lock = Lock()
#TODO: use regular expressions to parse the url
server = {}
if not bucket:
raise InvalidArgumentException("bucket can not be an empty string", parameters="bucket")
if not url:
raise InvalidArgumentException("url can not be an empty string", parameters="url")
if url.find("http://") != -1 and url.rfind(":") != -1 and url.find("/pools/default") != -1:
server["ip"] = url[url.find("http://") + len("http://"):url.rfind(":")]
server["port"] = url[url.rfind(":") + 1:url.find("/pools/default")]
server["username"] = self.rest_username
server["password"] = self.rest_password
self.servers = [server]
self.servers_lock = Lock()
self.rest = RestConnection(server)
self.reconfig_vbucket_map()
self.init_vbucket_connections()
self.dispatcher = CommandDispatcher(self)
self.dispatcher_thread = Thread(name="dispatcher-thread", target=self._start_dispatcher)
self.dispatcher_thread.daemon = True
self.dispatcher_thread.start()
self.streaming_thread = Thread(name="streaming", target=self._start_streaming, args=())
self.streaming_thread.daemon = True
self.streaming_thread.start()
self.verbose = verbose
def _start_dispatcher(self):
self.dispatcher.dispatch()
def _start_streaming(self):
# this will dynamically update vBucketMap, vBucketMapFastForward, servers
urlopener = urllib.FancyURLopener()
urlopener.prompt_user_passwd = lambda host, realm: (self.rest_username, self.rest_password)
current_servers = True
while current_servers:
self.servers_lock.acquire()
current_servers = deepcopy(self.servers)
self.servers_lock.release()
for server in current_servers:
response = urlopener.open("http://{0}:{1}/pools/default/bucketsStreaming/{2}".format(server["ip"], server["port"], self.bucket))
while response:
try:
line = response.readline()
if not line:
# try next server if we get an EOF
response.close()
break
except:
# try next server if we fail to read
response.close()
break
try:
data = json.loads(line)
except:
continue
serverlist = data['vBucketServerMap']['serverList']
vbucketmapfastforward = {}
index = 0
if 'vBucketMapForward' in data['vBucketServerMap']:
for vbucket in data['vBucketServerMap']['vBucketMapForward']:
vbucketmapfastforward[index] = serverlist[vbucket[0]]
index += 1
self._vBucketMapFastForward_lock.acquire()
self._vBucketMapFastForward = deepcopy(vbucketmapfastforward)
self._vBucketMapFastForward_lock.release()
vbucketmap = {}
index = 0
for vbucket in data['vBucketServerMap']['vBucketMap']:
vbucketmap[index] = serverlist[vbucket[0]]
index += 1
# only update vBucketMap if we don't have a fastforward
# on a not_mb_vbucket error, we already update the
# vBucketMap from the fastforward map
if not vbucketmapfastforward:
self._vBucketMap_lock.acquire()
self._vBucketMap = deepcopy(vbucketmap)
self._vBucketMap_lock.release()
new_servers = []
nodes = data["nodes"]
for node in nodes:
if node["clusterMembership"] == "active" and node["status"] == "healthy":
hostport = node["hostname"]
new_servers.append({"ip":hostport.split(":")[0],
"port":int(hostport.split(":")[1]),
"username":self.rest_username,
"password":self.rest_password})
new_servers.sort()
self.servers_lock.acquire()
self.servers = deepcopy(new_servers)
self.servers_lock.release()
def init_vbucket_connections(self):
# start up all vbucket connections
self._vBucketMap_lock.acquire()
vbucketcount = len(self._vBucketMap)
self._vBucketMap_lock.release()
for i in range(vbucketcount):
self.start_vbucket_connection(i)
def start_vbucket_connection(self,vbucket):
self._vBucketMap_lock.acquire()
server = deepcopy(self._vBucketMap[vbucket])
self._vBucketMap_lock.release()
serverIp, serverPort = server.split(":")
if not server in self._memcacheds:
self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket)
def start_vbucket_fastforward_connection(self,vbucket):
self._vBucketMapFastForward_lock.acquire()
if not vbucket in self._vBucketMapFastForward:
self._vBucketMapFastForward_lock.release()
return
server = deepcopy(self._vBucketMapFastForward[vbucket])
self._vBucketMapFastForward_lock.release()
serverIp, serverPort = server.split(":")
if not server in self._memcacheds:
self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket)
def restart_vbucket_connection(self,vbucket):
self._vBucketMap_lock.acquire()
server = deepcopy(self._vBucketMap[vbucket])
self._vBucketMap_lock.release()
serverIp, serverPort = server.split(":")
if server in self._memcacheds:
self._memcacheds[server].close()
self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket)
def reconfig_vbucket_map(self, vbucket=-1):
vb_ready = RestHelper(self.rest).vbucket_map_ready(self.bucket, 60)
if not vb_ready:
raise Exception("vbucket map is not ready for bucket {0}".format(self.bucket))
vBuckets = self.rest.get_vbuckets(self.bucket)
self.vbucket_count = len(vBuckets)
bucket_info = self.rest.get_bucket(self.bucket)
nodes = bucket_info.nodes
self._vBucketMap_lock.acquire()
for vBucket in vBuckets:
if vBucket.id == vbucket or vbucket == -1:
self._vBucketMap[vBucket.id] = vBucket.master
self._vBucketMap_lock.release()
def memcached(self, key, fastforward=False):
self._vBucketMap_lock.acquire()
self._vBucketMapFastForward_lock.acquire()
vBucketId = crc32.crc32_hash(key) & (len(self._vBucketMap) - 1)
if fastforward and vBucketId in self._vBucketMapFastForward:
# only try the fastforward if we have an entry
# otherwise we just wait for the main map to update
self.start_vbucket_fastforward_connection(vBucketId)
self._vBucketMap[vBucketId] = self._vBucketMapFastForward[vBucketId]
if vBucketId not in self._vBucketMap:
msg = "vbucket map does not have an entry for vb : {0}"
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
raise Exception(msg.format(vBucketId))
if self._vBucketMap[vBucketId] not in self._memcacheds:
msg = "smart client does not have a mc connection for server : {0}"
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
raise Exception(msg.format(self._vBucketMap[vBucketId]))
r = self._memcacheds[self._vBucketMap[vBucketId]]
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
return r
def vbucketid(self, key):
self._vBucketMap_lock.acquire()
r = crc32.crc32_hash(key) & (len(self._vBucketMap) - 1)
self._vBucketMap_lock.release()
return r
def done(self):
if self.dispatcher:
self.dispatcher.shutdown()
if self.verbose:
self.log.info("dispatcher shutdown invoked")
[self._memcacheds[ip].close() for ip in self._memcacheds]
if self.verbose:
self.log.info("closed all memcached open connections")
self.dispatcher = None
def _respond(self, item, event):
timeout = 30
event.wait(timeout)
if not event.is_set():
# if we timeout, then try to reconnect to the server
# responsible for this vbucket
self.restart_vbucket_connection(self.vbucketid(item['key']))
raise MemcachedTimeoutException(item, timeout)
if "error" in item["response"]:
raise item["response"]["error"]
return item["response"]["return"]
def get(self, key):
event = Event()
item = {"operation": "get", "key": key, "event": event, "response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def gat(self, key, expiry):
event = Event()
item = {"operation": "gat", "key": key, "expiry": expiry, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def touch(self, key, expiry):
event = Event()
item = {"operation": "touch", "key": key, "expiry": expiry, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def cas(self, key, expiry, flags, old_value, value):
event = Event()
item = {"operation": "cas", "key": key, "expiry": expiry, "flags": flags, "old_value": old_value, "value": value
, "event": event, "response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def decr(self, key, amount=1, init=0, expiry=0):
event = Event()
item = {"operation": "decr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def set(self, key, expiry, flags, value):
event = Event()
item = {"operation": "set", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def add(self, key, expiry, flags, value):
event = Event()
item = {"operation": "add", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def delete(self, key, cas=0):
event = Event()
item = {"operation": "delete", "key": key, "cas": cas, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def prepend(self, key, value, cas=0):
event = Event()
item = {"operation": "prepend", "key": key, "cas": cas, "value": value, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def getl(self, key, expiry=15):
event = Event()
item = {"operation": "getl", "key": key, "expiry": expiry, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def replace(self, key, expiry, flags, value):
event = Event()
item = {"operation": "replace", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def incr(self, key, amount=1, init=0, expiry=0):
event = Event()
item = {"operation": "incr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event,
"response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
class VBucketAwareCouchbaseClient(object):
#poll server every few seconds to see if the vbucket-map
#has changes
def __init__(self, url, bucket, password="", verbose=False):
self.log = logger.logger("VBucketAwareMemcachedClient")
self.bucket = bucket
self.rest_username = bucket
self.rest_password = password
self._memcacheds = {}
self._vBucketMap = {}
self._vBucketMap_lock = Lock()
self._vBucketMapFastForward = {}
self._vBucketMapFastForward_lock = Lock()
#TODO: use regular expressions to parse the url
server = {}
if not bucket:
raise InvalidArgumentException("bucket can not be an empty string",
parameters="bucket")
if not url:
raise InvalidArgumentException("url can not be an empty string",
parameters="url")
if url.find("http://") != -1 and url.rfind(":") != -1 and url.find(
"/pools/default") != -1:
server["ip"] = url[url.find("http://") +
len("http://"):url.rfind(":")]
server["port"] = url[url.rfind(":") + 1:url.find("/pools/default")]
server["username"] = self.rest_username
server["password"] = self.rest_password
self.servers = [server]
self.servers_lock = Lock()
self.rest = RestConnection(server)
self.reconfig_vbucket_map()
self.init_vbucket_connections()
self.dispatcher = CommandDispatcher(self)
self.dispatcher_thread = Thread(name="dispatcher-thread",
target=self._start_dispatcher)
self.dispatcher_thread.daemon = True
self.dispatcher_thread.start()
self.streaming_thread = Thread(name="streaming",
target=self._start_streaming,
args=())
self.streaming_thread.daemon = True
self.streaming_thread.start()
self.verbose = verbose
def _start_dispatcher(self):
self.dispatcher.dispatch()
def _start_streaming(self):
# this will dynamically update vBucketMap, vBucketMapFastForward, servers
urlopener = urllib.FancyURLopener()
urlopener.prompt_user_passwd = lambda host, realm: (self.rest_username,
self.rest_password)
current_servers = True
while current_servers:
self.servers_lock.acquire()
current_servers = deepcopy(self.servers)
self.servers_lock.release()
for server in current_servers:
response = urlopener.open(
"http://{0}:{1}/pools/default/bucketsStreaming/{2}".format(
server["ip"], server["port"], self.bucket))
while response:
try:
line = response.readline()
if not line:
# try next server if we get an EOF
response.close()
break
except:
# try next server if we fail to read
response.close()
break
try:
data = json.loads(line)
except:
continue
serverlist = data['vBucketServerMap']['serverList']
vbucketmapfastforward = {}
index = 0
if 'vBucketMapForward' in data['vBucketServerMap']:
for vbucket in data['vBucketServerMap'][
'vBucketMapForward']:
vbucketmapfastforward[index] = serverlist[
vbucket[0]]
index += 1
self._vBucketMapFastForward_lock.acquire()
self._vBucketMapFastForward = deepcopy(
vbucketmapfastforward)
self._vBucketMapFastForward_lock.release()
vbucketmap = {}
index = 0
for vbucket in data['vBucketServerMap']['vBucketMap']:
vbucketmap[index] = serverlist[vbucket[0]]
index += 1
# only update vBucketMap if we don't have a fastforward
# on a not_mb_vbucket error, we already update the
# vBucketMap from the fastforward map
if not vbucketmapfastforward:
self._vBucketMap_lock.acquire()
self._vBucketMap = deepcopy(vbucketmap)
self._vBucketMap_lock.release()
new_servers = []
nodes = data["nodes"]
for node in nodes:
if node["clusterMembership"] == "active" and node[
"status"] == "healthy":
hostport = node["hostname"]
new_servers.append({
"ip":
hostport.split(":")[0],
"port":
int(hostport.split(":")[1]),
"username":
self.rest_username,
"password":
self.rest_password
})
new_servers.sort()
self.servers_lock.acquire()
self.servers = deepcopy(new_servers)
self.servers_lock.release()
def init_vbucket_connections(self):
# start up all vbucket connections
self._vBucketMap_lock.acquire()
vbucketcount = len(self._vBucketMap)
self._vBucketMap_lock.release()
for i in range(vbucketcount):
self.start_vbucket_connection(i)
def start_vbucket_connection(self, vbucket):
self._vBucketMap_lock.acquire()
server = deepcopy(self._vBucketMap[vbucket])
self._vBucketMap_lock.release()
serverIp, serverPort = server.split(":")
if not server in self._memcacheds:
self._memcacheds[server] = MemcachedClientHelper.direct_client(
self.rest, serverIp, serverPort, self.bucket)
def start_vbucket_fastforward_connection(self, vbucket):
self._vBucketMapFastForward_lock.acquire()
if not vbucket in self._vBucketMapFastForward:
self._vBucketMapFastForward_lock.release()
return
server = deepcopy(self._vBucketMapFastForward[vbucket])
self._vBucketMapFastForward_lock.release()
serverIp, serverPort = server.split(":")
if not server in self._memcacheds:
self._memcacheds[server] = MemcachedClientHelper.direct_client(
self.rest, serverIp, serverPort, self.bucket)
def restart_vbucket_connection(self, vbucket):
self._vBucketMap_lock.acquire()
server = deepcopy(self._vBucketMap[vbucket])
self._vBucketMap_lock.release()
serverIp, serverPort = server.split(":")
if server in self._memcacheds:
self._memcacheds[server].close()
self._memcacheds[server] = MemcachedClientHelper.direct_client(
self.rest, serverIp, serverPort, self.bucket)
def reconfig_vbucket_map(self, vbucket=-1):
vb_ready = RestHelper(self.rest).vbucket_map_ready(self.bucket, 60)
if not vb_ready:
raise Exception("vbucket map is not ready for bucket {0}".format(
self.bucket))
vBuckets = self.rest.get_vbuckets(self.bucket)
self.vbucket_count = len(vBuckets)
bucket_info = self.rest.get_bucket(self.bucket)
nodes = bucket_info.nodes
self._vBucketMap_lock.acquire()
for vBucket in vBuckets:
if vBucket.id == vbucket or vbucket == -1:
self._vBucketMap[vBucket.id] = vBucket.master
self._vBucketMap_lock.release()
def memcached(self, key, fastforward=False):
self._vBucketMap_lock.acquire()
self._vBucketMapFastForward_lock.acquire()
vBucketId = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1)
if fastforward and vBucketId in self._vBucketMapFastForward:
# only try the fastforward if we have an entry
# otherwise we just wait for the main map to update
self.start_vbucket_fastforward_connection(vBucketId)
self._vBucketMap[vBucketId] = self._vBucketMapFastForward[
vBucketId]
if vBucketId not in self._vBucketMap:
msg = "vbucket map does not have an entry for vb : {0}"
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
raise Exception(msg.format(vBucketId))
if self._vBucketMap[vBucketId] not in self._memcacheds:
msg = "smart client does not have a mc connection for server : {0}"
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
raise Exception(msg.format(self._vBucketMap[vBucketId]))
r = self._memcacheds[self._vBucketMap[vBucketId]]
self._vBucketMapFastForward_lock.release()
self._vBucketMap_lock.release()
return r
def vbucketid(self, key):
self._vBucketMap_lock.acquire()
r = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1)
self._vBucketMap_lock.release()
return r
def done(self):
if self.dispatcher:
self.dispatcher.shutdown()
if self.verbose:
self.log.info("dispatcher shutdown invoked")
[self._memcacheds[ip].close() for ip in self._memcacheds]
if self.verbose:
self.log.info("closed all memcached open connections")
self.dispatcher = None
def _respond(self, item, event):
timeout = 30
event.wait(timeout)
if not event.is_set():
# if we timeout, then try to reconnect to the server
# responsible for this vbucket
self.restart_vbucket_connection(self.vbucketid(item['key']))
raise MemcachedTimeoutException(item, timeout)
if "error" in item["response"]:
raise item["response"]["error"]
return item["response"]["return"]
def get(self, key):
event = Event()
item = {"operation": "get", "key": key, "event": event, "response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def gat(self, key, expiry):
event = Event()
item = {
"operation": "gat",
"key": key,
"expiry": expiry,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def touch(self, key, expiry):
event = Event()
item = {
"operation": "touch",
"key": key,
"expiry": expiry,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def cas(self, key, expiry, flags, old_value, value):
event = Event()
item = {
"operation": "cas",
"key": key,
"expiry": expiry,
"flags": flags,
"old_value": old_value,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def decr(self, key, amount=1, init=0, expiry=0):
event = Event()
item = {
"operation": "decr",
"key": key,
"amount": amount,
"init": init,
"expiry": expiry,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def set(self, key, expiry, flags, value):
event = Event()
item = {
"operation": "set",
"key": key,
"expiry": expiry,
"flags": flags,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def add(self, key, expiry, flags, value):
event = Event()
item = {
"operation": "add",
"key": key,
"expiry": expiry,
"flags": flags,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def append(self, key, value, cas=0):
event = Event()
item = {
"operation": "append",
"key": key,
"cas": cas,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def delete(self, key, cas=0):
event = Event()
item = {
"operation": "delete",
"key": key,
"cas": cas,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def prepend(self, key, value, cas=0):
event = Event()
item = {
"operation": "prepend",
"key": key,
"cas": cas,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def getl(self, key, expiry=15):
event = Event()
item = {
"operation": "getl",
"key": key,
"expiry": expiry,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def replace(self, key, expiry, flags, value):
event = Event()
item = {
"operation": "replace",
"key": key,
"expiry": expiry,
"flags": flags,
"value": value,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
def incr(self, key, amount=1, init=0, expiry=0):
event = Event()
item = {
"operation": "incr",
"key": key,
"amount": amount,
"init": init,
"expiry": expiry,
"event": event,
"response": {}
}
self.dispatcher.put(item)
return self._respond(item, event)
