def __init__(self, hosts, secret=None): if secret: secret = secret[:16] secret += chr(0) * (16 - len(secret)) self.secret = struct.pack('16c', *secret) else: self.secret = None self.connections = {} self.input_buffer = [] if type(hosts) == str: self.nodes = parse_hosts_string(hosts) if not self.nodes: raise Exception('Can\'t parse the hosts string ' + hosts) elif type(hosts) == list: validate_hosts_array(hosts) self.nodes = hosts self.chash = CHash([node['label'] for node in self.nodes], 200) random.seed()
def __init__(self, hosts, secret=None): if secret: secret = secret[:16] secret += chr(0) * (16 - len(secret)) self.secret = struct.pack('16c', *secret) else: self.secret = None self.connections = { } self.input_buffer = [] if type(hosts) == str: self.nodes = parse_hosts_string(hosts) if not self.nodes: raise Exception('Can\'t parse the hosts string ' + hosts) elif type(hosts) == list: validate_hosts_array(hosts) self.nodes = hosts self.chash = CHash([ node['label'] for node in self.nodes ], 200) random.seed()
class ShardcacheClient: "Simple Python client for shardcache" def __init__(self, hosts, secret=None): if secret: secret = secret[:16] secret += chr(0) * (16 - len(secret)) self.secret = struct.pack('16c', *secret) else: self.secret = None self.connections = { } self.input_buffer = [] if type(hosts) == str: self.nodes = parse_hosts_string(hosts) if not self.nodes: raise Exception('Can\'t parse the hosts string ' + hosts) elif type(hosts) == list: validate_hosts_array(hosts) self.nodes = hosts self.chash = CHash([ node['label'] for node in self.nodes ], 200) random.seed() def get(self, key): records = self._send_message(message = MSG_GET, records = [key], node = self.chash.lookup(key)) if records: return records[0] return None def offset(self, key, offset=0, length=0): input_records = [key, struct.pack('!L', offset)] if length > 0: input_records.append(struct.pack('!L', length)) records = self._send_message(message = MSG_OFX, records = input_records, node = self.chash.lookup(key)) if records: return records[0] return None def stats(self, node=None): if node: records = self._send_message(message = MSG_STS, node = node) if records: return records[0] else: stats = [] for node in self.nodes: node_stats = { } records = self._send_message(message = MSG_STS, node = node['label']) node_stats = { 'node': node['label'], 'stats': records[0] } if records else { } stats.append(node_stats) return stats return None def check(self, node=None): if node: records = self._send_message(message = MSG_CHK, node = node) if records and records[0] == RES_OK: return True return False statuses = [] for node in self.nodes: records = self._send_message(message = MSG_CHK, node = node['label']) status = True if not records or records[0] != RES_OK: status = False statuses.append({ 'node':node['label'], 'status':status }) return statuses def set(self, key, value, expire=None): input_records = [key, value] if expire: input_records.append(struct.pack('!L', expire)) records = self._send_message(message = MSG_SET, records = input_records, node = self.chash.lookup(key)) if records and records[0] == RES_OK: return True; return False def add(self, key, value, expire=None): input_records = [key, value] if expire: input_records.append(struct.pack('!L', expire)) records = self._send_message(message = MSG_ADD, records = input_records, node = self.chash.lookup(key)) if records and records[0] == RES_OK: return True return False def delete(self, key): records = self.__send_message(message = MSG_DEL, records = [key], node = self.chash.lookup(key)) if records and records[0] == RES_OK: return True; return False def evict(self, key): records = self.__send_message(message = MSG_EVI, records = [key], node = self.chash.lookup(key)) if records and records[0] == RES_OK: return True; return False def _get_connection(self, host, port): host_key = host + ":" + str(port) fd = self.connections.get(host_key, None) if fd: fd.setblocking(1) # test if the socket is still valid if fd.send(MSG_NOOP) != 1: print >>sys.stderr, 'socket ' + str(fd) + ' is not valid anymore' fd = None retries = 0; # retry at most 10 times while not fd and retries < 10: fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if fd: try: fd.connect((host, int(port))) self.connections[host_key] = fd except Exception, e: print >>sys.stderr, 'Can\'t connect to %s:%d. Exception type is %s' % (host, int(port), `e`) else: # start sleeping 128 microsecs # and increase the sleep time by a power of two at every retry # (so 256 microsecs the seceond time, 512 the third and so on) sleep_time = (1<<(7+retries))/1e6 time.sleep(sleep_time) retries += 1 return fd
class ShardcacheClient: "Simple Python client for shardcache" def __init__(self, hosts, secret=None): if secret: secret = secret[:16] secret += chr(0) * (16 - len(secret)) self.secret = struct.pack('16c', *secret) else: self.secret = None self.connections = {} self.input_buffer = [] if type(hosts) == str: self.nodes = parse_hosts_string(hosts) if not self.nodes: raise Exception('Can\'t parse the hosts string ' + hosts) elif type(hosts) == list: validate_hosts_array(hosts) self.nodes = hosts self.chash = CHash([node['label'] for node in self.nodes], 200) random.seed() def get(self, key): records = self._send_message(message=MSG_GET, records=[key], node=self.chash.lookup(key)) if records: return records[0] return None def offset(self, key, offset=0, length=0): input_records = [key, struct.pack('!L', offset)] if length > 0: input_records.append(struct.pack('!L', length)) records = self._send_message(message=MSG_OFX, records=input_records, node=self.chash.lookup(key)) if records: return records[0] return None def stats(self, node=None): if node: records = self._send_message(message=MSG_STS, node=node) if records: return records[0] else: stats = [] for node in self.nodes: node_stats = {} records = self._send_message(message=MSG_STS, node=node['label']) node_stats = { 'node': node['label'], 'stats': records[0] } if records else {} stats.append(node_stats) return stats return None def check(self, node=None): if node: records = self._send_message(message=MSG_CHK, node=node) if records and records[0] == RES_OK: return True return False statuses = [] for node in self.nodes: records = self._send_message(message=MSG_CHK, node=node['label']) status = True if not records or records[0] != RES_OK: status = False statuses.append({'node': node['label'], 'status': status}) return statuses def set(self, key, value, expire=None): input_records = [key, value] if expire: input_records.append(struct.pack('!L', expire)) records = self._send_message(message=MSG_SET, records=input_records, node=self.chash.lookup(key)) if records and records[0] == RES_OK: return True return False def add(self, key, value, expire=None): input_records = [key, value] if expire: input_records.append(struct.pack('!L', expire)) records = self._send_message(message=MSG_ADD, records=input_records, node=self.chash.lookup(key)) if records and records[0] == RES_OK: return True return False def delete(self, key): records = self.__send_message(message=MSG_DEL, records=[key], node=self.chash.lookup(key)) if records and records[0] == RES_OK: return True return False def evict(self, key): records = self.__send_message(message=MSG_EVI, records=[key], node=self.chash.lookup(key)) if records and records[0] == RES_OK: return True return False def _get_connection(self, host, port): host_key = host + ":" + str(port) fd = self.connections.get(host_key, None) if fd: fd.setblocking(1) # test if the socket is still valid if fd.send(MSG_NOOP) != 1: print >> sys.stderr, 'socket ' + str( fd) + ' is not valid anymore' fd = None retries = 0 # retry at most 10 times while not fd and retries < 10: fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if fd: try: fd.connect((host, int(port))) self.connections[host_key] = fd except Exception, e: print >> sys.stderr, 'Can\'t connect to %s:%d. Exception type is %s' % ( host, int(port), ` e `) else: # start sleeping 128 microsecs # and increase the sleep time by a power of two at every retry # (so 256 microsecs the seceond time, 512 the third and so on) sleep_time = (1 << (7 + retries)) / 1e6 time.sleep(sleep_time) retries += 1 return fd