def __init__(self, cluster): """ Initializes the client """ self._cluster = ArakoonManagementEx().getCluster(cluster) self._client = self._cluster.getClient() self._identifier = int(round(random.random() * 10000000)) self._lock = Lock() self._batch_size = 100
def __init__(self, cluster): """ Initializes the client """ self._cluster = ArakoonManagementEx().getCluster(cluster) self._client = self._cluster.getClient() self._identifier = int(round(random.random() * 10000000)) self._lock = Lock() self._batch_size = 100
class ArakoonStore(object): """ Arakoon client wrapper: * Uses json serialisation * Raises generic exception """ def __init__(self, cluster): """ Initializes the client """ self._cluster = ArakoonManagementEx().getCluster(cluster) self._client = self._cluster.getClient() self._identifier = int(round(random.random() * 10000000)) self._lock = Lock() self._batch_size = 100 @locked() def get(self, key): """ Retrieves a certain value for a given key """ try: return json.loads( ArakoonStore._try(self._identifier, self._client.get, key)) except ValueError: raise KeyNotFoundException( 'Could not parse JSON stored for {0}'.format(key)) except ArakoonNotFound as field: raise KeyNotFoundException(field) @locked() def set(self, key, value): """ Sets the value for a key to a given value """ return ArakoonStore._try(self._identifier, self._client.set, key, json.dumps(value)) @locked() def prefix(self, prefix): """ Lists all keys starting with the given prefix """ next_prefix = ArakoonStore._next_key(prefix) batch = None while batch is None or len(batch) > 0: batch = ArakoonStore._try( self._identifier, self._client.range, beginKey=prefix if batch is None else batch[-1], beginKeyIncluded=batch is None, endKey=next_prefix, endKeyIncluded=False, maxElements=self._batch_size) for item in batch: yield item @locked() def delete(self, key): """ Deletes a given key from the store """ try: return ArakoonStore._try(self._identifier, self._client.delete, key) except ArakoonNotFound as field: raise KeyNotFoundException(field) @locked() def nop(self): """ Executes a nop command """ return ArakoonStore._try(self._identifier, self._client.nop) @locked() def exists(self, key): """ Check if key exists """ return ArakoonStore._try(self._identifier, self._client.exists, key) @staticmethod def _try(identifier, method, *args, **kwargs): """ Tries to call a given method, retry-ing if Arakoon is temporary unavailable """ try: last_exception = None tries = 5 while tries > 0: try: start = time.time() return_value = method(*args, **kwargs) duration = time.time() - start if duration > 0.5: logger.warning('Arakoon call {0} took {1}s'.format( method.__name__, round(duration, 2))) return return_value except ArakoonSockReadNoBytes as exception: logger.debug('Error during {0}, retry'.format( method.__name__)) last_exception = exception tries -= 1 time.sleep(1) raise last_exception except ArakoonNotFound: # No extra logging for ArakoonNotFound raise except Exception: logger.error( 'Error during {0}. Process {1}, thread {2}, clientid {3}'. format(method.__name__, os.getpid(), current_thread().ident, identifier)) raise @staticmethod def _next_key(key): """ Calculates the next key (to be used in range queries) """ encoding = 'ascii' # For future python 3 compatibility array = bytearray(str(key), encoding) for index in range(len(array) - 1, -1, -1): array[index] += 1 if array[index] < 128: while array[-1] == 0: array = array[:-1] return str(array.decode(encoding)) array[index] = 0 return '\xff'
class ArakoonStore(object): """ Arakoon client wrapper: * Uses json serialisation * Raises generic exception """ def __init__(self, cluster): """ Initializes the client """ self._cluster = ArakoonManagementEx().getCluster(cluster) self._client = self._cluster.getClient() self._identifier = int(round(random.random() * 10000000)) self._lock = Lock() self._batch_size = 100 @locked() def get(self, key): """ Retrieves a certain value for a given key """ try: return json.loads(ArakoonStore._try(self._identifier, self._client.get, key)) except ValueError: raise KeyNotFoundException('Could not parse JSON stored for {0}'.format(key)) except ArakoonNotFound as field: raise KeyNotFoundException(field) @locked() def set(self, key, value): """ Sets the value for a key to a given value """ return ArakoonStore._try(self._identifier, self._client.set, key, json.dumps(value)) @locked() def prefix(self, prefix): """ Lists all keys starting with the given prefix """ next_prefix = ArakoonStore._next_key(prefix) batch = None while batch is None or len(batch) > 0: batch = ArakoonStore._try(self._identifier, self._client.range, beginKey=prefix if batch is None else batch[-1], beginKeyIncluded=batch is None, endKey=next_prefix, endKeyIncluded=False, maxElements=self._batch_size) for item in batch: yield item @locked() def delete(self, key): """ Deletes a given key from the store """ try: return ArakoonStore._try(self._identifier, self._client.delete, key) except ArakoonNotFound as field: raise KeyNotFoundException(field) @locked() def nop(self): """ Executes a nop command """ return ArakoonStore._try(self._identifier, self._client.nop) @locked() def exists(self, key): """ Check if key exists """ return ArakoonStore._try(self._identifier, self._client.exists, key) @staticmethod def _try(identifier, method, *args, **kwargs): """ Tries to call a given method, retry-ing if Arakoon is temporary unavailable """ try: last_exception = None tries = 5 while tries > 0: try: start = time.time() return_value = method(*args, **kwargs) duration = time.time() - start if duration > 0.5: logger.warning('Arakoon call {0} took {1}s'.format(method.__name__, round(duration, 2))) return return_value except ArakoonSockReadNoBytes as exception: logger.debug('Error during {0}, retry'.format(method.__name__)) last_exception = exception tries -= 1 time.sleep(1) raise last_exception except ArakoonNotFound: # No extra logging for ArakoonNotFound raise except Exception: logger.error('Error during {0}. Process {1}, thread {2}, clientid {3}'.format( method.__name__, os.getpid(), current_thread().ident, identifier )) raise @staticmethod def _next_key(key): """ Calculates the next key (to be used in range queries) """ encoding = 'ascii' # For future python 3 compatibility array = bytearray(str(key), encoding) for index in range(len(array) - 1, -1, -1): array[index] += 1 if array[index] < 128: while array[-1] == 0: array = array[:-1] return str(array.decode(encoding)) array[index] = 0 return '\xff'