def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
host = config.get('host', '127.0.0.1')
port = config.get('port')
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
transport_options = config.get('transport_options', {})
client_args = dict(
host=host, prefix=prefix, mapred_prefix=mapred_prefix,
protocol=transport_type, client_id=client_id,
transport_options=transport_options)
if port is not None:
client_args['port'] = port
client = RiakClient(**client_args)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client, bucket_prefix, load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
class RiakStorage(AbstractStorage):
bucket_types = {
'users': 'egara-lww',
'users-current': 'egara-unique',
'imap-events': 'egara-lww',
'imap-folders': 'egara-lww',
'imap-folders-current': 'egara-unique',
'imap-message-timeline': 'egara-lww'
}
def __init__(self, *args, **kw):
riak_host = 'localhost'
riak_port = 8098
self.client = RiakClient(
protocol='http',
host=conf['STORAGE'].get('riak_host', riak_host),
http_port=conf['STORAGE'].get('riak_port', riak_port)
)
self.client.set_decoder('application/octet-stream', self._decode_binary)
self.users_cache = CachedDict(ttl=10)
def _decode_binary(self, data):
return str(data).encode("utf-8")
def _get_bucket(self, bucketname):
_type = self.bucket_types.get(bucketname, None)
if _type:
return self.client.bucket_type(_type).bucket(bucketname)
return None
def get(self, key, index, doctype=None, fields=None, **kw):
"""
Standard API for accessing key/value storage
"""
result = None
log.debug("Riak get key %r from %r", key, index)
try:
bucket = self._get_bucket(index)
res = bucket.get(key)
if res and res.data:
result = res.data
except Exception, e:
log.warning("Riak exception: %s", str(e))
result = None
return result
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
transport_class = {
'http': RiakHttpTransport,
'protocol_buffer': RiakPbcTransport,
}.get(transport_type, RiakHttpTransport)
host = config.get('host', '127.0.0.1')
port = config.get('port', 8098)
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
# NOTE: the current riak.RiakClient expects this parameter but
# internally doesn't do anything with it.
solr_transport_class = config.get('solr_transport_class', None)
transport_options = config.get('transport_options', None)
client = RiakClient(host=host,
port=port,
prefix=prefix,
mapred_prefix=mapred_prefix,
transport_class=transport_class,
client_id=client_id,
solr_transport_class=solr_transport_class,
transport_options=transport_options)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client,
bucket_prefix,
load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
transport_class = {
'http': RiakHttpTransport,
'protocol_buffer': RiakPbcTransport,
}.get(transport_type, RiakHttpTransport)
host = config.get('host', '127.0.0.1')
port = config.get('port', 8098)
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
# NOTE: the current riak.RiakClient expects this parameter but
# internally doesn't do anything with it.
solr_transport_class = config.get('solr_transport_class', None)
transport_options = config.get('transport_options', None)
client = RiakClient(host=host, port=port, prefix=prefix,
mapred_prefix=mapred_prefix, transport_class=transport_class,
client_id=client_id, solr_transport_class=solr_transport_class,
transport_options=transport_options)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client, bucket_prefix, load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
