def writeSinglePair(pair, bucketName, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) bucket = rc.bucket(bucketName) # create key value pairs to stock in riak key = str(str(pair[0]) + "_" + str(pair[1])) val = { "StockA": pair[0], "StockB": pair[1], "Date": pair[2], "CloseA": pair[3], "CloseB": pair[4], "ZScore": pair[5], "Beta": pair[6], "SignalMean": pair[7], "SignalSD": pair[8], } myDate = pair[2].split("-") obj = RiakObject(rc, bucket, key) # add 2i tags obj.add_index("stocka_bin", str(pair[0])) obj.add_index("stockb_bin", str(pair[3])) obj.add_index("year_int", int(myDate[0])) obj.add_index("month_int", int(myDate[1])) obj.add_index("day_int", int(myDate[2])) obj.content_type = "text/json" obj.data = val obj.data = json.dumps(val) # store obj.store() # return a list of written pairs return pair
def writeHistory(ticker, data, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) bucket = rc.bucket('stocks') gtemp = data if len(gtemp) == 0: return 0 else: for j in range(0, len(gtemp.index)): #upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + '_' + str(date)) riakVal = {'OPEN': gtemp.values[j,0],\ 'HIGH': gtemp.values[j,1],\ 'LOW': gtemp.values[j,2], \ 'CLOSE': gtemp.values[j,3], \ 'VOLUME': gtemp.values[j,4],\ 'DATE': str(date),\ 'TICKER': str(ticker)} obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = 'text/json' #obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def writeSinglePair(pair, bucketName, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) bucket = rc.bucket(bucketName) #create key value pairs to stock in riak key = str(str(pair[0]) + '_' + str(pair[1])) val = {'StockA': pair[0], \ 'StockB': pair[1], \ 'Date': pair[2],\ 'CloseA': pair[3], \ 'CloseB': pair[4], \ 'ZScore': pair[5],\ 'Beta': pair[6],\ 'SignalMean': pair[7],\ 'SignalSD': pair[8]} myDate = pair[2].split('-') obj = RiakObject(rc, bucket, key) #add 2i tags obj.add_index("stocka_bin", str(pair[0])) obj.add_index("stockb_bin", str(pair[3])) obj.add_index("year_int", int(myDate[0])) obj.add_index("month_int", int(myDate[1])) obj.add_index("day_int", int(myDate[2])) obj.content_type = 'text/json' obj.data = val obj.data = json.dumps(val) #store obj.store() #return a list of written pairs return pair
def test_close_underlying_socket_retry(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, transport_options={"max_attempts": 2}) bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('barbaz', rand) obj.store() obj = bucket.get('barbaz') self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns conns[0].sock.close() # This should work, since we have a retry obj = bucket.get('barbaz') self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand)
class DS_Riak(DS): """ Riak DS """ def __init__(self, host=PB_HOST, port=PB_PORT, bucketname='_scads'): """ Params: host: Hostname or IP Address of the riak database port: Port of the riak database bucketname: Where to store stuff """ self.client = RiakClient(host, port, transport_class=RiakPbcTransport, transport_options={'max_attempts': 3}) self.bucket = self.client.bucket(bucketname) def store(self, key, value, indexes=None): obj = self.bucket.get(key) obj.set_data(value) if indexes is not None: for index in indexes: obj.add_index('scads_bin', index) obj.store() def get_index(self, index): return self.bucket.get_index('scads_bin', index) def get(self, key): d = self.bucket.get(key).get_data() if d is None: return None return d def clear(self): for key in self.bucket.get_keys(): self.bucket.get(key).delete()
def selection_function(request): RiakClient() RiakClient(protocol='http', host='127.0.0.1', http_port=8098) RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}]) myClient = RiakClient() myBucket = myClient.bucket('users') keys = myBucket.get_keys() usuarios = Clientes() lista2 = [] for i in range(len(keys)): lista = { 'name': myBucket.get(keys[i]).data['name'], 'location': myBucket.get(keys[i]).data['location'], 'role': myBucket.get(keys[i]).data['role'], 'fullname': myBucket.get(keys[i]).data['fullname'] } lista2.append(lista) # for i in range(len(lista1)): # #lista.append(myBucket.get(keys[i]).data['name']) # usuarios.nome = str(lista1[i][0]) # #lista.append(myBucket.get(keys[i]).data['location']) # usuarios.locacao = str(lista1[i][1]) # #lista.append(myBucket.get(keys[i]).data['role']) # usuarios.profissao = str(lista1[i][2]) # #lista.append(myBucket.get(keys[i]).data['fulname']) # usuarios.apelido = str(lista1[i][3]) # usuarios.save() usuarios = Clientes.objects.all() return render(request, 'usuario.html', {'usuarios': lista2})
def test_close_underlying_socket_retry(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, transport_options={"max_attempts": 2}) bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('barbaz', rand) obj.store() obj = bucket.get('barbaz') self.assertTrue(obj.exists) self.assertEqual(obj.bucket.name, 'bucket_test_close') self.assertEqual(obj.key, 'barbaz') self.assertEqual(obj.data, rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns conns[0].sock.close() # This should work, since we have a retry obj = bucket.get('barbaz') self.assertTrue(obj.exists) self.assertEqual(obj.bucket.name, 'bucket_test_close') self.assertEqual(obj.key, 'barbaz') self.assertEqual(obj.data, rand)
def register(host,user_id,time): currenttld = extractdom(host) riakclient = RiakClient() #bucket : trackers, key:user_id, object:json map str_bucket = 'trackers' bucket = riakclient.bucket(str_bucket) str_key = user_id obj = RiakObject(riakclient, bucket, str_key)
def conn(self): """Open connection on Riak DataBase""" client = RiakClient(protocol=self.conf.get("protocol"), http_port=self.conf.get("port"), host=self.conf.get("host")) conn = client.bucket(self.conf.get("db")) conn.enable_search() return conn
class UserService: def __init__(self): self.users_bucket = settings.RIAK_USERS_BUCKET self.riak = RiakClient(host=settings.RIAK_HOST, port=settings.RIAK_PORT) def save(self, user_data): key = self.generate_key() password = user_data['password'] user_data['encrypted_password'] = make_password('sha1', password) del user_data['password'] user = self.riak.bucket(self.users_bucket).new(key, user_data) user.store() return key def get(self, user_id): user = self.riak.bucket(settings.RIAK_USERS_BUCKET).get(str(user_id)) if user.exists(): return user.get_data() else: return None def login(self, username, password): try: user = self.riak.search(settings.RIAK_USERS_BUCKET, "username:%s" % username).run() except ValueError: user = None if user: user = user[0] user.set_bucket(str(user.get_bucket())) user.set_key(str(user.get_key())) user_data = user.get().get_data() if check_password(password, user_data['encrypted_password']): return user else: return False else: return False def generate_key(self): return uuid.uuid1().hex
def conn(self): """Open connection on Riak DataBase""" client = RiakClient( protocol=self.conf.get("protocol"), http_port=self.conf.get("port"), host=self.conf.get("host")) conn = client.bucket(self.conf.get("db")) conn.enable_search() return conn
def clear_expired(cls): # TODO: get_keys() method is not efficent operation for searching sessions, use search feature or mapreduce or ... client = RiakClient(host=RIAK_DB_HOST, port=RIAK_DB_PORT, transport_class=RiakPbcTransport) sessions = client.bucket(RIAK_DB_BUCKET) now = int(timezone.now().strftime("%s")) for key in sessions.get_keys(): data = sessions.get(key) if int(data.get_usermeta()['expire_age']) < now: data.delete()
def getDataByTicker(ticker, dataSource, start, end, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) # get daily data for each ticker gtemp = pd.DataFrame() bucket = rc.bucket("stocks") try: gtemp = DataReader(ticker, dataSource, start, end) print ticker except: pass # didnt get any data if len(gtemp) == 0: return 0 # got data else: for j in range(0, len(gtemp.index)): # upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + "_" + str(date)) riakVal = { "OPEN": gtemp.values[j, 0], "HIGH": gtemp.values[j, 1], "LOW": gtemp.values[j, 2], "CLOSE": gtemp.values[j, 3], "VOLUME": gtemp.values[j, 4], "DATE": str(date), "TICKER": str(ticker), } obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = "text/json" # obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def get_flower(lang, user, repo_name): client = RiakClient(host=riak_ip, pb_port=riak_port, protocol="pbc") bucket = client.bucket("%s/flowers" % lang) repo = "%s/%s" % (user, repo_name) graph = bucket.get(repo).data if graph is None: abort(404) _nodes, _links = graph nodes = [{"id": repo, "group": degree} for (repo, degree) in _nodes] del _nodes links = [{"source": repo_a, "target": repo_b} for \ (repo_a, repo_b) in _links] del _links return jsonify({"nodes": nodes, "links": links})
def run_cluster_server(riak_conf): riak_hosts = [] with open(riak_conf, "r") as fin: for line in fin: riak_hosts.append({"host": line.replace("\n", ""), "http_port": RIAK_PORT}) cluster = RiakClient(nodes=riak_hosts) bucket = cluster.bucket("HashedData") random_str_arr = [] random_str_arr = run_exp_init(socket.gethostname()) insert_throuput = run_exp_insert(bucket, random_str_arr) lookup_throuput = run_exp_lookup(bucket, random_str_arr) delete_throuput = run_exp_delete(bucket, random_str_arr) overall_throuput = (insert_throuput + lookup_throuput + delete_throuput) / 3 print("Overall Throughput: " + str(overall_throuput) + " OPs/s")
def getDataByTicker(ticker, dataSource, start, end, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) #get daily data for each ticker gtemp = pd.DataFrame() bucket = rc.bucket('stocks') try: gtemp = DataReader(ticker, dataSource, start, end) print ticker except: pass #didnt get any data if len(gtemp) == 0: return 0 #got data else: for j in range(0, len(gtemp.index)): #upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + '_' + str(date)) riakVal = {'OPEN': gtemp.values[j,0],\ 'HIGH': gtemp.values[j,1],\ 'LOW': gtemp.values[j,2], \ 'CLOSE': gtemp.values[j,3], \ 'VOLUME': gtemp.values[j,4],\ 'DATE': str(date),\ 'TICKER': str(ticker)} obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = 'text/json' #obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def test_close_underlying_socket_fails(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns conns[0].sock.close() # This shoud fail with a socket error now self.assertRaises(socket.error, bucket.get, 'foo')
def test_close_underlying_socket_fails(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') self.assertTrue(obj.exists) self.assertEqual(obj.bucket.name, 'bucket_test_close') self.assertEqual(obj.key, 'foo') self.assertEqual(obj.data, rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns conns[0].sock.close() # This shoud fail with a socket error now self.assertRaises(socket.error, bucket.get, 'foo')
def insert_function(request): RiakClient() RiakClient(protocol='http', host='127.0.0.1', http_port=8098) RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}]) myClient = RiakClient() myBucket = myClient.bucket('users') getNome = request.POST.get("name") getNome = str(getNome) getLocation = request.POST.get("location") getRole = request.POST.get("role") getFulname = request.POST.get("fullname") nome = str('user') + str(randint(0, 9)) nome = { 'name': getNome, 'location': getLocation, 'role': getRole, 'fullname': getFulname } obj = myBucket.new(str(nome), data=nome) obj.store() return redirect('/home')
class MaternityService: def __init__(self): self.maternity_bucket = settings.RIAK_MATERNITY_BUCKET self.riak = RiakClient(host = settings.RIAK_DATABASE['HOST'], port = settings.RIAK_DATABASE['PORT']) def get(self, pk): entity = self.riak.bucket(settings.RIAK_MATERNITY_BUCKET).get(str(pk)) if entity.exists(): data = json.loads(entity.get_data()) data['statistics'] = sorted(data['statistics'].iteritems()) return data else: return None def get_authorities(self): query = self.riak.add(settings.RIAK_MATERNITY_BUCKET) query.map(''' function(v) { var data = JSON.parse(v.values[0].data); if(data.type == 'AUTHORITY') { return [[v.key, data]]; } return []; } ''') return query.run() #data = [] #for result in query.run(): # d = result[1] #dj = json.loads(d) # data.append((result[0], d)) #return data #return [ # (result[0], json.loads(result[1])) # for result in query.run() # ]
def writeHistory(ticker, data, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) bucket = rc.bucket("stocks") gtemp = data if len(gtemp) == 0: return 0 else: for j in range(0, len(gtemp.index)): # upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + "_" + str(date)) riakVal = { "OPEN": gtemp.values[j, 0], "HIGH": gtemp.values[j, 1], "LOW": gtemp.values[j, 2], "CLOSE": gtemp.values[j, 3], "VOLUME": gtemp.values[j, 4], "DATE": str(date), "TICKER": str(ticker), } obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = "text/json" # obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
class ImageService: def __init__(self): self.riak = RiakClient(host=settings.RIAK_HOST, port=settings.RIAK_PORT) self._metadata_bucket = self.riak.bucket(settings.RIAK_METADATA_BUCKET) self._image_bucket = self.riak.bucket(settings.RIAK_IMAGE_BUCKET) self._thumbs_bucket = self.riak.bucket(settings.RIAK_THUMBS_BUCKET) def store_from_url(self, address, user): url = urlparse(address) if str(url.scheme) != 'http': raise ImageError('Not a valid URL') if not re.search(r'(jpg|jpeg|png|gif)$', url.path): raise ImageError('Unsupported file format') http = httplib.HTTPConnection(url.netloc) http.request('GET', url.path) response = http.getresponse() image = RemoteImage(response) return self.store(image, user, response.getheader("Content-Type", "image/jpg")) def store(self, image, user, content_type): key = self.create_unique_key() filename = self.filename_for_image(key, content_type) data = {"user": user, "uploaded_at": datetime.utcnow().isoformat(), "filename": filename} thumbnail = self.create_thumbnail(image) metadata = self._metadata_bucket.new(key, data, content_type="application/json") metadata.store() image = self._image_bucket.new_binary(key, image.read(), content_type) image.store() thumb = self._thumbs_bucket.new_binary(key, thumbnail, content_type="image/jpeg") thumb.store() return key def create_thumbnail(self, image): thumbnail = Image.open(image.temporary_file_path()) thumbnail.thumbnail((128, 128), Image.ANTIALIAS) thumbio = StringIO() thumbnail.save(thumbio, format="jpeg") return thumbio.getvalue() def find_metadata(self, image_id): image = self._metadata_bucket.get(image_id) if image.exists(): return image.get_data() else: return None def find(self, image_id, thumb=False): if thumb: image = self._thumbs_bucket.get_binary(image_id) else: image = self._image_bucket.get_binary(image_id) if image.exists(): return image else: return None def find_all(self, user): images = self.riak.search(settings.RIAK_METADATA_BUCKET, "user:%s" % user).map(MAP_DOCID).run() return images def create_unique_key(self, length=6): unique = False while not unique: id = self.unique_key(length) if not self._image_bucket.get(id).exists(): return id def unique_key(self, length=6): return shortuuid.uuid()[0:length] def filename_for_image(self, key, content_type): if content_type in ['image/jpg', 'image/jpeg']: extension = 'jpg' elif content_type == "image/png": extension = 'png' elif content_type == 'image/gif': extension = 'gif' return "%s.%s" % (key, extension)
# This is only needed if this script is outside of the Riak Python library folder import sys sys.path.insert(0, '/root/riak-python-client') import riak from riak import RiakClient from riak.security import SecurityCreds print "Initializing certificates." creds = SecurityCreds(username='******', cacert_file='/home/vagrant/certs/rootCA.crt', cert_file='/home/vagrant/certs/riakuser.crt', pkey_file='/home/vagrant/certs/riakuser.key') print "Testing Riak connectivity." myClient = RiakClient(host="riak1.myserver.com", pb_port=8087, credentials=creds) myBucket = myClient.bucket('RiakTestBucket') val1 = 1 key1 = myBucket.new('KeyOne', data=val1) fetched1 = myBucket.get('KeyOne') if fetched1.data is None: print "Ok." else: print fetched1.data
client = RiakClient(protocol='pbc') bkeys = [('multiget', str(key)) for key in xrange(10000)] data = open(__file__).read() print "Benchmarking multiget:" print " CPUs: {0}".format(cpu_count()) print " Threads: {0}".format(POOL_SIZE) print " Keys: {0}".format(len(bkeys)) print with benchmark.measure() as b: with b.report('populate'): for bucket, key in bkeys: client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() for b in benchmark.measure_with_rehearsal(): client.protocol = 'http' with b.report('http seq'): for bucket, key in bkeys: client.bucket(bucket).get(key) with b.report('http multi'): multiget(client, bkeys) client.protocol = 'pbc' with b.report('pbc seq'): for bucket, key in bkeys: client.bucket(bucket).get(key)
#!/usr/bin/env python3 from riak import RiakClient, RiakObject, RiakBucket riak_client = RiakClient(nodes=[{'host':'192.168.50.11','http_port':8098}, {'host':'192.168.50.12','http_port':8098}, {'host':'192.168.50.13','http_port':8098}]) bucket = riak_client.bucket('testbucket') for object_id in range(100000): riak_object = RiakObject(riak_client, bucket, str(object_id)) riak_object.data = {'id': object_id, 'desc': 'Description for ' + str(object_id)} riak_object.store()
import os import json import hashlib from riak import RiakClient from riak import RiakPbcTransport riak_client = RiakClient(os.getenv("RUNZ_RIAK_HOST") or "127.0.0.1", 8087, transport_class=RiakPbcTransport) proj_bucket = riak_client.bucket("projects") for key in proj_bucket.get_keys(): proj_bucket.get_binary(key).delete()
class RiakBackend(BaseBackend): custom_epoch = datetime.datetime(month=1, day=1, year=2013) def __init__( self, protocol="pbc", nodes=[], objects_bucket_name="objects", activities_bucket_name="activities", **kwargs): self._riak_backend = RiakClient(protocol=protocol, nodes=nodes) r_value = kwargs.get("r") w_value = kwargs.get("w") dw_value = kwargs.get("dw") pr_value = kwargs.get("pr") pw_value = kwargs.get("pw") self._objects = self._riak_backend.bucket(objects_bucket_name) self._activities = self._riak_backend.bucket(activities_bucket_name) if r_value: self._objects.r = r_value self._activities.r = r_value if w_value: self._objects.w = w_value self._activities.w = w_value if dw_value: self._objects.dw = dw_value self._activities.dw = dw_value if pr_value: self._objects.pr = pr_value self._activities.pr = pr_value if pw_value: self._objects.pw = pw_value self._activities.pw = pw_value def clear_all(self, **kwargs): """ Deletes all activity stream data from riak """ self.clear_all_activities() self.clear_all_objects() def clear_all_objects(self, **kwargs): """ Deletes all objects data from riak """ for key in self._objects.get_keys(): self._objects.get(key).delete(r='all', w='all', dw='all') assert not self._objects.get(key).exists def clear_all_activities(self, **kwargs): """ Deletes all activities data from riak """ for key in self._activities.get_keys(): self._activities.get(key).delete(r='all', w='all', dw='all') assert not self._activities.get(key).exists def obj_exists(self, obj, **kwargs): obj_id = self._extract_id(obj) return self._objects.get(obj_id).exists def activity_exists(self, activity, **kwargs): activity_id = self._extract_id(activity) return self._activities.get(activity_id).exists def obj_create(self, obj, **kwargs): obj = Object(obj, backend=self) obj.validate() obj_dict = obj.get_parsed_dict() key = self._extract_id(obj_dict) riak_obj = self._objects.new(key=key) riak_obj.data = obj_dict riak_obj = self.set_general_indexes(riak_obj) riak_obj.store() #finally save the data return obj_dict def set_general_indexes(self, riak_object): """ Sets the default riak 2Is. :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes): riak_object.add_index("timestamp_int", self._get_timestamp()) riak_object.remove_index('modified_int') riak_object.add_index("modified_int", self._get_timestamp()) return riak_object def obj_update(self, obj, **kwargs): self.obj_create(obj, **kwargs) def obj_get(self, obj, **kwargs): """ Given a list of object ids, returns a list of objects """ if not obj: return obj object_bucket_name = self._objects.name objects = self._riak_backend for o in obj: objects = objects.add(object_bucket_name, self._extract_id(o)) results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run() return results or [] def obj_delete(self, obj, **kwargs): obj_id = self._extract_id(obj) self._objects.new(key=obj_id).delete() def activity_create(self, activity, **kwargs): """ Creates an activity. You can provide objects for activities as dictionaries or as ids for already existing objects. If you provide a dictionary for an object, it is saved as a new object. If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty dictionary when retriving the activity later. """ activity = Activity(activity, backend=self) activity.validate() activity_dict = activity.get_parsed_dict() key = self._extract_id(activity_dict) riak_obj = self._activities.new(key=key) riak_obj.data = activity_dict riak_obj = self.set_activity_indexes(self.set_general_indexes(riak_obj)) if activity_dict['verb'] in SUB_ACTIVITY_MAP: riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs) riak_obj.store() return self.dehydrate_activities([riak_obj.data])[0] def set_activity_indexes(self, riak_object): """ Store indexes specific to an ``Activity``. Stores the following indexes: 1. ``verb`` of the ``Activity`` 2. ``actor`` of the ``Activity`` 3. ``object`` of the ``Activity`` 4. if target is defined, verb for the ``target`` of the Activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ _dict = riak_object.data riak_object.remove_index('verb_bin') riak_object.remove_index('actor_bin') riak_object.remove_index('object_bin') riak_object.add_index("verb_bin", self._extract_id(_dict['verb'])) riak_object.add_index("actor_bin", self._extract_id(_dict['actor'])) riak_object.add_index("object_bin", self._extract_id(_dict['object'])) if 'target' in _dict and _dict.get("target"): riak_object.remove_index('target_bin') riak_object.add_index("target_bin", self._extract_id(_dict['target'])) return riak_object def activity_delete(self, activity, **kwargs): """ Deletes an activity item and all associated sub items """ activity_dict = self.get_activity(activity, **kwargs)[0] for response_field in Activity._response_fields: if response_field in activity_dict: for response_item in activity_dict[response_field]['items']: self._activities.get(response_item['id']).delete() self._activities.get(self._extract_id(activity_dict)).delete() def activity_update(self, activity, **kwargs): return self.activity_create(activity, **kwargs) def activity_get( self, activity_ids=[], raw_filter="", filters={}, include_public=False, audience_targeting={}, aggregation_pipeline=[], **kwargs): """ Gets a list of activities. You can also group activities by providing a list of attributes to group by. :type activity_ids: list :param activity_ids: The list of activities you want to retrieve :type filters: dict :param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``. Filters do not work for nested dictionaries. :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on the ``audience_targeting`` parameter. :type include_public: boolean :param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are not targeted towards anyone are included in the results :type audience_targeting: dict :param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``. The values are an array of object ids :type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator`` :param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline :return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set. Activities are returned in the order of ids provided. """ activity_ids = map(self._extract_id, activity_ids) if not activity_ids: return [] activities = self._get_many_activities( activity_ids, raw_filter=raw_filter, filters=filters, include_public=include_public, audience_targeting=audience_targeting) activities = self.dehydrate_activities(activities) if aggregation_pipeline: original_activities = copy.deepcopy(activities) for aggregator in aggregation_pipeline: activities = aggregator.process(activities, original_activities, aggregation_pipeline) return activities def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs): if sub_activity_verb.lower() not in SUB_ACTIVITY_MAP: raise Exception('Verb not supported') return super(RiakBackend, self).create_sub_activity( activity, actor, content, extra=extra, sub_activity_verb=sub_activity_verb, **kwargs) def sub_activity_create( self, activity, actor, content, extra={}, sub_activity_verb="", published=None, **kwargs): sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] sub_activity_attribute = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1] object_type = kwargs.get('object_type', sub_activity_verb) activity_id = self._extract_id(activity) activity_model = Activity(self._activities.get(key=activity_id).data, backend=self) sub_activity_obj, original_activity_obj = activity_model\ .get_parsed_sub_activity_dict( actor=actor, content=content, verb=sub_activity_verb, object_type=object_type, collection=sub_activity_attribute, activity_class=sub_activity_model, published=published, extra=extra) sub_activity_obj = self.create_activity(sub_activity_obj, activity_id=original_activity_obj['id']) original_activity_obj[sub_activity_attribute]['items'][0]['actor'] = sub_activity_obj['actor']['id'] original_activity_obj[sub_activity_attribute]['items'][0]['id'] = sub_activity_obj['id'] original_activity_obj[sub_activity_attribute]['items'][0]['published'] = sub_activity_obj['published'] original_activity_obj[sub_activity_attribute]['items'][0]['object']['id'] = sub_activity_obj['id'] original_activity_obj = self.activity_update(original_activity_obj) return sub_activity_obj, original_activity_obj def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs): """ Deletes a sub_activity made on an activity. This will also update the corresponding parent activity. :type sub_activity: string :param sub_activity: the id of the reply activity to delete. :type sub_activity_verb: string :param sub_activity_verb: the verb of the sub activity :return: a dict representing the updated parent activity """ sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] sub_activity_id = self._extract_id(sub_activity) sub_activity_riak_model = self._activities.get(sub_activity_id) if sub_activity_riak_model.data['verb'] != sub_activity_model.sub_item_verb: raise SunspearValidationException("Trying to delete something that is not a {}.".format(sub_activity_model.sub_item_verb)) #clean up the reference from the original activity in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin', sub_activity_riak_model.indexes)[0][1] activity = self._activities.get(key=in_reply_to_key) activity_data = activity.data activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1 activity_data[sub_activity_model.sub_item_key]['items'] = filter( lambda x: x["id"] != sub_activity_id, activity_data[sub_activity_model.sub_item_key]['items']) updated_activity = self.update_activity(activity_data, **kwargs) self.delete_activity(sub_activity_id) return updated_activity def set_sub_item_indexes(self, riak_object, **kwargs): """ Store indexes specific to a sub-activity. Stores the following indexes: 1. id of the the parent ``Activity`` of this sub-activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ original_activity_id = kwargs.get('activity_id') if not original_activity_id: raise SunspearValidationException() riak_object.remove_index('inreplyto_bin') riak_object.add_index("inreplyto_bin", str(original_activity_id)) return riak_object def dehydrate_activities(self, activities): """ Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object """ activities = self._extract_sub_activities(activities) #collect a list of unique object ids. We only iterate through the fields that we know #for sure are objects. User is responsible for hydrating all other fields. object_ids = set() for activity in activities: object_ids.update(self._extract_object_keys(activity)) #Get the objects for the ids we have collected objects = self.get_obj(object_ids) objects_dict = dict(((obj["id"], obj,) for obj in objects)) #We also need to extract any activities that were diguised as objects. IE activities with #objectType=activity activities_in_objects_ids = set() #replace the object ids with the hydrated objects for activity in activities: activity = self._dehydrate_object_keys(activity, objects_dict) #Extract keys of any activities that were objects activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True)) #If we did have activities that were objects, we need to hydrate those activities and #the objects for those activities if activities_in_objects_ids: sub_activities = self._get_many_activities(activities_in_objects_ids) activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities)) for activity in activities: activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True) #we have to do one more round of object dehydration for our new sub-activities object_ids.update(self._extract_object_keys(activity)) #now get all the objects we don't already have and for sub-activities and and hydrate them into #our list of activities object_ids -= set(objects_dict.keys()) objects = self.get_obj(object_ids) for obj in objects: objects_dict[obj["id"]] = obj for activity in activities: activity = self._dehydrate_object_keys(activity, objects_dict) return activities def _extract_sub_activities(self, activities): """ Extract all objects that have an objectType of activity as an activity """ #We might also have to get sub activities for things like replies and likes activity_ids = set() activities_dict = dict(((activity["id"], activity,) for activity in activities)) for activity in activities: activity_ids.update(self._extract_activity_keys(activity)) if activity_ids: #don't bother fetching the activities we already have activity_ids -= set(activities_dict.keys()) if activity_ids: sub_activities = self._get_many_activities(activity_ids) for sub_activity in sub_activities: activities_dict[sub_activity["id"]] = sub_activity #Dehydrate out any subactivities we may have for activity in activities: activity = self._dehydrate_sub_activity(activity, activities_dict) return activities def _extract_activity_keys(self, activity, skip_sub_activities=False): keys = [] for activity_key in Model._object_fields + ['inReplyTo']: if activity_key not in activity: continue obj = activity.get(activity_key) if isinstance(obj, dict): if obj.get('objectType', None) == 'activity': keys.append(obj['id']) if obj.get('inReplyTo', None): [keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo']] if not skip_sub_activities: for collection in Activity._response_fields: if collection in activity and activity[collection]['items']: for item in activity[collection]['items']: keys.extend(self._extract_activity_keys(item)) return keys def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False): for activity_key in Model._object_fields: if activity_key not in sub_activity: continue if isinstance(sub_activity[activity_key], dict): if sub_activity[activity_key].get('objectType', None) == 'activity': sub_activity[activity_key].update(obj_list[sub_activity[activity_key]['id']]) if sub_activity[activity_key].get('inReplyTo', None): for i, in_reply_to_obj in enumerate(sub_activity[activity_key]['inReplyTo']): sub_activity[activity_key]['inReplyTo'][i]\ .update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']]) if not skip_sub_activities: for collection in Activity._response_fields: if collection in sub_activity and sub_activity[collection]['items']: dehydrated_sub_items = [] for i, item in enumerate(sub_activity[collection]['items']): try: dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list)) except KeyError, e: pass sub_activity[collection]['items'] = dehydrated_sub_items sub_activity[collection]['totalItems'] = len(dehydrated_sub_items) return sub_activity
return [line.strip() for line in open(keyListFilename, 'r')] # unused helper method to save an image to the disk if you want that def saveImage(riakObj, filename): with open(os.path.join(baseDir, filename), 'w+') as f: f.write(riakObj.encoded_data) # Read in redis-keys imgs = readRiakKeys() print 'Riak Keys: \n' + '\n'.join(imgs) # connect to live riak rcLive = RiakClient(protocol='http', host=srcRiakHost, http_port=8098) rbLive = rcLive.bucket('ez') # connect to integration riak rcInt = RiakClient(protocol='http', host=destRiakHost, http_port=8098) rbInt = rcInt.bucket('ez') # save image in integration riak def saveToIntRiak(key, riakObj): if riakObj.content_type is not None: img = rbInt.new(key, encoded_data=riakObj.encoded_data, content_type=riakObj.content_type) img.store() # get and save all images for img in imgs: print img local = rbInt.get(img)
class RiakBackend(BaseBackend): custom_epoch = datetime.datetime(month=1, day=1, year=2013) def __init__(self, protocol="pbc", nodes=[], objects_bucket_name="objects", activities_bucket_name="activities", **kwargs): self._riak_backend = RiakClient(protocol=protocol, nodes=nodes) r_value = kwargs.get("r") w_value = kwargs.get("w") dw_value = kwargs.get("dw") pr_value = kwargs.get("pr") pw_value = kwargs.get("pw") self._objects = self._riak_backend.bucket(objects_bucket_name) self._activities = self._riak_backend.bucket(activities_bucket_name) if r_value: self._objects.r = r_value self._activities.r = r_value if w_value: self._objects.w = w_value self._activities.w = w_value if dw_value: self._objects.dw = dw_value self._activities.dw = dw_value if pr_value: self._objects.pr = pr_value self._activities.pr = pr_value if pw_value: self._objects.pw = pw_value self._activities.pw = pw_value def clear_all(self, **kwargs): """ Deletes all activity stream data from riak """ self.clear_all_activities() self.clear_all_objects() def clear_all_objects(self, **kwargs): """ Deletes all objects data from riak """ for key in self._objects.get_keys(): self._objects.get(key).delete(rw='all', r='all', w='all', dw='all') assert not self._objects.get(key).exists def clear_all_activities(self, **kwargs): """ Deletes all activities data from riak """ for key in self._activities.get_keys(): self._activities.get(key).delete(rw='all', r='all', w='all', dw='all') assert not self._activities.get(key).exists def obj_exists(self, obj, **kwargs): obj_id = self._extract_id(obj) return self._objects.get(obj_id).exists def activity_exists(self, activity, **kwargs): activity_id = self._extract_id(activity) return self._activities.get(activity_id).exists def obj_create(self, obj, **kwargs): obj = Object(obj, backend=self) obj.validate() obj_dict = obj.get_parsed_dict() key = self._extract_id(obj_dict) riak_obj = self._objects.new(key=key) riak_obj.data = obj_dict riak_obj = self.set_general_indexes(riak_obj) riak_obj.store() #finally save the data return obj_dict def set_general_indexes(self, riak_object): """ Sets the default riak 2Is. :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes): riak_object.add_index("timestamp_int", self._get_timestamp()) riak_object.remove_index('modified_int') riak_object.add_index("modified_int", self._get_timestamp()) return riak_object def obj_update(self, obj, **kwargs): self.obj_create(obj, **kwargs) def obj_get(self, obj, **kwargs): """ Given a list of object ids, returns a list of objects """ if not obj: return obj object_bucket_name = self._objects.name objects = self._riak_backend for o in obj: objects = objects.add(object_bucket_name, self._extract_id(o)) results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run() return results or [] def obj_delete(self, obj, **kwargs): obj_id = self._extract_id(obj) self._objects.new(key=obj_id).delete() def activity_create(self, activity, **kwargs): """ Creates an activity. You can provide objects for activities as dictionaries or as ids for already existing objects. If you provide a dictionary for an object, it is saved as a new object. If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty dictionary when retriving the activity later. """ activity = Activity(activity, backend=self) activity.validate() activity_dict = activity.get_parsed_dict() key = self._extract_id(activity_dict) riak_obj = self._activities.new(key=key) riak_obj.data = activity_dict riak_obj = self.set_activity_indexes( self.set_general_indexes(riak_obj)) if activity_dict['verb'] in SUB_ACTIVITY_MAP: riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs) riak_obj.store() return self.dehydrate_activities([riak_obj.data])[0] def set_activity_indexes(self, riak_object): """ Store indexes specific to an ``Activity``. Stores the following indexes: 1. ``verb`` of the ``Activity`` 2. ``actor`` of the ``Activity`` 3. ``object`` of the ``Activity`` 4. if target is defined, verb for the ``target`` of the Activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ _dict = riak_object.data riak_object.remove_index('verb_bin') riak_object.remove_index('actor_bin') riak_object.remove_index('object_bin') riak_object.add_index("verb_bin", self._extract_id(_dict['verb'])) riak_object.add_index("actor_bin", self._extract_id(_dict['actor'])) riak_object.add_index("object_bin", self._extract_id(_dict['object'])) if 'target' in _dict and _dict.get("target"): riak_object.remove_index('target_bin') riak_object.add_index("target_bin", self._extract_id(_dict['target'])) return riak_object def activity_delete(self, activity, **kwargs): """ Deletes an activity item and all associated sub items """ activity_dict = self.get_activity(activity, **kwargs)[0] for response_field in Activity._response_fields: if response_field in activity_dict: for response_item in activity_dict[response_field]['items']: self._activities.get(response_item['id']).delete() self._activities.get(self._extract_id(activity_dict)).delete() def activity_update(self, activity, **kwargs): return self.activity_create(activity, **kwargs) def activity_get(self, activity_ids=[], raw_filter="", filters={}, include_public=False, audience_targeting={}, aggregation_pipeline=[], **kwargs): """ Gets a list of activities. You can also group activities by providing a list of attributes to group by. :type activity_ids: list :param activity_ids: The list of activities you want to retrieve :type filters: dict :param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``. Filters do not work for nested dictionaries. :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on the ``audience_targeting`` parameter. :type include_public: boolean :param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are not targeted towards anyone are included in the results :type audience_targeting: dict :param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``. The values are an array of object ids :type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator`` :param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline :return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set. Activities are returned in the order of ids provided. """ activity_ids = map(self._extract_id, activity_ids) if not activity_ids: return [] activities = self._get_many_activities( activity_ids, raw_filter=raw_filter, filters=filters, include_public=include_public, audience_targeting=audience_targeting) activities = self.dehydrate_activities(activities) original_activities = copy.deepcopy(activities) for aggregator in aggregation_pipeline: activities = aggregator.process(activities, original_activities, aggregation_pipeline) return activities def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs): if sub_activity_verb.lower() not in SUB_ACTIVITY_MAP: raise Exception('Verb not supported') return super(RiakBackend, self).create_sub_activity( activity, actor, content, extra=extra, sub_activity_verb=sub_activity_verb, **kwargs) def sub_activity_create(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs): sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] sub_activity_attribute = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1] object_type = kwargs.get('object_type', sub_activity_verb) activity_id = self._extract_id(activity) activity_model = Activity(self._activities.get(key=activity_id).data, backend=self) sub_activity_obj, original_activity_obj = activity_model\ .get_parsed_sub_activity_dict( actor=actor, content=content, verb=sub_activity_verb, object_type=object_type, collection=sub_activity_attribute, activity_class=sub_activity_model, extra=extra) sub_activity_obj = self.create_activity( sub_activity_obj, activity_id=original_activity_obj['id']) original_activity_obj[sub_activity_attribute]['items'][0][ 'actor'] = sub_activity_obj['actor']['id'] original_activity_obj[sub_activity_attribute]['items'][0][ 'id'] = sub_activity_obj['id'] original_activity_obj[sub_activity_attribute]['items'][0][ 'published'] = sub_activity_obj['published'] original_activity_obj[sub_activity_attribute]['items'][0]['object'][ 'id'] = sub_activity_obj['id'] original_activity_obj = self.activity_update(original_activity_obj) return sub_activity_obj, original_activity_obj def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs): """ Deletes a sub_activity made on an activity. This will also update the corresponding parent activity. :type sub_activity: string :param sub_activity: the id of the reply activity to delete. :type sub_activity_verb: string :param sub_activity_verb: the verb of the sub activity :return: a dict representing the updated parent activity """ sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] sub_activity_id = self._extract_id(sub_activity) sub_activity_riak_model = self._activities.get(sub_activity_id) if sub_activity_riak_model.data[ 'verb'] != sub_activity_model.sub_item_verb: raise SunspearValidationException( "Trying to delete something that is not a {}.".format( sub_activity_model.sub_item_verb)) #clean up the reference from the original activity in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin', sub_activity_riak_model.indexes)[0][1] activity = self._activities.get(key=in_reply_to_key) activity_data = activity.data activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1 activity_data[sub_activity_model.sub_item_key]['items'] = filter( lambda x: x["id"] != sub_activity_id, activity_data[sub_activity_model.sub_item_key]['items']) updated_activity = self.update_activity(activity_data, **kwargs) self.delete_activity(sub_activity_id) return updated_activity def set_sub_item_indexes(self, riak_object, **kwargs): """ Store indexes specific to a sub-activity. Stores the following indexes: 1. id of the the parent ``Activity`` of this sub-activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ original_activity_id = kwargs.get('activity_id') if not original_activity_id: raise SunspearValidationException() riak_object.remove_index('inreplyto_bin') riak_object.add_index("inreplyto_bin", str(original_activity_id)) return riak_object def dehydrate_activities(self, activities): """ Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object """ activities = self._extract_sub_activities(activities) #collect a list of unique object ids. We only iterate through the fields that we know #for sure are objects. User is responsible for hydrating all other fields. object_ids = set() for activity in activities: object_ids.update(self._extract_object_keys(activity)) #Get the objects for the ids we have collected objects = self.get_obj(object_ids) objects_dict = dict((( obj["id"], obj, ) for obj in objects)) #We also need to extract any activities that were diguised as objects. IE activities with #objectType=activity activities_in_objects_ids = set() #replace the object ids with the hydrated objects for activity in activities: activity = self._dehydrate_object_keys(activity, objects_dict) #Extract keys of any activities that were objects activities_in_objects_ids.update( self._extract_activity_keys(activity, skip_sub_activities=True)) #If we did have activities that were objects, we need to hydrate those activities and #the objects for those activities if activities_in_objects_ids: sub_activities = self._get_many_activities( activities_in_objects_ids) activities_in_objects_dict = dict((( sub_activity["id"], sub_activity, ) for sub_activity in sub_activities)) for activity in activities: activity = self._dehydrate_sub_activity( activity, activities_in_objects_dict, skip_sub_activities=True) #we have to do one more round of object dehydration for our new sub-activities object_ids.update(self._extract_object_keys(activity)) #now get all the objects we don't already have and for sub-activities and and hydrate them into #our list of activities object_ids -= set(objects_dict.keys()) objects = self.get_obj(object_ids) for obj in objects: objects_dict[obj["id"]] = obj for activity in activities: activity = self._dehydrate_object_keys(activity, objects_dict) return activities def _extract_sub_activities(self, activities): """ Extract all objects that have an objectType of activity as an activity """ #We might also have to get sub activities for things like replies and likes activity_ids = set() activities_dict = dict((( activity["id"], activity, ) for activity in activities)) for activity in activities: activity_ids.update(self._extract_activity_keys(activity)) if activity_ids: #don't bother fetching the activities we already have activity_ids -= set(activities_dict.keys()) if activity_ids: sub_activities = self._get_many_activities(activity_ids) for sub_activity in sub_activities: activities_dict[sub_activity["id"]] = sub_activity #Dehydrate out any subactivities we may have for activity in activities: activity = self._dehydrate_sub_activity( activity, activities_dict) return activities def _extract_activity_keys(self, activity, skip_sub_activities=False): keys = [] for activity_key in Model._object_fields + ['inReplyTo']: if activity_key not in activity: continue obj = activity.get(activity_key) if isinstance(obj, dict): if obj.get('objectType', None) == 'activity': keys.append(obj['id']) if obj.get('inReplyTo', None): [ keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo'] ] if not skip_sub_activities: for collection in Activity._response_fields: if collection in activity and activity[collection]['items']: for item in activity[collection]['items']: keys.extend(self._extract_activity_keys(item)) return keys def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False): for activity_key in Model._object_fields: if activity_key not in sub_activity: continue if isinstance(sub_activity[activity_key], dict): if sub_activity[activity_key].get('objectType', None) == 'activity': sub_activity[activity_key].update( obj_list[sub_activity[activity_key]['id']]) if sub_activity[activity_key].get('inReplyTo', None): for i, in_reply_to_obj in enumerate( sub_activity[activity_key]['inReplyTo']): sub_activity[activity_key]['inReplyTo'][i]\ .update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']]) if not skip_sub_activities: for collection in Activity._response_fields: if collection in sub_activity and sub_activity[collection][ 'items']: dehydrated_sub_items = [] for i, item in enumerate( sub_activity[collection]['items']): try: dehydrated_sub_items.append( self._dehydrate_sub_activity(item, obj_list)) except KeyError, e: pass sub_activity[collection][ 'items'] = dehydrated_sub_items sub_activity[collection]['totalItems'] = len( dehydrated_sub_items) return sub_activity
def get_cooccurs(lang, dep): client = RiakClient(host=riak_ip, pb_port=riak_port, protocol='pbc') bucket = client.bucket('%s/cooccurs' % lang) return jsonify({"cooccurs": bucket.get(dep).data})
class ImageService: def __init__(self): self.riak = RiakClient(host=settings.RIAK_HOST, port=settings.RIAK_PORT) self._metadata_bucket = self.riak.bucket(settings.RIAK_METADATA_BUCKET) self._image_bucket = self.riak.bucket(settings.RIAK_IMAGE_BUCKET) self._thumbs_bucket = self.riak.bucket(settings.RIAK_THUMBS_BUCKET) def store_from_url(self, address, user): url = urlparse(address) if str(url.scheme) != 'http': raise ImageError('Not a valid URL') if not re.search(r'(jpg|jpeg|png|gif)$', url.path): raise ImageError('Unsupported file format') http = httplib.HTTPConnection(url.netloc) http.request('GET', url.path) response = http.getresponse() image = RemoteImage(response) return self.store(image, user, response.getheader("Content-Type", "image/jpg")) def store(self, image, user, content_type): key = self.create_unique_key() filename = self.filename_for_image(key, content_type) data = { "user": user, "uploaded_at": datetime.utcnow().isoformat(), "filename": filename } thumbnail = self.create_thumbnail(image) metadata = self._metadata_bucket.new(key, data, content_type="application/json") metadata.store() image = self._image_bucket.new_binary(key, image.read(), content_type) image.store() thumb = self._thumbs_bucket.new_binary(key, thumbnail, content_type="image/jpeg") thumb.store() return key def create_thumbnail(self, image): thumbnail = Image.open(image.temporary_file_path()) thumbnail.thumbnail((128, 128), Image.ANTIALIAS) thumbio = StringIO() thumbnail.save(thumbio, format="jpeg") return thumbio.getvalue() def find_metadata(self, image_id): image = self._metadata_bucket.get(image_id) if image.exists(): return image.get_data() else: return None def find(self, image_id, thumb=False): if thumb: image = self._thumbs_bucket.get_binary(image_id) else: image = self._image_bucket.get_binary(image_id) if image.exists(): return image else: return None def find_all(self, user): images = self.riak.search(settings.RIAK_METADATA_BUCKET, "user:%s" % user).map(MAP_DOCID).run() return images def create_unique_key(self, length=6): unique = False while not unique: id = self.unique_key(length) if not self._image_bucket.get(id).exists(): return id def unique_key(self, length=6): return shortuuid.uuid()[0:length] def filename_for_image(self, key, content_type): if content_type in ['image/jpg', 'image/jpeg']: extension = 'jpg' elif content_type == "image/png": extension = 'png' elif content_type == 'image/gif': extension = 'gif' return "%s.%s" % (key, extension)
import csv import riak from riak import RiakClient print("\nWelcome to Google Stock Archive") print("Ente name of document in current directory to be archived in Riak") y = raw_input() file1 = open(y, "r") reader = csv.reader(file1) client = RiakClient() my_bucket = client.bucket('Stock Data') for row in reader: list = { 'Date': row[0], 'Open': row[1], 'High': row[2], 'Low': row[3], 'Close': row[4], 'Volume': row[5], 'Adj Close': row[6] } x = my_bucket.new(data=list) x.store() print("Please Enter Date in following format\nYYYY-MM-DD:") val = raw_input()
#!/usr/bin/python from riak import RiakClient, RiakNode from riak.riak_object import RiakObject client = RiakClient(protocal='http', host='127.0.0.1', http_port=8098) res = client.ping() bucket = client.bucket('xxx') #res = client.get_keys(res[0]) #obj = RiakObject(client, res, 'xxx') obj = bucket.new('a', 'hello', 'text/plain') obj.store() res = bucket.get('a') n = 0
#!/usr/bin/python from workload import * import sys import time from riak import RiakClient, RiakNode rc = RiakClient(protocol='pbc', host='127.0.0.1', pb_port=8087) #rc = RiakClient(protocol='http', host='127.0.0.1', pb_port=8098) bucket = rc.bucket('benchmark') def benchmark_function(actions, f_name, workload): method = actions[f_name] t0 = time.time() if f_name in ['get', 'del']: for k, v in workload.items(): method(k) else: for k, v in workload.items(): method(k, v) t1 = time.time() delta = t1-t0 print("Riak %s benchmark finished in %.3fs." % (f_name, delta)) def benchmark(size): workload = gen_workload(size) actions = { 'put': lambda x,y: bucket.new(x, data=y).store(), 'get': bucket.get, 'del': bucket.delete }
create = """ CREATE TABLE {table} ( family varchar not null, series varchar not null, time timestamp not null, metric double, PRIMARY KEY ( (family, series, quantum(time, 1, 'd') ),family, series, time)) """ rc = RiakClient(host='localhost', pb_port=8087) create_q = create.format(table=table_name) print create_q secs = datetime.timedelta(0, 5) start = datetime.datetime.now() - datetime.timedelta(0,3600) try: rc.ts_query("","DESCRIBE %s" % table_name) except RiakError: rc.ts_query(table_name, create_q) try: rc.get_search_index(node_table_name) except RiakError: rc.create_search_index(node_table_name) bucket = rc.bucket(node_table_name) bucket.set_properties({'search_index':node_table_name})
def __init__(self, session_key=None): super(SessionStore, self).__init__(session_key) client = RiakClient(host=RIAK_DB_HOST, port=RIAK_DB_PORT, transport_class=RiakPbcTransport) self.sessions = client.bucket(RIAK_DB_BUCKET)
class RiakTSWriter(TimeSeriesDatabase): plugin_name = 'graphite_riakts' def __init__(self, option): print "Here we be initializing yon connections and things: %r" % option self.riak = RiakClient(host=option['RIAKTS_HOST'], pb_port=int(option['RIAKTS_PORT'])) self.nodes_table = option['RIAKTS_NODES_TABLE'] self.table_name = option['RIAKTS_TABLE_NAME'] self.family = option['RIAKTS_FAMILY'] self.nodes = self.riak.bucket(self.nodes_table) def write(self, metric, datapoints): try: #print "Here we be writing to %s the value of %r" % ( metric, datapoints ) table = self.riak.table(self.table_name) rows = [] for point in datapoints: time = datetime.datetime.utcfromtimestamp(point[0]) rows.append([self.family, metric, time, float(point[1])]) #print rows ts_obj = table.new(rows) #print vars(ts_obj) res = ts_obj.store() #print res return except Exception as e: raise RuntimeError("write error: %s" % (traceback.format_exc(e),)) def exists(self, metric): try: key = "node-%s" % metric #print "Here we be checking if yon metric %s is existing" % metric o = self.nodes.get(key) if o.exists: #print "Yar, there be existing the metric of %s" % metric return True else: #print "Lo! The metric of %s be not here!" % metric o = self.nodes.new(key) branch = '.'.join(metric.split('.')[:-1]) o.data = { "node_s": metric, "type_s": "node"} # print "Adding LEAF: %s" % metric o.store() while len(branch): bkey = "branch-%s" % branch b = self.nodes.get(bkey) if not b.exists: b.data = { "branch_s": branch, "type_s": "branch"} # print "Adding BRANCH: %s" % branch b.store() branch = '.'.join(branch.split('.')[:-1]) return True except Exception as e: raise RuntimeError("exists error: %s" % (traceback.format_exc(e),)) def create(self, metric, **options): try: #print "Here we be attempting yon creation of %s metric, with options: %r" % ( metric, options ) # o = self.nodes.new(metric) # o.data = { "node_t" : metric } # o.store() return metric except Exception as e: raise RuntimeError("create error: %s" % (traceback.format_exc(e),)) def get_metadata(self, metric, key): try: #print "Here we be getting yon metadata of metric %s and key %s" % ( metric, key ) return True except Exception as e: raise RuntimeError("get_metadata error: %s" % (traceback.format_exc(e),)) def set_metadata(self, metric, key, value): try: #print "Here we be setting yon metadata of metric %s and key %s with value %r" % ( metric, key, value ) return True except Exception as e: raise RuntimeError("set_metadata error: %s" % (traceback.format_exc(e),)) # def getFilesystemPath(self, metric): # try: # print "Here we be getting yon file system path of here metric %s" % metric # return metric # except (Exception) as e: # raise RuntimeError("getFilesystemPath error: %s" % (traceback.format_exc(e),))
import uuid from django.http import HttpResponseRedirect from django.shortcuts import render, redirect from .models import * from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required from django.views.decorators.http import require_POST # Create your views here. RiakClient() RiakClient(protocol='http', host='127.0.0.1', http_port=8098) RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}]) myClient = RiakClient() myBucket = myClient.bucket('users') def home(request): function = Functions.objects.all() return render(request, 'home.html', {'functions': function}) def salvar_function(request): titulo = request.POST.get('titulo') if titulo: functions = Functions() functions.titulo = titulo functions.save() return redirect('/functions')
from flask import Blueprint, request, jsonify from marshmallow import schema, ValidationError from riak import RiakClient # create crime blueprint from app.schema import CrimeSchema crime = Blueprint('crime', __name__) client = RiakClient() crime_bucket = client.bucket('crime') # bucket = client.bucket_type('crime').bucket('category') @crime.route("/crime", defaults={'crime_id': None}, methods=['GET', 'POST']) @crime.route("/crime/<crime_id>", methods=['GET', 'DELETE']) def crime_route(crime_id): if request.method == 'POST': # get crime_id and value from request # if request.json: try: crime_schema = CrimeSchema() result = crime_schema.load(request.json) except ValidationError as err: return jsonify({'message': err.messages}), 404 new_crime = request.json new_crime_id = new_crime['crime_id']
from riak import RiakClient, RiakNode client = RiakClient(protocol='pbc', host='127.0.0.1', http_port=8098) print(client.ping()) tweetBucket = client.bucket("tweets") print(tweetBucket.get_properties()) print(client.get_buckets()) print(client.bucket_type("tweets")) allKeysInTweets = client.get_keys(tweetBucket) print("Number of keys... ",len(allKeysInTweets)) stream = client.stream_keys(tweetBucket) for key_list in stream: print("key_list;" , key_list) stream.close()
class RiakBackend(BaseBackend): def __init__( self, protocol="pbc", nodes=[], objects_bucket_name="objects", activities_bucket_name="activities", **kwargs): self._riak_backend = RiakClient(protocol=protocol, nodes=nodes) r_value = kwargs.get("r") w_value = kwargs.get("w") dw_value = kwargs.get("dw") pr_value = kwargs.get("pr") pw_value = kwargs.get("pw") self._objects = self._riak_backend.bucket(objects_bucket_name) self._activities = self._riak_backend.bucket(activities_bucket_name) if r_value: self._objects.r = r_value self._activities.r = r_value if w_value: self._objects.w = w_value self._activities.w = w_value if dw_value: self._objects.dw = dw_value self._activities.dw = dw_value if pr_value: self._objects.pr = pr_value self._activities.pr = pr_value if pw_value: self._objects.pw = pw_value self._activities.pw = pw_value def clear_all(self, **kwargs): """ Deletes all activity stream data from riak """ self.clear_all_activities() self.clear_all_objects() def clear_all_objects(self, **kwargs): """ Deletes all objects data from riak """ for key in self._objects.get_keys(): self._objects.get(key).delete(r='all', w='all', dw='all') assert not self._objects.get(key).exists def clear_all_activities(self, **kwargs): """ Deletes all activities data from riak """ for key in self._activities.get_keys(): self._activities.get(key).delete(r='all', w='all', dw='all') assert not self._activities.get(key).exists def obj_exists(self, obj, **kwargs): obj_id = self._extract_id(obj) return self._objects.get(obj_id).exists def activity_exists(self, activity, **kwargs): activity_id = self._extract_id(activity) return self._activities.get(activity_id).exists def obj_create(self, obj, **kwargs): obj = Object(obj, backend=self) obj.validate() obj_dict = obj.get_parsed_dict() key = self._extract_id(obj_dict) riak_obj = self._objects.new(key=key) riak_obj.data = obj_dict riak_obj = self.set_general_indexes(riak_obj) riak_obj.store() #finally save the data return obj_dict def set_general_indexes(self, riak_object): """ Sets the default riak 2Is. :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes): riak_object.add_index("timestamp_int", self._get_timestamp()) riak_object.remove_index('modified_int') riak_object.add_index("modified_int", self._get_timestamp()) return riak_object def obj_update(self, obj, **kwargs): self.obj_create(obj, **kwargs) def obj_get(self, obj, **kwargs): """ Given a list of object ids, returns a list of objects """ if not obj: return obj object_bucket_name = self._objects.name objects = self._riak_backend for o in obj: objects = objects.add(object_bucket_name, self._extract_id(o)) results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run() return results or [] def obj_delete(self, obj, **kwargs): obj_id = self._extract_id(obj) self._objects.new(key=obj_id).delete() def activity_create(self, activity, **kwargs): """ Creates an activity. You can provide objects for activities as dictionaries or as ids for already existing objects. If you provide a dictionary for an object, it is saved as a new object. If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty dictionary when retriving the activity later. """ activity = Activity(activity, backend=self) activity.validate() activity_dict = activity.get_parsed_dict() key = self._extract_id(activity_dict) riak_obj = self._activities.new(key=key) riak_obj.data = activity_dict riak_obj = self.set_activity_indexes(self.set_general_indexes(riak_obj)) if activity_dict['verb'] in SUB_ACTIVITY_MAP: riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs) riak_obj.store() return self.dehydrate_activities([riak_obj.data])[0] def set_activity_indexes(self, riak_object): """ Store indexes specific to an ``Activity``. Stores the following indexes: 1. ``verb`` of the ``Activity`` 2. ``actor`` of the ``Activity`` 3. ``object`` of the ``Activity`` 4. if target is defined, verb for the ``target`` of the Activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ _dict = riak_object.data riak_object.remove_index('verb_bin') riak_object.remove_index('actor_bin') riak_object.remove_index('object_bin') riak_object.add_index("verb_bin", self._extract_id(_dict['verb'])) riak_object.add_index("actor_bin", self._extract_id(_dict['actor'])) riak_object.add_index("object_bin", self._extract_id(_dict['object'])) if 'target' in _dict and _dict.get("target"): riak_object.remove_index('target_bin') riak_object.add_index("target_bin", self._extract_id(_dict['target'])) return riak_object def activity_delete(self, activity, **kwargs): """ Deletes an activity item and all associated sub items """ activity_dict = self.get_activity(activity, **kwargs)[0] for response_field in Activity._response_fields: if response_field in activity_dict: for response_item in activity_dict[response_field]['items']: self._activities.get(response_item['id']).delete() self._activities.get(self._extract_id(activity_dict)).delete() def activity_update(self, activity, **kwargs): return self.activity_create(activity, **kwargs) def activity_get( self, activity_ids=[], raw_filter="", filters={}, include_public=False, audience_targeting={}, aggregation_pipeline=[], **kwargs): """ Gets a list of activities. You can also group activities by providing a list of attributes to group by. :type activity_ids: list :param activity_ids: The list of activities you want to retrieve :type filters: dict :param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``. Filters do not work for nested dictionaries. :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on the ``audience_targeting`` parameter. :type include_public: boolean :param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are not targeted towards anyone are included in the results :type audience_targeting: dict :param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``. The values are an array of object ids :type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator`` :param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline :return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set. Activities are returned in the order of ids provided. """ activity_ids = map(self._extract_id, activity_ids) if not activity_ids: return [] activities = self._get_many_activities( activity_ids, raw_filter=raw_filter, filters=filters, include_public=include_public, audience_targeting=audience_targeting) activities = self.dehydrate_activities(activities) if aggregation_pipeline: original_activities = copy.deepcopy(activities) for aggregator in aggregation_pipeline: activities = aggregator.process(activities, original_activities, aggregation_pipeline) return activities def sub_activity_create( self, activity, actor, content, extra={}, sub_activity_verb="", published=None, **kwargs): sub_activity_model = self.get_sub_activity_model(sub_activity_verb) sub_activity_attribute = self.get_sub_activity_attribute(sub_activity_verb) object_type = kwargs.get('object_type', sub_activity_verb) activity_id = self._extract_id(activity) activity_model = Activity(self._activities.get(key=activity_id).data, backend=self) sub_activity_obj, original_activity_obj = activity_model\ .get_parsed_sub_activity_dict( actor=actor, content=content, verb=sub_activity_verb, object_type=object_type, collection=sub_activity_attribute, activity_class=sub_activity_model, published=published, extra=extra) sub_activity_obj = self.create_activity(sub_activity_obj, activity_id=original_activity_obj['id']) original_activity_obj[sub_activity_attribute]['items'][0]['actor'] = sub_activity_obj['actor']['id'] original_activity_obj[sub_activity_attribute]['items'][0]['id'] = sub_activity_obj['id'] original_activity_obj[sub_activity_attribute]['items'][0]['published'] = sub_activity_obj['published'] original_activity_obj[sub_activity_attribute]['items'][0]['object']['id'] = sub_activity_obj['id'] original_activity_obj = self.activity_update(original_activity_obj) return sub_activity_obj, original_activity_obj def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs): """ Deletes a sub_activity made on an activity. This will also update the corresponding parent activity. :type sub_activity: string :param sub_activity: the id of the reply activity to delete. :type sub_activity_verb: string :param sub_activity_verb: the verb of the sub activity :return: a dict representing the updated parent activity """ sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] sub_activity_id = self._extract_id(sub_activity) sub_activity_riak_model = self._activities.get(sub_activity_id) if sub_activity_riak_model.data['verb'] != sub_activity_model.sub_item_verb: raise SunspearValidationException("Trying to delete something that is not a {}.".format(sub_activity_model.sub_item_verb)) #clean up the reference from the original activity in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin', sub_activity_riak_model.indexes)[0][1] activity = self._activities.get(key=in_reply_to_key) activity_data = activity.data activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1 activity_data[sub_activity_model.sub_item_key]['items'] = filter( lambda x: x["id"] != sub_activity_id, activity_data[sub_activity_model.sub_item_key]['items']) updated_activity = self.update_activity(activity_data, **kwargs) self.delete_activity(sub_activity_id) return updated_activity def set_sub_item_indexes(self, riak_object, **kwargs): """ Store indexes specific to a sub-activity. Stores the following indexes: 1. id of the the parent ``Activity`` of this sub-activity :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ original_activity_id = kwargs.get('activity_id') if not original_activity_id: raise SunspearValidationException() riak_object.remove_index('inreplyto_bin') riak_object.add_index("inreplyto_bin", str(original_activity_id)) return riak_object def _get_many_activities(self, activity_ids=[], raw_filter="", filters=None, include_public=False, audience_targeting={}): """ Given a list of activity ids, returns a list of activities from riak. :type activity_ids: list :param activity_ids: The list of activities you want to retrieve :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on the ``audience_targeting`` parameter. :type filters: dict :param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``. Filters do not work for nested dictionaries. :type include_public: boolean :param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are not targeted towards anyone are included in the results :type audience_targeting: dict :param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``. """ activity_bucket_name = self._activities.name activities = self._riak_backend for activity_id in activity_ids: activities = activities.add(activity_bucket_name, str(activity_id)) results = activities.map(JS_MAP) if audience_targeting: results = results.reduce(JS_REDUCE_FILTER_AUD_TARGETTING, options={'arg': {'public': include_public, 'filters': audience_targeting}}) if filters or raw_filter: # An empty `filters` dict would cause all activities to be filtered out. If you wanted that effect, you # wouldn't have to call this function, so let's assume that an empty dict is a typical default value and # should denote that there are no filters to apply. filters = filters or None results = results.reduce(JS_REDUCE_FILTER_PROP, options={'arg': {'raw_filter': raw_filter, 'filters': filters}}) results = results.reduce(JS_REDUCE).run() results = results or [] # riak does not return the results in any particular order (unless we sort). So, # we have to put the objects returned by riak back in order results_map = dict(map(lambda result: (result['id'], result,), results)) reordered_results = [results_map[id] for id in activity_ids if id in results_map] return reordered_results def _get_timestamp(self): """ returns a unix timestamp representing the ``datetime`` object """ dt_obj = datetime.datetime.utcnow() return long((calendar.timegm(dt_obj.utctimetuple()) * 1000)) + (dt_obj.microsecond / 1000) def get_new_id(self): """ Generates a new unique ID. The default implementation uses uuid1 to generate a unique ID. :return: a new id """ return uuid.uuid1().hex
client = RiakClient(protocol='pbc') bkeys = [('multiget', str(key)) for key in xrange(10000)] data = open(__file__).read() print "Benchmarking multiget:" print " CPUs: {0}".format(cpu_count()) print " Threads: {0}".format(POOL_SIZE) print " Keys: {0}".format(len(bkeys)) print with benchmark.measure() as b: with b.report('populate'): for bucket, key in bkeys: client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() for b in benchmark.measure_with_rehearsal(): client.protocol = 'http' with b.report('http seq'): for bucket, key in bkeys: client.bucket(bucket).get(key) with b.report('http multi'): multiget(client, bkeys) client.protocol = 'pbc' with b.report('pbc seq'): for bucket, key in bkeys: client.bucket(bucket).get(key)
return [line.strip() for line in open(keyListFilename, 'r')] # unused helper method to save an image to the disk if you want that def saveImage(riakObj, filename): with open(os.path.join(baseDir, filename), 'w+') as f: f.write(riakObj.encoded_data) # Read in redis-keys imgs = readRiakKeys() print 'Riak Keys: \n' + '\n'.join(imgs) # connect to live riak rcLive = RiakClient(protocol='http', host=srcRiakHost, http_port=8098) rbLive = rcLive.bucket('ez') # connect to integration riak rcInt = RiakClient(protocol='http', host=destRiakHost, http_port=8098) rbInt = rcInt.bucket('ez') # save image in integration riak def saveToIntRiak(key, riakObj): if riakObj.content_type is not None: img = rbInt.new(key, encoded_data=riakObj.encoded_data, content_type=riakObj.content_type) img.store()
from django.conf import settings from simplejson import loads, dumps from riak import RiakClient from datetime import datetime import re HOST = settings.RIAK_HOST PORT = settings.RIAK_PORT client = RiakClient(host=HOST, port=PORT) page_bucket = client.bucket('riakipage') version_bucket = client.bucket('riakiversion') tag_bucket = client.bucket('riakitag') # since it's inefficient to list all the items # in a bucket, we manually index some things index_bucket = client.bucket('riakiindex') # Create your models here. DTFORMAT = "%Y-%m-%dT%H:%M:%S" def slugify(s): return s.replace(" ", "-") class Page:
parser = argparse.ArgumentParser(description='Takes a list of redis keys and deletes them') parser.add_argument('--host', dest='dest_riak', default='10.228.39.181', help='The host we delete the riak data from') args = parser.parse_args() # map arguments dest_riak_host = args.dest_riak print 'Targeted Riak Host ' + dest_riak_host base_dir = os.path.dirname(os.path.realpath(__file__)) # connect to live riak riak_connection = RiakClient(protocol='http', host=dest_riak_host, http_port=8098) print riak_connection.ping() riak_bucket = riak_connection.bucket('ez') # Parses the keys.txt file and writes url def readRiakKeys(): keyListFilename = os.path.join(base_dir, 'keys.txt') return [line.strip() for line in open(keyListFilename, 'r')] imgs = readRiakKeys() print 'Deleting Riak Keys: \n' + '\n'.join(imgs) # get and save all images for img in imgs: obj = riak_bucket.get(img) print 'Image exists? %s' % str(obj.exists) if obj.exists is True: print 'Now deleting key: %s' % img
def get_bucket(): if 'RIAK_PORT_8098_TCP_ADDR' in os.environ: riak = RiakClient('http', host=os.environ['RIAK_PORT_8098_TCP_ADDR']) else: riak = RiakClient('http') return riak.bucket('apps')
from riak import RiakClient from flask import Flask, jsonify from docopt import docopt import sys sys.path.insert(0, '../redis') from RedisConfig import RedisConfig arguments = docopt(__doc__) language = arguments['<language>'] cfg = RedisConfig() # single threaded blocking... for now client = RiakClient(host=cfg.RIAK_IPS.split(',')[0], pb_port=cfg.RIAK_PORT, protocol="pbc") bucket = client.bucket("%s/top_flowers" % language) node_set = set() link_set = set() def keygen(): for keys in bucket.stream_keys(): for key in keys: if key == 'K': continue yield key for k in keygen(): nodes, links = bucket.get(k).data nodes = [tuple(lst) for lst in nodes] links = [tuple(lst) for lst in links]