def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
# create key value pairs to stock in riak
key = str(str(pair[0]) + "_" + str(pair[1]))
val = {
"StockA": pair[0],
"StockB": pair[1],
"Date": pair[2],
"CloseA": pair[3],
"CloseB": pair[4],
"ZScore": pair[5],
"Beta": pair[6],
"SignalMean": pair[7],
"SignalSD": pair[8],
}
myDate = pair[2].split("-")
obj = RiakObject(rc, bucket, key)
# add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = "text/json"
obj.data = val
obj.data = json.dumps(val)
# store
obj.store()
# return a list of written pairs
return pair
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket('stocks')
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
#create key value pairs to stock in riak
key = str(str(pair[0]) + '_' + str(pair[1]))
val = {'StockA': pair[0], \
'StockB': pair[1], \
'Date': pair[2],\
'CloseA': pair[3], \
'CloseB': pair[4], \
'ZScore': pair[5],\
'Beta': pair[6],\
'SignalMean': pair[7],\
'SignalSD': pair[8]}
myDate = pair[2].split('-')
obj = RiakObject(rc, bucket, key)
#add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = 'text/json'
obj.data = val
obj.data = json.dumps(val)
#store
obj.store()
#return a list of written pairs
return pair
def test_close_underlying_socket_retry(self):
c = RiakClient(PB_HOST,
PB_PORT,
transport_class=RiakPbcTransport,
transport_options={"max_attempts": 2})
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('barbaz', rand)
obj.store()
obj = bucket.get('barbaz')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'barbaz')
self.assertEqual(obj.get_data(), rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This should work, since we have a retry
obj = bucket.get('barbaz')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'barbaz')
self.assertEqual(obj.get_data(), rand)
class DS_Riak(DS):
""" Riak DS
"""
def __init__(self, host=PB_HOST, port=PB_PORT, bucketname='_scads'):
"""
Params:
host: Hostname or IP Address of the riak database
port: Port of the riak database
bucketname: Where to store stuff
"""
self.client = RiakClient(host, port,
transport_class=RiakPbcTransport,
transport_options={'max_attempts': 3})
self.bucket = self.client.bucket(bucketname)
def store(self, key, value, indexes=None):
obj = self.bucket.get(key)
obj.set_data(value)
if indexes is not None:
for index in indexes:
obj.add_index('scads_bin', index)
obj.store()
def get_index(self, index):
return self.bucket.get_index('scads_bin', index)
def get(self, key):
d = self.bucket.get(key).get_data()
if d is None:
return None
return d
def clear(self):
for key in self.bucket.get_keys():
self.bucket.get(key).delete()
def selection_function(request):
RiakClient()
RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}])
myClient = RiakClient()
myBucket = myClient.bucket('users')
keys = myBucket.get_keys()
usuarios = Clientes()
lista2 = []
for i in range(len(keys)):
lista = {
'name': myBucket.get(keys[i]).data['name'],
'location': myBucket.get(keys[i]).data['location'],
'role': myBucket.get(keys[i]).data['role'],
'fullname': myBucket.get(keys[i]).data['fullname']
}
lista2.append(lista)
# for i in range(len(lista1)):
# #lista.append(myBucket.get(keys[i]).data['name'])
# usuarios.nome = str(lista1[i][0])
# #lista.append(myBucket.get(keys[i]).data['location'])
# usuarios.locacao = str(lista1[i][1])
# #lista.append(myBucket.get(keys[i]).data['role'])
# usuarios.profissao = str(lista1[i][2])
# #lista.append(myBucket.get(keys[i]).data['fulname'])
# usuarios.apelido = str(lista1[i][3])
# usuarios.save()
usuarios = Clientes.objects.all()
return render(request, 'usuario.html', {'usuarios': lista2})
def test_close_underlying_socket_retry(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport,
transport_options={"max_attempts": 2})
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('barbaz', rand)
obj.store()
obj = bucket.get('barbaz')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'barbaz')
self.assertEqual(obj.data, rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This should work, since we have a retry
obj = bucket.get('barbaz')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'barbaz')
self.assertEqual(obj.data, rand)
def register(host,user_id,time): currenttld = extractdom(host) riakclient = RiakClient() #bucket : trackers, key:user_id, object:json map str_bucket = 'trackers' bucket = riakclient.bucket(str_bucket) str_key = user_id obj = RiakObject(riakclient, bucket, str_key)
def conn(self):
"""Open connection on Riak DataBase"""
client = RiakClient(protocol=self.conf.get("protocol"),
http_port=self.conf.get("port"),
host=self.conf.get("host"))
conn = client.bucket(self.conf.get("db"))
conn.enable_search()
return conn
class UserService:
def __init__(self):
self.users_bucket = settings.RIAK_USERS_BUCKET
self.riak = RiakClient(host=settings.RIAK_HOST,
port=settings.RIAK_PORT)
def save(self, user_data):
key = self.generate_key()
password = user_data['password']
user_data['encrypted_password'] = make_password('sha1', password)
del user_data['password']
user = self.riak.bucket(self.users_bucket).new(key, user_data)
user.store()
return key
def get(self, user_id):
user = self.riak.bucket(settings.RIAK_USERS_BUCKET).get(str(user_id))
if user.exists():
return user.get_data()
else:
return None
def login(self, username, password):
try:
user = self.riak.search(settings.RIAK_USERS_BUCKET,
"username:%s" % username).run()
except ValueError:
user = None
if user:
user = user[0]
user.set_bucket(str(user.get_bucket()))
user.set_key(str(user.get_key()))
user_data = user.get().get_data()
if check_password(password, user_data['encrypted_password']):
return user
else:
return False
else:
return False
def generate_key(self):
return uuid.uuid1().hex
def conn(self):
"""Open connection on Riak DataBase"""
client = RiakClient(
protocol=self.conf.get("protocol"),
http_port=self.conf.get("port"),
host=self.conf.get("host"))
conn = client.bucket(self.conf.get("db"))
conn.enable_search()
return conn
def clear_expired(cls):
# TODO: get_keys() method is not efficent operation for searching sessions, use search feature or mapreduce or ...
client = RiakClient(host=RIAK_DB_HOST,
port=RIAK_DB_PORT,
transport_class=RiakPbcTransport)
sessions = client.bucket(RIAK_DB_BUCKET)
now = int(timezone.now().strftime("%s"))
for key in sessions.get_keys():
data = sessions.get(key)
if int(data.get_usermeta()['expire_age']) < now:
data.delete()
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
# get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket("stocks")
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
# didnt get any data
if len(gtemp) == 0:
return 0
# got data
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def get_flower(lang, user, repo_name):
client = RiakClient(host=riak_ip, pb_port=riak_port, protocol="pbc")
bucket = client.bucket("%s/flowers" % lang)
repo = "%s/%s" % (user, repo_name)
graph = bucket.get(repo).data
if graph is None:
abort(404)
_nodes, _links = graph
nodes = [{"id": repo, "group": degree} for (repo, degree) in _nodes]
del _nodes
links = [{"source": repo_a, "target": repo_b} for \
(repo_a, repo_b) in _links]
del _links
return jsonify({"nodes": nodes, "links": links})
def run_cluster_server(riak_conf):
riak_hosts = []
with open(riak_conf, "r") as fin:
for line in fin:
riak_hosts.append({"host": line.replace("\n", ""), "http_port": RIAK_PORT})
cluster = RiakClient(nodes=riak_hosts)
bucket = cluster.bucket("HashedData")
random_str_arr = []
random_str_arr = run_exp_init(socket.gethostname())
insert_throuput = run_exp_insert(bucket, random_str_arr)
lookup_throuput = run_exp_lookup(bucket, random_str_arr)
delete_throuput = run_exp_delete(bucket, random_str_arr)
overall_throuput = (insert_throuput + lookup_throuput + delete_throuput) / 3
print("Overall Throughput: " + str(overall_throuput) + " OPs/s")
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def test_close_underlying_socket_fails(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport)
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('foo', rand)
obj.store()
obj = bucket.get('foo')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'foo')
self.assertEqual(obj.get_data(), rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This shoud fail with a socket error now
self.assertRaises(socket.error, bucket.get, 'foo')
def test_close_underlying_socket_fails(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport)
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('foo', rand)
obj.store()
obj = bucket.get('foo')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'foo')
self.assertEqual(obj.data, rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This shoud fail with a socket error now
self.assertRaises(socket.error, bucket.get, 'foo')
def insert_function(request):
RiakClient()
RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}])
myClient = RiakClient()
myBucket = myClient.bucket('users')
getNome = request.POST.get("name")
getNome = str(getNome)
getLocation = request.POST.get("location")
getRole = request.POST.get("role")
getFulname = request.POST.get("fullname")
nome = str('user') + str(randint(0, 9))
nome = {
'name': getNome,
'location': getLocation,
'role': getRole,
'fullname': getFulname
}
obj = myBucket.new(str(nome), data=nome)
obj.store()
return redirect('/home')
class MaternityService:
def __init__(self):
self.maternity_bucket = settings.RIAK_MATERNITY_BUCKET
self.riak = RiakClient(host = settings.RIAK_DATABASE['HOST'],
port = settings.RIAK_DATABASE['PORT'])
def get(self, pk):
entity = self.riak.bucket(settings.RIAK_MATERNITY_BUCKET).get(str(pk))
if entity.exists():
data = json.loads(entity.get_data())
data['statistics'] = sorted(data['statistics'].iteritems())
return data
else:
return None
def get_authorities(self):
query = self.riak.add(settings.RIAK_MATERNITY_BUCKET)
query.map('''
function(v) {
var data = JSON.parse(v.values[0].data);
if(data.type == 'AUTHORITY') {
return [[v.key, data]];
}
return [];
}
''')
return query.run()
#data = []
#for result in query.run():
# d = result[1]
#dj = json.loads(d)
# data.append((result[0], d))
#return data
#return [
# (result[0], json.loads(result[1]))
# for result in query.run()
# ]
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket("stocks")
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
class ImageService:
def __init__(self):
self.riak = RiakClient(host=settings.RIAK_HOST,
port=settings.RIAK_PORT)
self._metadata_bucket = self.riak.bucket(settings.RIAK_METADATA_BUCKET)
self._image_bucket = self.riak.bucket(settings.RIAK_IMAGE_BUCKET)
self._thumbs_bucket = self.riak.bucket(settings.RIAK_THUMBS_BUCKET)
def store_from_url(self, address, user):
url = urlparse(address)
if str(url.scheme) != 'http':
raise ImageError('Not a valid URL')
if not re.search(r'(jpg|jpeg|png|gif)$', url.path):
raise ImageError('Unsupported file format')
http = httplib.HTTPConnection(url.netloc)
http.request('GET', url.path)
response = http.getresponse()
image = RemoteImage(response)
return self.store(image, user,
response.getheader("Content-Type", "image/jpg"))
def store(self, image, user, content_type):
key = self.create_unique_key()
filename = self.filename_for_image(key, content_type)
data = {"user": user,
"uploaded_at": datetime.utcnow().isoformat(),
"filename": filename}
thumbnail = self.create_thumbnail(image)
metadata = self._metadata_bucket.new(key, data,
content_type="application/json")
metadata.store()
image = self._image_bucket.new_binary(key, image.read(), content_type)
image.store()
thumb = self._thumbs_bucket.new_binary(key, thumbnail,
content_type="image/jpeg")
thumb.store()
return key
def create_thumbnail(self, image):
thumbnail = Image.open(image.temporary_file_path())
thumbnail.thumbnail((128, 128), Image.ANTIALIAS)
thumbio = StringIO()
thumbnail.save(thumbio, format="jpeg")
return thumbio.getvalue()
def find_metadata(self, image_id):
image = self._metadata_bucket.get(image_id)
if image.exists():
return image.get_data()
else:
return None
def find(self, image_id, thumb=False):
if thumb:
image = self._thumbs_bucket.get_binary(image_id)
else:
image = self._image_bucket.get_binary(image_id)
if image.exists():
return image
else:
return None
def find_all(self, user):
images = self.riak.search(settings.RIAK_METADATA_BUCKET,
"user:%s" % user).map(MAP_DOCID).run()
return images
def create_unique_key(self, length=6):
unique = False
while not unique:
id = self.unique_key(length)
if not self._image_bucket.get(id).exists():
return id
def unique_key(self, length=6):
return shortuuid.uuid()[0:length]
def filename_for_image(self, key, content_type):
if content_type in ['image/jpg', 'image/jpeg']:
extension = 'jpg'
elif content_type == "image/png":
extension = 'png'
elif content_type == 'image/gif':
extension = 'gif'
return "%s.%s" % (key, extension)
# This is only needed if this script is outside of the Riak Python library folder
import sys
sys.path.insert(0, '/root/riak-python-client')
import riak
from riak import RiakClient
from riak.security import SecurityCreds
print "Initializing certificates."
creds = SecurityCreds(username='******',
cacert_file='/home/vagrant/certs/rootCA.crt',
cert_file='/home/vagrant/certs/riakuser.crt',
pkey_file='/home/vagrant/certs/riakuser.key')
print "Testing Riak connectivity."
myClient = RiakClient(host="riak1.myserver.com",
pb_port=8087,
credentials=creds)
myBucket = myClient.bucket('RiakTestBucket')
val1 = 1
key1 = myBucket.new('KeyOne', data=val1)
fetched1 = myBucket.get('KeyOne')
if fetched1.data is None:
print "Ok."
else:
print fetched1.data
client = RiakClient(protocol='pbc')
bkeys = [('multiget', str(key)) for key in xrange(10000)]
data = open(__file__).read()
print "Benchmarking multiget:"
print " CPUs: {0}".format(cpu_count())
print " Threads: {0}".format(POOL_SIZE)
print " Keys: {0}".format(len(bkeys))
print
with benchmark.measure() as b:
with b.report('populate'):
for bucket, key in bkeys:
client.bucket(bucket).new(key, encoded_data=data,
content_type='text/plain'
).store()
for b in benchmark.measure_with_rehearsal():
client.protocol = 'http'
with b.report('http seq'):
for bucket, key in bkeys:
client.bucket(bucket).get(key)
with b.report('http multi'):
multiget(client, bkeys)
client.protocol = 'pbc'
with b.report('pbc seq'):
for bucket, key in bkeys:
client.bucket(bucket).get(key)
#!/usr/bin/env python3
from riak import RiakClient, RiakObject, RiakBucket
riak_client = RiakClient(nodes=[{'host':'192.168.50.11','http_port':8098},
{'host':'192.168.50.12','http_port':8098},
{'host':'192.168.50.13','http_port':8098}])
bucket = riak_client.bucket('testbucket')
for object_id in range(100000):
riak_object = RiakObject(riak_client, bucket, str(object_id))
riak_object.data = {'id': object_id, 'desc': 'Description for ' + str(object_id)}
riak_object.store()
import os
import json
import hashlib
from riak import RiakClient
from riak import RiakPbcTransport
riak_client = RiakClient(os.getenv("RUNZ_RIAK_HOST") or "127.0.0.1", 8087, transport_class=RiakPbcTransport)
proj_bucket = riak_client.bucket("projects")
for key in proj_bucket.get_keys():
proj_bucket.get_binary(key).delete()
class RiakBackend(BaseBackend):
custom_epoch = datetime.datetime(month=1, day=1, year=2013)
def __init__(
self, protocol="pbc", nodes=[], objects_bucket_name="objects",
activities_bucket_name="activities", **kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
def clear_all(self, **kwargs):
"""
Deletes all activity stream data from riak
"""
self.clear_all_activities()
self.clear_all_objects()
def clear_all_objects(self, **kwargs):
"""
Deletes all objects data from riak
"""
for key in self._objects.get_keys():
self._objects.get(key).delete(r='all', w='all', dw='all')
assert not self._objects.get(key).exists
def clear_all_activities(self, **kwargs):
"""
Deletes all activities data from riak
"""
for key in self._activities.get_keys():
self._activities.get(key).delete(r='all', w='all', dw='all')
assert not self._activities.get(key).exists
def obj_exists(self, obj, **kwargs):
obj_id = self._extract_id(obj)
return self._objects.get(obj_id).exists
def activity_exists(self, activity, **kwargs):
activity_id = self._extract_id(activity)
return self._activities.get(activity_id).exists
def obj_create(self, obj, **kwargs):
obj = Object(obj, backend=self)
obj.validate()
obj_dict = obj.get_parsed_dict()
key = self._extract_id(obj_dict)
riak_obj = self._objects.new(key=key)
riak_obj.data = obj_dict
riak_obj = self.set_general_indexes(riak_obj)
riak_obj.store()
#finally save the data
return obj_dict
def set_general_indexes(self, riak_object):
"""
Sets the default riak 2Is.
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes):
riak_object.add_index("timestamp_int", self._get_timestamp())
riak_object.remove_index('modified_int')
riak_object.add_index("modified_int", self._get_timestamp())
return riak_object
def obj_update(self, obj, **kwargs):
self.obj_create(obj, **kwargs)
def obj_get(self, obj, **kwargs):
"""
Given a list of object ids, returns a list of objects
"""
if not obj:
return obj
object_bucket_name = self._objects.name
objects = self._riak_backend
for o in obj:
objects = objects.add(object_bucket_name, self._extract_id(o))
results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run()
return results or []
def obj_delete(self, obj, **kwargs):
obj_id = self._extract_id(obj)
self._objects.new(key=obj_id).delete()
def activity_create(self, activity, **kwargs):
"""
Creates an activity. You can provide objects for activities as dictionaries or as ids for already
existing objects.
If you provide a dictionary for an object, it is saved as a new object.
If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty
dictionary when retriving the activity later.
"""
activity = Activity(activity, backend=self)
activity.validate()
activity_dict = activity.get_parsed_dict()
key = self._extract_id(activity_dict)
riak_obj = self._activities.new(key=key)
riak_obj.data = activity_dict
riak_obj = self.set_activity_indexes(self.set_general_indexes(riak_obj))
if activity_dict['verb'] in SUB_ACTIVITY_MAP:
riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs)
riak_obj.store()
return self.dehydrate_activities([riak_obj.data])[0]
def set_activity_indexes(self, riak_object):
"""
Store indexes specific to an ``Activity``. Stores the following indexes:
1. ``verb`` of the ``Activity``
2. ``actor`` of the ``Activity``
3. ``object`` of the ``Activity``
4. if target is defined, verb for the ``target`` of the Activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
_dict = riak_object.data
riak_object.remove_index('verb_bin')
riak_object.remove_index('actor_bin')
riak_object.remove_index('object_bin')
riak_object.add_index("verb_bin", self._extract_id(_dict['verb']))
riak_object.add_index("actor_bin", self._extract_id(_dict['actor']))
riak_object.add_index("object_bin", self._extract_id(_dict['object']))
if 'target' in _dict and _dict.get("target"):
riak_object.remove_index('target_bin')
riak_object.add_index("target_bin", self._extract_id(_dict['target']))
return riak_object
def activity_delete(self, activity, **kwargs):
"""
Deletes an activity item and all associated sub items
"""
activity_dict = self.get_activity(activity, **kwargs)[0]
for response_field in Activity._response_fields:
if response_field in activity_dict:
for response_item in activity_dict[response_field]['items']:
self._activities.get(response_item['id']).delete()
self._activities.get(self._extract_id(activity_dict)).delete()
def activity_update(self, activity, **kwargs):
return self.activity_create(activity, **kwargs)
def activity_get(
self, activity_ids=[], raw_filter="", filters={}, include_public=False,
audience_targeting={}, aggregation_pipeline=[], **kwargs):
"""
Gets a list of activities. You can also group activities by providing a list of attributes to group
by.
:type activity_ids: list
:param activity_ids: The list of activities you want to retrieve
:type filters: dict
:param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``.
Filters do not work for nested dictionaries.
:type raw_filter: string
:param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set
or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on
the ``audience_targeting`` parameter.
:type include_public: boolean
:param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are
not targeted towards anyone are included in the results
:type audience_targeting: dict
:param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``.
The values are an array of object ids
:type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator``
:param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline
:return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set.
Activities are returned in the order of ids provided.
"""
activity_ids = map(self._extract_id, activity_ids)
if not activity_ids:
return []
activities = self._get_many_activities(
activity_ids, raw_filter=raw_filter, filters=filters, include_public=include_public,
audience_targeting=audience_targeting)
activities = self.dehydrate_activities(activities)
if aggregation_pipeline:
original_activities = copy.deepcopy(activities)
for aggregator in aggregation_pipeline:
activities = aggregator.process(activities, original_activities, aggregation_pipeline)
return activities
def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs):
if sub_activity_verb.lower() not in SUB_ACTIVITY_MAP:
raise Exception('Verb not supported')
return super(RiakBackend, self).create_sub_activity(
activity, actor, content, extra=extra,
sub_activity_verb=sub_activity_verb, **kwargs)
def sub_activity_create(
self, activity, actor, content, extra={}, sub_activity_verb="",
published=None, **kwargs):
sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0]
sub_activity_attribute = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1]
object_type = kwargs.get('object_type', sub_activity_verb)
activity_id = self._extract_id(activity)
activity_model = Activity(self._activities.get(key=activity_id).data, backend=self)
sub_activity_obj, original_activity_obj = activity_model\
.get_parsed_sub_activity_dict(
actor=actor, content=content, verb=sub_activity_verb,
object_type=object_type, collection=sub_activity_attribute,
activity_class=sub_activity_model, published=published, extra=extra)
sub_activity_obj = self.create_activity(sub_activity_obj, activity_id=original_activity_obj['id'])
original_activity_obj[sub_activity_attribute]['items'][0]['actor'] = sub_activity_obj['actor']['id']
original_activity_obj[sub_activity_attribute]['items'][0]['id'] = sub_activity_obj['id']
original_activity_obj[sub_activity_attribute]['items'][0]['published'] = sub_activity_obj['published']
original_activity_obj[sub_activity_attribute]['items'][0]['object']['id'] = sub_activity_obj['id']
original_activity_obj = self.activity_update(original_activity_obj)
return sub_activity_obj, original_activity_obj
def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs):
"""
Deletes a sub_activity made on an activity. This will also update the corresponding
parent activity.
:type sub_activity: string
:param sub_activity: the id of the reply activity to delete.
:type sub_activity_verb: string
:param sub_activity_verb: the verb of the sub activity
:return: a dict representing the updated parent activity
"""
sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0]
sub_activity_id = self._extract_id(sub_activity)
sub_activity_riak_model = self._activities.get(sub_activity_id)
if sub_activity_riak_model.data['verb'] != sub_activity_model.sub_item_verb:
raise SunspearValidationException("Trying to delete something that is not a {}.".format(sub_activity_model.sub_item_verb))
#clean up the reference from the original activity
in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin', sub_activity_riak_model.indexes)[0][1]
activity = self._activities.get(key=in_reply_to_key)
activity_data = activity.data
activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1
activity_data[sub_activity_model.sub_item_key]['items'] = filter(
lambda x: x["id"] != sub_activity_id,
activity_data[sub_activity_model.sub_item_key]['items'])
updated_activity = self.update_activity(activity_data, **kwargs)
self.delete_activity(sub_activity_id)
return updated_activity
def set_sub_item_indexes(self, riak_object, **kwargs):
"""
Store indexes specific to a sub-activity. Stores the following indexes:
1. id of the the parent ``Activity`` of this sub-activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
original_activity_id = kwargs.get('activity_id')
if not original_activity_id:
raise SunspearValidationException()
riak_object.remove_index('inreplyto_bin')
riak_object.add_index("inreplyto_bin", str(original_activity_id))
return riak_object
def dehydrate_activities(self, activities):
"""
Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object
"""
activities = self._extract_sub_activities(activities)
#collect a list of unique object ids. We only iterate through the fields that we know
#for sure are objects. User is responsible for hydrating all other fields.
object_ids = set()
for activity in activities:
object_ids.update(self._extract_object_keys(activity))
#Get the objects for the ids we have collected
objects = self.get_obj(object_ids)
objects_dict = dict(((obj["id"], obj,) for obj in objects))
#We also need to extract any activities that were diguised as objects. IE activities with
#objectType=activity
activities_in_objects_ids = set()
#replace the object ids with the hydrated objects
for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)
#Extract keys of any activities that were objects
activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True))
#If we did have activities that were objects, we need to hydrate those activities and
#the objects for those activities
if activities_in_objects_ids:
sub_activities = self._get_many_activities(activities_in_objects_ids)
activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities))
for activity in activities:
activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True)
#we have to do one more round of object dehydration for our new sub-activities
object_ids.update(self._extract_object_keys(activity))
#now get all the objects we don't already have and for sub-activities and and hydrate them into
#our list of activities
object_ids -= set(objects_dict.keys())
objects = self.get_obj(object_ids)
for obj in objects:
objects_dict[obj["id"]] = obj
for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)
return activities
def _extract_sub_activities(self, activities):
"""
Extract all objects that have an objectType of activity as an activity
"""
#We might also have to get sub activities for things like replies and likes
activity_ids = set()
activities_dict = dict(((activity["id"], activity,) for activity in activities))
for activity in activities:
activity_ids.update(self._extract_activity_keys(activity))
if activity_ids:
#don't bother fetching the activities we already have
activity_ids -= set(activities_dict.keys())
if activity_ids:
sub_activities = self._get_many_activities(activity_ids)
for sub_activity in sub_activities:
activities_dict[sub_activity["id"]] = sub_activity
#Dehydrate out any subactivities we may have
for activity in activities:
activity = self._dehydrate_sub_activity(activity, activities_dict)
return activities
def _extract_activity_keys(self, activity, skip_sub_activities=False):
keys = []
for activity_key in Model._object_fields + ['inReplyTo']:
if activity_key not in activity:
continue
obj = activity.get(activity_key)
if isinstance(obj, dict):
if obj.get('objectType', None) == 'activity':
keys.append(obj['id'])
if obj.get('inReplyTo', None):
[keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo']]
if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in activity and activity[collection]['items']:
for item in activity[collection]['items']:
keys.extend(self._extract_activity_keys(item))
return keys
def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False):
for activity_key in Model._object_fields:
if activity_key not in sub_activity:
continue
if isinstance(sub_activity[activity_key], dict):
if sub_activity[activity_key].get('objectType', None) == 'activity':
sub_activity[activity_key].update(obj_list[sub_activity[activity_key]['id']])
if sub_activity[activity_key].get('inReplyTo', None):
for i, in_reply_to_obj in enumerate(sub_activity[activity_key]['inReplyTo']):
sub_activity[activity_key]['inReplyTo'][i]\
.update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']])
if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in sub_activity and sub_activity[collection]['items']:
dehydrated_sub_items = []
for i, item in enumerate(sub_activity[collection]['items']):
try:
dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list))
except KeyError, e:
pass
sub_activity[collection]['items'] = dehydrated_sub_items
sub_activity[collection]['totalItems'] = len(dehydrated_sub_items)
return sub_activity
return [line.strip() for line in open(keyListFilename, 'r')]
# unused helper method to save an image to the disk if you want that
def saveImage(riakObj, filename):
with open(os.path.join(baseDir, filename), 'w+') as f:
f.write(riakObj.encoded_data)
# Read in redis-keys
imgs = readRiakKeys()
print 'Riak Keys: \n' + '\n'.join(imgs)
# connect to live riak
rcLive = RiakClient(protocol='http', host=srcRiakHost, http_port=8098)
rbLive = rcLive.bucket('ez')
# connect to integration riak
rcInt = RiakClient(protocol='http', host=destRiakHost, http_port=8098)
rbInt = rcInt.bucket('ez')
# save image in integration riak
def saveToIntRiak(key, riakObj):
if riakObj.content_type is not None:
img = rbInt.new(key, encoded_data=riakObj.encoded_data, content_type=riakObj.content_type)
img.store()
# get and save all images
for img in imgs:
print img
local = rbInt.get(img)
class RiakBackend(BaseBackend):
custom_epoch = datetime.datetime(month=1, day=1, year=2013)
def __init__(self,
protocol="pbc",
nodes=[],
objects_bucket_name="objects",
activities_bucket_name="activities",
**kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
def clear_all(self, **kwargs):
"""
Deletes all activity stream data from riak
"""
self.clear_all_activities()
self.clear_all_objects()
def clear_all_objects(self, **kwargs):
"""
Deletes all objects data from riak
"""
for key in self._objects.get_keys():
self._objects.get(key).delete(rw='all', r='all', w='all', dw='all')
assert not self._objects.get(key).exists
def clear_all_activities(self, **kwargs):
"""
Deletes all activities data from riak
"""
for key in self._activities.get_keys():
self._activities.get(key).delete(rw='all',
r='all',
w='all',
dw='all')
assert not self._activities.get(key).exists
def obj_exists(self, obj, **kwargs):
obj_id = self._extract_id(obj)
return self._objects.get(obj_id).exists
def activity_exists(self, activity, **kwargs):
activity_id = self._extract_id(activity)
return self._activities.get(activity_id).exists
def obj_create(self, obj, **kwargs):
obj = Object(obj, backend=self)
obj.validate()
obj_dict = obj.get_parsed_dict()
key = self._extract_id(obj_dict)
riak_obj = self._objects.new(key=key)
riak_obj.data = obj_dict
riak_obj = self.set_general_indexes(riak_obj)
riak_obj.store()
#finally save the data
return obj_dict
def set_general_indexes(self, riak_object):
"""
Sets the default riak 2Is.
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes):
riak_object.add_index("timestamp_int", self._get_timestamp())
riak_object.remove_index('modified_int')
riak_object.add_index("modified_int", self._get_timestamp())
return riak_object
def obj_update(self, obj, **kwargs):
self.obj_create(obj, **kwargs)
def obj_get(self, obj, **kwargs):
"""
Given a list of object ids, returns a list of objects
"""
if not obj:
return obj
object_bucket_name = self._objects.name
objects = self._riak_backend
for o in obj:
objects = objects.add(object_bucket_name, self._extract_id(o))
results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run()
return results or []
def obj_delete(self, obj, **kwargs):
obj_id = self._extract_id(obj)
self._objects.new(key=obj_id).delete()
def activity_create(self, activity, **kwargs):
"""
Creates an activity. You can provide objects for activities as dictionaries or as ids for already
existing objects.
If you provide a dictionary for an object, it is saved as a new object.
If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty
dictionary when retriving the activity later.
"""
activity = Activity(activity, backend=self)
activity.validate()
activity_dict = activity.get_parsed_dict()
key = self._extract_id(activity_dict)
riak_obj = self._activities.new(key=key)
riak_obj.data = activity_dict
riak_obj = self.set_activity_indexes(
self.set_general_indexes(riak_obj))
if activity_dict['verb'] in SUB_ACTIVITY_MAP:
riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs)
riak_obj.store()
return self.dehydrate_activities([riak_obj.data])[0]
def set_activity_indexes(self, riak_object):
"""
Store indexes specific to an ``Activity``. Stores the following indexes:
1. ``verb`` of the ``Activity``
2. ``actor`` of the ``Activity``
3. ``object`` of the ``Activity``
4. if target is defined, verb for the ``target`` of the Activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
_dict = riak_object.data
riak_object.remove_index('verb_bin')
riak_object.remove_index('actor_bin')
riak_object.remove_index('object_bin')
riak_object.add_index("verb_bin", self._extract_id(_dict['verb']))
riak_object.add_index("actor_bin", self._extract_id(_dict['actor']))
riak_object.add_index("object_bin", self._extract_id(_dict['object']))
if 'target' in _dict and _dict.get("target"):
riak_object.remove_index('target_bin')
riak_object.add_index("target_bin",
self._extract_id(_dict['target']))
return riak_object
def activity_delete(self, activity, **kwargs):
"""
Deletes an activity item and all associated sub items
"""
activity_dict = self.get_activity(activity, **kwargs)[0]
for response_field in Activity._response_fields:
if response_field in activity_dict:
for response_item in activity_dict[response_field]['items']:
self._activities.get(response_item['id']).delete()
self._activities.get(self._extract_id(activity_dict)).delete()
def activity_update(self, activity, **kwargs):
return self.activity_create(activity, **kwargs)
def activity_get(self,
activity_ids=[],
raw_filter="",
filters={},
include_public=False,
audience_targeting={},
aggregation_pipeline=[],
**kwargs):
"""
Gets a list of activities. You can also group activities by providing a list of attributes to group
by.
:type activity_ids: list
:param activity_ids: The list of activities you want to retrieve
:type filters: dict
:param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``.
Filters do not work for nested dictionaries.
:type raw_filter: string
:param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set
or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on
the ``audience_targeting`` parameter.
:type include_public: boolean
:param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are
not targeted towards anyone are included in the results
:type audience_targeting: dict
:param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``.
The values are an array of object ids
:type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator``
:param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline
:return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set.
Activities are returned in the order of ids provided.
"""
activity_ids = map(self._extract_id, activity_ids)
if not activity_ids:
return []
activities = self._get_many_activities(
activity_ids,
raw_filter=raw_filter,
filters=filters,
include_public=include_public,
audience_targeting=audience_targeting)
activities = self.dehydrate_activities(activities)
original_activities = copy.deepcopy(activities)
for aggregator in aggregation_pipeline:
activities = aggregator.process(activities, original_activities,
aggregation_pipeline)
return activities
def create_sub_activity(self,
activity,
actor,
content,
extra={},
sub_activity_verb="",
**kwargs):
if sub_activity_verb.lower() not in SUB_ACTIVITY_MAP:
raise Exception('Verb not supported')
return super(RiakBackend, self).create_sub_activity(
activity,
actor,
content,
extra=extra,
sub_activity_verb=sub_activity_verb,
**kwargs)
def sub_activity_create(self,
activity,
actor,
content,
extra={},
sub_activity_verb="",
**kwargs):
sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0]
sub_activity_attribute = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1]
object_type = kwargs.get('object_type', sub_activity_verb)
activity_id = self._extract_id(activity)
activity_model = Activity(self._activities.get(key=activity_id).data,
backend=self)
sub_activity_obj, original_activity_obj = activity_model\
.get_parsed_sub_activity_dict(
actor=actor, content=content, verb=sub_activity_verb,
object_type=object_type, collection=sub_activity_attribute,
activity_class=sub_activity_model, extra=extra)
sub_activity_obj = self.create_activity(
sub_activity_obj, activity_id=original_activity_obj['id'])
original_activity_obj[sub_activity_attribute]['items'][0][
'actor'] = sub_activity_obj['actor']['id']
original_activity_obj[sub_activity_attribute]['items'][0][
'id'] = sub_activity_obj['id']
original_activity_obj[sub_activity_attribute]['items'][0][
'published'] = sub_activity_obj['published']
original_activity_obj[sub_activity_attribute]['items'][0]['object'][
'id'] = sub_activity_obj['id']
original_activity_obj = self.activity_update(original_activity_obj)
return sub_activity_obj, original_activity_obj
def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs):
"""
Deletes a sub_activity made on an activity. This will also update the corresponding
parent activity.
:type sub_activity: string
:param sub_activity: the id of the reply activity to delete.
:type sub_activity_verb: string
:param sub_activity_verb: the verb of the sub activity
:return: a dict representing the updated parent activity
"""
sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0]
sub_activity_id = self._extract_id(sub_activity)
sub_activity_riak_model = self._activities.get(sub_activity_id)
if sub_activity_riak_model.data[
'verb'] != sub_activity_model.sub_item_verb:
raise SunspearValidationException(
"Trying to delete something that is not a {}.".format(
sub_activity_model.sub_item_verb))
#clean up the reference from the original activity
in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin',
sub_activity_riak_model.indexes)[0][1]
activity = self._activities.get(key=in_reply_to_key)
activity_data = activity.data
activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1
activity_data[sub_activity_model.sub_item_key]['items'] = filter(
lambda x: x["id"] != sub_activity_id,
activity_data[sub_activity_model.sub_item_key]['items'])
updated_activity = self.update_activity(activity_data, **kwargs)
self.delete_activity(sub_activity_id)
return updated_activity
def set_sub_item_indexes(self, riak_object, **kwargs):
"""
Store indexes specific to a sub-activity. Stores the following indexes:
1. id of the the parent ``Activity`` of this sub-activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
original_activity_id = kwargs.get('activity_id')
if not original_activity_id:
raise SunspearValidationException()
riak_object.remove_index('inreplyto_bin')
riak_object.add_index("inreplyto_bin", str(original_activity_id))
return riak_object
def dehydrate_activities(self, activities):
"""
Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object
"""
activities = self._extract_sub_activities(activities)
#collect a list of unique object ids. We only iterate through the fields that we know
#for sure are objects. User is responsible for hydrating all other fields.
object_ids = set()
for activity in activities:
object_ids.update(self._extract_object_keys(activity))
#Get the objects for the ids we have collected
objects = self.get_obj(object_ids)
objects_dict = dict(((
obj["id"],
obj,
) for obj in objects))
#We also need to extract any activities that were diguised as objects. IE activities with
#objectType=activity
activities_in_objects_ids = set()
#replace the object ids with the hydrated objects
for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)
#Extract keys of any activities that were objects
activities_in_objects_ids.update(
self._extract_activity_keys(activity,
skip_sub_activities=True))
#If we did have activities that were objects, we need to hydrate those activities and
#the objects for those activities
if activities_in_objects_ids:
sub_activities = self._get_many_activities(
activities_in_objects_ids)
activities_in_objects_dict = dict(((
sub_activity["id"],
sub_activity,
) for sub_activity in sub_activities))
for activity in activities:
activity = self._dehydrate_sub_activity(
activity,
activities_in_objects_dict,
skip_sub_activities=True)
#we have to do one more round of object dehydration for our new sub-activities
object_ids.update(self._extract_object_keys(activity))
#now get all the objects we don't already have and for sub-activities and and hydrate them into
#our list of activities
object_ids -= set(objects_dict.keys())
objects = self.get_obj(object_ids)
for obj in objects:
objects_dict[obj["id"]] = obj
for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)
return activities
def _extract_sub_activities(self, activities):
"""
Extract all objects that have an objectType of activity as an activity
"""
#We might also have to get sub activities for things like replies and likes
activity_ids = set()
activities_dict = dict(((
activity["id"],
activity,
) for activity in activities))
for activity in activities:
activity_ids.update(self._extract_activity_keys(activity))
if activity_ids:
#don't bother fetching the activities we already have
activity_ids -= set(activities_dict.keys())
if activity_ids:
sub_activities = self._get_many_activities(activity_ids)
for sub_activity in sub_activities:
activities_dict[sub_activity["id"]] = sub_activity
#Dehydrate out any subactivities we may have
for activity in activities:
activity = self._dehydrate_sub_activity(
activity, activities_dict)
return activities
def _extract_activity_keys(self, activity, skip_sub_activities=False):
keys = []
for activity_key in Model._object_fields + ['inReplyTo']:
if activity_key not in activity:
continue
obj = activity.get(activity_key)
if isinstance(obj, dict):
if obj.get('objectType', None) == 'activity':
keys.append(obj['id'])
if obj.get('inReplyTo', None):
[
keys.append(in_reply_to_obj['id'])
for in_reply_to_obj in obj['inReplyTo']
]
if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in activity and activity[collection]['items']:
for item in activity[collection]['items']:
keys.extend(self._extract_activity_keys(item))
return keys
def _dehydrate_sub_activity(self,
sub_activity,
obj_list,
skip_sub_activities=False):
for activity_key in Model._object_fields:
if activity_key not in sub_activity:
continue
if isinstance(sub_activity[activity_key], dict):
if sub_activity[activity_key].get('objectType',
None) == 'activity':
sub_activity[activity_key].update(
obj_list[sub_activity[activity_key]['id']])
if sub_activity[activity_key].get('inReplyTo', None):
for i, in_reply_to_obj in enumerate(
sub_activity[activity_key]['inReplyTo']):
sub_activity[activity_key]['inReplyTo'][i]\
.update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']])
if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in sub_activity and sub_activity[collection][
'items']:
dehydrated_sub_items = []
for i, item in enumerate(
sub_activity[collection]['items']):
try:
dehydrated_sub_items.append(
self._dehydrate_sub_activity(item, obj_list))
except KeyError, e:
pass
sub_activity[collection][
'items'] = dehydrated_sub_items
sub_activity[collection]['totalItems'] = len(
dehydrated_sub_items)
return sub_activity
def get_cooccurs(lang, dep):
client = RiakClient(host=riak_ip, pb_port=riak_port, protocol='pbc')
bucket = client.bucket('%s/cooccurs' % lang)
return jsonify({"cooccurs": bucket.get(dep).data})
class ImageService:
def __init__(self):
self.riak = RiakClient(host=settings.RIAK_HOST,
port=settings.RIAK_PORT)
self._metadata_bucket = self.riak.bucket(settings.RIAK_METADATA_BUCKET)
self._image_bucket = self.riak.bucket(settings.RIAK_IMAGE_BUCKET)
self._thumbs_bucket = self.riak.bucket(settings.RIAK_THUMBS_BUCKET)
def store_from_url(self, address, user):
url = urlparse(address)
if str(url.scheme) != 'http':
raise ImageError('Not a valid URL')
if not re.search(r'(jpg|jpeg|png|gif)$', url.path):
raise ImageError('Unsupported file format')
http = httplib.HTTPConnection(url.netloc)
http.request('GET', url.path)
response = http.getresponse()
image = RemoteImage(response)
return self.store(image, user,
response.getheader("Content-Type", "image/jpg"))
def store(self, image, user, content_type):
key = self.create_unique_key()
filename = self.filename_for_image(key, content_type)
data = {
"user": user,
"uploaded_at": datetime.utcnow().isoformat(),
"filename": filename
}
thumbnail = self.create_thumbnail(image)
metadata = self._metadata_bucket.new(key,
data,
content_type="application/json")
metadata.store()
image = self._image_bucket.new_binary(key, image.read(), content_type)
image.store()
thumb = self._thumbs_bucket.new_binary(key,
thumbnail,
content_type="image/jpeg")
thumb.store()
return key
def create_thumbnail(self, image):
thumbnail = Image.open(image.temporary_file_path())
thumbnail.thumbnail((128, 128), Image.ANTIALIAS)
thumbio = StringIO()
thumbnail.save(thumbio, format="jpeg")
return thumbio.getvalue()
def find_metadata(self, image_id):
image = self._metadata_bucket.get(image_id)
if image.exists():
return image.get_data()
else:
return None
def find(self, image_id, thumb=False):
if thumb:
image = self._thumbs_bucket.get_binary(image_id)
else:
image = self._image_bucket.get_binary(image_id)
if image.exists():
return image
else:
return None
def find_all(self, user):
images = self.riak.search(settings.RIAK_METADATA_BUCKET,
"user:%s" % user).map(MAP_DOCID).run()
return images
def create_unique_key(self, length=6):
unique = False
while not unique:
id = self.unique_key(length)
if not self._image_bucket.get(id).exists():
return id
def unique_key(self, length=6):
return shortuuid.uuid()[0:length]
def filename_for_image(self, key, content_type):
if content_type in ['image/jpg', 'image/jpeg']:
extension = 'jpg'
elif content_type == "image/png":
extension = 'png'
elif content_type == 'image/gif':
extension = 'gif'
return "%s.%s" % (key, extension)
import csv
import riak
from riak import RiakClient
print("\nWelcome to Google Stock Archive")
print("Ente name of document in current directory to be archived in Riak")
y = raw_input()
file1 = open(y, "r")
reader = csv.reader(file1)
client = RiakClient()
my_bucket = client.bucket('Stock Data')
for row in reader:
list = {
'Date': row[0],
'Open': row[1],
'High': row[2],
'Low': row[3],
'Close': row[4],
'Volume': row[5],
'Adj Close': row[6]
}
x = my_bucket.new(data=list)
x.store()
print("Please Enter Date in following format\nYYYY-MM-DD:")
val = raw_input()
#!/usr/bin/python
from riak import RiakClient, RiakNode
from riak.riak_object import RiakObject
client = RiakClient(protocal='http', host='127.0.0.1', http_port=8098)
res = client.ping()
bucket = client.bucket('xxx')
#res = client.get_keys(res[0])
#obj = RiakObject(client, res, 'xxx')
obj = bucket.new('a', 'hello', 'text/plain')
obj.store()
res = bucket.get('a')
n = 0
#!/usr/bin/python
from workload import *
import sys
import time
from riak import RiakClient, RiakNode
rc = RiakClient(protocol='pbc', host='127.0.0.1', pb_port=8087)
#rc = RiakClient(protocol='http', host='127.0.0.1', pb_port=8098)
bucket = rc.bucket('benchmark')
def benchmark_function(actions, f_name, workload):
method = actions[f_name]
t0 = time.time()
if f_name in ['get', 'del']:
for k, v in workload.items():
method(k)
else:
for k, v in workload.items():
method(k, v)
t1 = time.time()
delta = t1-t0
print("Riak %s benchmark finished in %.3fs." % (f_name, delta))
def benchmark(size):
workload = gen_workload(size)
actions = {
'put': lambda x,y: bucket.new(x, data=y).store(),
'get': bucket.get,
'del': bucket.delete
}
create = """
CREATE TABLE {table} (
family varchar not null,
series varchar not null,
time timestamp not null,
metric double,
PRIMARY KEY (
(family, series, quantum(time, 1, 'd')
),family, series, time))
"""
rc = RiakClient(host='localhost', pb_port=8087)
create_q = create.format(table=table_name)
print create_q
secs = datetime.timedelta(0, 5)
start = datetime.datetime.now() - datetime.timedelta(0,3600)
try:
rc.ts_query("","DESCRIBE %s" % table_name)
except RiakError:
rc.ts_query(table_name, create_q)
try:
rc.get_search_index(node_table_name)
except RiakError:
rc.create_search_index(node_table_name)
bucket = rc.bucket(node_table_name)
bucket.set_properties({'search_index':node_table_name})
def __init__(self, session_key=None):
super(SessionStore, self).__init__(session_key)
client = RiakClient(host=RIAK_DB_HOST,
port=RIAK_DB_PORT,
transport_class=RiakPbcTransport)
self.sessions = client.bucket(RIAK_DB_BUCKET)
class RiakTSWriter(TimeSeriesDatabase):
plugin_name = 'graphite_riakts'
def __init__(self, option):
print "Here we be initializing yon connections and things: %r" % option
self.riak = RiakClient(host=option['RIAKTS_HOST'], pb_port=int(option['RIAKTS_PORT']))
self.nodes_table = option['RIAKTS_NODES_TABLE']
self.table_name = option['RIAKTS_TABLE_NAME']
self.family = option['RIAKTS_FAMILY']
self.nodes = self.riak.bucket(self.nodes_table)
def write(self, metric, datapoints):
try:
#print "Here we be writing to %s the value of %r" % ( metric, datapoints )
table = self.riak.table(self.table_name)
rows = []
for point in datapoints:
time = datetime.datetime.utcfromtimestamp(point[0])
rows.append([self.family, metric, time, float(point[1])])
#print rows
ts_obj = table.new(rows)
#print vars(ts_obj)
res = ts_obj.store()
#print res
return
except Exception as e:
raise RuntimeError("write error: %s" % (traceback.format_exc(e),))
def exists(self, metric):
try:
key = "node-%s" % metric
#print "Here we be checking if yon metric %s is existing" % metric
o = self.nodes.get(key)
if o.exists:
#print "Yar, there be existing the metric of %s" % metric
return True
else:
#print "Lo! The metric of %s be not here!" % metric
o = self.nodes.new(key)
branch = '.'.join(metric.split('.')[:-1])
o.data = { "node_s": metric, "type_s": "node"}
# print "Adding LEAF: %s" % metric
o.store()
while len(branch):
bkey = "branch-%s" % branch
b = self.nodes.get(bkey)
if not b.exists:
b.data = { "branch_s": branch, "type_s": "branch"}
# print "Adding BRANCH: %s" % branch
b.store()
branch = '.'.join(branch.split('.')[:-1])
return True
except Exception as e:
raise RuntimeError("exists error: %s" % (traceback.format_exc(e),))
def create(self, metric, **options):
try:
#print "Here we be attempting yon creation of %s metric, with options: %r" % ( metric, options )
# o = self.nodes.new(metric)
# o.data = { "node_t" : metric }
# o.store()
return metric
except Exception as e:
raise RuntimeError("create error: %s" % (traceback.format_exc(e),))
def get_metadata(self, metric, key):
try:
#print "Here we be getting yon metadata of metric %s and key %s" % ( metric, key )
return True
except Exception as e:
raise RuntimeError("get_metadata error: %s" % (traceback.format_exc(e),))
def set_metadata(self, metric, key, value):
try:
#print "Here we be setting yon metadata of metric %s and key %s with value %r" % ( metric, key, value )
return True
except Exception as e:
raise RuntimeError("set_metadata error: %s" % (traceback.format_exc(e),))
# def getFilesystemPath(self, metric):
# try:
# print "Here we be getting yon file system path of here metric %s" % metric
# return metric
# except (Exception) as e:
# raise RuntimeError("getFilesystemPath error: %s" % (traceback.format_exc(e),))
import uuid
from django.http import HttpResponseRedirect
from django.shortcuts import render, redirect
from .models import *
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_POST
# Create your views here.
RiakClient()
RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}])
myClient = RiakClient()
myBucket = myClient.bucket('users')
def home(request):
function = Functions.objects.all()
return render(request, 'home.html', {'functions': function})
def salvar_function(request):
titulo = request.POST.get('titulo')
if titulo:
functions = Functions()
functions.titulo = titulo
functions.save()
return redirect('/functions')
from flask import Blueprint, request, jsonify
from marshmallow import schema, ValidationError
from riak import RiakClient
# create crime blueprint
from app.schema import CrimeSchema
crime = Blueprint('crime', __name__)
client = RiakClient()
crime_bucket = client.bucket('crime')
# bucket = client.bucket_type('crime').bucket('category')
@crime.route("/crime", defaults={'crime_id': None}, methods=['GET', 'POST'])
@crime.route("/crime/<crime_id>", methods=['GET', 'DELETE'])
def crime_route(crime_id):
if request.method == 'POST':
# get crime_id and value from request
# if request.json:
try:
crime_schema = CrimeSchema()
result = crime_schema.load(request.json)
except ValidationError as err:
return jsonify({'message': err.messages}), 404
new_crime = request.json
new_crime_id = new_crime['crime_id']
from riak import RiakClient, RiakNode
client = RiakClient(protocol='pbc', host='127.0.0.1', http_port=8098)
print(client.ping())
tweetBucket = client.bucket("tweets")
print(tweetBucket.get_properties())
print(client.get_buckets())
print(client.bucket_type("tweets"))
allKeysInTweets = client.get_keys(tweetBucket)
print("Number of keys... ",len(allKeysInTweets))
stream = client.stream_keys(tweetBucket)
for key_list in stream:
print("key_list;" , key_list)
stream.close()
class RiakBackend(BaseBackend):
def __init__(
self, protocol="pbc", nodes=[], objects_bucket_name="objects",
activities_bucket_name="activities", **kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
def clear_all(self, **kwargs):
"""
Deletes all activity stream data from riak
"""
self.clear_all_activities()
self.clear_all_objects()
def clear_all_objects(self, **kwargs):
"""
Deletes all objects data from riak
"""
for key in self._objects.get_keys():
self._objects.get(key).delete(r='all', w='all', dw='all')
assert not self._objects.get(key).exists
def clear_all_activities(self, **kwargs):
"""
Deletes all activities data from riak
"""
for key in self._activities.get_keys():
self._activities.get(key).delete(r='all', w='all', dw='all')
assert not self._activities.get(key).exists
def obj_exists(self, obj, **kwargs):
obj_id = self._extract_id(obj)
return self._objects.get(obj_id).exists
def activity_exists(self, activity, **kwargs):
activity_id = self._extract_id(activity)
return self._activities.get(activity_id).exists
def obj_create(self, obj, **kwargs):
obj = Object(obj, backend=self)
obj.validate()
obj_dict = obj.get_parsed_dict()
key = self._extract_id(obj_dict)
riak_obj = self._objects.new(key=key)
riak_obj.data = obj_dict
riak_obj = self.set_general_indexes(riak_obj)
riak_obj.store()
#finally save the data
return obj_dict
def set_general_indexes(self, riak_object):
"""
Sets the default riak 2Is.
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes):
riak_object.add_index("timestamp_int", self._get_timestamp())
riak_object.remove_index('modified_int')
riak_object.add_index("modified_int", self._get_timestamp())
return riak_object
def obj_update(self, obj, **kwargs):
self.obj_create(obj, **kwargs)
def obj_get(self, obj, **kwargs):
"""
Given a list of object ids, returns a list of objects
"""
if not obj:
return obj
object_bucket_name = self._objects.name
objects = self._riak_backend
for o in obj:
objects = objects.add(object_bucket_name, self._extract_id(o))
results = objects.map(JS_MAP_OBJS).reduce(JS_REDUCE_OBJS).run()
return results or []
def obj_delete(self, obj, **kwargs):
obj_id = self._extract_id(obj)
self._objects.new(key=obj_id).delete()
def activity_create(self, activity, **kwargs):
"""
Creates an activity. You can provide objects for activities as dictionaries or as ids for already
existing objects.
If you provide a dictionary for an object, it is saved as a new object.
If you provide an object id and the object does not exist, it is saved anyway, and returned as an empty
dictionary when retriving the activity later.
"""
activity = Activity(activity, backend=self)
activity.validate()
activity_dict = activity.get_parsed_dict()
key = self._extract_id(activity_dict)
riak_obj = self._activities.new(key=key)
riak_obj.data = activity_dict
riak_obj = self.set_activity_indexes(self.set_general_indexes(riak_obj))
if activity_dict['verb'] in SUB_ACTIVITY_MAP:
riak_obj = self.set_sub_item_indexes(riak_obj, **kwargs)
riak_obj.store()
return self.dehydrate_activities([riak_obj.data])[0]
def set_activity_indexes(self, riak_object):
"""
Store indexes specific to an ``Activity``. Stores the following indexes:
1. ``verb`` of the ``Activity``
2. ``actor`` of the ``Activity``
3. ``object`` of the ``Activity``
4. if target is defined, verb for the ``target`` of the Activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
_dict = riak_object.data
riak_object.remove_index('verb_bin')
riak_object.remove_index('actor_bin')
riak_object.remove_index('object_bin')
riak_object.add_index("verb_bin", self._extract_id(_dict['verb']))
riak_object.add_index("actor_bin", self._extract_id(_dict['actor']))
riak_object.add_index("object_bin", self._extract_id(_dict['object']))
if 'target' in _dict and _dict.get("target"):
riak_object.remove_index('target_bin')
riak_object.add_index("target_bin", self._extract_id(_dict['target']))
return riak_object
def activity_delete(self, activity, **kwargs):
"""
Deletes an activity item and all associated sub items
"""
activity_dict = self.get_activity(activity, **kwargs)[0]
for response_field in Activity._response_fields:
if response_field in activity_dict:
for response_item in activity_dict[response_field]['items']:
self._activities.get(response_item['id']).delete()
self._activities.get(self._extract_id(activity_dict)).delete()
def activity_update(self, activity, **kwargs):
return self.activity_create(activity, **kwargs)
def activity_get(
self, activity_ids=[], raw_filter="", filters={}, include_public=False,
audience_targeting={}, aggregation_pipeline=[], **kwargs):
"""
Gets a list of activities. You can also group activities by providing a list of attributes to group
by.
:type activity_ids: list
:param activity_ids: The list of activities you want to retrieve
:type filters: dict
:param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``.
Filters do not work for nested dictionaries.
:type raw_filter: string
:param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set
or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on
the ``audience_targeting`` parameter.
:type include_public: boolean
:param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are
not targeted towards anyone are included in the results
:type audience_targeting: dict
:param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``.
The values are an array of object ids
:type aggregation_pipeline: array of ``sunspear.aggregators.base.BaseAggregator``
:param aggregation_pipeline: modify the final list of activities. Exact results depends on the implementation of the aggregation pipeline
:return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set.
Activities are returned in the order of ids provided.
"""
activity_ids = map(self._extract_id, activity_ids)
if not activity_ids:
return []
activities = self._get_many_activities(
activity_ids, raw_filter=raw_filter, filters=filters, include_public=include_public,
audience_targeting=audience_targeting)
activities = self.dehydrate_activities(activities)
if aggregation_pipeline:
original_activities = copy.deepcopy(activities)
for aggregator in aggregation_pipeline:
activities = aggregator.process(activities, original_activities, aggregation_pipeline)
return activities
def sub_activity_create(
self, activity, actor, content, extra={}, sub_activity_verb="",
published=None, **kwargs):
sub_activity_model = self.get_sub_activity_model(sub_activity_verb)
sub_activity_attribute = self.get_sub_activity_attribute(sub_activity_verb)
object_type = kwargs.get('object_type', sub_activity_verb)
activity_id = self._extract_id(activity)
activity_model = Activity(self._activities.get(key=activity_id).data, backend=self)
sub_activity_obj, original_activity_obj = activity_model\
.get_parsed_sub_activity_dict(
actor=actor, content=content, verb=sub_activity_verb,
object_type=object_type, collection=sub_activity_attribute,
activity_class=sub_activity_model, published=published, extra=extra)
sub_activity_obj = self.create_activity(sub_activity_obj, activity_id=original_activity_obj['id'])
original_activity_obj[sub_activity_attribute]['items'][0]['actor'] = sub_activity_obj['actor']['id']
original_activity_obj[sub_activity_attribute]['items'][0]['id'] = sub_activity_obj['id']
original_activity_obj[sub_activity_attribute]['items'][0]['published'] = sub_activity_obj['published']
original_activity_obj[sub_activity_attribute]['items'][0]['object']['id'] = sub_activity_obj['id']
original_activity_obj = self.activity_update(original_activity_obj)
return sub_activity_obj, original_activity_obj
def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs):
"""
Deletes a sub_activity made on an activity. This will also update the corresponding
parent activity.
:type sub_activity: string
:param sub_activity: the id of the reply activity to delete.
:type sub_activity_verb: string
:param sub_activity_verb: the verb of the sub activity
:return: a dict representing the updated parent activity
"""
sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0]
sub_activity_id = self._extract_id(sub_activity)
sub_activity_riak_model = self._activities.get(sub_activity_id)
if sub_activity_riak_model.data['verb'] != sub_activity_model.sub_item_verb:
raise SunspearValidationException("Trying to delete something that is not a {}.".format(sub_activity_model.sub_item_verb))
#clean up the reference from the original activity
in_reply_to_key = filter(lambda x: x[0] == 'inreplyto_bin', sub_activity_riak_model.indexes)[0][1]
activity = self._activities.get(key=in_reply_to_key)
activity_data = activity.data
activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1
activity_data[sub_activity_model.sub_item_key]['items'] = filter(
lambda x: x["id"] != sub_activity_id,
activity_data[sub_activity_model.sub_item_key]['items'])
updated_activity = self.update_activity(activity_data, **kwargs)
self.delete_activity(sub_activity_id)
return updated_activity
def set_sub_item_indexes(self, riak_object, **kwargs):
"""
Store indexes specific to a sub-activity. Stores the following indexes:
1. id of the the parent ``Activity`` of this sub-activity
:type riak_object: RiakObject
:param riak_object: a RiakObject representing the model of the class
"""
original_activity_id = kwargs.get('activity_id')
if not original_activity_id:
raise SunspearValidationException()
riak_object.remove_index('inreplyto_bin')
riak_object.add_index("inreplyto_bin", str(original_activity_id))
return riak_object
def _get_many_activities(self, activity_ids=[], raw_filter="", filters=None, include_public=False, audience_targeting={}):
"""
Given a list of activity ids, returns a list of activities from riak.
:type activity_ids: list
:param activity_ids: The list of activities you want to retrieve
:type raw_filter: string
:param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set
or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on
the ``audience_targeting`` parameter.
:type filters: dict
:param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``.
Filters do not work for nested dictionaries.
:type include_public: boolean
:param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are
not targeted towards anyone are included in the results
:type audience_targeting: dict
:param audience_targeting: Filters the list of activities targeted towards a particular audience. The key for the dictionary is one of ``to``, ``cc``, ``bto``, or ``bcc``.
"""
activity_bucket_name = self._activities.name
activities = self._riak_backend
for activity_id in activity_ids:
activities = activities.add(activity_bucket_name, str(activity_id))
results = activities.map(JS_MAP)
if audience_targeting:
results = results.reduce(JS_REDUCE_FILTER_AUD_TARGETTING, options={'arg': {'public': include_public, 'filters': audience_targeting}})
if filters or raw_filter:
# An empty `filters` dict would cause all activities to be filtered out. If you wanted that effect, you
# wouldn't have to call this function, so let's assume that an empty dict is a typical default value and
# should denote that there are no filters to apply.
filters = filters or None
results = results.reduce(JS_REDUCE_FILTER_PROP, options={'arg': {'raw_filter': raw_filter, 'filters': filters}})
results = results.reduce(JS_REDUCE).run()
results = results or []
# riak does not return the results in any particular order (unless we sort). So,
# we have to put the objects returned by riak back in order
results_map = dict(map(lambda result: (result['id'], result,), results))
reordered_results = [results_map[id] for id in activity_ids if id in results_map]
return reordered_results
def _get_timestamp(self):
"""
returns a unix timestamp representing the ``datetime`` object
"""
dt_obj = datetime.datetime.utcnow()
return long((calendar.timegm(dt_obj.utctimetuple()) * 1000)) + (dt_obj.microsecond / 1000)
def get_new_id(self):
"""
Generates a new unique ID. The default implementation uses uuid1 to
generate a unique ID.
:return: a new id
"""
return uuid.uuid1().hex
client = RiakClient(protocol='pbc')
bkeys = [('multiget', str(key)) for key in xrange(10000)]
data = open(__file__).read()
print "Benchmarking multiget:"
print " CPUs: {0}".format(cpu_count())
print " Threads: {0}".format(POOL_SIZE)
print " Keys: {0}".format(len(bkeys))
print
with benchmark.measure() as b:
with b.report('populate'):
for bucket, key in bkeys:
client.bucket(bucket).new(key, encoded_data=data,
content_type='text/plain'
).store()
for b in benchmark.measure_with_rehearsal():
client.protocol = 'http'
with b.report('http seq'):
for bucket, key in bkeys:
client.bucket(bucket).get(key)
with b.report('http multi'):
multiget(client, bkeys)
client.protocol = 'pbc'
with b.report('pbc seq'):
for bucket, key in bkeys:
client.bucket(bucket).get(key)
return [line.strip() for line in open(keyListFilename, 'r')]
# unused helper method to save an image to the disk if you want that
def saveImage(riakObj, filename):
with open(os.path.join(baseDir, filename), 'w+') as f:
f.write(riakObj.encoded_data)
# Read in redis-keys
imgs = readRiakKeys()
print 'Riak Keys: \n' + '\n'.join(imgs)
# connect to live riak
rcLive = RiakClient(protocol='http', host=srcRiakHost, http_port=8098)
rbLive = rcLive.bucket('ez')
# connect to integration riak
rcInt = RiakClient(protocol='http', host=destRiakHost, http_port=8098)
rbInt = rcInt.bucket('ez')
# save image in integration riak
def saveToIntRiak(key, riakObj):
if riakObj.content_type is not None:
img = rbInt.new(key,
encoded_data=riakObj.encoded_data,
content_type=riakObj.content_type)
img.store()
from django.conf import settings
from simplejson import loads, dumps
from riak import RiakClient
from datetime import datetime
import re
HOST = settings.RIAK_HOST
PORT = settings.RIAK_PORT
client = RiakClient(host=HOST, port=PORT)
page_bucket = client.bucket('riakipage')
version_bucket = client.bucket('riakiversion')
tag_bucket = client.bucket('riakitag')
# since it's inefficient to list all the items
# in a bucket, we manually index some things
index_bucket = client.bucket('riakiindex')
# Create your models here.
DTFORMAT = "%Y-%m-%dT%H:%M:%S"
def slugify(s):
return s.replace(" ", "-")
class Page:
parser = argparse.ArgumentParser(description='Takes a list of redis keys and deletes them')
parser.add_argument('--host', dest='dest_riak', default='10.228.39.181', help='The host we delete the riak data from')
args = parser.parse_args()
# map arguments
dest_riak_host = args.dest_riak
print 'Targeted Riak Host ' + dest_riak_host
base_dir = os.path.dirname(os.path.realpath(__file__))
# connect to live riak
riak_connection = RiakClient(protocol='http', host=dest_riak_host, http_port=8098)
print riak_connection.ping()
riak_bucket = riak_connection.bucket('ez')
# Parses the keys.txt file and writes url
def readRiakKeys():
keyListFilename = os.path.join(base_dir, 'keys.txt')
return [line.strip() for line in open(keyListFilename, 'r')]
imgs = readRiakKeys()
print 'Deleting Riak Keys: \n' + '\n'.join(imgs)
# get and save all images
for img in imgs:
obj = riak_bucket.get(img)
print 'Image exists? %s' % str(obj.exists)
if obj.exists is True:
print 'Now deleting key: %s' % img
def get_bucket():
if 'RIAK_PORT_8098_TCP_ADDR' in os.environ:
riak = RiakClient('http', host=os.environ['RIAK_PORT_8098_TCP_ADDR'])
else:
riak = RiakClient('http')
return riak.bucket('apps')
from riak import RiakClient
from flask import Flask, jsonify
from docopt import docopt
import sys
sys.path.insert(0, '../redis')
from RedisConfig import RedisConfig
arguments = docopt(__doc__)
language = arguments['<language>']
cfg = RedisConfig()
# single threaded blocking... for now
client = RiakClient(host=cfg.RIAK_IPS.split(',')[0],
pb_port=cfg.RIAK_PORT, protocol="pbc")
bucket = client.bucket("%s/top_flowers" % language)
node_set = set()
link_set = set()
def keygen():
for keys in bucket.stream_keys():
for key in keys:
if key == 'K':
continue
yield key
for k in keygen():
nodes, links = bucket.get(k).data
nodes = [tuple(lst) for lst in nodes]
links = [tuple(lst) for lst in links]
