def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
#create key value pairs to stock in riak
key = str(str(pair[0]) + '_' + str(pair[1]))
val = {'StockA': pair[0], \
'StockB': pair[1], \
'Date': pair[2],\
'CloseA': pair[3], \
'CloseB': pair[4], \
'ZScore': pair[5],\
'Beta': pair[6],\
'SignalMean': pair[7],\
'SignalSD': pair[8]}
myDate = pair[2].split('-')
obj = RiakObject(rc, bucket, key)
#add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = 'text/json'
obj.data = val
obj.data = json.dumps(val)
#store
obj.store()
#return a list of written pairs
return pair
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
# create key value pairs to stock in riak
key = str(str(pair[0]) + "_" + str(pair[1]))
val = {
"StockA": pair[0],
"StockB": pair[1],
"Date": pair[2],
"CloseA": pair[3],
"CloseB": pair[4],
"ZScore": pair[5],
"Beta": pair[6],
"SignalMean": pair[7],
"SignalSD": pair[8],
}
myDate = pair[2].split("-")
obj = RiakObject(rc, bucket, key)
# add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = "text/json"
obj.data = val
obj.data = json.dumps(val)
# store
obj.store()
# return a list of written pairs
return pair
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
host = config.get('host', '127.0.0.1')
port = config.get('port')
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
transport_options = config.get('transport_options', {})
client_args = dict(
host=host, prefix=prefix, mapred_prefix=mapred_prefix,
protocol=transport_type, client_id=client_id,
transport_options=transport_options)
if port is not None:
client_args['port'] = port
client = RiakClient(**client_args)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client, bucket_prefix, load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
def test_close_underlying_socket_retry(self):
c = RiakClient(PB_HOST,
PB_PORT,
transport_class=RiakPbcTransport,
transport_options={"max_attempts": 2})
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('barbaz', rand)
obj.store()
obj = bucket.get('barbaz')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'barbaz')
self.assertEqual(obj.get_data(), rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This should work, since we have a retry
obj = bucket.get('barbaz')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'barbaz')
self.assertEqual(obj.get_data(), rand)
def riakGetStock(searchVal, riakIP):
myData = []
myBucket = RiakClient(protocol="pbc", host=riakIP, pb_port=8087).bucket("stocks")
for key in myBucket.get_index("ticker_bin", searchVal): # get all from 2002 to 2012
value = json.loads(myBucket.get(key).data)
myData.append([(value["CLOSE"]), (value["VOLUME"]), str(value["DATE"])])
return myData
def test_close_underlying_socket_retry(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport,
transport_options={"max_attempts": 2})
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('barbaz', rand)
obj.store()
obj = bucket.get('barbaz')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'barbaz')
self.assertEqual(obj.data, rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This should work, since we have a retry
obj = bucket.get('barbaz')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'barbaz')
self.assertEqual(obj.data, rand)
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket('stocks')
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def _get_client(self):
"""Get client connection."""
if self._client is None or not self._client.is_alive():
self._client = RiakClient(protocol=self.protocol,
host=self.host,
pb_port=self.port)
self._client.resolver = last_written_resolver
return self._client
def test_uses_client_id_if_given(self):
self.host = PB_HOST
self.port = PB_PORT
zero_client_id = "\0\0\0\0"
c = RiakClient(PB_HOST, PB_PORT,
transport_class = RiakPbcTransport,
client_id = zero_client_id)
self.assertEqual(zero_client_id, c.get_client_id()) #
def register(host,user_id,time): currenttld = extractdom(host) riakclient = RiakClient() #bucket : trackers, key:user_id, object:json map str_bucket = 'trackers' bucket = riakclient.bucket(str_bucket) str_key = user_id obj = RiakObject(riakclient, bucket, str_key)
def conn(self):
"""Open connection on Riak DataBase"""
client = RiakClient(protocol=self.conf.get("protocol"),
http_port=self.conf.get("port"),
host=self.conf.get("host"))
conn = client.bucket(self.conf.get("db"))
conn.enable_search()
return conn
def test_uses_client_id_if_given(self):
self.host = PB_HOST
self.port = PB_PORT
zero_client_id = "\0\0\0\0"
c = RiakClient(PB_HOST,
PB_PORT,
transport_class=RiakPbcTransport,
client_id=zero_client_id)
self.assertEqual(zero_client_id, c.get_client_id()) #
def riakGetStock(searchVal, riakIP):
myData = []
myBucket = RiakClient(protocol='pbc', host=riakIP,
pb_port=8087).bucket('stocks')
for key in myBucket.get_index('ticker_bin',
searchVal): # get all from 2002 to 2012
value = json.loads(myBucket.get(key).data)
myData.append([(value['CLOSE']), (value['VOLUME']),
str(value['DATE'])])
return myData
def conn(self):
"""Open connection on Riak DataBase"""
client = RiakClient(
protocol=self.conf.get("protocol"),
http_port=self.conf.get("port"),
host=self.conf.get("host"))
conn = client.bucket(self.conf.get("db"))
conn.enable_search()
return conn
def connect():
#nodes = []
#for ip in ["45.55.0.44", "107.170.224.168", "45.55.1.236"]:
# nodes.append(dict(host=ip, http_port=RIAK_HTTP_PORT, pb_port=RIAK_PBC_PORT))
nodes = [
dict(host=RIAK_HOST, http_port=RIAK_HTTP_PORT, pb_port=RIAK_PBC_PORT)
]
client = RiakClient(nodes=nodes)
assert client.ping(), "Failed to connect to client"
return client
def riakSearchData(searchBucket, searchTerm, searchVal1, searchVal2, riakIP):
myData = {} # empty dict
myBucket = RiakClient(protocol="pbc", host=riakIP, pb_port=8087).bucket(searchBucket)
# check wether 1 or 2 search terms
if searchVal2 != None:
for key in myBucket.get_index(searchTerm, searchVal1, searchVal2): # get all keys with 2i match
myData[key] = json.loads(myBucket.get(key).data) # store data for each key
else:
for key in myBucket.get_index(searchTerm, searchVal1): # get all keys with 2i match
myData[key] = json.loads(myBucket.get(key).data) # store data for each key
return myData
def clear_expired(cls):
# TODO: get_keys() method is not efficent operation for searching sessions, use search feature or mapreduce or ...
client = RiakClient(host=RIAK_DB_HOST,
port=RIAK_DB_PORT,
transport_class=RiakPbcTransport)
sessions = client.bucket(RIAK_DB_BUCKET)
now = int(timezone.now().strftime("%s"))
for key in sessions.get_keys():
data = sessions.get(key)
if int(data.get_usermeta()['expire_age']) < now:
data.delete()
class RiakStorage(AbstractStorage):
bucket_types = {
'users': 'egara-lww',
'users-current': 'egara-unique',
'imap-events': 'egara-lww',
'imap-folders': 'egara-lww',
'imap-folders-current': 'egara-unique',
'imap-message-timeline': 'egara-lww'
}
def __init__(self, *args, **kw):
riak_host = 'localhost'
riak_port = 8098
self.client = RiakClient(
protocol='http',
host=conf['STORAGE'].get('riak_host', riak_host),
http_port=conf['STORAGE'].get('riak_port', riak_port)
)
self.client.set_decoder('application/octet-stream', self._decode_binary)
self.users_cache = CachedDict(ttl=10)
def _decode_binary(self, data):
return str(data).encode("utf-8")
def _get_bucket(self, bucketname):
_type = self.bucket_types.get(bucketname, None)
if _type:
return self.client.bucket_type(_type).bucket(bucketname)
return None
def get(self, key, index, doctype=None, fields=None, **kw):
"""
Standard API for accessing key/value storage
"""
result = None
log.debug("Riak get key %r from %r", key, index)
try:
bucket = self._get_bucket(index)
res = bucket.get(key)
if res and res.data:
result = res.data
except Exception, e:
log.warning("Riak exception: %s", str(e))
result = None
return result
def __init__(
self, protocol="pbc", nodes=[], objects_bucket_name="objects",
activities_bucket_name="activities", **kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
class DS_Riak(DS):
""" Riak DS
"""
def __init__(self, host=PB_HOST, port=PB_PORT, bucketname='_scads'):
"""
Params:
host: Hostname or IP Address of the riak database
port: Port of the riak database
bucketname: Where to store stuff
"""
self.client = RiakClient(host, port,
transport_class=RiakPbcTransport,
transport_options={'max_attempts': 3})
self.bucket = self.client.bucket(bucketname)
def store(self, key, value, indexes=None):
obj = self.bucket.get(key)
obj.set_data(value)
if indexes is not None:
for index in indexes:
obj.add_index('scads_bin', index)
obj.store()
def get_index(self, index):
return self.bucket.get_index('scads_bin', index)
def get(self, key):
d = self.bucket.get(key).get_data()
if d is None:
return None
return d
def clear(self):
for key in self.bucket.get_keys():
self.bucket.get(key).delete()
def __init__(self, option): print "Here we be initializing yon connections and things: %r" % option self.riak = RiakClient(host=option['RIAKTS_HOST'], pb_port=int(option['RIAKTS_PORT'])) self.nodes_table = option['RIAKTS_NODES_TABLE'] self.table_name = option['RIAKTS_TABLE_NAME'] self.family = option['RIAKTS_FAMILY'] self.nodes = self.riak.bucket(self.nodes_table)
def create_client(self, host=None, port=None, transport_class=None):
host = host or self.host
port = port or self.port
transport_class = transport_class or self.transport_class
return RiakClient(self.host,
self.port,
transport_class=self.transport_class)
def client(request):
ids = subprocess.check_output(
['docker', 'ps', '-qaf', 'label=role=cluster']).strip().split('\n')
info = json.loads(subprocess.check_output(['docker', 'inspect'] + ids))
ips = [(i['NetworkSettings']['Networks']).values()[0]['IPAddress']
for i in info]
return RiakClient(nodes=[RiakNode(host=ip) for ip in ips])
def main(argv):
global gtable
global gclient
# Overhead to manage command line opts and config file
p = getCmdLineParser()
args = p.parse_args()
cfg.read(args.config_file)
# Get the logger going
glogname = cfg.get('logging', 'logName')
rightNow = time.strftime("%Y%m%d%H%M%S")
logger = initLog(rightNow)
logger.info('Starting Run: '+time.strftime("%Y%m%d%H%M%S")+' =========================')
# Get Riak going
gclient = RiakClient(protocol='pbc',nodes=[{ 'host': cfg.get('riakts', 'ip'), 'pb_port': int(cfg.get('riakts', 'port')) }])
gtable = cfg.get('riakts','tables').split(',')
# Get Rabbit going
rabbitChannel = setUpRabbit(cfg.get('rabbitmq', 'ip'), int(cfg.get('rabbitmq', 'port')),cfg.get('rabbitmq', 'login'),cfg.get('rabbitmq', 'password'),cfg.get('rabbitmq', 'queue'))
logger.info('RabbitMQ channel initialized...')
rabbitChannel.basic_qos(prefetch_count=1)
rabbitChannel.basic_consume(processData, queue=cfg.get('rabbitmq', 'queue'))
logger.info('RabbitMQ consumer initialized...')
rabbitChannel.start_consuming()
# Clean up
logger.info('Done! '+time.strftime("%Y%m%d%H%M%S")+' ==========================')
def write_flower_to_riak(repos):
"""
We could optimize this function via memoization. If part of a flower
already exists on the Riak cluster, then we just re-use that instead of
calculating it again. However, in the worst-case scenario, since Riak isn't
in strong consistency mode by default, we have no guarantee that the
memoized view is up-to-date.
Riak does have a strong consistency mode, but at the moment it isn't ready
for production.
"""
clients = [
RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS
]
neighborss = [client.bucket('%s/neighbors' % LANGUAGE) \
for client in clients]
flowers = [client.bucket('%s/flowers' % LANGUAGE) \
for client in clients]
for repo0, neighbor, flower in zip(repos, cycle(neighborss),
cycle(flowers)):
repo_set = set()
nodes = []
links = []
repos1 = neighbor.get(repo0).data
repos1 = repos1 if repos1 is not None else []
degree0 = len(repos1)
if repo0 not in repo_set:
repo_set.add(repo0)
nodes.append([repo0, degree0])
# Take up to N
for i in range(len(repos1)):
repos2 = neighbor.get(repos1[i]).data
repos2 = repos2 if repos2 is not None else []
repos1[i] = [repos1[i], len(repos2)]
top_repos1 = heapq.nlargest(N, repos1, lambda tup: tup[1])
# 1st order
for x in top_repos1:
repo1, degree1 = x
if repo1 not in repo_set:
repo_set.add(repo1)
nodes.append([repo1, degree1])
links.append([repo0, repo1])
# Take up to N
repos2 = neighbor.get(repo1).data
repos2 = repos2 if repos2 is not None else []
for j in range(len(repos2)):
repos3 = neighbor.get(repos2[j]).data
repos3 = repos3 if repos3 is not None else []
repos2[j] = [repos2[j], len(repos3)]
top_repos2 = heapq.nlargest(N, repos2, lambda tup: tup[1])
# 2nd order
for y in top_repos2:
repo2, degree2 = y
if repo2 not in repo_set:
repo_set.add(repo2)
nodes.append([repo2, degree2])
links.append([repo1, repo2])
flower.new(repo0, [nodes, links]).store()
def _get_client(self):
"""Get client connection."""
if self._client is None or not self._client.is_alive():
self._client = RiakClient(protocol=self.protocol,
host=self.host,
pb_port=self.port)
self._client.resolver = last_written_resolver
return self._client
def get_flower(lang, user, repo_name):
client = RiakClient(host=riak_ip, pb_port=riak_port, protocol="pbc")
bucket = client.bucket("%s/flowers" % lang)
repo = "%s/%s" % (user, repo_name)
graph = bucket.get(repo).data
if graph is None:
abort(404)
_nodes, _links = graph
nodes = [{"id": repo, "group": degree} for (repo, degree) in _nodes]
del _nodes
links = [{"source": repo_a, "target": repo_b} for \
(repo_a, repo_b) in _links]
del _links
return jsonify({"nodes": nodes, "links": links})
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
# get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket("stocks")
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
# didnt get any data
if len(gtemp) == 0:
return 0
# got data
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def populate():
cl = RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
cities = Cities(cl)
users_list = []
# create users
print('doing users')
for i in range(num_users):
usr = User(cl, fake.email())
users_list.append(usr)
print(usr.info.data)
# make users follow each others
print('doing follows')
for usr in users_list:
to_follow = choices(users_list, k=10)
for tg in to_follow:
usr.follow_user(tg.repr)
print(usr.info.data)
# create messages
print('doing messages')
for usr in users_list:
num_msg = randint(5, msg_per_user)
current = datetime.now()
for msg in range(num_msg):
current -= timedelta(randint(1, 10))
new_msg = usr.add_message(
fake.text(), timestamp=current.strftime(format_timestamp))
# figure out if make a share or a replica out of this
random_pick = randint(0, 5)
print('original', new_msg)
if random_pick == 2:
# its a replica
replicant = choices(users_list)[0]
reply_timestamp = current + timedelta(minutes=randint(3, 90))
original_msg_ref = '_'.join(
[new_msg['owner']['email'], new_msg['timestamp']])
new_reply = replicant.add_message(
fake.text(),
timestamp=reply_timestamp.strftime(format_timestamp),
reply_to={
'ref': original_msg_ref,
'content': new_msg['content']
})
print('reply', new_reply)
elif random_pick == 3:
# its a share
replicant = choices(users_list)[0]
reply_timestamp = current + timedelta(minutes=randint(3, 90))
original_msg_ref = '_'.join(
[new_msg['owner']['email'], new_msg['timestamp']])
new_reply = replicant.add_message(
'',
timestamp=reply_timestamp.strftime(format_timestamp),
reply_to={
'ref': original_msg_ref,
'content': new_msg['content']
})
print('share', new_reply)
def connect(self):
return RiakClient(host=self.host,
port=self.port,
prefix=self.prefix,
mapred_prefix=self.mapred_prefix,
client_id=self.client_id,
transport_class=self.transport_class,
solr_transport_class=self.solr_transport_class,
transport_options=self.transport_options)
def run_cluster_server(riak_conf):
riak_hosts = []
with open(riak_conf, "r") as fin:
for line in fin:
riak_hosts.append({"host": line.replace("\n", ""), "http_port": RIAK_PORT})
cluster = RiakClient(nodes=riak_hosts)
bucket = cluster.bucket("HashedData")
random_str_arr = []
random_str_arr = run_exp_init(socket.gethostname())
insert_throuput = run_exp_insert(bucket, random_str_arr)
lookup_throuput = run_exp_lookup(bucket, random_str_arr)
delete_throuput = run_exp_delete(bucket, random_str_arr)
overall_throuput = (insert_throuput + lookup_throuput + delete_throuput) / 3
print("Overall Throughput: " + str(overall_throuput) + " OPs/s")
def _write_neighbors(tups, RIAK_IPS, RIAK_PORT, LANGUAGE):
clients = [RiakClient(host=ip, protocol='pbc',
pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/neighbors' % LANGUAGE)
for client in clients]
for tup, bucket in izip(tups, cycle(buckets)):
repo, adj_repos, _ = tup
_ = bucket.new(repo, adj_repos).store()
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def write_top_cooccurs(RIAK_IPS, RIAK_PORT, LANGUAGE):
clients = [RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/top_cooccurs' % LANGUAGE) \
for client in clients]
for b, (dep1, dep2, count), bucket in izip(range(len(top_cooccurs)),
top_cooccurs,
cycle(buckets)):
bucket.new('%d' % b, [dep1, dep2, count]).store()
buckets[0].new('B', len(top_cooccurs)).store()
class UserService:
def __init__(self):
self.users_bucket = settings.RIAK_USERS_BUCKET
self.riak = RiakClient(host=settings.RIAK_HOST,
port=settings.RIAK_PORT)
def save(self, user_data):
key = self.generate_key()
password = user_data['password']
user_data['encrypted_password'] = make_password('sha1', password)
del user_data['password']
user = self.riak.bucket(self.users_bucket).new(key, user_data)
user.store()
return key
def get(self, user_id):
user = self.riak.bucket(settings.RIAK_USERS_BUCKET).get(str(user_id))
if user.exists():
return user.get_data()
else:
return None
def login(self, username, password):
try:
user = self.riak.search(settings.RIAK_USERS_BUCKET,
"username:%s" % username).run()
except ValueError:
user = None
if user:
user = user[0]
user.set_bucket(str(user.get_bucket()))
user.set_key(str(user.get_key()))
user_data = user.get().get_data()
if check_password(password, user_data['encrypted_password']):
return user
else:
return False
else:
return False
def generate_key(self):
return uuid.uuid1().hex
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
transport_class = {
'http': RiakHttpTransport,
'protocol_buffer': RiakPbcTransport,
}.get(transport_type, RiakHttpTransport)
host = config.get('host', '127.0.0.1')
port = config.get('port', 8098)
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
# NOTE: the current riak.RiakClient expects this parameter but
# internally doesn't do anything with it.
solr_transport_class = config.get('solr_transport_class', None)
transport_options = config.get('transport_options', None)
client = RiakClient(host=host,
port=port,
prefix=prefix,
mapred_prefix=mapred_prefix,
transport_class=transport_class,
client_id=client_id,
solr_transport_class=solr_transport_class,
transport_options=transport_options)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client,
bucket_prefix,
load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
def __init__(self, host=PB_HOST, port=PB_PORT, bucketname='_scads'):
"""
Params:
host: Hostname or IP Address of the riak database
port: Port of the riak database
bucketname: Where to store stuff
"""
self.client = RiakClient(host, port,
transport_class=RiakPbcTransport,
transport_options={'max_attempts': 3})
self.bucket = self.client.bucket(bucketname)
def __init__(self, **client_args):
self._closed = False
self._raw_client = RiakClient(**client_args)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
self._client.set_encoder("application/json", json.dumps)
self._client.set_encoder("text/json", json.dumps)
self._client.set_decoder("application/json", json.loads)
self._client.set_decoder("text/json", json.loads)
def __init__(self, *args, **kw):
riak_host = 'localhost'
riak_port = 8098
self.client = RiakClient(
protocol='http',
host=conf['STORAGE'].get('riak_host', riak_host),
http_port=conf['STORAGE'].get('riak_port', riak_port)
)
self.client.set_decoder('application/octet-stream', self._decode_binary)
self.users_cache = CachedDict(ttl=10)
def test_close_underlying_socket_fails(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport)
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('foo', rand)
obj.store()
obj = bucket.get('foo')
self.assertTrue(obj.exists)
self.assertEqual(obj.bucket.name, 'bucket_test_close')
self.assertEqual(obj.key, 'foo')
self.assertEqual(obj.data, rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This shoud fail with a socket error now
self.assertRaises(socket.error, bucket.get, 'foo')
def testCluster(IPs, login, password):
initial_attempts = 5
port = 8087
num_attempts = 0
nodes = []
for eachNode in IPs.split(","):
thisNode = {'host': eachNode, 'pb_port': port}
nodes.append(thisNode)
connected = False
client = RiakClient(protocol='pbc', nodes=nodes)
try:
with client.retry_count(initial_attempts):
connected = client.ping()
except Exception as e:
print('[EXCP] No Riak server found after ' + str(initial_attempts) + " tries. Error: " + str(e))
if connected:
print('[STATE] Successful PING! Connected to Riak!')
def test_close_underlying_socket_fails(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport)
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('foo', rand)
obj.store()
obj = bucket.get('foo')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'foo')
self.assertEqual(obj.get_data(), rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This shoud fail with a socket error now
self.assertRaises(socket.error, bucket.get, 'foo')
def _write_cooccurs(tups, RIAK_IPS, RIAK_PORT, B, LANGUAGE):
results = []
clients = [RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/cooccurs' % LANGUAGE) \
for client in clients]
for (dep, counter), bucket in izip(tups, cycle(buckets)):
top_items = heapq.nlargest(B, counter.iteritems(),
key=lambda tup: tup[1]) # WARNING: Python 2.7
bucket.new(dep, top_items).store()
results.append((dep, Counter(dict(top_items))))
return results
def selection_function(request):
RiakClient()
RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}])
myClient = RiakClient()
myBucket = myClient.bucket('users')
keys = myBucket.get_keys()
usuarios = Clientes()
lista2 = []
for i in range(len(keys)):
lista = {
'name': myBucket.get(keys[i]).data['name'],
'location': myBucket.get(keys[i]).data['location'],
'role': myBucket.get(keys[i]).data['role'],
'fullname': myBucket.get(keys[i]).data['fullname']
}
lista2.append(lista)
# for i in range(len(lista1)):
# #lista.append(myBucket.get(keys[i]).data['name'])
# usuarios.nome = str(lista1[i][0])
# #lista.append(myBucket.get(keys[i]).data['location'])
# usuarios.locacao = str(lista1[i][1])
# #lista.append(myBucket.get(keys[i]).data['role'])
# usuarios.profissao = str(lista1[i][2])
# #lista.append(myBucket.get(keys[i]).data['fulname'])
# usuarios.apelido = str(lista1[i][3])
# usuarios.save()
usuarios = Clientes.objects.all()
return render(request, 'usuario.html', {'usuarios': lista2})
def configureRiak(riakIPs, riakPort,logger):
initial_attempts = 5
num_attempts = 0
initial_delay = 5 # 5 seconds between between initial attempts
longer_delay = 5*60 # 5 minutes = 300 seconds
delay_time = initial_delay
nodes=[]
for eachNode in riakIPs.split(","):
thisNode= { 'host': eachNode, 'pb_port': riakPort}
nodes.append(thisNode)
riakIP=""
for node in nodes:
riakIP=json.dumps(node)+ " - " +riakIP
logger.info('[STATE] Connecting to Riak...')
connected = False
client = RiakClient(protocol='pbc',nodes=nodes)
while not connected:
try:
logger.info("Attempting to PING....")
client.ping()
connected = True
except:
num_attempts += 1
logger.error('EXCP: No Riak server found')
if num_attempts == initial_attempts:
delay_time = longer_delay
# Wait to issue next connection attempt
time.sleep(delay_time)
logger.info('[STATE] Connected to Riak. Successful PING')
return client
def cache_top_K():
clients = [
RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS
]
flowers = [client.bucket('%s/flowers' % LANGUAGE) \
for client in clients]
top_flowers = [client.bucket('%s/top_flowers' % LANGUAGE) \
for client in clients]
for k, tup, flower, top_flower in zip(range(K), top_K_repos,
cycle(flowers), cycle(top_flowers)):
dst_repo = tup[0]
top_flower.new('%d' % k, flower.get(dst_repo).data).store()
top_flowers[0].new('K', K).store()
def get_hosts(hosts,servers):
results=list()
riak_reply={}
for server in servers:
mcodata=RiakClient(host=server).bucket_type('puppet').bucket('nodes')
if hosts is not None:
for host in hosts:
#then get specific host
riak_reply[server] = mcodata.search("facts.fqdn:%s" % host,index='mco_nodes')
if riak_reply[server]['docs']:
results.extend(riak_reply[server]['docs'])
else:
#then get all hosts
riak_reply[server] = mcodata.get_keys()
if riak_reply[server]:
results.extend(riak_reply[server])
results.sort()
reply = {'servers':'%s'%servers,'results':results} or None
return reply
class MaternityService:
def __init__(self):
self.maternity_bucket = settings.RIAK_MATERNITY_BUCKET
self.riak = RiakClient(host = settings.RIAK_DATABASE['HOST'],
port = settings.RIAK_DATABASE['PORT'])
def get(self, pk):
entity = self.riak.bucket(settings.RIAK_MATERNITY_BUCKET).get(str(pk))
if entity.exists():
data = json.loads(entity.get_data())
data['statistics'] = sorted(data['statistics'].iteritems())
return data
else:
return None
def get_authorities(self):
query = self.riak.add(settings.RIAK_MATERNITY_BUCKET)
query.map('''
function(v) {
var data = JSON.parse(v.values[0].data);
if(data.type == 'AUTHORITY') {
return [[v.key, data]];
}
return [];
}
''')
return query.run()
#data = []
#for result in query.run():
# d = result[1]
#dj = json.loads(d)
# data.append((result[0], d))
#return data
#return [
# (result[0], json.loads(result[1]))
# for result in query.run()
# ]
def __init__(self,
protocol="pbc",
nodes=[],
objects_bucket_name="objects",
activities_bucket_name="activities",
**kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
def get_hosts(hosts, servers):
results = list()
riak_reply = {}
for server in servers:
mcodata = RiakClient(host=server).bucket_type('puppet').bucket('nodes')
if hosts is not None:
for host in hosts:
#then get specific host
riak_reply[server] = mcodata.search("facts.fqdn:%s" % host,
index='mco_nodes')
if riak_reply[server]['docs']:
results.extend(riak_reply[server]['docs'])
else:
#then get all hosts
riak_reply[server] = mcodata.get_keys()
if riak_reply[server]:
results.extend(riak_reply[server])
results.sort()
reply = {'servers': '%s' % servers, 'results': results} or None
return reply
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket("stocks")
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def _connect(self):
nodes = []
security = None
for host, port in self.hosts:
node = {'host': host}
if self.protocol == 'pbc':
node['pb_port'] = port
else:
node['http_port'] = port
nodes.append(node)
kwargs = {}
if self.user is not None:
kwargs['username'] = self.user
if self.pwd:
kwargs['password'] = self.pwd
if self.pkey is not None:
kwargs['pkey_file'] = self.pkey
if self.cert is not None:
kwargs['cert_file'] = self.cert
if self.cacert is not None:
kwargs['cacert_file'] = self.cacert
if self.crl is not None:
kwargs['crl_file'] = self.crl
if self.ciphers is not None:
kwargs['ciphers'] = self.ciphers
if kwargs:
kwargs['ssl_version'] = self.sslver
security = SecurityCreds(**kwargs)
else:
security = None
return RiakClient(nodes=nodes, credentials=security)
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
transport_class = {
'http': RiakHttpTransport,
'protocol_buffer': RiakPbcTransport,
}.get(transport_type, RiakHttpTransport)
host = config.get('host', '127.0.0.1')
port = config.get('port', 8098)
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
# NOTE: the current riak.RiakClient expects this parameter but
# internally doesn't do anything with it.
solr_transport_class = config.get('solr_transport_class', None)
transport_options = config.get('transport_options', None)
client = RiakClient(host=host, port=port, prefix=prefix,
mapred_prefix=mapred_prefix, transport_class=transport_class,
client_id=client_id, solr_transport_class=solr_transport_class,
transport_options=transport_options)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client, bucket_prefix, load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
def getData(tickerFile, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP,
pb_port=8087) #set up riak connection
added = [] #list of successful adds
notAdded = [] #list of unsuccessful adds
stock = pd.read_csv(tickerFile, sep='\t',
header=None) #read in stock tickers
#loop over all stock tickers
for i in range(0, len(stock.head(100))):
ticker = stock.ix[i, 0]
if getDataByTicker(ticker, dataSource, start, end, riakIP) == 0:
notAdded.append(ticker)
else:
added.append(ticker)
return added, notAdded
class RiakTSFinder(object):
def __init__(self, config):
self.config = config
self.riak = RiakClient(host=config['riak_ts']['host'],port=config['riak_ts']['port'])
def find_nodes(self, query):
#print vars(query)
bucket = self.riak.bucket_type('default').bucket('metric_nodes')
exact = bucket.get("node-%s" % query.pattern)
nodes = []
if exact.exists:
yield LeafNode(query.pattern,RiakTSReader(query.pattern, self.riak, self.config))
else:
pattern = query.pattern
pattern = re.sub('\.select metric','',pattern)
# If there's not a star in the pattern:
if re.match('^[^*]*$', pattern):
# remove any possible trailing .'s, and add a .* at the end
pattern = re.sub('\.*$', '.*', pattern)
# Replace all embeded *'s with a Solr regex compatible wildcard, scoped between .'s: [^.]*
pattern = re.sub('\*','[^.]*', pattern)
print "Solr pattern: %s" % pattern
results = bucket.search("branch_s:/%s/" % pattern, index='metric_nodes', rows=1000000)
print(results)
if results['num_found'] > 0:
print "Branch search results"
for doc in results['docs']:
branch = bucket.get(doc['_yz_rk'])
branch_node = BranchNode(branch.data['branch_s'])
print "BranchNode: name: %s, path: %s" % (branch_node.name, branch_node.path)
yield branch_node
else:
node_results = bucket.search("node_s:/%s/" % pattern, index='metric_nodes', rows=1000000)
print "Node search results"
print(node_results['docs'])
for doc in node_results['docs']:
node = bucket.get(doc['_yz_rk'])
node_name = node.data['node_s']
print "Node: %s" % node_name
yield LeafNode(node_name, RiakTSReader(node_name, self.riak, self.config))
def get_bucket():
if 'RIAK_PORT_8098_TCP_ADDR' in os.environ:
riak = RiakClient('http', host=os.environ['RIAK_PORT_8098_TCP_ADDR'])
else:
riak = RiakClient('http')
return riak.bucket('apps')
class RiakBackend(KeyValueStoreBackend):
# TODO: allow using other protocols than protobuf ?
#: default protocol used to connect to Riak, might be `http` or `pbc`
protocol = 'pbc'
#: default Riak bucket name (`default`)
bucket_name = 'celery'
#: default Riak server hostname (`localhost`)
host = 'localhost'
#: default Riak server port (8087)
port = 8087
# supports_autoexpire = False
def __init__(self, host=None, port=None, bucket_name=None, protocol=None,
url=None, *args, **kwargs):
"""Initialize Riak backend instance.
:raises celery.exceptions.ImproperlyConfigured: if
module :mod:`riak` is not available.
"""
super(RiakBackend, self).__init__(*args, **kwargs)
if not riak:
raise ImproperlyConfigured(
'You need to install the riak library to use the '
'Riak backend.')
uhost = uport = uname = upass = ubucket = None
if url:
uprot, uhost, uport, uname, upass, ubucket, _ = _parse_url(url)
if ubucket:
ubucket = ubucket.strip('/')
config = self.app.conf.get('CELERY_RIAK_BACKEND_SETTINGS', None)
if config is not None:
if not isinstance(config, dict):
raise ImproperlyConfigured(
'Riak backend settings should be grouped in a dict')
else:
config = {}
self.host = uhost or config.get('host', self.host)
self.port = int(uport or config.get('port', self.port))
self.bucket_name = ubucket or config.get('bucket', self.bucket_name)
self.protocol = protocol or config.get('protocol', self.protocol)
# riak bucket must be ascii letters or numbers only
if not is_ascii(self.bucket_name):
raise ValueError(E_BUCKET_NAME.format(self.bucket_name))
self._client = None
def _get_client(self):
"""Get client connection."""
if self._client is None or not self._client.is_alive():
self._client = RiakClient(protocol=self.protocol,
host=self.host,
pb_port=self.port)
self._client.resolver = last_written_resolver
return self._client
def _get_bucket(self):
"""Connect to our bucket."""
if (
self._client is None or not self._client.is_alive()
or not self._bucket
):
self._bucket = self.client.bucket(self.bucket_name)
return self._bucket
@property
def client(self):
return self._get_client()
@property
def bucket(self):
return self._get_bucket()
def get(self, key):
return self.bucket.get(key).data
def set(self, key, value):
_key = self.bucket.new(key, data=value)
_key.store()
def mget(self, keys):
return [self.get(key).data for key in keys]
def delete(self, key):
self.bucket.delete(key)
def test_mapred_bucket_exception(self):
c = RiakClient()
with self.assertRaises(ListError):
c.add('bucket')
