def connect(self):
return RiakClient(host=self.host,
port=self.port,
prefix=self.prefix,
mapred_prefix=self.mapred_prefix,
client_id=self.client_id,
transport_class=self.transport_class,
solr_transport_class=self.solr_transport_class,
transport_options=self.transport_options)
def _write_neighbors(tups, RIAK_IPS, RIAK_PORT, LANGUAGE):
clients = [RiakClient(host=ip, protocol='pbc',
pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/neighbors' % LANGUAGE)
for client in clients]
for tup, bucket in izip(tups, cycle(buckets)):
repo, adj_repos, _ = tup
_ = bucket.new(repo, adj_repos).store()
def test_uses_client_id_if_given(self):
self.host = PB_HOST
self.port = PB_PORT
zero_client_id = "\0\0\0\0"
c = RiakClient(PB_HOST,
PB_PORT,
transport_class=RiakPbcTransport,
client_id=zero_client_id)
self.assertEqual(zero_client_id, c.get_client_id()) #
def conn(self):
"""Open connection on Riak DataBase"""
client = RiakClient(protocol=self.conf.get("protocol"),
http_port=self.conf.get("port"),
host=self.conf.get("host"))
conn = client.bucket(self.conf.get("db"))
conn.enable_search()
return conn
def write_top_cooccurs(RIAK_IPS, RIAK_PORT, LANGUAGE):
clients = [RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/top_cooccurs' % LANGUAGE) \
for client in clients]
for b, (dep1, dep2, count), bucket in izip(range(len(top_cooccurs)),
top_cooccurs,
cycle(buckets)):
bucket.new('%d' % b, [dep1, dep2, count]).store()
buckets[0].new('B', len(top_cooccurs)).store()
def connect():
#nodes = []
#for ip in ["45.55.0.44", "107.170.224.168", "45.55.1.236"]:
# nodes.append(dict(host=ip, http_port=RIAK_HTTP_PORT, pb_port=RIAK_PBC_PORT))
nodes = [
dict(host=RIAK_HOST, http_port=RIAK_HTTP_PORT, pb_port=RIAK_PBC_PORT)
]
client = RiakClient(nodes=nodes)
assert client.ping(), "Failed to connect to client"
return client
def riakGetStock(searchVal, riakIP):
myData = []
myBucket = RiakClient(protocol='pbc', host=riakIP,
pb_port=8087).bucket('stocks')
for key in myBucket.get_index('ticker_bin',
searchVal): # get all from 2002 to 2012
value = json.loads(myBucket.get(key).data)
myData.append([(value['CLOSE']), (value['VOLUME']),
str(value['DATE'])])
return myData
def _write_cooccurs(tups, RIAK_IPS, RIAK_PORT, B, LANGUAGE):
results = []
clients = [RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS]
buckets = [client.bucket('%s/cooccurs' % LANGUAGE) \
for client in clients]
for (dep, counter), bucket in izip(tups, cycle(buckets)):
top_items = heapq.nlargest(B, counter.iteritems(),
key=lambda tup: tup[1]) # WARNING: Python 2.7
bucket.new(dep, top_items).store()
results.append((dep, Counter(dict(top_items))))
return results
def insert_function(request):
RiakClient()
RiakClient(protocol='http', host='127.0.0.1', http_port=8098)
RiakClient(nodes=[{'host': '127.0.0.1', 'http_port': 8098}])
myClient = RiakClient()
myBucket = myClient.bucket('users')
getNome = request.POST.get("name")
getNome = str(getNome)
getLocation = request.POST.get("location")
getRole = request.POST.get("role")
getFulname = request.POST.get("fullname")
nome = str('user') + str(randint(0, 9))
nome = {
'name': getNome,
'location': getLocation,
'role': getRole,
'fullname': getFulname
}
obj = myBucket.new(str(nome), data=nome)
obj.store()
return redirect('/home')
def cache_top_K():
clients = [
RiakClient(host=ip, protocol='pbc', pb_port=RIAK_PORT)
for ip in RIAK_IPS
]
flowers = [client.bucket('%s/flowers' % LANGUAGE) \
for client in clients]
top_flowers = [client.bucket('%s/top_flowers' % LANGUAGE) \
for client in clients]
for k, tup, flower, top_flower in zip(range(K), top_K_repos,
cycle(flowers), cycle(top_flowers)):
dst_repo = tup[0]
top_flower.new('%d' % k, flower.get(dst_repo).data).store()
top_flowers[0].new('K', K).store()
def riakSearchData(searchBucket, searchTerm, searchVal1, searchVal2, riakIP):
myData = {} #empty dict
myBucket = RiakClient(protocol='pbc', host=riakIP,
pb_port=8087).bucket(searchBucket)
#check wether 1 or 2 search terms
if searchVal2 != None:
for key in myBucket.get_index(searchTerm, searchVal1,
searchVal2): #get all keys with 2i match
myData[key] = json.loads(
myBucket.get(key).data) #store data for each key
else:
for key in myBucket.get_index(searchTerm,
searchVal1): #get all keys with 2i match
myData[key] = json.loads(
myBucket.get(key).data) #store data for each key
return myData
def _connect(self):
nodes = []
security = None
for host, port in self.hosts:
node = {'host': host}
if self.protocol == 'pbc':
node['pb_port'] = port
else:
node['http_port'] = port
nodes.append(node)
kwargs = {}
if self.user is not None:
kwargs['username'] = self.user
if self.pwd:
kwargs['password'] = self.pwd
if self.pkey is not None:
kwargs['pkey_file'] = self.pkey
if self.cert is not None:
kwargs['cert_file'] = self.cert
if self.cacert is not None:
kwargs['cacert_file'] = self.cacert
if self.crl is not None:
kwargs['crl_file'] = self.crl
if self.ciphers is not None:
kwargs['ciphers'] = self.ciphers
if kwargs:
kwargs['ssl_version'] = self.sslver
security = SecurityCreds(**kwargs)
else:
security = None
return RiakClient(nodes=nodes, credentials=security)
def get_flower(lang, user, repo_name):
client = RiakClient(host=riak_ip, pb_port=riak_port, protocol="pbc")
bucket = client.bucket("%s/flowers" % lang)
repo = "%s/%s" % (user, repo_name)
graph = bucket.get(repo).data
if graph is None:
abort(404)
_nodes, _links = graph
nodes = [{"id": repo, "group": degree} for (repo, degree) in _nodes]
del _nodes
links = [{"source": repo_a, "target": repo_b} for \
(repo_a, repo_b) in _links]
del _links
return jsonify({"nodes": nodes, "links": links})
def run_cluster_server(riak_conf):
riak_hosts = []
with open(riak_conf, "r") as fin:
for line in fin:
riak_hosts.append({"host": line.replace("\n", ""), "http_port": RIAK_PORT})
cluster = RiakClient(nodes=riak_hosts)
bucket = cluster.bucket("HashedData")
random_str_arr = []
random_str_arr = run_exp_init(socket.gethostname())
insert_throuput = run_exp_insert(bucket, random_str_arr)
lookup_throuput = run_exp_lookup(bucket, random_str_arr)
delete_throuput = run_exp_delete(bucket, random_str_arr)
overall_throuput = (insert_throuput + lookup_throuput + delete_throuput) / 3
print("Overall Throughput: " + str(overall_throuput) + " OPs/s")
def getData(tickerFile, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP,
pb_port=8087) #set up riak connection
added = [] #list of successful adds
notAdded = [] #list of unsuccessful adds
stock = pd.read_csv(tickerFile, sep='\t',
header=None) #read in stock tickers
#loop over all stock tickers
for i in range(0, len(stock.head(100))):
ticker = stock.ix[i, 0]
if getDataByTicker(ticker, dataSource, start, end, riakIP) == 0:
notAdded.append(ticker)
else:
added.append(ticker)
return added, notAdded
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
transport_class = {
'http': RiakHttpTransport,
'protocol_buffer': RiakPbcTransport,
}.get(transport_type, RiakHttpTransport)
host = config.get('host', '127.0.0.1')
port = config.get('port', 8098)
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
# NOTE: the current riak.RiakClient expects this parameter but
# internally doesn't do anything with it.
solr_transport_class = config.get('solr_transport_class', None)
transport_options = config.get('transport_options', None)
client = RiakClient(host=host,
port=port,
prefix=prefix,
mapred_prefix=mapred_prefix,
transport_class=transport_class,
client_id=client_id,
solr_transport_class=solr_transport_class,
transport_options=transport_options)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client,
bucket_prefix,
load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout)
def from_config(cls, config):
config = config.copy()
bucket_prefix = config.pop('bucket_prefix')
load_bunch_size = config.pop('load_bunch_size',
cls.DEFAULT_LOAD_BUNCH_SIZE)
mapreduce_timeout = config.pop('mapreduce_timeout',
cls.DEFAULT_MAPREDUCE_TIMEOUT)
transport_type = config.pop('transport_type', 'http')
store_versions = config.pop('store_versions', None)
host = config.get('host', '127.0.0.1')
port = config.get('port')
prefix = config.get('prefix', 'riak')
mapred_prefix = config.get('mapred_prefix', 'mapred')
client_id = config.get('client_id')
transport_options = config.get('transport_options', {})
client_args = dict(host=host,
prefix=prefix,
mapred_prefix=mapred_prefix,
protocol=transport_type,
client_id=client_id,
transport_options=transport_options)
if port is not None:
client_args['port'] = port
client = RiakClient(**client_args)
# Some versions of the riak client library use simplejson by
# preference, which breaks some of our unicode assumptions. This makes
# sure we're using stdlib json which doesn't sometimes return
# bytestrings instead of unicode.
client.set_encoder('application/json', json.dumps)
client.set_encoder('text/json', json.dumps)
client.set_decoder('application/json', json.loads)
client.set_decoder('text/json', json.loads)
return cls(client,
bucket_prefix,
load_bunch_size=load_bunch_size,
mapreduce_timeout=mapreduce_timeout,
store_versions=store_versions)
def main(argv):
global gtable
global gclient
# Overhead to manage command line opts and config file
p = getCmdLineParser()
args = p.parse_args()
cfg.read(args.config_file)
# Get the logger going
glogname = cfg.get('logging', 'logName')
rightNow = time.strftime("%Y%m%d%H%M%S")
logger = initLog(rightNow)
logger.info('Starting Run: ' + time.strftime("%Y%m%d%H%M%S") +
' =========================')
# Get Riak going
gclient = RiakClient(protocol='pbc',
nodes=[{
'host': cfg.get('riakts', 'ip'),
'pb_port': int(cfg.get('riakts', 'port'))
}])
gtable = cfg.get('riakts', 'table')
# Get Rabbit going
rabbitChannel = setUpRabbit(cfg.get('rabbitmq', 'ip'),
int(cfg.get('rabbitmq', 'port')),
cfg.get('rabbitmq', 'login'),
cfg.get('rabbitmq', 'password'),
cfg.get('rabbitmq', 'queue'))
logger.info('RabbitMQ channel initialized...')
rabbitChannel.basic_qos(prefetch_count=1)
rabbitChannel.basic_consume(processAssignment,
queue=cfg.get('rabbitmq', 'queue'))
logger.info('RabbitMQ consumer initialized...')
rabbitChannel.start_consuming()
# Clean up
logger.info('Done! ' + time.strftime("%Y%m%d%H%M%S") +
' ==========================')
def executeScenario(scenario):
client = RiakClient(protocol='pbc',nodes=[{ 'host': scenario["server"]["ip"], 'pb_port': scenario["server"]["port"] }])
table, maxInterval, precision = getSource(scenario["dataSource"])
asset = scenario["id"]
destinationLoc = '{0}{1}{2}'.format('/tmp/',scenario["name"],'.csv')
destination = open(destinationLoc,'w')
if isinstance(scenario["interval"], dict):
startTimeInt = toUnixTime(scenario["interval"]["startTime"], precision)
endTimeInt = toUnixTime(scenario["interval"]["endTime"], precision)
now = startTimeInt
offset = getOffset(scenario["interval"]["value"],scenario["interval"]["units"], precision)
expectedPayloadSize = getPayload(scenario["interval"]["expected"]["value"], scenario["interval"]["expected"]["units"])
while now < endTimeInt:
now = now + offset
executeQuery(client, table, asset, now, now + offset, maxInterval, expectedPayloadSize, destination)
elif isinstance(scenario["interval"], list):
for interval in scenario["interval"]:
expectedPayload = interval[2]
expectedPayloadSize = getPayload(expectedPayload['value'], expectedPayload['units'])
executeQuery(client, table, asset, toUnixTime(interval[0], precision), toUnixTime(interval[1], precision), maxInterval, expectedPayloadSize, destination)
destination.close()
def test_close_underlying_socket_fails(self):
c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport)
bucket = c.bucket('bucket_test_close')
rand = self.randint()
obj = bucket.new('foo', rand)
obj.store()
obj = bucket.get('foo')
self.assertTrue(obj.exists())
self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close')
self.assertEqual(obj.get_key(), 'foo')
self.assertEqual(obj.get_data(), rand)
# Close the underlying socket. This gets a bit sketchy,
# since we are reaching into the internals, but there is
# no other way to get at the socket
conns = c._cm.conns
conns[0].sock.close()
# This shoud fail with a socket error now
self.assertRaises(socket.error, bucket.get, 'foo')
def run(self):
"""
Runs the commands in the input file, writing results to the output file.
"""
(host, http_port, pb_port) = self.h.split(':')
self.client = RiakClient(host=host, port=http_port)
commands = json.load(self.f)
results = []
errors = 0
for command in commands:
name = command.pop('command')
try:
if hasattr(self, name):
method = getattr(self, name)
results.append(method(command))
else:
errors = errors + 1
results.append(UNIMPLEMENTED)
except Exception as error:
errors = errors + 1
results.append(dict(error=str(error)))
self.report(results)
return errors
def __init__(self,
protocol="pbc",
nodes=[],
objects_bucket_name="objects",
activities_bucket_name="activities",
**kwargs):
self._riak_backend = RiakClient(protocol=protocol, nodes=nodes)
r_value = kwargs.get("r")
w_value = kwargs.get("w")
dw_value = kwargs.get("dw")
pr_value = kwargs.get("pr")
pw_value = kwargs.get("pw")
self._objects = self._riak_backend.bucket(objects_bucket_name)
self._activities = self._riak_backend.bucket(activities_bucket_name)
if r_value:
self._objects.r = r_value
self._activities.r = r_value
if w_value:
self._objects.w = w_value
self._activities.w = w_value
if dw_value:
self._objects.dw = dw_value
self._activities.dw = dw_value
if pr_value:
self._objects.pr = pr_value
self._activities.pr = pr_value
if pw_value:
self._objects.pw = pw_value
self._activities.pw = pw_value
def get_hosts(hosts, servers):
results = list()
riak_reply = {}
for server in servers:
mcodata = RiakClient(host=server).bucket_type('puppet').bucket('nodes')
if hosts is not None:
for host in hosts:
#then get specific host
riak_reply[server] = mcodata.search("facts.fqdn:%s" % host,
index='mco_nodes')
if riak_reply[server]['docs']:
results.extend(riak_reply[server]['docs'])
else:
#then get all hosts
riak_reply[server] = mcodata.get_keys()
if riak_reply[server]:
results.extend(riak_reply[server])
results.sort()
reply = {'servers': '%s' % servers, 'results': results} or None
return reply
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from riak import RiakClient
client = RiakClient(protocol='pbc',
nodes=[{
'host': '172.17.0.2',
'http_port': 8098,
'pb_port': 8087
}])
#client.retries = 3
bucket = client.bucket_type('mfn_counter_trigger').bucket('counter_triggers')
# Use the code below when mapper wants to create a k-of-n parallel execution
# topic, key, value are separated by ";", and are used for trigger to publish <key, value> to kafka's topic.
# topic and key must not contain the character ";". value can contain the character ";".
counter_name = 'topic;key;value'
k = 3
counter = bucket.new(counter_name)
counter.increment(k)
counter.store()
def __init__(self):
self.client = RiakClient(protocol='http',
host='127.0.0.1',
http_port=8098)
def test_mapred_bucket_exception(self):
c = RiakClient()
with self.assertRaises(ListError):
c.add('bucket')
#!/usr/bin/python
from riak import RiakClient, RiakNode
from riak.riak_object import RiakObject
client = RiakClient(protocal='http', host='127.0.0.1', http_port=8098)
res = client.ping()
bucket = client.bucket('xxx')
#res = client.get_keys(res[0])
#obj = RiakObject(client, res, 'xxx')
obj = bucket.new('a', 'hello', 'text/plain')
obj.store()
res = bucket.get('a')
n = 0
def __init__(self):
self.users_bucket = settings.RIAK_USERS_BUCKET
self.riak = RiakClient(host=settings.RIAK_HOST,
port=settings.RIAK_PORT)
print u'search ' + term + u' on ' + buck
return riak.search(buck, term)
def export(var):
if isinstance(var, basestring):
print(var)
return
if isinstance(var, dict):
for k, v in var.iteritems():
print k, ':',
export(v)
return
if isinstance(var, (list, tuple, set)):
for v in var:
export(v)
return
print var
riak = RiakClient(**config.getRiak())
query = search(*sys.argv[1:])
count = 0
for record in query.run():
res = record.get().get_data()
export(res)
count += 1
if count > 10:
break
