def tearDownModule():
global testrun_search_bucket, testrun_props_bucket, \
testrun_sibs_bucket, testrun_yz_bucket
c = RiakClient(host=HTTP_HOST,
http_port=HTTP_PORT,
pb_port=PB_PORT,
credentials=SECURITY_CREDS)
c.bucket(testrun_sibs_bucket).clear_properties()
c.bucket(testrun_props_bucket).clear_properties()
if not SKIP_SEARCH and not RUN_YZ:
b = c.bucket(testrun_search_bucket)
b.clear_properties()
if RUN_YZ:
for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr):
if yz['btype'] is not None:
t = c.bucket_type(yz['btype'])
b = t.bucket(yz['bucket'])
else:
b = c.bucket(yz['bucket'])
b.set_property('search_index', '_dont_index_')
c.delete_search_index(yz['index'])
for keys in b.stream_keys():
for key in keys:
b.delete(key)
def create_client(cls,
host=None,
http_port=None,
pb_port=None,
protocol=None,
credentials=None,
**client_args):
host = host or HOST
http_port = http_port or HTTP_PORT
pb_port = pb_port or PB_PORT
if protocol is None:
if hasattr(cls, 'protocol') and (cls.protocol is not None):
protocol = cls.protocol
else:
protocol = PROTOCOL
cls.protocol = protocol
credentials = credentials or SECURITY_CREDS
if hasattr(cls, 'logging_enabled') and cls.logging_enabled:
cls.logger.debug(
"RiakClient(protocol='%s', host='%s', " +
"pb_port='%d', http_port='%d', " +
"credentials='%s', client_args='%s')", protocol, host, pb_port,
http_port, credentials, client_args)
return RiakClient(protocol=protocol,
host=host,
http_port=http_port,
credentials=credentials,
pb_port=pb_port,
**client_args)
def tearDownModule():
c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT)
if not int(os.environ.get('SKIP_SEARCH', '0')):
b = c.bucket(testrun_search_bucket)
b.clear_properties()
b = c.bucket(testrun_sibs_bucket)
b.clear_properties()
b = c.bucket(testrun_props_bucket)
b.clear_properties()
def create_client(self, host=None, http_port=None, pb_port=None,
protocol=None, **client_args):
host = host or self.host or HOST
http_port = http_port or self.http_port or HTTP_PORT
pb_port = pb_port or self.pb_port or PB_PORT
protocol = protocol or self.protocol
return RiakClient(protocol=protocol,
host=host,
http_port=http_port,
pb_port=pb_port, **client_args)
def setUpModule():
global testrun_search_bucket, testrun_props_bucket, \
testrun_sibs_bucket
c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT)
testrun_props_bucket = 'propsbucket'
testrun_sibs_bucket = 'sibsbucket'
c.bucket(testrun_sibs_bucket).allow_mult = True
if not int(os.environ.get('SKIP_SEARCH', '0')):
testrun_search_bucket = 'searchbucket'
b = c.bucket(testrun_search_bucket)
b.enable_search()
def setUpModule():
global testrun_search_bucket, testrun_props_bucket, \
testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr
c = RiakClient(host=PB_HOST,
http_port=HTTP_PORT,
pb_port=PB_PORT,
credentials=SECURITY_CREDS)
testrun_props_bucket = 'propsbucket'
testrun_sibs_bucket = 'sibsbucket'
c.bucket(testrun_sibs_bucket).allow_mult = True
if (not SKIP_SEARCH and not RUN_YZ):
testrun_search_bucket = 'searchbucket'
b = c.bucket(testrun_search_bucket)
b.enable_search()
if RUN_YZ:
# YZ index on bucket of the same name
testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'}
# YZ index on bucket of a different name
testrun_yz_index = {
'btype': None,
'bucket': 'yzindexbucket',
'index': 'yzindex'
}
# Add bucket and type for Search 2.0 -> MapReduce
testrun_yz_mr = {
'btype': 'pytest-mr',
'bucket': 'mrbucket',
'index': 'mrbucket'
}
for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr):
c.create_search_index(yz['index'])
if yz['btype'] is not None:
t = c.bucket_type(yz['btype'])
b = t.bucket(yz['bucket'])
else:
b = c.bucket(yz['bucket'])
# Keep trying to set search bucket property until it succeeds
index_set = False
while not index_set:
try:
b.set_property('search_index', yz['index'])
index_set = True
except RiakError:
pass
def create_client(self,
host=None,
http_port=None,
pb_port=None,
protocol=None,
credentials=None,
**client_args):
host = host or self.host or HOST
http_port = http_port or self.http_port or HTTP_PORT
pb_port = pb_port or self.pb_port or PB_PORT
protocol = protocol or self.protocol
credentials = credentials or SECURITY_CREDS
return RiakClient(protocol=protocol,
host=host,
http_port=http_port,
credentials=credentials,
pb_port=pb_port,
**client_args)
def main(argv):
# Overhead to manage command line opts and config file
p = getCmdLineParser()
args = p.parse_args()
cfg.read(args.config_file)
# Get the logger going
glogname = cfg.get('logging', 'logName')
rightNow = time.strftime("%Y%m%d%H%M%S")
logger = initLog(rightNow)
logger.info('Starting Run: ' + time.strftime("%Y%m%d%H%M%S") +
' =========================')
startDate = datetime.strptime(cfg.get('coverage', 'startTime'), '%Y-%m-%d')
t1 = unix_time_millis(startDate)
qid = cfg.get('coverage', 'qid')
table = cfg.get('coverage', 'table')
client = RiakClient(host=cfg.get('riak', 'ip'),
pb_port=int(cfg.get('riak', 'port')))
epoch = datetime.utcfromtimestamp(0)
counter = 0
for pid in range(int(cfg.get('coverage', 'startID')),
int(cfg.get('coverage', 'numIDs'))):
for dayOffset in range(1, int(cfg.get('coverage', 'numDays')) + 1):
counter = counter + 1
t2 = unix_time_millis(startDate + timedelta(days=dayOffset))
startTime = time.time()
coverage = getCoverage(client, table, pid, qid, t1, t2)
duration = round((time.time() - startTime), 3)
if (dayOffset - 1) * 2 == len(coverage["coverage"]):
pass1 = "PASS!"
else:
pass1 = "FAIL!"
results = "Test #{counter}: ID: {x}: total: {dur1}s, riak: {dur2}s, numdays: {numDays}, result: {result}".format(
counter=counter,
x=pid,
dur1=duration,
dur2=coverage["duration"],
numDays=dayOffset,
result=pass1)
logger.info(results)
def create_client(cls,
host=None,
http_port=None,
pb_port=None,
protocol=None,
credentials=None,
**kwargs):
host = host or HOST
http_port = http_port or HTTP_PORT
pb_port = pb_port or PB_PORT
if protocol is None:
if hasattr(cls, 'protocol') and (cls.protocol is not None):
protocol = cls.protocol
else:
protocol = PROTOCOL
cls.protocol = protocol
credentials = credentials or SECURITY_CREDS
if hasattr(cls, 'client_options'):
kwargs.update(cls.client_options)
logger = logging.getLogger()
logger.debug(
"RiakClient(protocol='%s', host='%s', pb_port='%d', "
"http_port='%d', credentials='%s', kwargs='%s')", protocol, host,
pb_port, http_port, credentials, kwargs)
return RiakClient(protocol=protocol,
host=host,
http_port=http_port,
credentials=credentials,
pb_port=pb_port,
**kwargs)
import pika
import datetime
from riak.client import RiakClient
import os
riak_client = RiakClient(host=os.environ['RIAK_HOST'], pb_port=8087)
table_temperature = riak_client.table('Temperature')
table_cpu = riak_client.table('CPU')
table_mem = riak_client.table('Mem')
credentials = pika.PlainCredentials('guest', 'guest')
parameters = (
pika.ConnectionParameters(host=os.environ['RABBIT_1'],credentials=credentials),
pika.ConnectionParameters(host=os.environ['RABBIT_2'],
connection_attempts=5, retry_delay=1,credentials=credentials))
def callback(ch, method, properties, body):
message = body.decode('utf-8')
message = message.split(',')
print(message)
producer = int(message[0])
date = datetime.datetime.strptime(message[1], '%Y-%m-%dT%H:%M:%S-07:00')
temp = float(message[4])
cpu_usage = float(message[2])
mem_usage = float(message[3])
ts_obj_temp = table_temperature.new([[producer,producer,date,temp]])
print(ts_obj_temp.store())
ts_obj_cpu = table_cpu.new([[producer,producer,date,cpu_usage]])
print(ts_obj_cpu.store())
from riak.client import RiakClient
from riak.riak_error import RiakError
import sys
import time
client = RiakClient(host=sys.argv[1], pb_port=8087)
table_name = sys.argv[2]
table = client.table(table_name)
query = "SELECT COUNT(*) FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'"
print "Query: ", query
time1 = int(round(time.time() * 1000))
ts_obj = client.ts_query(table_name, query)
time2 = int(round(time.time() * 1000))
print ts_obj.rows
print "Time elapsed (ms): ", str(time2 - time1)
print "\n"
query = "SELECT * FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'"
print "Query: ", query
time1 = int(round(time.time() * 1000))
ts_obj = client.ts_query(table_name, query)
time2 = int(round(time.time() * 1000))
print ts_obj.rows
print "Time elapsed (ms): ", str(time2 - time1)
print "\n"
query = "SELECT MAX(avgSpeed) FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'"
print "Query: ", query
time1 = int(round(time.time() * 1000))
ts_obj = client.ts_query(table_name, query)
@timeit
def test_riak_read(test_bucket, n):
for i in range(n):
r = test_bucket.get(str(i))
# stable version
# data = r.get_data()
# master_version
data = r.data
if data != i:
raise
if __name__ == '__main__':
# master_version
client = RiakClient(host='219.224.135.60', pb_port=10017, protocol='pbc')
# master_version use nodes param
# 此场景下性能差不多，和上面指定具体port相比
# client = RiakClient(host='219.224.135.60', nodes=Nodes, protocol='pbc')
# test
test_bucket = client.bucket('lijun_test')
"""
new_node = test_bucket.new('hehe', data='hehe')
new_node.store()
r = test_bucket.get('hehe')
print r.get_data(), type(r.get_data())
new_node = test_bucket.new('hehe1', data=1)
new_node.store()
r = test_bucket.get('hehe1')
