def tearDownModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz_bucket c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) c.bucket(testrun_sibs_bucket).clear_properties() c.bucket(testrun_props_bucket).clear_properties() if not SKIP_SEARCH and not RUN_YZ: b = c.bucket(testrun_search_bucket) b.clear_properties() if RUN_YZ: for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): if yz['btype'] is not None: t = c.bucket_type(yz['btype']) b = t.bucket(yz['bucket']) else: b = c.bucket(yz['bucket']) b.set_property('search_index', '_dont_index_') c.delete_search_index(yz['index']) for keys in b.stream_keys(): for key in keys: b.delete(key)
def tearDownModule(): c = RiakClient(transport='http', http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): b = c.bucket(testrun_search_bucket) b.clear_properties() b = c.bucket(testrun_sibs_bucket) b.clear_properties() b = c.bucket(testrun_props_bucket) b.clear_properties()
def tearDownModule(): c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) if not SKIP_SEARCH and not RUN_YZ: b = c.bucket(testrun_search_bucket) b.clear_properties() b = c.bucket(testrun_sibs_bucket) b.clear_properties() b = c.bucket(testrun_props_bucket) b.clear_properties()
def tearDownModule(): c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): b = c.bucket(testrun_search_bucket) b.clear_properties() b = c.bucket(testrun_sibs_bucket) b.clear_properties() b = c.bucket(testrun_props_bucket) b.clear_properties()
def tearDownModule(): c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')) and not int(os.environ.get('RUN_YZ', '1')): b = c.bucket(testrun_search_bucket) b.clear_properties() b = c.bucket(testrun_sibs_bucket) b.clear_properties() b = c.bucket(testrun_props_bucket) b.clear_properties()
def tearDownModule(): c = RiakClient(protocol="http", host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get("SKIP_SEARCH", "0")): b = c.bucket(testrun_search_bucket) b.clear_properties() b = c.bucket(testrun_sibs_bucket) b.clear_properties() b = c.bucket(testrun_props_bucket) b.clear_properties()
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, testrun_sibs_bucket c = RiakClient(protocol="http", host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = "propsbucket" testrun_sibs_bucket = "sibsbucket" c.bucket(testrun_sibs_bucket).allow_mult = True if not int(os.environ.get("SKIP_SEARCH", "0")): testrun_search_bucket = "searchbucket" b = c.bucket(testrun_search_bucket) b.enable_search()
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if not int(os.environ.get('SKIP_SEARCH', '0')) and not int(os.environ.get('RUN_YZ', '1')): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search()
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if not int(os.environ.get('SKIP_SEARCH', '0')): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search()
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if (not SKIP_SEARCH and not RUN_YZ): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search()
def create_client(cls, host=None, http_port=None, pb_port=None, protocol=None, credentials=None, **client_args): host = host or HOST http_port = http_port or HTTP_PORT pb_port = pb_port or PB_PORT if protocol is None: if hasattr(cls, 'protocol') and (cls.protocol is not None): protocol = cls.protocol else: protocol = PROTOCOL cls.protocol = protocol credentials = credentials or SECURITY_CREDS if hasattr(cls, 'logging_enabled') and cls.logging_enabled: cls.logger.debug( "RiakClient(protocol='%s', host='%s', " + "pb_port='%d', http_port='%d', " + "credentials='%s', client_args='%s')", protocol, host, pb_port, http_port, credentials, client_args) return RiakClient(protocol=protocol, host=host, http_port=http_port, credentials=credentials, pb_port=pb_port, **client_args)
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz_bucket c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT, pb_port=PB_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if (not SKIP_SEARCH and not RUN_YZ): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search() if RUN_YZ: c.protocol = 'pbc' testrun_yz_bucket = 'yzbucket' c.create_search_index(testrun_yz_bucket) b = c.bucket(testrun_yz_bucket) index_set = False while not index_set: try: b.set_property('search_index', testrun_yz_bucket) index_set = True except RiakError: pass
def create_client(self, host=None, http_port=None, pb_port=None, protocol=None, **client_args): host = host or self.host or HOST http_port = http_port or self.http_port or HTTP_PORT pb_port = pb_port or self.pb_port or PB_PORT protocol = protocol or self.protocol return RiakClient(protocol=protocol, host=host, http_port=http_port, pb_port=pb_port, **client_args)
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz_bucket, testrun_mr_btype, \ testrun_mr_bucket c = RiakClient(protocol='pbc', host=PB_HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if (not SKIP_SEARCH and not RUN_YZ): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search() if RUN_YZ: c.protocol = 'pbc' testrun_yz_bucket = 'yzbucket' c.create_search_index(testrun_yz_bucket) b = c.bucket(testrun_yz_bucket) index_set = False while not index_set: try: b.set_property('search_index', testrun_yz_bucket) index_set = True except RiakError: pass # Add bucket and type for Search -> MapReduce testrun_mr_btype = 'pytest-mr' testrun_mr_bucket = 'mrbucket' c.create_search_index(testrun_mr_bucket, '_yz_default') t = c.bucket_type(testrun_mr_btype) b = t.bucket(testrun_mr_bucket) index_set = False while not index_set: try: b.set_property('search_index', testrun_mr_bucket) index_set = True except RiakError: pass
def tearDownModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz_bucket c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT, pb_port=PB_PORT) c.bucket(testrun_sibs_bucket).clear_properties() c.bucket(testrun_props_bucket).clear_properties() if not SKIP_SEARCH and not RUN_YZ: b = c.bucket(testrun_search_bucket) b.clear_properties() if RUN_YZ: c.protocol = 'pbc' yzbucket = c.bucket(testrun_yz_bucket) yzbucket.set_property('search_index', '_dont_index_') c.delete_search_index(testrun_yz_bucket) for keys in yzbucket.stream_keys(): for key in keys: yzbucket.delete(key)
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if (not SKIP_SEARCH and not RUN_YZ): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search() if RUN_YZ: # YZ index on bucket of the same name testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} # YZ index on bucket of a different name testrun_yz_index = { 'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex' } # Add bucket and type for Search 2.0 -> MapReduce testrun_yz_mr = { 'btype': 'pytest-mr', 'bucket': 'mrbucket', 'index': 'mrbucket' } for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): c.create_search_index(yz['index']) if yz['btype'] is not None: t = c.bucket_type(yz['btype']) b = t.bucket(yz['bucket']) else: b = c.bucket(yz['bucket']) # Keep trying to set search bucket property until it succeeds index_set = False while not index_set: try: b.set_property('search_index', yz['index']) index_set = True except RiakError: pass
def create_client(self, host=None, http_port=None, pb_port=None, protocol=None, credentials=None, **client_args): host = host or self.host or HOST http_port = http_port or self.http_port or HTTP_PORT pb_port = pb_port or self.pb_port or PB_PORT protocol = protocol or self.protocol credentials = credentials or SECURITY_CREDS return RiakClient(protocol=protocol, host=host, http_port=http_port, credentials=credentials, pb_port=pb_port, **client_args)
def main(argv): # Overhead to manage command line opts and config file p = getCmdLineParser() args = p.parse_args() cfg.read(args.config_file) # Get the logger going glogname = cfg.get('logging', 'logName') rightNow = time.strftime("%Y%m%d%H%M%S") logger = initLog(rightNow) logger.info('Starting Run: ' + time.strftime("%Y%m%d%H%M%S") + ' =========================') startDate = datetime.strptime(cfg.get('coverage', 'startTime'), '%Y-%m-%d') t1 = unix_time_millis(startDate) qid = cfg.get('coverage', 'qid') table = cfg.get('coverage', 'table') client = RiakClient(host=cfg.get('riak', 'ip'), pb_port=int(cfg.get('riak', 'port'))) epoch = datetime.utcfromtimestamp(0) counter = 0 for pid in range(int(cfg.get('coverage', 'startID')), int(cfg.get('coverage', 'numIDs'))): for dayOffset in range(1, int(cfg.get('coverage', 'numDays')) + 1): counter = counter + 1 t2 = unix_time_millis(startDate + timedelta(days=dayOffset)) startTime = time.time() coverage = getCoverage(client, table, pid, qid, t1, t2) duration = round((time.time() - startTime), 3) if (dayOffset - 1) * 2 == len(coverage["coverage"]): pass1 = "PASS!" else: pass1 = "FAIL!" results = "Test #{counter}: ID: {x}: total: {dur1}s, riak: {dur2}s, numdays: {numDays}, result: {result}".format( counter=counter, x=pid, dur1=duration, dur2=coverage["duration"], numDays=dayOffset, result=pass1) logger.info(results)
def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' c.bucket(testrun_sibs_bucket).allow_mult = True if (not SKIP_SEARCH and not RUN_YZ): testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search() if RUN_YZ: # YZ index on bucket of the same name testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} # YZ index on bucket of a different name testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex'} # Add bucket and type for Search 2.0 -> MapReduce testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', 'index': 'mrbucket'} for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): c.create_search_index(yz['index']) if yz['btype'] is not None: t = c.bucket_type(yz['btype']) b = t.bucket(yz['bucket']) else: b = c.bucket(yz['bucket']) # Keep trying to set search bucket property until it succeeds index_set = False while not index_set: try: b.set_property('search_index', yz['index']) index_set = True except RiakError: pass
def create_client(cls, host=None, http_port=None, pb_port=None, protocol=None, credentials=None, **kwargs): host = host or HOST http_port = http_port or HTTP_PORT pb_port = pb_port or PB_PORT if protocol is None: if hasattr(cls, 'protocol') and (cls.protocol is not None): protocol = cls.protocol else: protocol = PROTOCOL cls.protocol = protocol credentials = credentials or SECURITY_CREDS if hasattr(cls, 'client_options'): kwargs.update(cls.client_options) logger = logging.getLogger() logger.debug( "RiakClient(protocol='%s', host='%s', pb_port='%d', " "http_port='%d', credentials='%s', kwargs='%s')", protocol, host, pb_port, http_port, credentials, kwargs) return RiakClient(protocol=protocol, host=host, http_port=http_port, credentials=credentials, pb_port=pb_port, **kwargs)
from riak.client import RiakClient from riak.riak_error import RiakError import sys import os import time from datetime import datetime import calendar import csv client = RiakClient(host=sys.argv[1], pb_port=8087) table_name = sys.argv[2] # Create table print "Creating table..." print "Schema: CREATE TABLE " + table_name + " (status varchar not null, extid varchar not null, ts timestamp not null, avgMeasuredTime sint64 not null, avgSpeed sint64 not null, medianMeasuredTime sint64 not null, vehicleCount sint64 not null, PRIMARY KEY ((status, extid, quantum(ts, 30, 'd')), status, extid, ts))" time1 = int(round(time.time() * 1000)) try: print "Create table return: ", str( client.ts_query( table_name, "CREATE TABLE {table} (status varchar not null, extid varchar not null, ts timestamp not null, avgMeasuredTime sint64 not null, avgSpeed sint64 not null, medianMeasuredTime sint64 not null, vehicleCount sint64 not null, PRIMARY KEY ((status, extid, quantum(ts, 30, 'd')), status, extid, ts))" )) except RiakError as e: print "Table " + table_name + " already exists." time2 = int(round(time.time() * 1000)) print "Time elapsed (ms): ", str(time2 - time1) print "\n" table = client.table(table_name)
def tearDownModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz_bucket c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) c.bucket(testrun_sibs_bucket).clear_properties() c.bucket(testrun_props_bucket).clear_properties() if not SKIP_SEARCH and not RUN_YZ: b = c.bucket(testrun_search_bucket) b.clear_properties() if RUN_YZ: yzbucket = c.bucket(testrun_yz_bucket) yzbucket.set_property('search_index', '_dont_index_') c.delete_search_index(testrun_yz_bucket) for keys in yzbucket.stream_keys(): for key in keys: yzbucket.delete(key) mrtype = c.bucket_type(testrun_mr_btype) mrbucket = mrtype.bucket(testrun_mr_bucket) mrbucket.set_property('search_index', '_dont_index_') c.delete_search_index(testrun_mr_bucket) for keys in mrbucket.stream_keys(): for key in keys: mrbucket.delete(key)
@timeit def test_riak_read(test_bucket, n): for i in range(n): r = test_bucket.get(str(i)) # stable version # data = r.get_data() # master_version data = r.data if data != i: raise if __name__ == '__main__': # master_version client = RiakClient(host='219.224.135.60', pb_port=10017, protocol='pbc') # master_version use nodes param # 此场景下性能差不多,和上面指定具体port相比 # client = RiakClient(host='219.224.135.60', nodes=Nodes, protocol='pbc') # test test_bucket = client.bucket('lijun_test') """ new_node = test_bucket.new('hehe', data='hehe') new_node.store() r = test_bucket.get('hehe') print r.get_data(), type(r.get_data()) new_node = test_bucket.new('hehe1', data=1) new_node.store() r = test_bucket.get('hehe1')
from riak.client import RiakClient from riak.riak_error import RiakError import sys import os import time from datetime import datetime import calendar import csv client = RiakClient(host=sys.argv[1], pb_port=8087) table_name = sys.argv[2] # Create table print "Creating table..." print "Schema: CREATE TABLE " + table_name + " (status varchar not null, extid varchar not null, ts timestamp not null, avgMeasuredTime sint64 not null, avgSpeed sint64 not null, medianMeasuredTime sint64 not null, vehicleCount sint64 not null, PRIMARY KEY ((status, extid, quantum(ts, 30, 'd')), status, extid, ts))" time1 = int(round(time.time() * 1000)) try: print "Create table return: ", str(client.ts_query(table_name, "CREATE TABLE {table} (status varchar not null, extid varchar not null, ts timestamp not null, avgMeasuredTime sint64 not null, avgSpeed sint64 not null, medianMeasuredTime sint64 not null, vehicleCount sint64 not null, PRIMARY KEY ((status, extid, quantum(ts, 30, 'd')), status, extid, ts))")) except RiakError as e: print "Table " + table_name + " already exists." time2 = int(round(time.time() * 1000)) print "Time elapsed (ms): ", str(time2 - time1) print "\n" table = client.table(table_name) def changetime(stime): dt=datetime.strptime(stime,'%Y-%m-%dT%H:%M:%S') #print dt return calendar.timegm(datetime.timetuple(dt))*1000
from riak.client import RiakClient import sys import time import datetime import random from checkin import Checkin checkin = Checkin(sys.argv[1], sys.argv[2], sys.argv[3]) checkin_probability = [60, 55, 25, 0, 0, 0, 0, 0, 0, 0, 0, 20, 40, 30, 25, 15, 15, 45, 60, 70, 80, 75, 65, 60] rating_comment = ["", "Lousy", "Ok", "Decent", "Pretty good", "Wow!"] client = RiakClient(host=sys.argv[1], pb_port=8087) bucket_type_default = client.bucket_type("bucket_type_default") location_bucket = bucket_type_default.bucket('location') f = open("users.txt") users = f.read().splitlines() user_set = set(users) # remove dupes user_list = list(user_set) user_count = len(user_list) f.close() f = open("wines.txt") wines = f.read().splitlines() wine_count = len(wines) f.close() f = open("locations.txt") locations = f.read().splitlines() location_count = len(locations)
from riak.riak_error import RiakError from riak.client import RiakClient from riak.datatypes import Map import sys import time import random client = RiakClient(host=sys.argv[1], pb_port=8087) bucket_type_default = client.bucket_type("bucket_type_default") bucket_type_map = client.bucket_type("bucket_type_map") def create_users(user_bucket, friends_bucket): with open("users.txt") as f: users = f.read().splitlines() user_set = set(users) # remove dupes user_list = list(user_set) # Add user for user in user_list: user = user.strip() record = user_bucket.new(user, data={}) print "User " + user + " added... " + str(record.store()) # Add 10 random friends for i in range(10): friend = user_list[random.randint(0, len(user_list) - 1)].strip() if friend != user:
from riak.client import RiakClient from riak.riak_error import RiakError import sys import time client = RiakClient(host=sys.argv[1], pb_port=8087) table_name = sys.argv[2] table = client.table(table_name) query = "SELECT COUNT(*) FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'" print "Query: ", query time1 = int(round(time.time() * 1000)) ts_obj = client.ts_query(table_name, query) time2 = int(round(time.time() * 1000)) print ts_obj.rows print "Time elapsed (ms): ", str(time2 - time1) print "\n" query = "SELECT * FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'" print "Query: ", query time1 = int(round(time.time() * 1000)) ts_obj = client.ts_query(table_name, query) time2 = int(round(time.time() * 1000)) print ts_obj.rows print "Time elapsed (ms): ", str(time2 - time1) print "\n" query = "SELECT MAX(avgSpeed) FROM {table} where ts > 1392249600000 and ts < 1397347199000 and status='OK' and extid='668'" print "Query: ", query time1 = int(round(time.time() * 1000)) ts_obj = client.ts_query(table_name, query)
import pika import datetime from riak.client import RiakClient import os riak_client = RiakClient(host=os.environ['RIAK_HOST'], pb_port=8087) table_temperature = riak_client.table('Temperature') table_cpu = riak_client.table('CPU') table_mem = riak_client.table('Mem') credentials = pika.PlainCredentials('guest', 'guest') parameters = ( pika.ConnectionParameters(host=os.environ['RABBIT_1'],credentials=credentials), pika.ConnectionParameters(host=os.environ['RABBIT_2'], connection_attempts=5, retry_delay=1,credentials=credentials)) def callback(ch, method, properties, body): message = body.decode('utf-8') message = message.split(',') print(message) producer = int(message[0]) date = datetime.datetime.strptime(message[1], '%Y-%m-%dT%H:%M:%S-07:00') temp = float(message[4]) cpu_usage = float(message[2]) mem_usage = float(message[3]) ts_obj_temp = table_temperature.new([[producer,producer,date,temp]]) print(ts_obj_temp.store()) ts_obj_cpu = table_cpu.new([[producer,producer,date,cpu_usage]]) print(ts_obj_cpu.store())