v_sum = 0

bins = histo.keys()

bins.sort()

for bin in bins:

v_sum += histo[bin]

v_sum = float(v_sum)

v_cur = 0 # Running total.

rv = []

for bin in bins:

if not percentiles:

return rv

v_cur += histo[bin]

while percentiles and (v_cur / v_sum) >= percentiles[0]:

rv.append((percentiles[0], bin))

percentiles.pop(0)

_task = {}

_verbosity = True

def __init__(self, verbosity):

self._verbosity = verbosity

self.is_leader = False

self.log = logger.Logger.get_logger()

#this function starts collecting stats from all nodes with the given

#frequency

def start(self, nodes, bucket, pnames, name, frequency, client_id='',

collect_server_stats=True):

self._task = {"state": "running", "threads": [],

"name": name, "time": time.time(),

"ops": [], "totalops": [],

"ops-temp": [], "latency": {},

"data_size_stats": []}

rest = RestConnection(nodes[0])

info = rest.get_nodes_self()

self.data_path = info.storage[0].get_data_path()

self.client_id = str(client_id)

if collect_server_stats:

mbstats_thread = Thread(target=self.membase_stats,

args=(nodes, bucket, frequency, self._verbosity))

mbstats_thread.start()

sysstats_thread = Thread(target=self.system_stats,

args=(nodes, pnames, frequency, self._verbosity))

sysstats_thread.start()

diskstats_thread = Thread(target=self.disk_stats,

args=(nodes, frequency, self._verbosity))

diskstats_thread.start()

ns_server_stats_thread = Thread(target=self.ns_server_stats,

args=([nodes[0]], bucket, 10, self._verbosity))

ns_server_stats_thread.start()

rest = RestConnection(nodes[0])

bucket_size_thead = Thread(target=self.get_bucket_size,

args=(bucket, rest, frequency))

bucket_size_thead.start()

#data_size_thread = Thread(target=self.get_data_file_size,

# args=(nodes, 60, bucket))

#data_size_thread.start()

self._task["threads"] = [sysstats_thread, ns_server_stats_thread, bucket_size_thead,

mbstats_thread]

#data_size_thread ]

# Getting build/machine stats from only one node in the cluster

self.build_stats(nodes)

self.machine_stats(nodes)

def stop(self):

self.log.info("stopping the stat collection")

self._task["state"] = "stopped"

for t in self._task["threads"]:

t.join()

self._task["time"] = time.time() - self._task["time"]

def sample(self, cur):

pass

def export(self, name, test_params):

for latency in self._task["latency"].keys():

histos = self._task["latency"].get(latency, [])

key = 'percentile-' + latency

self._task["latency"][key] = []

for histo in histos:

temp = []

delta = histo['delta']

del histo['delta']

p = histo_percentile(histo, [0.90, 0.95, 0.99])

# p is list of tuples

for val in p:

temp.append(val[-1])

temp.append(self.client_id)

temp.append(delta)

self._task["latency"][key].append(temp)

obj = {"buildinfo": self._task.get("buildstats", {}),

"machineinfo": self._task.get("machinestats", {}),

"membasestats": self._task.get("membasestats", []),

"systemstats": self._task.get("systemstats", []),

"name": name,

"totalops": self._task["totalops"],

"ops": self._task["ops"],

"time": self._task["time"],

"info": test_params,

"ns_server_data": self._task.get("ns_server_stats", []),

"ns_server_data_system": self._task.get("ns_server_stats_system", []),

"timings": self._task.get("timings", []),

"dispatcher": self._task.get("dispatcher", []),

"tap": self._task.get("tap", []),

"checkpoint": self._task.get("checkpoint", []),

"bucket-size": self._task.get("bucket_size", []),

"data-size": self._task.get("data_size_stats", []),

"latency-set": self._task["latency"].get('percentile-latency-set', []),

"latency-set-recent": self._task["latency"].get('percentile-latency-set-recent', []),

"latency-get": self._task["latency"].get('percentile-latency-get', []),

"latency-get-recent": self._task["latency"].get('percentile-latency-get-recent', []),

"latency-delete": self._task["latency"].get('percentile-latency-delete', []),

"latency-delete-recent": self._task["latency"].get('percentile-latency-delete-recent', []),

}

if self.client_id:

filename = str(self.client_id) + '.loop'

if re.search('load$', self._task["name"]):

filename = str(self.client_id) + '.load'

file = gzip.open("{0}.json.gz".format(filename), 'wb')

file.write("{0}".format(json.dumps(obj)))

file.close()

else:

file = gzip.open("{0}.json.gz".format(name), 'wb')

file.write("{0}".format(json.dumps(obj)))

file.close()

def get_bucket_size(self, bucket, rest, frequency):

self._task["bucket_size"] = []

d = []

while not self._aborted():

self.log.info("collecting bucket size stats from {0}:{1}".format(rest.ip, rest.port))

status, db_size = rest.get_database_disk_size(bucket)

if status:

d.append(db_size)

else:

print "Enable to read bucket stats"

time.sleep(frequency)

self._task["bucket_size"] = d

self.log.info("the thread finished collecting bucket stats")

def get_data_file_size(self, nodes, frequency, bucket):

shells = []

for node in nodes:

try:

shells.append(RemoteMachineShellConnection(node))

except:

pass

paths = []

if shells[0].is_membase_installed():

paths.append(self.data_path + '/{0}-data'.format(bucket))

else:

bucket_path = self.data_path + '/{0}'.format(bucket)

paths.append(bucket_path)

view_path = bucket_path + '/set_view_{0}_design'.format(bucket)

paths.append(view_path)

d = {"snapshots": []}

start_time = str(self._task["time"])

while not self._aborted():

time.sleep(frequency)

current_time = time.time()

i = 0

for shell in shells:

node = nodes[i]

unique_id = node.ip + '-' + start_time

value = {}

for path in paths:

size = shell.get_data_file_size(path)

value["file"] = path.split('/')[-1]

value["size"] = size

value["unique_id"] = unique_id

value["time"] = current_time

value["ip"] = node.ip

d["snapshots"].append(value.copy())

i += 1

self._task["data_size_stats"] = d["snapshots"]

self.log.info(" finished data_size_stats")

#ops stats

#{'tot-sets': 899999, 'tot-gets': 1, 'tot-items': 899999, 'tot-creates': 899999}

def ops_stats(self, ops_stat):

ops_stat["time"] = time.time()

self._task["ops-temp"].append(ops_stat)

if len(self._task["ops-temp"]) >= 100:

merged = self._merge()

self._task["ops"].append(merged)

self._task["ops-temp"] = []

#if self._task["ops"] has more than 1000 elements try to aggregate them ?

def latency_stats(self, latency_cmd, latency_stat):

if self._task["latency"].get(latency_cmd) is None:

self._task["latency"][latency_cmd] = []

temp_latency_stat = latency_stat.copy()

temp_latency_stat['delta'] = time.time() - self._task['time']

self._task["latency"][latency_cmd].append(temp_latency_stat)

def _merge(self):

first = self._task["ops-temp"][0]

merged = {"startTime": first["start-time"]}

totalgets = 0

totalsets = 0

delta = 0

i = 0

for i in range(len(self._task["ops-temp"]) - 1):

current = self._task["ops-temp"][i]

next = self._task["ops-temp"][i + 1]

totalgets += current["tot-gets"]

totalsets += current["tot-sets"]

delta += (next["start-time"] - current["start-time"])

merged["endTime"] = merged["startTime"] + delta

merged["totalSets"] = totalsets

merged["totalGets"] = totalgets

return merged

def total_stats(self, ops_stat):

ops_stat["time"] = time.time()

self._task["totalops"].append(ops_stat)

def build_stats(self, nodes):

json_response = StatUtil.build_info(nodes[0])

self._task["buildstats"] = json_response

def machine_stats(self, nodes):

try:

machine_stats = StatUtil.machine_info(nodes[0])

self._task["machinestats"] = machine_stats

except:

self._aborted()

def _extract_proc_info(self, shell, pid):

o, r = shell.execute_command("cat /proc/{0}/stat".format(pid))

fields = ('pid comm state ppid pgrp session tty_nr tpgid flags minflt '

'cminflt majflt cmajflt utime stime cutime cstime priority '

'nice num_threads itrealvalue starttime vsize rss rsslim '

'startcode endcode startstack kstkesp kstkeip signal blocked '

'sigignore sigcatch wchan nswap cnswap exit_signal '

'processor rt_priority policy delayacct_blkio_ticks '

'guest_time cguest_time ').split(' ')

d = dict(zip(fields, o[0].split(' ')))

return d

def _extrace_disk_states(self, shell):

o, r = shell.execute_command("iostat -dkx 1 2 /dev/sda2")

fields = ('rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util').split(' ')

y = []

for x in o[6]:

if x and x != ' ':

y.extend(x)

d = dict(zip(fields, y))

return d

def disk_stats(self, nodes, frequency, verbosity=False):

shells = []

for node in nodes:

try:

bucket = RestConnection(node).get_buckets()[0].name

MemcachedClientHelper.direct_client(node, bucket)

shells.append(RemoteMachineShellConnection(node))

except:

pass

d = {"snapshots": []}

# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]

start_time = str(self._task["time"])

while not self._aborted():

time.sleep(frequency)

i = 0

for shell in shells:

node = nodes[i]

unique_id = node.ip + '-' + start_time

value = self._extrace_disk_states(shell)

value["unique_id"] = unique_id

value["time"] = time.time()

value["ip"] = node.ip

d["snapshots"].append(value)

i += 1

self._task["diskstats"] = d["snapshots"]

self.log.info(" finished diskstats")

def system_stats(self, nodes, pnames, frequency, verbosity=False):

try:

shells = []

for node in nodes:

try:

bucket = RestConnection(node).get_buckets()[0].name

MemcachedClientHelper.direct_client(node, bucket)

shells.append(RemoteMachineShellConnection(node))

except:

pass

d = {"snapshots": []}

# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]

start_time = str(self._task["time"])

while not self._aborted():

time.sleep(frequency)

current_time = time.time()

i = 0

for shell in shells:

node = nodes[i]

unique_id = node.ip + '-' + start_time

for pname in pnames:

obj = RemoteMachineHelper(shell).is_process_running(pname)

if obj and obj.pid:

value = self._extract_proc_info(shell, obj.pid)

value["name"] = pname

value["id"] = obj.pid

value["unique_id"] = unique_id

value["time"] = current_time

value["ip"] = node.ip

d["snapshots"].append(value)

i += 1

self._task["systemstats"] = d["snapshots"]

self.log.info(" finished system_stats")

except:

self._aborted()

def couchdb_stats(nodes):

pass

def membase_stats(self, nodes, bucket, frequency, verbose=False):

try:

mcs = []

for node in nodes:

try:

bucket = RestConnection(node).get_buckets()[0].name

mcs.append(MemcachedClientHelper.direct_client(node, bucket))

except:

pass

self._task["membasestats"] = []

self._task["timings"] = []

self._task["dispatcher"] = []

self._task["tap"] = []

self._task["checkpoint"] = []

d = {}

# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]

for mc in mcs:

d[mc.host] = {"snapshots": [], "timings": [], "dispatcher": [], "tap": [], "checkpoint": []}

while not self._aborted():

time_left = frequency

# at minimum we want to check for aborted every minute

while not self._aborted() and time_left > 0:

time.sleep(min(time_left, 60))

if time_left >= 60:

time_left -= 60

else:

time_left = 0

for mc in mcs:

stats = mc.stats()

stats["time"] = time.time()

stats["ip"] = mc.host

d[mc.host]["snapshots"].append(stats)

timings = mc.stats('timings')

d[mc.host]["timings"].append(timings)

dispatcher = mc.stats('dispatcher')

d[mc.host]["dispatcher"].append(dispatcher)

tap = mc.stats('tap')

d[mc.host]["tap"].append(tap)

checkpoint = mc.stats('checkpoint')

d[mc.host]["checkpoint"].append(checkpoint)

print len(d[mc.host]["checkpoint"])

start_time = str(self._task["time"])

for mc in mcs:

ip = mc.host

unique_id = ip + '-' + start_time

current_time = time.time()

for snapshot in d[mc.host]["snapshots"]:

snapshot['unique_id'] = unique_id

snapshot['time'] = current_time

snapshot['ip'] = ip

self._task["membasestats"].append(snapshot)

for timing in d[mc.host]["timings"]:

timing['unique_id'] = unique_id

timing['time'] = current_time

timing['ip'] = ip

self._task["timings"].append(timing)

for dispatcher in d[mc.host]["dispatcher"]:

dispatcher['unique_id'] = unique_id

dispatcher['time'] = current_time

dispatcher['ip'] = ip

self._task["dispatcher"].append(dispatcher)

for tap in d[mc.host]["tap"]:

tap['unique_id'] = unique_id

tap['time'] = current_time

tap['ip'] = ip

self._task["tap"].append(tap)

for checkpoint in d[mc.host]["checkpoint"]:

checkpoint['unique_id'] = unique_id

checkpoint['time'] = current_time

checkpoint['ip'] = ip

self._task["checkpoint"].append(checkpoint)

self.log.info(" finished membase_stats")

except:

self._aborted()

def ns_server_stats(self, nodes, bucket, frequency, verbose=False):

try:

self._task["ns_server_stats"] = []

self._task["ns_server_stats_system"] = []

d = {}

for node in nodes:

d[node] = {"snapshots": [], "system_snapshots": []}

while not self._aborted():

time.sleep(frequency)

self.log.info("collecting ns_server_stats from all nodes in the cluster")

for node in nodes:

self.log.info("running ")

os.system(

"curl -X GET http://{2}:{3}@{1}:8091/pools/{0}/buckets/{0}/stats?zoom=minute -o ns_server_data".format(

bucket, node.ip,nodes[0].rest_username,nodes[0].rest_password))

#f.close()

dict = open("./ns_server_data", "r").read()

data_json = json.loads(dict)

d[node]["snapshots"].append(data_json)

os.system(

"curl -X GET http://{2}:{3}@{1}:8091/pools/{0} -o ns_server_data_system_stats".format(

bucket, node.ip,nodes[0].rest_username,nodes[0].rest_password))

#f.close()

dict = open("./ns_server_data_system_stats", "r").read()

data_json = json.loads(dict)

d[node]["system_snapshots"].append(data_json)

for node in nodes:

for snapshot in d[node]["snapshots"]:

self._task["ns_server_stats"].append(snapshot)

for snapshot in d[node]["system_snapshots"]:

self._task["ns_server_stats_system"].append(snapshot)

self.log.info(" finished ns_server_stats")

except:

self._aborted()

def _aborted(self):

def __init__(self, verbosity):

# Tuples of level_name, level, callback.

self.level_callbacks = []

super(CallbackStatsCollector, self).__init__(verbosity)

def sample(self, cur):

for level_name, level, callback in self.level_callbacks:

if level < cur.get(level_name, -1):

callback(cur)

@staticmethod

def build_info(node):

rest = RestConnection(node)

api = rest.baseUrl + 'nodes/self'

status, content = rest._http_request(api)

json_parsed = json.loads(content)

return json_parsed

@staticmethod

def machine_info(node):

shell = RemoteMachineShellConnection(node)

info = shell.extract_remote_info()

return {"type": info.type, "distribution": info.distribution_type,

"version": info.distribution_version, "ram": info.ram,