def MEDIAN_GROUPED(df, n, price='Close', interval=1):
"""
Median, or 50th percentile, of grouped data
Returns: list of floats = jhta.MEDIAN_GROUPED(df, n, price='Close', interval=1)
"""
median_grouped_list = []
if n == len(df[price]):
start = None
for i in range(len(df[price])):
if df[price][i] != df[price][i]:
median_grouped = float('NaN')
else:
if start is None:
start = i
end = i + 1
median_grouped = statistics.median_grouped(
df[price][start:end], interval)
median_grouped_list.append(median_grouped)
else:
for i in range(len(df[price])):
if i + 1 < n:
median_grouped = float('NaN')
else:
start = i + 1 - n
end = i + 1
median_grouped = statistics.median_grouped(
df[price][start:end], interval)
median_grouped_list.append(median_grouped)
return median_grouped_list
def MEDIAN_GROUPED(df, n, price='Close', interval=1):
"""
Median, or 50th percentile, of grouped data
"""
median_grouped_list = []
i = 0
if n == len(df[price]):
start = None
while i < len(df[price]):
if df[price][i] != df[price][i]:
median_grouped = float('NaN')
else:
if start is None:
start = i
end = i + 1
median_grouped = statistics.median_grouped(
df[price][start:end], interval)
median_grouped_list.append(median_grouped)
i += 1
else:
while i < len(df[price]):
if i + 1 < n:
median_grouped = float('NaN')
else:
start = i + 1 - n
end = i + 1
median_grouped = statistics.median_grouped(
df[price][start:end], interval)
median_grouped_list.append(median_grouped)
i += 1
return median_grouped_list
def run501_01():
data = [1, 2, 2, 5, 10, 12]
print('{:0.2f}'.format(statistics.mean(data)))
print('{:0.2f}'.format(statistics.mode(data))) # 众数
print('{:0.2f}'.format(statistics.median(data))) # average 3.5
print('{:0.2f}'.format(statistics.median_low(data))) # low 2
print('{:0.2f}'.format(statistics.median_high(data))) # high 5
data = [10, 20, 30, 40]
print('{:0.2f}'.format(statistics.median_grouped(data, interval=1)))
print('{:0.2f}'.format(statistics.median_grouped(data, interval=2)))
print('{:0.2f}'.format(statistics.median_grouped(data, interval=3)))
def statistics_mean():
data = [1, 2, 2, 5, 10, 12]
print('data :', data)
print('mean : {:0.2f}'.format(statistics.mean(data)))
print('mode : {:0.2f}'.format(statistics.mode(data)))
print('median : {:0.2f}'.format(statistics.median(data)))
print('median_low : {:0.2f}'.format(statistics.median_low(data)))
print('median_high : {:0.2f}'.format(statistics.median_high(data)))
data = [10, 20, 30, 40]
print('data :', data)
print('median_grouped :')
print('1: {:0.2f}'.format(statistics.median_grouped(data, interval=1)))
print('2: {:0.2f}'.format(statistics.median_grouped(data, interval=2)))
print('3: {:0.2f}'.format(statistics.median_grouped(data, interval=3)))
def read_and_save_data():
while True:
with SHT1x(data_pin=DATA_PIN, sck_pin=SCX_PIN,
gpio_mode=GPIO.BCM) as sensor:
temperature_readings = []
humidity_readings = []
for reading_num in range(NUM_OF_READINGS):
temperature_readings.append(sensor.read_temperature())
humidity_readings.append(sensor.read_humidity())
time.sleep(1)
redis_conn = get_redis_connection()
save_readings(redis_conn,
statistics.median_grouped(temperature_readings),
statistics.median_grouped(humidity_readings))
time.sleep(READINGS_INTERVAL)
def print_data_stats(datalist, name):
""" console print of a datalists's statistical measures
"""
print("=============================================================")
print("\t{}".format(name))
print("--------------------- averages & measures of central location")
print("mean (average) : {}".format(round(stat.mean(datalist), 3)))
print("median (middle value) : {}".format(round(stat.median(datalist),
3)))
print("median_low : {}".format(
round(stat.median_low(datalist), 3)))
print("median_high : {}".format(
round(stat.median_high(datalist), 3)))
print("median_grouped : {}".format(
round(stat.median_grouped(datalist), 3)))
try:
print("mode (most common value): {}".format(
round(stat.mode(datalist), 3)))
except Exception as err:
print("mode: {}".format(err))
print("------------------------------------------ measures of spread")
print("population standard deviation: {}".format(
round(stat.pstdev(datalist), 3)))
print("population variance : {}".format(
round(stat.pvariance(datalist), 3)))
print("sample standard deviation : {}".format(
round(stat.stdev(datalist), 3)))
print("sample variance : {}".format(
round(stat.variance(datalist), 3)))
print("----------------------------------------------------- extends")
print("sample min: {}".format(round(min(datalist), 3)))
print("sample max: {}".format(round(max(datalist), 3)))
print("=============================================================")
def parseDstat(fname):
csvfile = open(fname, "r")
idle_list = []
wai_list = []
read_list = []
write_list = []
recv_list = []
send_list = []
for i in range(6):
next(csvfile)
reader = csv.DictReader(csvfile)
header = reader.fieldnames
for row in reader:
idle_list.append(float(row["idl"]))
wai_list.append(float(row["wai"]))
read_list.append(float(row["read"]) / 1024 / 1024) #in MB
write_list.append(float(row["writ"]) / 1024 / 1024)
recv_list.append(float(row["recv"]) / 1024 / 1024)
send_list.append(float(row["send"]) / 1024 / 1024)
mean_idle = statistics.mean(idle_list)
mean_wait = statistics.mean(wai_list)
mean_read = statistics.median_grouped(read_list)
mean_write = statistics.mean(write_list)
mean_recv = statistics.mean(recv_list)
mean_send = statistics.mean(send_list)
#total_read = sum(read_list)
#total_recv = sum(recv_list)
#total_send = sum(send_list)
#print(mean_read)
return (mean_idle, mean_wait, mean_read, mean_write, mean_recv, mean_send)
def stats_helper(list):
"""
https://docs.python.org/3/library/statistics.html#statistics.pvariance
:param list:
:return:
"""
mean = statistics.mean(list)
mode = None
try:
mode = statistics.mode(list)
except statistics.StatisticsError:
# no unique mode
pass
return {
'mean': mean,
'variance': statistics.pvariance(list, mu=mean),
'standard_deviation': statistics.pstdev(list, mu=mean),
'median': statistics.median(list),
'median_low': statistics.median_low(list),
'median_high': statistics.median_high(list),
'median_grouped': statistics.median_grouped(list),
'mode': mode
}
def calculate_stats():
print("STAT:", type(grade_value), grade_value)
try:
stat_file = open(path + '/CP_statistics.txt', 'w')
stat_file.write(
'Mean:\t {} - Arithmetic mean (“average”) of data.\n'.format(
statistics.mean(grade_value)))
stat_file.write(
'Median:\t {} - Median (middle value) of data.\n'.format(
statistics.median(grade_value)))
stat_file.write('Median_low:\t {} - Low median of data.\n'.format(
statistics.median_low(grade_value)))
stat_file.write('Median_high:\t {} - High median of data.\n'.format(
statistics.median_low(grade_value)))
stat_file.write(
'Median_grouped:\t {} - Median, or 50th percentile, of grouped data.\n'
.format(statistics.median_grouped(grade_value)))
stat_file.write(
'Population standard deviation:\t {} - Population standard deviation of data.\n'
.format(statistics.pstdev(grade_value)))
stat_file.write(
'Population variance:\t {} - Population variance of data.\n'.
format(statistics.pvariance(grade_value)))
stat_file.write(
'Standard deviation:\t {} - Sample standard deviation of data.\n'.
format(statistics.stdev(grade_value)))
stat_file.write('Variance:\t {} - Sample variance of data.\n'.format(
statistics.variance(grade_value)))
stat_file.close()
log(whoami(), 'Done calculated stats')
except TypeError as msg:
log(whoami(), msg)
def csvRead(csv_file):
x = []
time_csv = []
rtt = []
delay = []
delay_ms = []
jitter = []
time_r = []
time_s = []
delay_r = []
delay_s = []
if not path.exists(csv_file):
print("The file does not exist")
sys.exit()
with open(csv_file,'r') as csvfile:
plots = csv.DictReader(csvfile)
i = 0
for row in plots:
try:
i+=1
x.append(i)
rtt.append(float(row["iRTT"])*1000)
time_csv.append(float(row["Time"]))
except Exception as e:
#print ('Line {i} is corrupt!'.format(i = i), " File ", Path(csv_file).name)
pass
time_r = time_csv[1:]
time_s = time_csv[:-1]
for i in range(len(time_s)):
delay.append(time_r[i] - time_s[i])
delay_ms.append((time_r[i] - time_s[i])*1000)
delay_r = delay[1:]
delay_s = delay[:-1]
for j in range(len(delay_s)):
jitter.append(round((delay_r[j] - delay_s[j]), 3))
rtt_accuracy = (median_grouped(rtt) * 100)/0.68
#ipro = 0.63
#print("Mean: ",round(mean(jitter), 5)*1000, round(sum(jitter), 3), Path(csv_file).name)
print(median_grouped(rtt), mean(delay_ms),median_grouped(jitter), Path(csv_file).name)
return [abs(median_grouped(jitter)*1000), mean(delay_ms), rtt_accuracy]
def stats_plot_2d_graph(x_label, y_label, x_cordinate_data = [],y_cordinate_data = []):
#map(float,x_cordinate_data)
#map(float,y_cordinate_data)
x_range_values = []
y_range_values = []
j=0
for i in range (len (x_cordinate_data)):
if i == 0:
print "skipping for error round off"
else:
x_range_values.insert(j,float(x_cordinate_data[i]))
y_range_values.insert(j,float(y_cordinate_data[i]))
j=j.__int__()+1
mean_x = mean(x_range_values)
mean_y = mean(y_range_values)
median_x = median(x_range_values)
median_y = median(y_range_values)
median_group_x = statistics.median_grouped(x_range_values)
median_group_y = statistics.median_grouped(y_range_values)
print x_range_values
print y_range_values
p_variance_value_x = statistics.pvariance(x_range_values)
p_variance_value_y = statistics.pvariance(y_range_values)
xlabel(x_label)
ylabel(y_label)
plot(x_range_values,y_range_values,'ro')
text(mean_x,mean_y,"<-- That's the mean value of x and y")
text(median_group_x,median_group_y,"<-- Median Group value of x and y after interpolation")
x_range_values.sort()
y_range_values.sort()
x_range_values.reverse()
y_range_values.reverse()
value_for_graph_info_xlabels = x_range_values[0] - 2
value_for_graph_info_ylabels = y_range_values[0] + 2
text(value_for_graph_info_xlabels,value_for_graph_info_ylabels,"Pvariance_x --> %.3d"% p_variance_value_x)
value_for_graph_info_ylabels = value_for_graph_info_ylabels - 1
text(value_for_graph_info_xlabels,value_for_graph_info_ylabels,"Pvariance_y --> %.3d"% p_variance_value_y)
value_for_graph_info_ylabels = value_for_graph_info_ylabels + 2
text(value_for_graph_info_xlabels,value_for_graph_info_ylabels,"Mean_x --> %.3d"% mean_x)
value_for_graph_info_ylabels = value_for_graph_info_ylabels + 1
text(value_for_graph_info_xlabels,value_for_graph_info_ylabels,"Mean_y --> %.3d"% mean_y)
show()
def getAverage(self):
distance = self.getFeet()
if (len(self.popping) > 9):
self.popping.pop(0) # remove first measurement if there are 10 measurements
self.popping.append(distance) # append measurement to end of list
return statistics.median_grouped(self.popping)
def median(self, data=None, data_type=None, **kwargs):
if not data_type:
return statistics.median(data)
elif data_type == 'low':
return statistics.median_low(data)
elif data_type == 'high':
return statistics.median_high(data)
else:
return statistics.median_grouped(kwargs.get('interval', 1))
def moving_median_grouped(df, sliding_window=1):
medians = []
terminal_condition = False
start = 0
end = sliding_window
data = [df.ix[ind] for ind in df.index]
while end != len(data):
medians.append(stat.median_grouped(data[start:end]))
start += 1
end += 1
return medians
def get_median_estimate(data):
est = list()
i = 0
while i < len(data[0]):
tmp = list()
for line in data:
tmp.append(line[i])
m = statistics.median_grouped(tmp)
est.append(m)
i += 1
return est
def read_prices():
filename = 'C920'
with open(filename, 'rb') as file:
item_prices = pickle.load(file)
for i, items in enumerate(item_prices):
print(i, sum(items) / 50)
item_prices = [item for items in item_prices[:8] for item in items]
item_prices = sorted(item_prices)[40:-10]
print(statistics.mean(item_prices), statistics.median(item_prices),
statistics.median_grouped(item_prices), statistics.mode(item_prices))
def med_apply(data, simple=False):
if simple:
med = median_grouped(data)
data -= med
return data
d = historic_med()
ns = pd.Series(index=data.index, dtype='float64')
for item in data.items():
date = item[0].month
ns.loc[item[0]] = item[1] - d[date]
return ns
def get_stats(self):
with self.lock:
if len(self.samples):
self.samples.sort()
mean = statistics.mean(self.samples)
fiftieth_percentile = statistics.median_grouped(
self.samples, interval=0.01
)
return Stats(
mean=mean, fiftieth_percentile=fiftieth_percentile, slow=self.slow
)
else:
return Stats(mean=0.0, fiftieth_percentile=0.0, slow=0.0)
def print_stats(lst):
print('List: ', lst)
lst = sorted(lst)
print('Sorted list: ', lst)
print('Mean: ', mean(lst))
print('Mode: ', mode(lst))
print('Median: ', median(lst))
print('Median low: ', median_low(lst))
print('Median high: ', median_high(lst))
print('Median grouped: ', median_grouped(lst))
print('Range: ', max(lst)- min(lst))
print('Standart deviation: ', pstdev(lst))
print('Variance: ', pvariance(lst))
def show_stack(args):
latencies = list()
lat_source = list()
index = list()
for nodes in args.nodes:
n1 = collect(nodes[0])
n2 = collect(nodes[1])
lat,timeline = calc_e2e_inter(n1,n2,args)
index.append(nodes[2])
node_num = len(lat)
if node_num != 1:
print("Error: the number of topic must be 1. ",nodes[2])
sys.exit(0)
for i,(node_name, node_time) in enumerate(lat.items(), start=0):
assert reduce(lambda x,y: x if len(x) == len(y) else False, node_time.values())
try:
demangler_out = subprocess.run(["./demangler", node_name], capture_output=True)
node_name = demangler_out.stdout.decode('utf-8')
except:
print("error: demangler")
print()
print(node_name)
print("num: ", len(list(node_time.values())[0]))
lat = list()
lat_source = list()
for j,(lat_name, time) in enumerate(node_time.items(), start=1):
if args.kind == "mean":
lat.append(mean(time))
elif args.kind == "median":
lat.append(median_grouped(time))
elif args.kind == "tail":
lat.append(max(time))
elif args.kind == "stdev":
lat.append(stdev(time))
elif args.kind == "99percentile":
lat.append(np.percentile(time,99))
lat_source.append(lat_name)
latencies.append(lat)
fig, ax = plt.subplots()
data = pd.DataFrame(latencies, index=index, columns=lat_source)
data.plot(kind='bar', stacked=True, ax=ax)
plt.show()
def stats():
nums = []
for i in range(1, 101):
nums.append(random.uniform(0, 100))
nums.sort()
print(f"numbers: {nums}")
medgrpd = statistics.median_grouped(nums)
medlow = statistics.median_low(nums)
medhigh = statistics.median_low(nums)
print(f"\ngrouped median of numbers = {medgrpd}")
print(f"low median of numbers = {medlow}")
print(f"high median of numbers = {medhigh}")
def findStatistics2(queryResults):
interW = list(queryResults['roi_connectivity_weight'])
interWMean = statistics.mean(interW)
interWMedian = statistics.median(interW)
interWMedianGrouped = statistics.median_grouped(interW)
interWMode = statistics.mode(interW)
print('\nStats for roi_connectivity_weight')
statsDict = {
'interMean': interWMean,
'interMedian': interWMedian,
'interMedianGrouped': interWMedianGrouped,
'interMode': interWMode
}
return statsDict
def findStatistics(queryResults, inputBodies):
roiW = list(inputBodies['roi_connectivity_weight'])
roiMean = statistics.mean(roiW)
roiMedian = statistics.median(roiW)
roiMedianGrouped = statistics.median_grouped(roiW)
roiMode = statistics.mode(roiW)
interW = list(queryResults['interneuron_weight'])
interWMean = statistics.mean(interW)
interWMedian = statistics.median(interW)
interWMedianGrouped = statistics.median_grouped(interW)
interWMode = statistics.mode(interW)
statsDict = {
"roiMean": roiMean,
"roiMedian": roiMedian,
"roiMedianGrouped": roiMedianGrouped,
'roiMode': roiMode,
'interMean': interWMean,
'interMedian': interWMedian,
'interMedianGrouped': interWMedianGrouped,
'interMode': interWMode
}
return statsDict
def main():
print(stats.mean(range(6)))
print(stats.median(range(6)))
print(stats.median_low(range(6)))
print(stats.median_high(range(6)))
print(stats.median_grouped(range(6)))
try:
print(stats.mode(range(6)))
except Exception as e:
print(e)
print(stats.mode(list(range(6)) + [3]))
print(stats.pstdev(list(range(6)) + [3]))
print(stats.stdev(list(range(6)) + [3]))
print(stats.pvariance(list(range(6)) + [3]))
print(stats.variance(list(range(6)) + [3]))
def calculate_mode_depth_real(bbox, depth, mask):
if depth is None:
return 0
person_area = depth[ bbox[2]:bbox[3],bbox[0]:bbox[1]]
if len(mask)>0 and len(mask[0])>0:
person_area=person_area[mask]
if person_area.size:
person_area=person_area.flatten()
person_area=person_area[np.nonzero(person_area)]
if len(person_area)==0:
return 0
return statistics.median_grouped(person_area)
else:
return 0
def calculate_stats(numbers):
mean = statistics.mean(numbers)
print("mean = ", mean)
median = statistics.median(numbers)
print("median = ", median)
median_low = statistics.median_low(numbers)
print("median_low = ", median_low)
median_high = statistics.median_high(numbers)
print("median_high = ", median_high)
median_grouped = statistics.median_grouped(numbers)
print("median_grouped = ", median_grouped)
stdev = statistics.stdev(numbers)
print("stdev = ", stdev)
variance = statistics.variance(numbers)
print("variance = ", variance)
def calculate_stats(times):
try:
min_time = str(min(times))
max_time = str(max(times))
mean_time = str(mean(times))
median_time = str(median_grouped(times))
stddev_time = str(round(stdev(times), 1))
print()
print("Lowest time: " + min_time)
print("Highest time: " + max_time)
print("Average time: " + mean_time)
print("Median time: " + median_time)
print("Standard deviation: " + stddev_time)
print()
except ValueError:
print("Insufficient data to show statistics.")
def CalcularMediana(
datos): #Funcion para calcular la mediana mediante la libreria statics
try: #Intenta
M = stats.median(
datos
) #Calcula la mediana con la funcion stats.median pasandole los datos y los almacena dentro de M
MG = stats.median_grouped(
datos
) #Dentro de MG se almacena la mediana agrupada mediante la funcion stats.median_grouped pasandole los datos
MA = stats.median_high(
datos
) #Dentro de MA se almacena la mediana alta mediante la funcion stats.median_high pasandole los datos
ML = stats.median_low(
datos
) #Dentro de ML se almacena la mediana baja mediante la funcion stats.median_log pasandole los datos
return [["Mediana", M], ["Mediana Agrupada", MG], ["Mediana Alta", MA],
["Mediana Baja", ML]]
except: #De lo contrario arroja error
print("ERROR - No se pueden ejecutar la funcion con los datos")
def median_watched_time(chunk):
# initializing DataFrame
video_id_watched_df = pd.DataFrame()
# initialize chunk
chunks_df_tmp = pd.DataFrame()
chunks_df_tmp = chunks_df[chunk]
for video_id in pd.unique(chunks_df_tmp.loc[:, 'video_id']):
# sub-setting
video_id_tmp = chunks_df_tmp.loc[chunks_df_tmp['video_id'] == str(
video_id)].reset_index(drop=True)
# build 'video_id' column
video_id_series = pd.Series(str(video_id))
# compute median of total views for each video
median_single_series = pd.Series(
stats.median_grouped(video_id_tmp.loc[:, 'tot_seconds']))
# concatenate across columns the two DataFrames
video_id_watched = pd.concat([video_id_series, median_single_series],
axis=1,
sort=False)
# rename columns
video_id_watched = video_id_watched.rename(columns={
0: 'video_id',
1: 'median_sec_watched'
},
inplace=False)
# append DataFrame
video_id_watched_df = video_id_watched_df.append(video_id_watched,
ignore_index=True,
sort=False)
# function's output
return (video_id_watched_df)
def get_current_Meat_temp():
try:
counter = 0
totalHarmonic = 0
arrayOfTemps = [None] * MAX_SAMPLES
while counter < MAX_SAMPLES:
data = bus.read_i2c_block_data(THERMOCOUPLE_2_ADDRESS, 1, 2)
val = (data[0] << 8) + data[1]
arrayOfTemps[counter] = val / 5.00 * 9.00 / 5.00 + 32.00
totalHarmonic = totalHarmonic + (1 / arrayOfTemps[counter])
counter = counter + 1
harmonicMean = MAX_SAMPLES / totalHarmonic
return float(
"%.2f" %
((statistics.median_grouped(arrayOfTemps) + harmonicMean) / 2))
except Exception as e:
print(
"***** Warning: Failed to gather data from device (Meat Temperature). Exception: %s"
% str(e))
raise
def export_runtime_kpi(self):
self.rares = []
self.timestamps = {}
ids = []
times = []
for v in self.R.smembers("COMPLETE:" + self.bucket):
times.append(self.__run_secs(v.decode('utf8')))
ids.append(int(v.decode('utf8')))
self.id_time_dict = dict(zip(ids, times))
self.ids = ids
self.times = times
self.total = len(times)
self.max_run = max(times)
self.mean = mean(times)
self.median = median(times)
self.median_grouped = median_grouped(times)
self.std = pstdev(times)
self.totalSeconds = sum(times)
self.__find_outliers()
def simple_stats():
mean_list = statistics.mean(list_of_values)
print("mean_list : ", mean_list)
geometric_mean_list = statistics.geometric_mean(list_of_values)
print("geometric_mean_list : ", geometric_mean_list)
harmonic_mean_list = statistics.harmonic_mean(list_of_values)
print("harmonic_mean_list : ", harmonic_mean_list)
median_list = statistics.median(list_of_values)
print("median_list : ", median_list)
median_low_list = statistics.median_low(list_of_values)
print("median_low_list : ", median_low_list)
median_high_list = statistics.median_high(list_of_values)
print("median_high_list : ", median_high_list)
median_grouped_list = statistics.median_grouped(list_of_values)
print("median_grouped_list : ", median_grouped_list)
mode_list = statistics.mode(list_of_values)
print("mode_list : ", mode_list)
multimode_list = statistics.multimode(list_of_values)
print("multimode_list : ", multimode_list)
quantiles_list = statistics.quantiles(list_of_values)
print("quantiles_list : ", quantiles_list)
return mean_list, geometric_mean_list, harmonic_mean_list, median_list, median_low_list, median_high_list, median_grouped_list, mode_list, multimode_list, quantiles_list
def stats(values):
"""Summarize the sequence of latency values.
"""
mean = statistics.mean(values)
stdev = statistics.stdev(values, xbar=mean)
se = stdev / math.sqrt(len(values)) # Standard error of the mean.
svalues = sorted(values)
return {
'mean': mean,
'stdev': stdev,
'se': se,
'mean_minus_stdev': mean - stdev,
'mean_plus_stdev': mean + stdev,
'mean_minus_se': mean - se,
'mean_plus_se': mean + se,
'q50': quantile(svalues, 0.5),
'q90': quantile(svalues, 0.9),
'q95': quantile(svalues, 0.95),
'q99': quantile(svalues, 0.99),
'median': statistics.median_grouped(svalues),
'min': svalues[0],
'max': svalues[-1],
}
def countStatisticsData(self, jsonFile):
#uniqueKeys = [j[0] for i in jsonFile for j in i.items()]
uniqueKeys = []
for key in jsonFile:
uniqueKeys.append(key)
data = []
if "RSSI_DATA" in uniqueKeys: # "FILTERED_RSSI_DATA"
data = jsonFile["RSSI_DATA"] #"FILTERED_RSSI_DATA"
elif "MAGNETIC_DATA" in uniqueKeys: #"FILTERED_MAGNETIC_DATA"
data = jsonFile["MAGNETIC_DATA"] #"FILTERED_MAGNETIC_DATA"
meanV = statistics.mean(data)
standardDeviation = statistics.pstdev(data)
maxV = max(data)
minV = min(data)
medianaV = statistics.median_grouped(data)
modeV = statistics.mode(data)
array = np.array(data)
percentile10 = np.percentile(array, 10)
percentile20 = np.percentile(array, 20)
percentile50 = np.percentile(array, 50)
percentile70 = np.percentile(array, 70)
percentile90 = np.percentile(array, 90)
statisticsDict = {"MEAN" : meanV,"STANDARD_DEVIATION" : standardDeviation, "MAX" : maxV, "MIN" : minV, "MEDIANA" : medianaV, "MODE" : modeV, "PERCENTILE - 10" : percentile10,"PERCENTILE - 20" : percentile20, "PERCENTILE - 50" : percentile50, "PERCENTILE - 70" : percentile70, "PERCENTILE - 90" : percentile90 }
jsonFile["STATISTICS"] = statisticsDict
def median_grouped(self):
return stats.median_grouped(self.values())
__author__ = 'luowen' """ the statisic demo """ import statistics list1 = [1, 2, 3, 4, 5, 6] midNum = statistics.mean(list1) print(midNum) # get the average data of list1 medianNum = statistics.median(list1) print(medianNum) # get median data of list1 medianNumLow = statistics.median_low(list1) print(medianNumLow) # get median lower data of list1 medianNumHigh = statistics.median_high(list1) print(medianNumHigh) # get median hight data of list1 medianNumGroup = statistics.median_grouped(list1, 10) print(medianNumGroup) # get detail information from https://docs.python.org/3/library/statistics.html
def stats(self):
self.mean = statistics.mean(self._results)
self.median = statistics.median(self._results)
self.median_grouped = statistics.median_grouped(self._results)
def update_data(event_id, gen_fake_data = False):
year = int(event_id[0:4])
year_config = get_year_config(year)
if not os.path.isdir("./data/" + event_id):
os.path.mkdir("./data/" + event_id)
open("./data/" + event_id + "/raw_data2.json", "w+").write("")
open("./data/" + event_id + "/avg_data2.json", "w+").write("")
open("./data/" + event_id + "/error_data2.json", "w+").write("")
if gen_fake_data and not os.path.isfile("./data/" + event_id + "/" + event_id + ".csv"):
FakeDataGenerator.gen_data(event_id)
raw = open("./data/" + event_id + "/" + event_id + ".csv").read().strip("\n").split("\n")
raw = map(lambda x: x.replace("\'", "\"").strip("\r"), raw)
data = []
headers = map(lambda x: str(x).encode('ascii', 'ignore'), year_config["headers"])
types = year_config["type"]
representation = year_config["representation"]
for i in range(len(raw)):
data += [{}]
line = json.loads("[" + raw[i].strip("\r") + "]")
for j in range(len(line)):
element = line[j]
if types[j] == "int":
data[i][headers[j]] = int(element)
elif types[j] == "bool":
data[i][headers[j]] = bool(element)
elif types[j] == "list":
data[i][headers[j]] = list(element)
elif types[j] == "dict":
data[i][headers[j]] = dict(element)
elif types[j] == "list":
data[i][headers[j]] = list(element)
else:
print "Error: Unrecognized Type - " + types[i]
json.dump(data, open("./data/" + event_id + "/raw_data2.json", "w"))
event_info = get_event_info(event_id)
teams = event_info["team_list"]
error_data = []
data_accum = {}
avg_data = {}
for t in teams:
data_accum[str(t)] = []
avg_data[str(t)] = {}
for i in range(len(data)):
line = data[i]
team = line["Team Number"]
try:
data_accum[str(team)] += [line]
except:
error_data.append(line)
for team in data_accum.keys():
team = str(team).encode('ascii', 'ignore')
matches = data_accum[team]
for i in range(len(headers)):
header = str(headers[i]).encode('ascii', 'ignore')
rep = representation[i]
avg_data[team][header] = { "value": "0%" if rep == "perc" else 0 }
if float(len(map(lambda x: x[header], matches))) <= 0:
continue
if rep == "hide":
continue
elif rep == "avg":
avg_data[team][header]["value"] = round(sum(map(lambda x: float(x[header]), matches)) / float(len(map(lambda x: x[header], matches))), 2)
avg_data[team][header]["stdev"] = round(statistics.stdev(map(lambda x: float(x[header]), matches)), 2)
avg_data[team][header]["median_grouped"] = round(statistics.median_grouped(map(lambda x: float(x[header]), matches)), 2)
elif rep == "mode":
avg_data[team][header]["value"] = modes(map(lambda x: x[header], matches))
elif rep == "perc":
avg_data[team][header] = { "value": "0%" }
avg_data[team][header]["value"] = str(int(round(map(lambda x: x[header] > 0, matches).count(True) / float(len(map(lambda x: x[header], matches))), 2) * 100)) + "%"
elif rep == "count":
avg_data[team][header]["value"] = len(map(lambda x: x[header], matches))
elif rep == "dict_count":
dicts = map(lambda x: x[header], matches)
cross_counts = {}
for defense in dicts[0].keys():
cross_counts[defense] = 0
for match in dicts:
for defense in match.keys():
if '2' in match[defense]:
cross_counts[defense] += 2
elif '1' in match[defense]:
cross_counts[defense] += 1
avg_data[team][header]["value"] = round(1.0 * sum(cross_counts.values()) / len(matches), 2)
avg_data[team][header]["ind"] = cross_counts
else:
print "Error: Unrecognized Representation - " + rep
for h in ["Auto Crossed", "Crossed", "Auton High", "Auton Low", "Scored High", "Scored Low"]:
if h not in avg_data[team].keys():
avg_data[team][h] = { "value": 0 }
if "value" in avg_data[team]["Team Number"].keys() and avg_data[team]["Team Number"]["value"] == '':
del avg_data[team]
continue
avg_data[team]["Total Crossed"] = { "value": avg_data[team]["Auto Crossed"]["value"] + avg_data[team]["Crossed"]["value"] }
avg_data[team]["Total Balls"] = { "value": avg_data[team]["Auton High"]["value"] + avg_data[team]["Scored High"]["value"] + avg_data[team]["Auton Low"]["value"] + avg_data[team]["Scored Low"]["value"] }
avg_data[team]["Qual Score"] = { "value": round(5 * avg_data[team]["Scored High"]["value"] + 2 * avg_data[team]["Scored Low"]["value"] + 10 * avg_data[team]["Auton High"]["value"] + 5 * avg_data[team]["Auton Low"]["value"] + 10 * avg_data[team]["Auto Crossed"]["value"] + 5 * avg_data[team]["Crossed"]["value"] + 15 * (int(avg_data[team]["Scale"]["value"][:-1]) / 100.0) + 5 * (int(avg_data[team]["Challenged"]["value"][:-1]) / 100.0), 1) }
avg_data[team]["Elim Score"] = { "value": round(avg_data[team]["Qual Score"]["value"] + Util.limit_value(20.0 * avg_data[team]["Total Crossed"]["value"] / 8, 20) + Util.limit_value(25.0 * avg_data[team]["Total Balls"]["value"] / 8, 25), 1) }
json.dump(avg_data, open("./data/" + event_id + "/avg_data2.json", "w"))
json.dump(avg_data, open("./data/" + event_id + "/error_data2.json", "w"))
conn_cnt = []
if process:
cpu_usage(process)
for _ in range(10):
results.append(run_wrk(loop, args.endpoint))
cpu_usages.append(psutil.cpu_percent())
if process:
process_cpu_usages.append(cpu_usage(process))
conn_cnt.append(connections(process))
mem_usages.append(round(memory(process), 2))
print('.', end='')
sys.stdout.flush()
if not args.endpoint:
server.terminate()
loop.run_until_complete(server.wait())
if args.cpu_change:
cpu.change('ondemand')
print()
print('RPS', results)
print('Mem', mem_usages)
print('Conn', conn_cnt)
print('Server', process_cpu_usages)
print('System', cpu_usages)
median = statistics.median_grouped(results)
stdev = round(statistics.stdev(results), 2)
p = round((stdev / median) * 100, 2)
print('median:', median, 'stdev:', stdev, '%', p)