def compute_friedman_test(best_test_score):
"""
"""
## test pair combinations
pairs_test = [('individual', 'union', 'mtl'), ('individual', 'union', 'mtmkl'), \
('individual', 'mtl', 'mtmkl'), ('union', 'mtl', 'mtmkl')]
org_names = best_test_score.keys()
##ttest_p_val = numpy.zeros((len(org_names), len(pairs_test)))
ttest_p_val = numpy.zeros((len(pairs_test), len(org_names)))
for org_idx, org_code in enumerate(org_names):
meth_perf = best_test_score[org_code]
for pair_idx, rel_pair in enumerate(pairs_test):
t_stats, p_val = stats.friedmanchisquare(meth_perf[rel_pair[0]], meth_perf[rel_pair[1]], meth_perf[rel_pair[2]])
##ttest_p_val[org_idx, pair_idx] = p_val
ttest_p_val[pair_idx, org_idx] = p_val
##df_pval = pandas.DataFrame(ttest_p_val, columns=pairs_test, index=org_names)
df_pval = pandas.DataFrame(ttest_p_val, columns=org_names, index=pairs_test)
return df_pval
def friedman_chi(data):
"""
non parametric
many samples
dependent
"""
F, pval = st.friedmanchisquare(*data)
return (F, pval)
def task_friedman(dataOne, dataTwo, dataThree):
"""
http://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.friedmanchisquare.html
"""
chi, pvalue = friedmanchisquare(array(dataOne), array(dataTwo), array(dataThree))
return chi, pvalue
def test_friedmanchisquare():
# see ticket:113
# verified with matlab and R
#From Demsar "Statistical Comparisons of Classifiers over Multiple Data Sets"
#2006, Xf=9.28 (no tie handling, tie corrected Xf >=9.28)
x1 = [array([0.763, 0.599, 0.954, 0.628, 0.882, 0.936, 0.661, 0.583,
0.775, 1.0, 0.94, 0.619, 0.972, 0.957]),
array([0.768, 0.591, 0.971, 0.661, 0.888, 0.931, 0.668, 0.583,
0.838, 1.0, 0.962, 0.666, 0.981, 0.978]),
array([0.771, 0.590, 0.968, 0.654, 0.886, 0.916, 0.609, 0.563,
0.866, 1.0, 0.965, 0.614, 0.9751, 0.946]),
array([0.798, 0.569, 0.967, 0.657, 0.898, 0.931, 0.685, 0.625,
0.875, 1.0, 0.962, 0.669, 0.975, 0.970])]
#From "Bioestadistica para las ciencias de la salud" Xf=18.95 p<0.001:
x2 = [array([4,3,5,3,5,3,2,5,4,4,4,3]),
array([2,2,1,2,3,1,2,3,2,1,1,3]),
array([2,4,3,3,4,3,3,4,4,1,2,1]),
array([3,5,4,3,4,4,3,3,3,4,4,4])]
#From Jerrorl H. Zar, "Biostatistical Analysis"(example 12.6), Xf=10.68, 0.005 < p < 0.01:
#Probability from this example is inexact using Chisquare aproximation of Friedman Chisquare.
x3 = [array([7.0,9.9,8.5,5.1,10.3]),
array([5.3,5.7,4.7,3.5,7.7]),
array([4.9,7.6,5.5,2.8,8.4]),
array([8.8,8.9,8.1,3.3,9.1])]
assert_array_almost_equal(stats.friedmanchisquare(x1[0],x1[1],x1[2],x1[3]),(10.2283464566929, 0.0167215803284414))
assert_array_almost_equal(stats.friedmanchisquare(x2[0],x2[1],x2[2],x2[3]),(18.9428571428571, 0.000280938375189499))
assert_array_almost_equal(stats.friedmanchisquare(x3[0],x3[1],x3[2],x3[3]),(10.68, 0.0135882729582176))
np.testing.assert_raises(ValueError, stats.friedmanchisquare,x3[0],x3[1])
# test using mstats
assert_array_almost_equal(stats.mstats.friedmanchisquare(x1[0],x1[1],x1[2],x1[3]),(10.2283464566929, 0.0167215803284414))
# the following fails
#assert_array_almost_equal(stats.mstats.friedmanchisquare(x2[0],x2[1],x2[2],x2[3]),(18.9428571428571, 0.000280938375189499))
assert_array_almost_equal(stats.mstats.friedmanchisquare(x3[0],x3[1],x3[2],x3[3]),(10.68, 0.0135882729582176))
np.testing.assert_raises(ValueError,stats.mstats.friedmanchisquare,x3[0],x3[1])
clasificar_HistGradientBoostingClassifier(
X_labor, y_labor, df_labor, trI_labor, trO_labor, teI_labor, teO_labor,
'P4_labor_HistGradientBoostingClassifier.svg'),
clasificar_HistGradientBoostingClassifier(
X_vote, y_vote, df_vote, trI_vote, trO_vote, teI_vote, teO_vote,
'P4_vote_HistGradientBoostingClassifier.svg'),
clasificar_HistGradientBoostingClassifier(
X_car, y_car, df_car, trI_car, trO_car, teI_car, teO_car,
'P4_car_HistGradientBoostingClassifier.svg'),
clasificar_HistGradientBoostingClassifier(
X_bank, y_bank, df_bank, trI_bank, trO_bank, teI_bank, teO_bank,
'P4_bank_HistGradientBoostingClassifier.svg')
])
friedman = friedmanchisquare(arrayScores_dt, arrayScores_knn, arrayScores_svm,
arrayScores_bagging,
arrayScores_GradientBoostingClassifier,
arrayScores_HistGradientBoostingClassifier)
print("\n FRIEDMAN => " + str(friedman) + "\n")
iman_davenport = ((10 - 1) * friedman[0]) / (10 * (6 - 1) - friedman[0])
print("\n IMAN-DAVENPORT => " + str(iman_davenport) + "\n")
wilcoxon_KNN_Bagging = wilcoxon(arrayScores_knn, arrayScores_bagging)
print("\n WILCOXON [KNN VS. BAGGING] => " + str(wilcoxon_KNN_Bagging) + "\n")
wilcoxon_KNN_GradientBoostingClassifier = wilcoxon(
arrayScores_knn, arrayScores_GradientBoostingClassifier)
sns.set(color_codes=True)
plt.boxplot(means_acc_knn)
plt.show()
plt.boxplot(means_acc_gauss)
plt.show()
plt.boxplot(means_acc_parzen)
plt.show()
sns.distplot(means_acc_knn)
plt.show()
sns.distplot(means_acc_gauss)
plt.show()
sns.distplot(means_acc_parzen)
plt.show()
sns.kdeplot(means_acc_knn, label="KNN")
sns.kdeplot(means_acc_gauss, label="Gauss")
sns.kdeplot(means_acc_parzen, label="Parzen")
plt.legend()
# Friedman test
stat, p = friedmanchisquare(means_acc_knn, means_acc_gauss, means_acc_parzen)
print('Statistics=%.3f, p=%.3f' % (stat, p))
alpha = 0.05
if p > alpha:
print('Same distributions (fail to reject H0)')
else:
print('Different distributions (reject H0)')
def print_full_analysis(df, field, h=1, by="tid", nt=stats.shapiro,
gd=True, sd=True, ffa=True, ovr=True, ovo=True):
"""Performs a full analysis of the data
:df: THE data frame
:field: field of interest such as 'time_ms' or 'success'
:by: perform grouping on this attribute (the higher level split on
"navigation" is always performed)
:h: header level to start with
:returns: nothing.
"""
if by is None:
sd = ffa = ovr = False
nav_methods = df.groupby("navigation")
if ffa or sd:
groups = df.groupby(["navigation", by])[field]
print_md_header(h, "Descriptions ({})".format(field))
# Global Descriptions
if gd:
print_md_header(h+1, "Global Descriptions ({})".format(field))
print_full_description(nav_methods[field], h=h+2, norm_test=nt)
# Sample Descriptions
if sd:
print_md_header(h+1, "Repeated measures ({})".format(field))
for desc, group in nav_methods:
subgroups = [values for _, values in group.groupby(by)[field]]
print_md_header(h+2, desc)
print_md_paragraph(stats.kruskal(*subgroups),
stats.friedmanchisquare(*subgroups),
# stats.f_oneway(*subgroups),
lineblock=True)
print_md_header(h+1, "Descriptions per {} ({})".format(by, field))
print_full_description(groups, h=h+2, norm_test=nt)
# free for all
if ffa:
print_md_header(h, "Cross-compare Tests per {} ({})".format(by, field))
for desc, result in cross_compare(*groups, norm_test=nt):
print_md_header(h+1, "{} vs {}".format(*desc))
print_md_paragraph(result)
# one vs rest
if ovr:
print_md_header(
h, "Global Burger vs Swipe per {} Tests ({})".format(
by, field))
burger = "burger", nav_methods.get_group("burger")[field]
swipe_tasks = nav_methods.get_group("swipe").groupby(by)[field]
for desc, result in one_vs_rest(burger, *swipe_tasks,
norm_test=nt):
print_md_header(h+1, "{} vs swipe {}".format(*desc))
print_md_paragraph(result)
# one vs one
if ovo:
print_md_header(h,
"Global Burger vs Global Swipe Test ({})"
.format(field))
for desc, result in cross_compare(*nav_methods[field],
norm_test=nt):
print_md_header(h+1, "{} vs {}".format(*desc))
print_md_paragraph(result)
'P5_car_ECOC.svg'
),
clasificar_ECOC(
X_iris_2D,
y_iris_2D,
df_iris_2D,
trI_iris_2D,
trO_iris_2D,
teI_iris_2D,
teO_iris_2D,
'P5_iris_2D_ECOC.svg'
)
]
)
friedman = friedmanchisquare(arrayScores_dt, arrayScores_OVO, arrayScores_OVA, arrayScores_ECOC)
print("\n FRIEDMAN => " + str(friedman) + "\n")
iman_davenport = ((10-1)*friedman[0])/(10*(4-1)-friedman[0])
print("\n IMAN-DAVENPORT => " + str(iman_davenport) + "\n")
wilcoxon_DT_OVO = wilcoxon(arrayScores_dt, arrayScores_OVO)
print("\n WILCOXON [DT VS. OVO] => " + str(wilcoxon_DT_OVO) + "\n")
wilcoxon_DT_OVA = wilcoxon(arrayScores_dt, arrayScores_OVA)
print("\n WILCOXON [DT VS. OVA] => " + str(wilcoxon_DT_OVA) + "\n")
wilcoxon_DT_ECOC = wilcoxon(arrayScores_dt, arrayScores_ECOC)
print("\n WILCOXON [DT VS. ECOC] => " + str(wilcoxon_DT_ECOC) + "\n")
wilcoxon_OVO_OVA = wilcoxon(arrayScores_OVO, arrayScores_OVA)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--infile', required=True, help='Tabular file.')
parser.add_argument('-o', '--outfile', required=True, help='Path to the output file.')
parser.add_argument("--sample_one_cols", help="Input format, like smi, sdf, inchi")
parser.add_argument("--sample_two_cols", help="Input format, like smi, sdf, inchi")
parser.add_argument("--sample_cols", help="Input format, like smi, sdf, inchi,separate arrays using ;")
parser.add_argument("--test_id", help="statistical test method")
parser.add_argument("--mwu_use_continuity", action="store_true", default = False,
help="Whether a continuity correction (1/2.) should be taken into account.")
parser.add_argument("--equal_var", action="store_true", default = False,
help="If set perform a standard independent 2 sample test that assumes equal population variances. If not set, perform Welch's t-test, which does not assume equal population variance.")
parser.add_argument("--reta", action="store_true", default = False,
help="Whether or not to return the internally computed a values.")
parser.add_argument("--fisher", action="store_true", default = False,
help="if true then Fisher definition is used")
parser.add_argument("--bias", action="store_true", default = False,
help="if false,then the calculations are corrected for statistical bias")
parser.add_argument("--inclusive1", action="store_true", default= False ,
help="if false,lower_limit will be ignored")
parser.add_argument("--inclusive2", action="store_true", default = False,
help="if false,higher_limit will be ignored")
parser.add_argument("--inclusive", action="store_true", default = False,
help="if false,limit will be ignored")
parser.add_argument("--printextras", action="store_true", default = False,
help="If True, if there are extra points a warning is raised saying how many of those points there are")
parser.add_argument("--initial_lexsort", action="store_true", default="False",
help="Whether to use lexsort or quicksort as the sorting method for the initial sort of the inputs.")
parser.add_argument("--correction", action="store_true", default = False,
help="continuity correction ")
parser.add_argument("--axis", type=int, default=0,
help="Axis can equal None (ravel array first), or an integer (the axis over which to operate on a and b)")
parser.add_argument("--n", type=int, default=0,
help="the number of trials. This is ignored if x gives both the number of successes and failures")
parser.add_argument("--b", type=int, default=0,
help="The number of bins to use for the histogram")
parser.add_argument("--N", type=int, default=0,
help="Score that is compared to the elements in a.")
parser.add_argument("--ddof", type=int, default=0,
help="Degrees of freedom correction")
parser.add_argument("--score", type=int, default=0,
help="Score that is compared to the elements in a.")
parser.add_argument("--m", type=float, default=0.0,
help="limits")
parser.add_argument("--mf", type=float, default=2.0,
help="lower limit")
parser.add_argument("--nf", type=float, default=99.9,
help="higher_limit")
parser.add_argument("--p", type=float, default=0.5,
help="The hypothesized probability of success. 0 <= p <= 1. The default value is p = 0.5")
parser.add_argument("--alpha", type=float, default=0.9,
help="probability")
parser.add_argument("--new", type=float, default=0.0,
help="Value to put in place of values in a outside of bounds")
parser.add_argument("--proportiontocut", type=float, default=0.0,
help="Proportion (in range 0-1) of total data set to trim of each end.")
parser.add_argument("--lambda_", type=float, default=1.0,
help="lambda_ gives the power in the Cressie-Read power divergence statistic")
parser.add_argument("--imbda", type=float, default=0,
help="If lmbda is not None, do the transformation for that value.If lmbda is None, find the lambda that maximizes the log-likelihood function and return it as the second output argument.")
parser.add_argument("--base", type=float, default=1.6,
help="The logarithmic base to use, defaults to e")
parser.add_argument("--dtype", help="dtype")
parser.add_argument("--med", help="med")
parser.add_argument("--cdf", help="cdf")
parser.add_argument("--zero_method", help="zero_method options")
parser.add_argument("--dist", help="dist options")
parser.add_argument("--ties", help="ties options")
parser.add_argument("--alternative", help="alternative options")
parser.add_argument("--mode", help="mode options")
parser.add_argument("--method", help="method options")
parser.add_argument("--md", help="md options")
parser.add_argument("--center", help="center options")
parser.add_argument("--kind", help="kind options")
parser.add_argument("--tail", help="tail options")
parser.add_argument("--interpolation", help="interpolation options")
parser.add_argument("--statistic", help="statistic options")
args = parser.parse_args()
infile = args.infile
outfile = open(args.outfile, 'w+')
test_id = args.test_id
nf = args.nf
mf = args.mf
imbda = args.imbda
inclusive1 = args.inclusive1
inclusive2 = args.inclusive2
sample0 = 0
sample1 = 0
sample2 = 0
if args.sample_cols != None:
sample0 = 1
barlett_samples = []
for sample in args.sample_cols.split(';'):
barlett_samples.append( map(int, sample.split(',')) )
if args.sample_one_cols != None:
sample1 = 1
sample_one_cols = args.sample_one_cols.split(',')
if args.sample_two_cols != None:
sample_two_cols = args.sample_two_cols.split(',')
sample2 = 1
for line in open( infile ):
sample_one = []
sample_two = []
cols = line.strip().split('\t')
if sample0 == 1:
b_samples = columns_to_values( barlett_samples,line )
if sample1 == 1:
for index in sample_one_cols:
sample_one.append( cols[ int(index) -1 ] )
if sample2 == 1:
for index in sample_two_cols:
sample_two.append( cols[ int(index) -1 ] )
if test_id.strip() == 'describe':
size, min_max,mean,uv,bs,bk = stats.describe( map(float, sample_one) )
cols.append( size )
cols.append( min_max )
cols.append( mean )
cols.append( uv )
cols.append( bs )
cols.append( bk )
elif test_id.strip() == 'mode':
vals, counts = stats.mode( map(float, sample_one) )
cols.append( vals )
cols.append( counts )
elif test_id.strip() == 'nanmean':
m = stats.nanmean( map(float, sample_one))
cols.append( m )
elif test_id.strip() == 'nanmedian':
m = stats.nanmedian( map(float, sample_one))
cols.append( m )
elif test_id.strip() == 'kurtosistest':
z_value, p_value = stats.kurtosistest( map(float, sample_one) )
cols.append( z_value )
cols.append( p_value )
elif test_id.strip() == 'variation':
ra = stats.variation( map(float, sample_one))
cols.append( ra )
elif test_id.strip() == 'itemfreq':
freq = stats.itemfreq( map(float, sample_one))
for list in freq:
elements = ','.join( map(str, list) )
cols.append( elements )
elif test_id.strip() == 'nanmedian':
m = stats.nanmedian( map(float, sample_one))
cols.append( m )
elif test_id.strip() == 'variation':
ra = stats.variation( map(float, sample_one))
cols.append( ra )
elif test_id.strip() == 'boxcox_llf':
IIf = stats.boxcox_llf( imbda,map(float, sample_one) )
cols.append( IIf )
elif test_id.strip() == 'tiecorrect':
fa = stats.tiecorrect( map(float, sample_one) )
cols.append( fa )
elif test_id.strip() == 'rankdata':
r = stats.rankdata( map(float, sample_one),method=args.md )
cols.append( r )
elif test_id.strip() == 'nanstd':
s = stats.nanstd( map(float, sample_one),bias=args.bias )
cols.append( s )
elif test_id.strip() == 'anderson':
A2, critical, sig = stats.anderson( map(float, sample_one), dist=args.dist )
cols.append( A2 )
for list in critical:
cols.append( list )
cols.append( ',' )
for list in sig:
cols.append( list )
elif test_id.strip() == 'binom_test':
p_value = stats.binom_test( map(float, sample_one), n=args.n, p=args.p )
cols.append( p_value )
elif test_id.strip() == 'gmean':
gm = stats.gmean( map(float, sample_one), dtype=args.dtype )
cols.append( gm )
elif test_id.strip() == 'hmean':
hm = stats.hmean( map(float, sample_one), dtype=args.dtype )
cols.append( hm )
elif test_id.strip() == 'kurtosis':
k = stats.kurtosis( map(float, sample_one),axis=args.axis, fisher=args.fisher, bias=args.bias )
cols.append( k )
elif test_id.strip() == 'moment':
n_moment = stats.moment( map(float, sample_one),n=args.n )
cols.append( n_moment )
elif test_id.strip() == 'normaltest':
k2, p_value = stats.normaltest( map(float, sample_one) )
cols.append( k2 )
cols.append( p_value )
elif test_id.strip() == 'skew':
skewness = stats.skew( map(float, sample_one),bias=args.bias )
cols.append( skewness )
elif test_id.strip() == 'skewtest':
z_value, p_value = stats.skewtest( map(float, sample_one))
cols.append( z_value )
cols.append( p_value )
elif test_id.strip() == 'sem':
s = stats.sem( map(float, sample_one),ddof=args.ddof )
cols.append( s )
elif test_id.strip() == 'zscore':
z = stats.zscore( map(float, sample_one),ddof=args.ddof )
for list in z:
cols.append( list )
elif test_id.strip() == 'signaltonoise':
s2n = stats.signaltonoise( map(float, sample_one),ddof=args.ddof )
cols.append( s2n )
elif test_id.strip() == 'percentileofscore':
p = stats.percentileofscore( map(float, sample_one),score=args.score,kind=args.kind )
cols.append( p )
elif test_id.strip() == 'bayes_mvs':
c_mean, c_var,c_std = stats.bayes_mvs( map(float, sample_one),alpha=args.alpha )
cols.append( c_mean )
cols.append( c_var )
cols.append( c_std )
elif test_id.strip() == 'sigmaclip':
c, c_low,c_up = stats.sigmaclip( map(float, sample_one),low=args.m,high=args.n )
cols.append( c )
cols.append( c_low )
cols.append( c_up )
elif test_id.strip() == 'kstest':
d, p_value = stats.kstest(map(float, sample_one), cdf=args.cdf , N=args.N,alternative=args.alternative,mode=args.mode )
cols.append(d)
cols.append(p_value)
elif test_id.strip() == 'chi2_contingency':
chi2, p, dof, ex = stats.chi2_contingency( map(float, sample_one), correction=args.correction ,lambda_=args.lambda_)
cols.append( chi2 )
cols.append( p )
cols.append( dof )
cols.append( ex )
elif test_id.strip() == 'tmean':
if nf is 0 and mf is 0:
mean = stats.tmean( map(float, sample_one))
else:
mean = stats.tmean( map(float, sample_one),( mf, nf ),( inclusive1, inclusive2 ))
cols.append( mean )
elif test_id.strip() == 'tmin':
if mf is 0:
min = stats.tmin( map(float, sample_one))
else:
min = stats.tmin( map(float, sample_one),lowerlimit=mf,inclusive=args.inclusive)
cols.append( min )
elif test_id.strip() == 'tmax':
if nf is 0:
max = stats.tmax( map(float, sample_one))
else:
max = stats.tmax( map(float, sample_one),upperlimit=nf,inclusive=args.inclusive)
cols.append( max )
elif test_id.strip() == 'tvar':
if nf is 0 and mf is 0:
var = stats.tvar( map(float, sample_one))
else:
var = stats.tvar( map(float, sample_one),( mf, nf ),( inclusive1, inclusive2 ))
cols.append( var )
elif test_id.strip() == 'tstd':
if nf is 0 and mf is 0:
std = stats.tstd( map(float, sample_one))
else:
std = stats.tstd( map(float, sample_one),( mf, nf ),( inclusive1, inclusive2 ))
cols.append( std )
elif test_id.strip() == 'tsem':
if nf is 0 and mf is 0:
s = stats.tsem( map(float, sample_one))
else:
s = stats.tsem( map(float, sample_one),( mf, nf ),( inclusive1, inclusive2 ))
cols.append( s )
elif test_id.strip() == 'scoreatpercentile':
if nf is 0 and mf is 0:
s = stats.scoreatpercentile( map(float, sample_one),map(float, sample_two),interpolation_method=args.interpolation )
else:
s = stats.scoreatpercentile( map(float, sample_one),map(float, sample_two),( mf, nf ),interpolation_method=args.interpolation )
for list in s:
cols.append( list )
elif test_id.strip() == 'relfreq':
if nf is 0 and mf is 0:
rel, low_range, binsize, ex = stats.relfreq( map(float, sample_one),args.b)
else:
rel, low_range, binsize, ex = stats.relfreq( map(float, sample_one),args.b,( mf, nf ))
for list in rel:
cols.append( list )
cols.append( low_range )
cols.append( binsize )
cols.append( ex )
elif test_id.strip() == 'binned_statistic':
if nf is 0 and mf is 0:
st, b_edge, b_n = stats.binned_statistic( map(float, sample_one),map(float, sample_two),statistic=args.statistic,bins=args.b )
else:
st, b_edge, b_n = stats.binned_statistic( map(float, sample_one),map(float, sample_two),statistic=args.statistic,bins=args.b,range=( mf, nf ) )
cols.append( st )
cols.append( b_edge )
cols.append( b_n )
elif test_id.strip() == 'threshold':
if nf is 0 and mf is 0:
o = stats.threshold( map(float, sample_one),newval=args.new )
else:
o = stats.threshold( map(float, sample_one),mf,nf,newval=args.new )
for list in o:
cols.append( list )
elif test_id.strip() == 'trimboth':
o = stats.trimboth( map(float, sample_one),proportiontocut=args.proportiontocut )
for list in o:
cols.append( list )
elif test_id.strip() == 'trim1':
t1 = stats.trim1( map(float, sample_one),proportiontocut=args.proportiontocut,tail=args.tail )
for list in t1:
cols.append( list )
elif test_id.strip() == 'histogram':
if nf is 0 and mf is 0:
hi, low_range, binsize, ex = stats.histogram( map(float, sample_one),args.b)
else:
hi, low_range, binsize, ex = stats.histogram( map(float, sample_one),args.b,( mf, nf ))
cols.append( hi )
cols.append( low_range )
cols.append( binsize )
cols.append( ex )
elif test_id.strip() == 'cumfreq':
if nf is 0 and mf is 0:
cum, low_range, binsize, ex = stats.cumfreq( map(float, sample_one),args.b)
else:
cum, low_range, binsize, ex = stats.cumfreq( map(float, sample_one),args.b,( mf, nf ))
cols.append( cum )
cols.append( low_range )
cols.append( binsize )
cols.append( ex )
elif test_id.strip() == 'boxcox_normmax':
if nf is 0 and mf is 0:
ma = stats.boxcox_normmax( map(float, sample_one))
else:
ma = stats.boxcox_normmax( map(float, sample_one),( mf, nf ),method=args.method)
cols.append( ma )
elif test_id.strip() == 'boxcox':
if imbda is 0:
box, ma, ci = stats.boxcox( map(float, sample_one),alpha=args.alpha )
cols.append( box )
cols.append( ma )
cols.append( ci )
else:
box = stats.boxcox( map(float, sample_one),imbda,alpha=args.alpha )
cols.append( box )
elif test_id.strip() == 'histogram2':
h2 = stats.histogram2( map(float, sample_one), map(float, sample_two) )
for list in h2:
cols.append( list )
elif test_id.strip() == 'ranksums':
z_statistic, p_value = stats.ranksums( map(float, sample_one), map(float, sample_two) )
cols.append(z_statistic)
cols.append(p_value)
elif test_id.strip() == 'ttest_1samp':
t, prob = stats.ttest_1samp( map(float, sample_one), map(float, sample_two) )
for list in t:
cols.append( list )
for list in prob:
cols.append( list )
elif test_id.strip() == 'ansari':
AB, p_value = stats.ansari( map(float, sample_one), map(float, sample_two) )
cols.append(AB)
cols.append(p_value)
elif test_id.strip() == 'linregress':
slope, intercept, r_value, p_value, stderr = stats.linregress( map(float, sample_one), map(float, sample_two) )
cols.append(slope)
cols.append(intercept)
cols.append(r_value)
cols.append(p_value)
cols.append(stderr)
elif test_id.strip() == 'pearsonr':
cor, p_value = stats.pearsonr( map(float, sample_one), map(float, sample_two) )
cols.append(cor)
cols.append(p_value)
elif test_id.strip() == 'pointbiserialr':
r, p_value = stats.pointbiserialr( map(float, sample_one), map(float, sample_two) )
cols.append(r)
cols.append(p_value)
elif test_id.strip() == 'ks_2samp':
d, p_value = stats.ks_2samp( map(float, sample_one), map(float, sample_two) )
cols.append(d)
cols.append(p_value)
elif test_id.strip() == 'mannwhitneyu':
mw_stats_u, p_value = stats.mannwhitneyu( map(float, sample_one), map(float, sample_two), use_continuity=args.mwu_use_continuity )
cols.append( mw_stats_u )
cols.append( p_value )
elif test_id.strip() == 'zmap':
z = stats.zmap( map(float, sample_one),map(float, sample_two),ddof=args.ddof )
for list in z:
cols.append( list )
elif test_id.strip() == 'ttest_ind':
mw_stats_u, p_value = stats.ttest_ind( map(float, sample_one), map(float, sample_two), equal_var=args.equal_var )
cols.append( mw_stats_u )
cols.append( p_value )
elif test_id.strip() == 'ttest_rel':
t, prob = stats.ttest_rel( map(float, sample_one), map(float, sample_two), axis=args.axis )
cols.append( t )
cols.append( prob )
elif test_id.strip() == 'mood':
z, p_value = stats.mood( map(float, sample_one), map(float, sample_two), axis=args.axis )
cols.append( z )
cols.append( p_value )
elif test_id.strip() == 'shapiro':
W, p_value, a = stats.shapiro( map(float, sample_one), map(float, sample_two), args.reta )
cols.append( W )
cols.append( p_value )
for list in a:
cols.append( list )
elif test_id.strip() == 'kendalltau':
k, p_value = stats.kendalltau( map(float, sample_one), map(float, sample_two), initial_lexsort=args.initial_lexsort )
cols.append(k)
cols.append(p_value)
elif test_id.strip() == 'entropy':
s = stats.entropy( map(float, sample_one), map(float, sample_two), base=args.base )
cols.append( s )
elif test_id.strip() == 'spearmanr':
if sample2 == 1 :
rho, p_value = stats.spearmanr( map(float, sample_one), map(float, sample_two) )
else:
rho, p_value = stats.spearmanr( map(float, sample_one))
cols.append( rho )
cols.append( p_value )
elif test_id.strip() == 'wilcoxon':
if sample2 == 1 :
T, p_value = stats.wilcoxon( map(float, sample_one), map(float, sample_two),zero_method=args.zero_method,correction=args.correction )
else:
T, p_value = stats.wilcoxon( map(float, sample_one),zero_method=args.zero_method,correction=args.correction )
cols.append( T )
cols.append( p_value )
elif test_id.strip() == 'chisquare':
if sample2 == 1 :
rho, p_value = stats.chisquare( map(float, sample_one), map(float, sample_two),ddof=args.ddof )
else:
rho, p_value = stats.chisquare( map(float, sample_one),ddof=args.ddof)
cols.append( rho )
cols.append( p_value )
elif test_id.strip() == 'power_divergence':
if sample2 == 1 :
stat, p_value = stats.power_divergence( map(float, sample_one), map(float, sample_two),ddof=args.ddof,lambda_=args.lambda_ )
else:
stat, p_value = stats.power_divergence( map(float, sample_one),ddof=args.ddof,lambda_=args.lambda_)
cols.append( stat )
cols.append( p_value )
elif test_id.strip() == 'theilslopes':
if sample2 == 1 :
mpe, met, lo, up = stats.theilslopes( map(float, sample_one), map(float, sample_two),alpha=args.alpha )
else:
mpe, met, lo, up = stats.theilslopes( map(float, sample_one),alpha=args.alpha)
cols.append( mpe )
cols.append( met )
cols.append( lo )
cols.append( up )
elif test_id.strip() == 'combine_pvalues':
if sample2 == 1 :
stat, p_value = stats.combine_pvalues( map(float, sample_one),method=args.med,weights=map(float, sample_two) )
else:
stat, p_value = stats.combine_pvalues( map(float, sample_one),method=args.med)
cols.append( stat )
cols.append( p_value )
elif test_id.strip() == 'obrientransform':
ob = stats.obrientransform( *b_samples )
for list in ob:
elements = ','.join( map(str, list) )
cols.append( elements )
elif test_id.strip() == 'f_oneway':
f_value, p_value = stats.f_oneway( *b_samples )
cols.append(f_value)
cols.append(p_value)
elif test_id.strip() == 'kruskal':
h, p_value = stats.kruskal( *b_samples )
cols.append(h)
cols.append(p_value)
elif test_id.strip() == 'friedmanchisquare':
fr, p_value = stats.friedmanchisquare( *b_samples )
cols.append(fr)
cols.append(p_value)
elif test_id.strip() == 'fligner':
xsq, p_value = stats.fligner( center=args.center,proportiontocut=args.proportiontocut,*b_samples )
cols.append(xsq)
cols.append(p_value)
elif test_id.strip() == 'bartlett':
T, p_value = stats.bartlett( *b_samples )
cols.append(T)
cols.append(p_value)
elif test_id.strip() == 'levene':
w, p_value = stats.levene( center=args.center,proportiontocut=args.proportiontocut,*b_samples )
cols.append(w)
cols.append(p_value)
elif test_id.strip() == 'median_test':
stat, p_value, m, table = stats.median_test( ties=args.ties,correction=args.correction ,lambda_=args.lambda_,*b_samples )
cols.append(stat)
cols.append(p_value)
cols.append(m)
cols.append(table)
for list in table:
elements = ','.join( map(str, list) )
cols.append( elements )
outfile.write( '%s\n' % '\t'.join( map(str, cols) ) )
outfile.close()
def wilcoxon_holm(alpha=0.05, df_perf=None):
"""
Applies the wilcoxon signed rank test between each pair of algorithm and then use Holm
to reject the null's hypothesis
"""
print(pd.unique(df_perf['classifier_name']))
# count the number of tested datasets per classifier
df_counts = pd.DataFrame({
'count':
df_perf.groupby(['classifier_name']).size()
}).reset_index()
# get the maximum number of tested datasets
max_nb_datasets = df_counts['count'].max()
# get the list of classifiers who have been tested on nb_max_datasets
classifiers = list(df_counts.loc[df_counts['count'] == max_nb_datasets]
['classifier_name'])
# test the null hypothesis using friedman before doing a post-hoc analysis
friedman_p_value = friedmanchisquare(
*(np.array(df_perf.loc[df_perf['classifier_name'] == c]['accuracy'])
for c in classifiers))[1]
if friedman_p_value >= alpha:
# then the null hypothesis over the entire classifiers cannot be rejected
print(
'the null hypothesis over the entire classifiers cannot be rejected'
)
exit()
# get the number of classifiers
m = len(classifiers)
# init array that contains the p-values calculated by the Wilcoxon signed rank test
p_values = []
# loop through the algorithms to compare pairwise
for i in range(m - 1):
# get the name of classifier one
classifier_1 = classifiers[i]
# get the performance of classifier one
perf_1 = np.array(df_perf.loc[df_perf['classifier_name'] ==
classifier_1]['accuracy'],
dtype=np.float64)
for j in range(i + 1, m):
# get the name of the second classifier
classifier_2 = classifiers[j]
# get the performance of classifier one
perf_2 = np.array(df_perf.loc[df_perf['classifier_name'] ==
classifier_2]['accuracy'],
dtype=np.float64)
# calculate the p_value
p_value = wilcoxon(perf_1, perf_2, zero_method='pratt')[1]
# appen to the list
p_values.append((classifier_1, classifier_2, p_value, False))
# get the number of hypothesis
k = len(p_values)
# sort the list in acsending manner of p-value
p_values.sort(key=operator.itemgetter(2))
# loop through the hypothesis
for i in range(k):
# correct alpha with holm
new_alpha = float(alpha / (k - i))
# test if significant after holm's correction of alpha
if p_values[i][2] <= new_alpha:
p_values[i] = (p_values[i][0], p_values[i][1], p_values[i][2],
True)
else:
# stop
break
# compute the average ranks to be returned (useful for drawing the cd diagram)
# sort the dataframe of performances
sorted_df_perf = df_perf.loc[df_perf['classifier_name'].isin(classifiers)]. \
sort_values(['classifier_name', 'dataset_name'])
# get the rank data
rank_data = np.array(sorted_df_perf['accuracy']).reshape(
m, max_nb_datasets)
# create the data frame containg the accuracies
df_ranks = pd.DataFrame(data=rank_data,
index=np.sort(classifiers),
columns=np.unique(sorted_df_perf['dataset_name']))
# number of wins
dfff = df_ranks.rank(ascending=False)
print(dfff[dfff == 1.0].sum(axis=1))
# average the ranks
average_ranks = df_ranks.rank(ascending=False).mean(axis=1).sort_values(
ascending=False)
# return the p-values and the average ranks
return p_values, average_ranks, max_nb_datasets
Pairwise1 = np.stack((np.array(TableMCC["LR"]), np.array(TableMCC["C-SVM"])))
PairwiseTotal = np.stack(
(np.array(TableMCC["LR"]), np.array(TableMCC["C-SVMRBF"]),
np.array(TableMCC["C-SVM"]), np.array(TableMCC["RF"]),
np.array(TableMCC["ANN"]), np.array(TableMCC["KNN"]),
np.array(TableMCC["Naive"])))
nr, nc = PairwiseTotal.shape
BR = ro.r.matrix(PairwiseTotal, nrow=nr, ncol=nc)
BR = BR.transpose()
print(BR)
friedmanchisquare(TableMCC["LR"], TableMCC["C-SVMRBF"], TableMCC["C-SVM"],
TableMCC["RF"], TableMCC["ANN"], TableMCC["KNN"],
TableMCC["Naive"])
print(PMCMRplus.friedmanTest(BR))
print(PMCMRplus.friedmanTest(BR).rx2("statistic"))
print(PMCMRplus.friedmanTest(BR).rx2("p.value"))
print(PMCMRplus.friedmanTest(BR, dist="FDist"))
print(PMCMRplus.friedmanTest(BR, dist="FDist").rx2("statistic"))
print(PMCMRplus.friedmanTest(BR, dist="FDist").rx2("p.value"))
BRNum = ro.r.matrix(ro.r.c(BR),
nrow=12,
ncol=7,
dimnames=ro.r.list(
ro.r.c("Adult", "Alzheimer", "Bank", "Breast", "Cell",
"Contra", "Aus", "Ger", "Iris", "Sonar", "Wine",
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 28 22:06:49 2018
@author: Delgado
"""
import numpy as np
import scipy as sp
import scipy.stats as sps
sol = sps.friedmanchisquare([79.52, 92.06, 79.59], [43.38, 54.54, 46.82],
[79.43, 88.60, 79.57])
'dataset': dataset_names
} # will contain one row per dataset and models over columns
groups_by_model = df_results.groupby('Algorithm Name')
for model_name in model_names:
df_model = groups_by_model.get_group(model_name)
groups_by_dataset = df_model.groupby('Dataset Name')
model_mean = []
for dataset_name in dataset_names:
model_mean.append(
groups_by_dataset.get_group(dataset_name)
[metric].mean()) # average over folds
average_results[model_name] = model_mean
df_results = pd.DataFrame(average_results)
df_results.to_csv('results/average_results.csv', index=False)
# save ranks of algorithms (1 is best, |models| is worst)
df = df_results.drop(columns='dataset')
ranks = rankdata(df.to_numpy(), method='dense', axis=1)
df = pd.DataFrame(ranks, columns=df.columns)
df['dataset'] = df_results['dataset']
df.to_csv('results/ranks.csv')
# friedman and post hoc tests
t_stat, p_val = friedmanchisquare(*[df_results[i] for i in model_names])
print('\nfriedman test p-val = %s' % p_val)
post_hoc_p_vals = posthoc_nemenyi_friedman(
df_results.drop(columns='dataset').to_numpy())
post_hoc_p_vals.columns = model_names
print('\npost hoc p-vals:\n%s' % post_hoc_p_vals)
post_hoc_p_vals.to_csv('results/post_hoc.csv', index=False)
-118.46336159, -118.48487121, -118.44997418, -118.45399572, -118.47371284
]
#Rastrigin Function
cost_hill = [
3757.74744873, 3136.04744873, 3544.18744873, 3470.84744873, 3560.48744873,
3318.34744873, 3208.80744873, 2954.96744873, 3324.58744873, 2725.18744873
]
cost_simulated = [
3160.80744873, 2728.52744873, 3899.78744873, 3540.60744873, 3322.78744873,
3687.14744873, 2733.22744873, 3421.12744873, 3619.34744873, 3037.52744873
]
cost_genetico = [
1518.74744873, 1458.26744873, 1573.86744873, 1680.00744873, 1253.10744873,
1675.40744873, 1542.90744873, 1516.92744873, 1409.98744873, 1603.82744873
]
cost_pso = [
2635.83, 2423.98, 2598.48, 2635.53, 2533.04, 2627.55, 2590.66, 2634.54,
2578.27, 2550.38
]
print(friedmanchisquare(cost_hill, cost_simulated, cost_genetico, cost_pso))
print(kruskal(cost_hill, cost_simulated, cost_genetico, cost_pso))
print(wilcoxon(cost_genetico, cost_pso))
# Creating plot
data = ([cost_hill, cost_simulated, cost_genetico, cost_pso])
plt.boxplot(data)
# show plot
plt.show()
def precision_plot_table():
'''
Precision bar plots for all folds together with friedman test.
Prints precision information to be used in R
'''
tables = []
for d in DIMS:
pBetter=[]
pVote=[]
pWeigh=[]
pFvote=[]
for fold in range(NFOLDS):
out = loadfile(RESULTSFOLDER+'u-100k-fold-'+str(d)+'-d'+str(fold)+'-top%d-results.out'%TOPK)
#ordem: maioria, ponderado, best
#print out[-1]
pBetter.append(out[E_ID['best']][2])
pWeigh.append(out[E_ID['weighted']][1])
pVote.append(out[E_ID['vote']][1])
pFvote.append(out[E_ID['filtered']][1])
print 'Friedman', friedmanchisquare(np.array(pBetter),
np.array(pWeigh),
np.array(pVote),
np.array(pFvote))
print 'Precisions d=%d'%d
Rprint = lambda alist : ' = c(' + \
", ".join([str(it) for it in alist]) + ')'
folds = [i for i in range(NFOLDS)]*3
precisions = pBetter + pWeigh + pVote + pFvote
ids = ["'best'"]*NFOLDS + ["'weight'"]*NFOLDS + \
["'vote'"]*NFOLDS + ["'filtered'"]*NFOLDS
print 'datafold', Rprint(folds)
print 'precision', Rprint(precisions)
print 'algo', Rprint(ids)
tables.append(np.vstack((np.array(pBetter),
np.array(pWeigh),
np.array(pVote),
np.array(pFvote) )).T)
x=np.arange(1,NFOLDS+1,1)
y = [4, 9, 2,5,6]
z=[1,2,3,5,7]
k=[11,12,13,5,9]
plt.figure()
ax = plt.subplot(111)
w = 0.2
ax.bar(x-2*w, pVote,width=w,color=COLORS[0],align='center', label='Vote')
ax.bar(x-w, pFvote,width=w,color=COLORS[1],align='center', label='Filtered')
ax.bar(x, pWeigh,width=w,color=COLORS[2],align='center', label='Weighted')
ax.bar(x+w, pBetter,width=w,color=COLORS[3],align='center', label='Best')
ax.autoscale(tight=True)
ax.legend(loc=4)
plt.ylim((0.7,1.0))
#plt.ylim((0,0.9))
plt.xticks(range(1,NFOLDS+1),['Fold %d'%i for i in range(1,NFOLDS+1)])
plt.title('Precision (d=%d)'%d)
plt.savefig(RESULTSFOLDER+'precisionat%d_bars_d%d.png'%(TOPK, d))
table = np.hstack((tables[0],tables[1]))
table = np.vstack((table,table.mean(axis=0), table.std(axis=0)))
np.savetxt(RESULTSFOLDER+'table_precisions.txt', table, fmt='%.4f', delimiter=' & ', newline='\\\\ \hline \n')
from scipy.stats import friedmanchisquare, f
import pandas as pd
df = pd.read_csv('decisiontree.csv', header=0, index_col=0)
treeclassifier = df.to_numpy().reshape((10, ))
df1 = pd.read_csv('multiclass.csv', header=0, index_col=0)
multiclass = df1.to_numpy()
ovo = multiclass[:, 0]
ovr = multiclass[:, 1]
ecoc = multiclass[:, 2]
friedman_stat, pvalue = friedmanchisquare(treeclassifier, ovo, ovr, ecoc)
datasets = 10
algorithms = 4
davenport_stat = (
(datasets - 1) * friedman_stat) / (datasets *
(algorithms - 1) - friedman_stat)
print(davenport_stat)
print(
f.ppf(q=1 - 0.05,
dfn=algorithms - 1,
dfd=(algorithms - 1) * (datasets - 1)))
if davenport_stat > f.ppf(
q=1 - 0.05, dfn=algorithms - 1, dfd=(algorithms - 1) * (datasets - 1)):
print("Hay diferencias entre los algoritmos")
else:
# coding=utf-8
from scipy.stats import friedmanchisquare
import scikit_posthocs as sp #pip3 install scikit-posthocs
import pandas as pd
df = pd.read_csv('csv/comparativo_tecnicas.csv', encoding='utf-8')
print('\nFriedman')
stat, p = friedmanchisquare(df['dt'].tolist(), df['nb'].tolist(),
df['mlp'].tolist())
print('p=%.3f' % (p))
if p > 0.05:
print('Não há diferença significativa')
else:
print('Há diferença significativa')
print('\n Posthoc')
posthoc = sp.posthoc_nemenyi(
[df['dt'].tolist(), df['nb'].tolist(), df['mlp'].tolist()])
print(posthoc)
# creo un vector muestra para guardar los resultados (representa una mudestra)
muesta = []
# Abro el archivo
with open("Muestras/" + str(nombreArchivo) + ".out") as archivo:
leido = archivo.readlines()
leido = [x.strip() for x in leido]
for x in leido:
muesta.append(float(x))
print("\tSolucion: " + str(muesta))
# para cada linea
# leo cada linea agreagando el valor de la solucion al vector muestra
# cierro el archivo
archivo.close()
print("Muestra: " + str(muesta))
# agrego el vector muestra al conjunto de muestras
conjuntoMuestras.append(muesta)
print
print("Conjunto de Muestas: " + str(conjuntoMuestras))
conjuntoMuestras = np.asarray(conjuntoMuestras)
resultado = stats.friedmanchisquare(
*(conjuntoMuestras[i, :] for i in range(conjuntoMuestras.shape[0])))
# conjuntoMuestras[0],conjuntoMuestras[1],conjuntoMuestras[2])
print(resultado)
# ejecuto el test de friedman para el conjunto de muestras
# conjuntoMuestras = np.array([[2, 2, 3], [1, 1, 1] ,[4, 5, 6]])
# resultado=stats.friedmanchisquare(*(conjuntoMuestras[i, :] for i in range(conjuntoMuestras.shape[0])))
# print resultado
filename_, file_extension = os.path.splitext(filename)
if file_extension == ".xlsx":
data = pd.read_excel(dirname+'\\'+filename,usecols=['dataset_name', 'algorithm_name', 'roc_auc'])
else:
data = pd.read_csv(dirname+'\\'+filename, usecols=['dataset_name', 'algorithm_name', 'roc_auc'])
# average auc for each dataset
avg_auc = data.groupby(['dataset_name', 'algorithm_name'], as_index= False).mean()
res_df = res_df.append(avg_auc)
res_df.reset_index(inplace=True, drop=True)
# get all datasets names with results from all four algorithms
dataset_names = res_df.groupby('dataset_name', as_index = False).count()
dataset_names = list(dataset_names[dataset_names['algorithm_name'] == 4]['dataset_name'])
# filter result to contain only datasets that are in dataset_names
res_df = res_df[res_df['dataset_name'].apply(lambda x: x in dataset_names)]
alog_names = list(res_df['algorithm_name'].unique())
# Friedman test
stat, p = stats.friedmanchisquare(res_df[res_df['algorithm_name'] == alog_names[0]].sort_values(by='dataset_name')['roc_auc'],
res_df[res_df['algorithm_name'] == alog_names[1]].sort_values(by='dataset_name')['roc_auc'],
res_df[res_df['algorithm_name'] == alog_names[2]].sort_values(by='dataset_name')['roc_auc'],
res_df[res_df['algorithm_name'] == alog_names[3]].sort_values(by='dataset_name')['roc_auc'])
# interpret results
alpha = 0.05
print('Statistics=%.3f, p=%.3f' % (stat, p))
if(p < alpha):
print('null hypothesis rejected')
# perform post-hoc test
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
print(sp.posthoc_nemenyi_friedman(res_df, y_col='roc_auc',block_col='dataset_name', group_col='algorithm_name',melted=True))
math.sqrt(total_obtain_answers * 8 * 2) for i in range(7)
]
print("Task1 Accuracy:\n", task1_acc_avg)
print("Task1 Accuracy 95\% CI:\n", task1_acc_ci)
print("Task1 Completion Time:\n", task1_time_avg)
print("Task1 Completion Time 95\% CI:\n", task1_time_ci)
task1_acc_detail = [[
np.average(task1_acc[:, i, k, j]) for i in range(8) for j in range(2)
] for k in range(7)]
task1_time_detail = [[
np.average(task1_time[:, i, k, j]) for i in range(8) for j in range(2)
] for k in range(7)]
acc_fm_res = stats.friedmanchisquare(*task1_acc_detail)
time_fm_res = stats.friedmanchisquare(*task1_time_detail)
print(acc_fm_res)
print(
sp.posthoc_conover_friedman(np.array(task1_acc_detail).T,
p_adjust=None))
print(time_fm_res)
print(sp.posthoc_conover_friedman(np.array(task1_time_detail).T))
print()
print("E2")
# Task 2 - density_2
task2_acc = np.zeros((total_obtain_answers, 8, 7, 2)).astype(np.int)
task2_time = np.zeros((total_obtain_answers, 8, 7, 2)).astype(np.float)
learning_effect_acc = np.zeros(
def non_parametric_test(self,
x: str,
y: str,
meth: str = 'kruskal-wallis',
continuity_correction: bool = True,
alternative: str = 'two-sided',
zero_meth: str = 'pratt',
*args):
"""
:param x:
:param y:
:param meth: String defining the hypothesis test method for non-parametric tests
-> kruskal-wallis: Kruskal-Wallis H test to test whether the distributions of two or more
independent samples are equal or not
-> mann-whitney: Mann-Whitney U test to test whether the distributions of two independent
samples are equal or not
-> wilcoxon: Wilcoxon Signed-Rank test for test whether the distributions of two paired samples
are equal or not
-> friedman: Friedman test for test whether the distributions of two or more paired samples
are equal or not
:param continuity_correction:
:param alternative: String defining the type of hypothesis test
-> two-sided:
-> less:
-> greater:
:param zero_meth: String defining the method to handle zero differences in the ranking process (Wilcoxon test)
-> pratt: Pratt treatment that includes zero-differences (more conservative)
-> wilcox: Wilcox tratment that discards all zero-differences
-> zsplit: Zero rank split, just like Pratt, but spliting the zero rank between positive
and negative ones
:param args:
:return:
"""
_reject = None
if meth == 'kruskal-wallis':
_non_parametric_test = kruskal(args, self.nan_policy)
elif meth == 'mann-whitney':
_non_parametric_test = mannwhitneyu(
x=self.data_set[x],
y=self.data_set[y],
use_continuity=continuity_correction,
alternative=alternative)
elif meth == 'wilcoxon':
_non_parametric_test = wilcoxon(x=self.data_set[x],
y=self.data_set[y],
zero_method=zero_meth,
correction=continuity_correction)
elif meth == 'friedman':
_non_parametric_test = friedmanchisquare(args)
else:
raise HappyLearningUtilsException('No non-parametric test found !')
if _non_parametric_test[1] <= self.p:
_reject = False
else:
_reject = True
return {
'feature': ''.join(self.data_set.keys()),
'cases': len(self.data_set.values),
'test_statistic': _non_parametric_test[0],
'p_value': _non_parametric_test[1],
'reject': _reject
}
def get_test(self):
"""
:param model_type: for which we want to extract
:return:
"""
print(f"Calculating sign test for subset: {self.subset_type}")
# Load all models predictions for each phase
for phase in self.phase_list:
phase_all_models_ndcg_list = [] # ndcg list
model_type_list = [] # Name of the model
for model_type in self.models_list:
ndcg_path = self._get_ndcg_path(self.model_preds_root,
model_type,
phase=phase,
subset_type=self.subset_type)
ndcg_list = self.read_file_as_list(ndcg_path)
print(f"Total samples {model_type}: {len(ndcg_list)}")
phase_all_models_ndcg_list.append(ndcg_list)
model_type_list.append(model_type)
# We form combinations of all indices
index_models_list = list(range(len(model_type_list)))
# pairwise
combination_set = combinations(index_models_list, 2)
for combination_indices in combination_set:
model1_preds = phase_all_models_ndcg_list[
combination_indices[0]]
model2_preds = phase_all_models_ndcg_list[
combination_indices[1]]
model1_name = model_type_list[combination_indices[0]]
model2_name = model_type_list[combination_indices[1]]
stat, p = mannwhitneyu(model1_preds, model2_preds)
print(
f'Mannwhitneyu - For phase: {phase} - models: {model1_name} vs'
f' {model2_name} : stat={stat:.4f}, p={p:.4f}')
stat, p = wilcoxon(model1_preds, model2_preds)
print(
f'Wilcoxon - For phase: {phase} - models: {model1_name} vs'
f' {model2_name} : stat={stat:.4f}, p={p:.4f}')
# Checking for equivalence of *args
# stat, p = f_oneway(phase_all_models_ndcg_list[0],
# phase_all_models_ndcg_list[1], phase_all_models_ndcg_list[2],
# phase_all_models_ndcg_list[3])
# stat, p = f_oneway(*phase_all_models_ndcg_list)
# stat, p = mannwhitneyu(*phase_all_models_ndcg_list)
# stat, p = wilcoxon(*phase_all_models_ndcg_list)
stat, p = kruskal(*phase_all_models_ndcg_list)
print(f'Kruskal - For phase: {phase}: stat={stat:.4f}, p={p:.4f}')
bonferroni_correction = multipletests(p, method='bonferroni')
# print(bonferroni_correction)
# (reject, pvals_corrected, alphacSidak, alphacBonf)
action = str(bonferroni_correction[0][0]) # np array
new_p_value = bonferroni_correction[1][0]
print(
f'Kruskal - bonferroni - For phase: {phase}: p={new_p_value:.4f}, '
f'action: {str(action)}')
stat, p = friedmanchisquare(*phase_all_models_ndcg_list)
print(
f'Friedmanchisquare - For phase: {phase}: stat={stat:.4f}, p={p:.4f}'
)
max_iter=500,
momentum=0.9)
SVM = svm.SVC()
GNB = GaussianNB()
cv = StratifiedKFold(n_splits=10)
scores_net = cross_val_score(net, X, y, cv=cv, scoring='accuracy')
scores_SVM = cross_val_score(SVM, X, y, cv=cv, scoring='accuracy')
scores_GNB = cross_val_score(GNB, X, y, cv=cv, scoring='accuracy')
ic_net = stats.norm.interval(0.95,
loc=1 - scores_net.mean(),
scale=scores_net.std() / sqrt(len(X)))
ic_SVM = stats.norm.interval(0.95,
loc=1 - scores_SVM.mean(),
scale=scores_SVM.std() / sqrt(len(X)))
ic_GNB = stats.norm.interval(0.95,
loc=1 - scores_GNB.mean(),
scale=scores_GNB.std() / sqrt(len(X)))
print("Error MLP: %0.2f (+/- %0.2f)" %
(1 - scores_net.mean(), scores_net.mean() - ic_net[0]))
print("Error SVM: %0.2f (+/- %0.2f)" %
(1 - scores_SVM.mean(), scores_SVM.mean() - ic_SVM[0]))
print("Error GNB: %0.2f (+/- %0.2f)" %
(1 - scores_GNB.mean(), scores_GNB.mean() - ic_GNB[0]))
print stats.friedmanchisquare(scores_net, scores_SVM, scores_GNB)
def friedman_test(experiment_pivot_df):
"""Returns p-value for Friedman test."""
pivot_df_as_matrix = experiment_pivot_df.values
_, p_value = ss.friedmanchisquare(*pivot_df_as_matrix)
return p_value
cds["Attrition"].replace(to_replace=("Yes","No"),value=(1,0),inplace=True)
print(cds.head())
#Processing wilcoxon test to check if a particular variable
#H0= There is no significiant attrition in the company caused by the age of the employees and the distance from home.
#H1= There is significiant attrition in the company caused by the age of the employees and the distance from home.
from scipy.stats import wilcoxon
print("\n")
print(wilcoxon(cds.Age,cds.DistanceFromHome))
#Output: WilcoxonResult(statistic=2892.0, pvalue=0.0)
#Friedman test
from scipy.stats import friedmanchisquare
print("\n")
print(friedmanchisquare(cds.Age,cds.DistanceFromHome,cds.JobLevel))
#output: FriedmanchisquareResult(statistic=7478.114178094286, pvalue=0.0)
#Mann Whitney test
from scipy.stats import mannwhitneyu
print("\n")
print(mannwhitneyu(cds.JobLevel,cds.NumCompaniesWorked))
# Moving on to Krushkal Wallis Test
from scipy.stats import kruskal
print("\n")
print(kruskal(cds.TotalWorkingYears,cds.TrainingTimesLastYear,cds.YearsAtCompany,cds.YearsSinceLastPromotion))
#ChiSquare Test
from scipy.stats import chi2_contingency
print("\n")
network_dict[n]['entropy'] = np.array(network_dict[n]['entropy'])
amsd_results[n].append(np.mean(network_dict[n]['dist']))
t_dist[n].append(network_dict[n]['dist'])
t_entropy[n].append(network_dict[n]['entropy'])
print('AMSD :', amsd_results[n][-1], '+-',
np.std(network_dict[n]['dist']))
if n not in ame_results:
ame_results[n] = []
ame_results[n].append(np.mean(network_dict[n]['entropy']))
print('AMUD :', ame_results[n][-1], '+-',
np.std(network_dict[n]['entropy']))
print('')
for n in ['1', '3']:
print('Nd:', n)
_, p = friedmanchisquare(t_dist[n][0], t_dist[n][1], t_dist[n][2])
data = np.array(t_dist[n])
# data = np.argsort(data, axis=0)
_, pk = f_oneway(data[0], data[2])
print('ANOVA AMSD :', pk)
_, p_12 = wilcoxon(t_dist[n][0], t_dist[n][3], zero_method="pratt")
_, p_13 = wilcoxon(t_dist[n][1], t_dist[n][4], zero_method="pratt")
_, p_23 = wilcoxon(t_dist[n][2], t_dist[n][5], zero_method="pratt")
print('FRIEDMAN AMSD :', p, p_12, p_13, p_23)
print('MEAN AMSD : ', np.mean(data, axis=1), np.std(data, axis=1))
print('MEAN AMSD : ', np.argsort(data[:2], axis=0).mean(axis=1))
print('MEAN AMSD : ', np.argsort(data[1:], axis=0).mean(axis=1))
_, p = friedmanchisquare(t_entropy[n][0], t_entropy[n][1],
t_entropy[n][2])
data = np.array(t_entropy[n])
# data = np.argsort(data, axis=0)
# ### Statistical analysis to select the best K-IBL algorithm
# In[10]:
import json
with open('accuracies_sick.json') as json_file:
accuracies_json = json.load(json_file)
accs = [accuracies_json[k] for k in accuracies_json]
len(accs)
from scipy.stats import friedmanchisquare
friedmanchisquare(*accs)
get_ipython().system(' pip install scikit-posthocs')
from scikit_posthocs import posthoc_dunn
p_values = posthoc_dunn(a=accs, p_adjust='holm', sort=True)
print('Post hocs dunn p-values: ')
p_values.head()
# In[11]:
pvalues = p_values.values
classifiers = accs.copy()
best = 0
best_acc = np.mean(accs[0])
def friedman_test(df: pd.DataFrame):
cols = [x for x in df.columns if x != 'dataset']
data = [df[col] for col in cols]
res = friedmanchisquare(*data)
return res
#https://machinelearningmastery.com/statistical-hypothesis-tests-in-python-cheat-sheet/
"Friedman Test ------------------------------------------------------------------------------------###"
""" Tests if the distribuitions of two or more paired samples are equal or not """
# Example of the Friedman Test
from scipy.stats import friedmanchisquare
# N of data must be equal!
data1 = [
0.873, 2.817, 0.121, -0.945, -0.055, -1.436, 0.360, -1.478, -1.637, -1.869
]
data2 = [
1.142, -0.432, -0.938, -0.729, -0.846, -0.157, 0.500, 1.183, -1.075, -0.169
]
data3 = [
-0.208, 0.696, 0.928, -1.148, -0.213, 0.229, 0.137, 0.269, -0.870, -1.204
]
stat, p = friedmanchisquare(data1, data2, data3)
print('stat=%.3f, p=%.3f' % (stat, p))
alpha = 0.05
if p > alpha:
print('Probably the same distribution')
else:
print('Probably different distributions')
for group, title in zip(data, titles):
endog.extend(group)
groups.extend([title] * len(group))
m_comp = pairwise_tukeyhsd(np.asarray(endog), np.asarray(groups), 0.05)
print(m_comp)
data_seg = [
results.rf, results.lf, results.rt, results.lt, results.rs, results.ls
]
data_act = [
results.walking, results.running, results.standing, results.sitting,
results.bicycling
]
f_comp = stats.friedmanchisquare(*data_seg)
print(f_comp)
#data = [proposed_dist.rice, proposed_dist.spline, proposed_dist.fifth, proposed_dist.fourth, proposed_dist.third, proposed_dist.second, proposed_dist.linear, proposed_dist.diff]
data = [
proposed_dist.fifth, proposed_dist.fourth, proposed_dist.third,
proposed_dist.second, proposed_dist.spline, proposed_dist.linear,
proposed_dist.diff
]
n_comp = sp.posthoc_nemenyi(data)
print(n_comp)
#n_comp = sp.posthoc_nemenyi_friedman(data)
#print(n_comp)
num_cat = len(data)
],
[
3131.5099999999993, 3147.58, 3147.7299999999996, 3133.35,
3148.0299999999997, 3147.2799999999997, 3145.2099999999996,
3126.78, 3145.2099999999996, 3141.46
]]
# df = pd.DataFrame(rastringin).T
# df.to_excel(excel_writer = "C:/Users/Tulio/Desktop/Mestrado/Busca_e_Otimizacao/search_and_optmization/src/utils/test.xlsx")
data = a280
# print(statistics.pstdev(data[2]))
# Friedman de grupo
print(stats.friedmanchisquare(*data))
# Kruskal-Wallis de grupo
print(stats.kruskal(*data))
#Teste de Conover baseado em Kruskal-Wallis
pc = sp.posthoc_conover(data)
#Caso precise mudar os indices e colunas do DataFrame
pc.columns = ['GRASP 2-opt', 'GRASP mBUC', 'HC mBUC']
pc.index = ['GRASP 2-opt', 'GRASP mBUC', 'HC mBUC']
print(pc)
#Heatmap do Teste de Conover
cmap = ['1', '#fb6a4a', '#08306b', '#4292c6', '#c6dbef']
table.to_csv(os.path.join(results_path, 'main_results.csv'))
table.to_latex(os.path.join(results_path, 'main_results.tex'))
remove_list = [[], ['isotonic'], ['beta2'], ['beta05'], ['beta', 'beta05'],
['beta2', 'beta05'], [None, 'None', 'isotonic', 'sigmoid']]
for rem in remove_list:
df_rem = df_all[np.logical_not(np.in1d(df_all.method, rem))]
methods_rem = [method for method in methods if method not in rem]
print methods_rem
print("-#-#-#-#-#-#-#-#-#-#-#-#-ACC-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-")
table = df_rem.pivot_table(index=['dataset'], columns=['method'],
values=['acc'], aggfunc=[np.mean, np.std])
table_to_latex(dataset_names, methods_rem, table, max_is_better=True)
accs = table.as_matrix()[:, :len(methods_rem)]
print friedmanchisquare(*[accs[:, x] for x in np.arange(accs.shape[1])])
table.to_csv(os.path.join(results_path, 'main_acc' + str(methods_rem) + '.csv'))
print("-#-#-#-#-#-#-#-#-#-#-#-LOSS-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-")
table = df_rem.pivot_table(index=['dataset'], columns=['method'],
values=['loss'], aggfunc=[np.mean, np.std])
table_to_latex(dataset_names, methods_rem, table, max_is_better=False)
losses = table.as_matrix()[:, :len(methods_rem)]
print friedmanchisquare(*[losses[:, x] for x in np.arange(losses.shape[1])])
table.to_csv(os.path.join(results_path, 'main_loss' + str(methods_rem) + '.csv'))
print("-#-#-#-#-#-#-#-#-#-#-#-BRIER-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-")
table = df_rem.pivot_table(index=['dataset'], columns=['method'],
values=['brier'], aggfunc=[np.mean, np.std])
table_to_latex(dataset_names, methods_rem, table, max_is_better=False)
briers = table.as_matrix()[:, :len(methods_rem)]
# except Exception as e:
# print('En ',ds,'_',tip,'_',l,'_',det)
# print (e.__doc__)
# print (e.message)
######################## FRIEDMAN TESTS ########################
from scipy.stats import friedmanchisquare
alpha = 0.05
#FOR PREQ.ACC
stat_preqacc, p_preqacc = friedmanchisquare(friedman_DDM_preqacc,
friedman_EDDM_preqacc,
friedman_ADWIN_preqacc,
friedman_PH_preqacc,
friedman_CURIE_preqacc)
print('---- PREQ. ACC ----')
print('Statistics=%.3f, p=%.3f' % (stat_preqacc, p_preqacc))
if p_preqacc > alpha:
print('Same distributions (fail to reject H0)')
else:
print('Different distributions (reject H0)')
print('')
#FOR RAM-HOURS
stat_ramhours, p_ramhours = friedmanchisquare(friedman_DDM_ramhours,
friedman_EDDM_ramhours,
friedman_ADWIN_ramhours,
friedman_PH_ramhours,
def time_friedmanchisquare(self):
stats.friedmanchisquare(self.a, self.b, self.c)
from scipy import stats
# Generate the data
nd = stats.norm(5,3)
data = nd.rvs((100,3))
# Check if the three are from the same distribution
alpha = 0.05
_,p = stats.friedmanchisquare(data[:,0], data[:,1], data[:,2])
if p > alpha:
print('First datasets come from the same distribution')
# Modify one of the data
data[:,1] += 3
_,p = stats.friedmanchisquare(data[:,0], data[:,1], data[:,2])
if p < alpha:
print('After the modification, one of the datasets is different')
# %%
"""
## Positive affect
"""
# %%
"""
### Friedman test
"""
# %%
# %%
# Compare groups
_, p = stats.friedmanchisquare(smile_pos, neutral_pos, sad_pos)
print('p={:.5f}'.format(p))
if p > ALPHA:
print('Same distributions')
exit()
else:
print('Different distributions. You can do a post-hoc test.')
# %%
"""
### Wilcoxon test (as a post-hoc test)
"""
# %%
# Smiley face vs Neutral face
def run(file_name):
results = util.from_file(file_name)
# Calculate accuracies for each classifier based on the true and
# predicted labels
accuracies = {}
for classifier in results.keys():
accuracies[classifier] = []
for info in results[classifier]:
y_true = info['true']
y_pred = info['pred']
accuracies[classifier].append(evaluate.get_accuracy_score(y_true, y_pred))
# Create plot of scores
if not os.path.isdir('figs'):
os.mkdir('figs')
df = create_df(accuracies)
sns.barplot(x='classifiers', y='scores', data=df)
plt.savefig('figs/scores.png', transparent=True)
create_plot(accuracies)
# Print mean accuracies and standard deviations
REPORT.info()
REPORT.info('Accuracies:')
for classifier in accuracies.keys():
REPORT.info('{}: {} +/- {}'.format(classifier, np.mean(accuracies[classifier]), np.std(accuracies[classifier])))
REPORT.info()
# Run matched-samples, equal-variance t-test
REPORT.info('Pairwise t-test:')
p_value_min = 1.0
classifiers_min = []
for classifier1 in accuracies.keys():
for classifier2 in accuracies.keys():
scores1 = accuracies[classifier1]
scores2 = accuracies[classifier2]
p_value = ttest_ind(scores1, scores2, equal_var=True)[1]
if p_value < p_value_min:
p_value_min = p_value
classifiers_min = [classifier1, classifier2]
REPORT.info('{} <-> {}: {}'.format(classifier1, classifier2, p_value))
REPORT.info('Min. p-value: {} between {} and {}'.format(p_value_min, classifiers_min[0], classifiers_min[1]))
REPORT.info()
# Run pairwise Wilcoxon ranked-sign test
REPORT.info('Pairwise Wilcoxon ranked-sign test:')
p_value_min = 1.0
classifiers_min = []
p_values = []
classifiers = []
for classifier1 in accuracies.keys():
for classifier2 in accuracies.keys():
if classifier1 == classifier2:
continue
scores1 = accuracies[classifier1]
scores2 = accuracies[classifier2]
p_value = wilcoxon(scores1, scores2, zero_method='wilcox')[1]
if p_value < p_value_min:
p_value_min = p_value
classifiers_min = [classifier1, classifier2]
if p_value < 0.05:
p_values.append(p_value)
classifiers.append([classifier1, classifier2])
REPORT.info('{} <-> {}: wilcoxon p-value: {}'.format(classifier1, classifier2, p_value))
REPORT.info('Min. p-value: {} between {} and {}'.format(p_value_min, classifiers_min[0], classifiers_min[1]))
REPORT.info('Below 0.05:')
for i in range(len(p_values)):
REPORT.info(' p-value: {} between {} and {}'.format(p_values[i], classifiers[i][0], classifiers[i][1]))
REPORT.info()
# Run Friedman test
REPORT.info('Friedman\'s Chi-Square test:')
data = []
for classifier in accuracies.keys():
row = []
for item in chunk(accuracies[classifier], 10):
row.append(np.mean(item))
data.append(row)
data = pd.DataFrame(
np.array(data).T,
columns=accuracies.keys(),
index=['D' + str(i) for i in range(10)])
p_value = friedmanchisquare(
data['svm-rbf'],
data['svm-poly'],
data['svm-sigmoid'],
data['random-forest'],
data['gp'])[1]
REPORT.info()
REPORT.info('p_value: {}'.format(p_value))
# The p-value tells us IF there's a significant difference between
# performance scores. The Q statistic allows us to determine which
# classifiers are best.
ranks = []
for row in data.index:
ranked = np.array(data.loc[row].tolist()).argsort()[::-1]
ranks.append(ranked)
ranks = pd.DataFrame(ranks, columns=data.columns, index=data.index)
n = ranks.shape[0]
k = ranks.shape[1]
mean_ranks = []
for column in ranks.columns:
mean_ranks.append(np.mean(ranks[column]))
mean_ranks = pd.DataFrame([mean_ranks], columns=ranks.columns, index=['mean_ranks'])
print(mean_ranks)
# Define q threshold at alpha = 0.05 and df = (n-1)*(k-1)
qq_threshold = 3.85 / np.sqrt(2)
REPORT.info('Q-threshold: {}'.format(qq_threshold))
tuples = []
for column1 in ranks.columns:
for column2 in ranks.columns:
if column1 == column2:
continue
if (column1, column2) not in tuples:
tuples.append((column1, column2))
if (column2, column1) in tuples:
continue
r1 = mean_ranks[column1][0]
r2 = mean_ranks[column2][0]
qq = (r1 - r2) / np.sqrt((k * (k + 1.0)) / (6.0 * n))
REPORT.info('{} <-> {}: q = {}'.format(column1, column2, qq))
if qq > qq_threshold:
REPORT.info(' q-statistic significant: {} > {}'.format(qq, qq_threshold))
# Close the report file
REPORT.append_file(__file__)
REPORT.close()
X, y = ClassesB.ReadCongress()
acc4 = ClassesB.KfoldAcc(X, y)
a = np.zeros([4, 4])
classfier = ['naive', 'neigh', 'rule', 'tree']
for index, classifier in enumerate(classfier):
a[0, index] = np.mean(acc1[classifier])
a[1, index] = np.mean(acc2[classifier])
a[2, index] = np.mean(acc3[classifier])
a[3, index] = np.mean(acc4[classifier])
from scipy.stats import friedmanchisquare
statistic, p_value = friedmanchisquare(a[:, 0], a[:, 1], a[:, 2], a[:, 3])
for index, classifier in enumerate(classfier):
p_value[index, 0] = friedmanchisquare(acc1[classifier],
acc2[classifier])[1]
p_value[index, 1] = friedmanchisquare(acc1[classifier],
acc3[classifier])[1]
p_value[index, 2] = friedmanchisquare(acc1[classifier],
acc4[classifier])[1]
p_value[index, 3] = friedmanchisquare(acc2[classifier],
acc3[classifier])[1]
p_value[index, 4] = friedmanchisquare(acc2[classifier],
acc4[classifier])[1]
p_value[index, 5] = friedmanchisquare(acc3[classifier],
acc4[classifier])[1]
"""
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--infile", required=True, help="Tabular file.")
parser.add_argument("-o", "--outfile", required=True, help="Path to the output file.")
parser.add_argument("--sample_one_cols", help="Input format, like smi, sdf, inchi")
parser.add_argument("--sample_two_cols", help="Input format, like smi, sdf, inchi")
parser.add_argument("--sample_cols", help="Input format, like smi, sdf, inchi,separate arrays using ;")
parser.add_argument("--test_id", help="statistical test method")
parser.add_argument(
"--mwu_use_continuity",
action="store_true",
default=False,
help="Whether a continuity correction (1/2.) should be taken into account.",
)
parser.add_argument(
"--equal_var",
action="store_true",
default=False,
help="If set perform a standard independent 2 sample test that assumes equal population variances. If not set, perform Welch's t-test, which does not assume equal population variance.",
)
parser.add_argument(
"--reta", action="store_true", default=False, help="Whether or not to return the internally computed a values."
)
parser.add_argument("--fisher", action="store_true", default=False, help="if true then Fisher definition is used")
parser.add_argument(
"--bias",
action="store_true",
default=False,
help="if false,then the calculations are corrected for statistical bias",
)
parser.add_argument("--inclusive1", action="store_true", default=False, help="if false,lower_limit will be ignored")
parser.add_argument(
"--inclusive2", action="store_true", default=False, help="if false,higher_limit will be ignored"
)
parser.add_argument("--inclusive", action="store_true", default=False, help="if false,limit will be ignored")
parser.add_argument(
"--printextras",
action="store_true",
default=False,
help="If True, if there are extra points a warning is raised saying how many of those points there are",
)
parser.add_argument(
"--initial_lexsort",
action="store_true",
default="False",
help="Whether to use lexsort or quicksort as the sorting method for the initial sort of the inputs.",
)
parser.add_argument("--correction", action="store_true", default=False, help="continuity correction ")
parser.add_argument(
"--axis",
type=int,
default=0,
help="Axis can equal None (ravel array first), or an integer (the axis over which to operate on a and b)",
)
parser.add_argument(
"--n",
type=int,
default=0,
help="the number of trials. This is ignored if x gives both the number of successes and failures",
)
parser.add_argument("--b", type=int, default=0, help="The number of bins to use for the histogram")
parser.add_argument("--N", type=int, default=0, help="Score that is compared to the elements in a.")
parser.add_argument("--ddof", type=int, default=0, help="Degrees of freedom correction")
parser.add_argument("--score", type=int, default=0, help="Score that is compared to the elements in a.")
parser.add_argument("--m", type=float, default=0.0, help="limits")
parser.add_argument("--mf", type=float, default=2.0, help="lower limit")
parser.add_argument("--nf", type=float, default=99.9, help="higher_limit")
parser.add_argument(
"--p",
type=float,
default=0.5,
help="The hypothesized probability of success. 0 <= p <= 1. The default value is p = 0.5",
)
parser.add_argument("--alpha", type=float, default=0.9, help="probability")
parser.add_argument("--new", type=float, default=0.0, help="Value to put in place of values in a outside of bounds")
parser.add_argument(
"--proportiontocut",
type=float,
default=0.0,
help="Proportion (in range 0-1) of total data set to trim of each end.",
)
parser.add_argument(
"--lambda_",
type=float,
default=1.0,
help="lambda_ gives the power in the Cressie-Read power divergence statistic",
)
parser.add_argument(
"--imbda",
type=float,
default=0,
help="If lmbda is not None, do the transformation for that value.If lmbda is None, find the lambda that maximizes the log-likelihood function and return it as the second output argument.",
)
parser.add_argument("--base", type=float, default=1.6, help="The logarithmic base to use, defaults to e")
parser.add_argument("--dtype", help="dtype")
parser.add_argument("--med", help="med")
parser.add_argument("--cdf", help="cdf")
parser.add_argument("--zero_method", help="zero_method options")
parser.add_argument("--dist", help="dist options")
parser.add_argument("--ties", help="ties options")
parser.add_argument("--alternative", help="alternative options")
parser.add_argument("--mode", help="mode options")
parser.add_argument("--method", help="method options")
parser.add_argument("--md", help="md options")
parser.add_argument("--center", help="center options")
parser.add_argument("--kind", help="kind options")
parser.add_argument("--tail", help="tail options")
parser.add_argument("--interpolation", help="interpolation options")
parser.add_argument("--statistic", help="statistic options")
args = parser.parse_args()
infile = args.infile
outfile = open(args.outfile, "w+")
test_id = args.test_id
nf = args.nf
mf = args.mf
imbda = args.imbda
inclusive1 = args.inclusive1
inclusive2 = args.inclusive2
sample0 = 0
sample1 = 0
sample2 = 0
if args.sample_cols != None:
sample0 = 1
barlett_samples = []
for sample in args.sample_cols.split(";"):
barlett_samples.append(map(int, sample.split(",")))
if args.sample_one_cols != None:
sample1 = 1
sample_one_cols = args.sample_one_cols.split(",")
if args.sample_two_cols != None:
sample_two_cols = args.sample_two_cols.split(",")
sample2 = 1
for line in open(infile):
sample_one = []
sample_two = []
cols = line.strip().split("\t")
if sample0 == 1:
b_samples = columns_to_values(barlett_samples, line)
if sample1 == 1:
for index in sample_one_cols:
sample_one.append(cols[int(index) - 1])
if sample2 == 1:
for index in sample_two_cols:
sample_two.append(cols[int(index) - 1])
if test_id.strip() == "describe":
size, min_max, mean, uv, bs, bk = stats.describe(map(float, sample_one))
cols.append(size)
cols.append(min_max)
cols.append(mean)
cols.append(uv)
cols.append(bs)
cols.append(bk)
elif test_id.strip() == "mode":
vals, counts = stats.mode(map(float, sample_one))
cols.append(vals)
cols.append(counts)
elif test_id.strip() == "nanmean":
m = stats.nanmean(map(float, sample_one))
cols.append(m)
elif test_id.strip() == "nanmedian":
m = stats.nanmedian(map(float, sample_one))
cols.append(m)
elif test_id.strip() == "kurtosistest":
z_value, p_value = stats.kurtosistest(map(float, sample_one))
cols.append(z_value)
cols.append(p_value)
elif test_id.strip() == "variation":
ra = stats.variation(map(float, sample_one))
cols.append(ra)
elif test_id.strip() == "itemfreq":
freq = stats.itemfreq(map(float, sample_one))
for list in freq:
elements = ",".join(map(str, list))
cols.append(elements)
elif test_id.strip() == "nanmedian":
m = stats.nanmedian(map(float, sample_one))
cols.append(m)
elif test_id.strip() == "variation":
ra = stats.variation(map(float, sample_one))
cols.append(ra)
elif test_id.strip() == "boxcox_llf":
IIf = stats.boxcox_llf(imbda, map(float, sample_one))
cols.append(IIf)
elif test_id.strip() == "tiecorrect":
fa = stats.tiecorrect(map(float, sample_one))
cols.append(fa)
elif test_id.strip() == "rankdata":
r = stats.rankdata(map(float, sample_one), method=args.md)
cols.append(r)
elif test_id.strip() == "nanstd":
s = stats.nanstd(map(float, sample_one), bias=args.bias)
cols.append(s)
elif test_id.strip() == "anderson":
A2, critical, sig = stats.anderson(map(float, sample_one), dist=args.dist)
cols.append(A2)
for list in critical:
cols.append(list)
cols.append(",")
for list in sig:
cols.append(list)
elif test_id.strip() == "binom_test":
p_value = stats.binom_test(map(float, sample_one), n=args.n, p=args.p)
cols.append(p_value)
elif test_id.strip() == "gmean":
gm = stats.gmean(map(float, sample_one), dtype=args.dtype)
cols.append(gm)
elif test_id.strip() == "hmean":
hm = stats.hmean(map(float, sample_one), dtype=args.dtype)
cols.append(hm)
elif test_id.strip() == "kurtosis":
k = stats.kurtosis(map(float, sample_one), axis=args.axis, fisher=args.fisher, bias=args.bias)
cols.append(k)
elif test_id.strip() == "moment":
n_moment = stats.moment(map(float, sample_one), n=args.n)
cols.append(n_moment)
elif test_id.strip() == "normaltest":
k2, p_value = stats.normaltest(map(float, sample_one))
cols.append(k2)
cols.append(p_value)
elif test_id.strip() == "skew":
skewness = stats.skew(map(float, sample_one), bias=args.bias)
cols.append(skewness)
elif test_id.strip() == "skewtest":
z_value, p_value = stats.skewtest(map(float, sample_one))
cols.append(z_value)
cols.append(p_value)
elif test_id.strip() == "sem":
s = stats.sem(map(float, sample_one), ddof=args.ddof)
cols.append(s)
elif test_id.strip() == "zscore":
z = stats.zscore(map(float, sample_one), ddof=args.ddof)
for list in z:
cols.append(list)
elif test_id.strip() == "signaltonoise":
s2n = stats.signaltonoise(map(float, sample_one), ddof=args.ddof)
cols.append(s2n)
elif test_id.strip() == "percentileofscore":
p = stats.percentileofscore(map(float, sample_one), score=args.score, kind=args.kind)
cols.append(p)
elif test_id.strip() == "bayes_mvs":
c_mean, c_var, c_std = stats.bayes_mvs(map(float, sample_one), alpha=args.alpha)
cols.append(c_mean)
cols.append(c_var)
cols.append(c_std)
elif test_id.strip() == "sigmaclip":
c, c_low, c_up = stats.sigmaclip(map(float, sample_one), low=args.m, high=args.n)
cols.append(c)
cols.append(c_low)
cols.append(c_up)
elif test_id.strip() == "kstest":
d, p_value = stats.kstest(
map(float, sample_one), cdf=args.cdf, N=args.N, alternative=args.alternative, mode=args.mode
)
cols.append(d)
cols.append(p_value)
elif test_id.strip() == "chi2_contingency":
chi2, p, dof, ex = stats.chi2_contingency(
map(float, sample_one), correction=args.correction, lambda_=args.lambda_
)
cols.append(chi2)
cols.append(p)
cols.append(dof)
cols.append(ex)
elif test_id.strip() == "tmean":
if nf is 0 and mf is 0:
mean = stats.tmean(map(float, sample_one))
else:
mean = stats.tmean(map(float, sample_one), (mf, nf), (inclusive1, inclusive2))
cols.append(mean)
elif test_id.strip() == "tmin":
if mf is 0:
min = stats.tmin(map(float, sample_one))
else:
min = stats.tmin(map(float, sample_one), lowerlimit=mf, inclusive=args.inclusive)
cols.append(min)
elif test_id.strip() == "tmax":
if nf is 0:
max = stats.tmax(map(float, sample_one))
else:
max = stats.tmax(map(float, sample_one), upperlimit=nf, inclusive=args.inclusive)
cols.append(max)
elif test_id.strip() == "tvar":
if nf is 0 and mf is 0:
var = stats.tvar(map(float, sample_one))
else:
var = stats.tvar(map(float, sample_one), (mf, nf), (inclusive1, inclusive2))
cols.append(var)
elif test_id.strip() == "tstd":
if nf is 0 and mf is 0:
std = stats.tstd(map(float, sample_one))
else:
std = stats.tstd(map(float, sample_one), (mf, nf), (inclusive1, inclusive2))
cols.append(std)
elif test_id.strip() == "tsem":
if nf is 0 and mf is 0:
s = stats.tsem(map(float, sample_one))
else:
s = stats.tsem(map(float, sample_one), (mf, nf), (inclusive1, inclusive2))
cols.append(s)
elif test_id.strip() == "scoreatpercentile":
if nf is 0 and mf is 0:
s = stats.scoreatpercentile(
map(float, sample_one), map(float, sample_two), interpolation_method=args.interpolation
)
else:
s = stats.scoreatpercentile(
map(float, sample_one), map(float, sample_two), (mf, nf), interpolation_method=args.interpolation
)
for list in s:
cols.append(list)
elif test_id.strip() == "relfreq":
if nf is 0 and mf is 0:
rel, low_range, binsize, ex = stats.relfreq(map(float, sample_one), args.b)
else:
rel, low_range, binsize, ex = stats.relfreq(map(float, sample_one), args.b, (mf, nf))
for list in rel:
cols.append(list)
cols.append(low_range)
cols.append(binsize)
cols.append(ex)
elif test_id.strip() == "binned_statistic":
if nf is 0 and mf is 0:
st, b_edge, b_n = stats.binned_statistic(
map(float, sample_one), map(float, sample_two), statistic=args.statistic, bins=args.b
)
else:
st, b_edge, b_n = stats.binned_statistic(
map(float, sample_one),
map(float, sample_two),
statistic=args.statistic,
bins=args.b,
range=(mf, nf),
)
cols.append(st)
cols.append(b_edge)
cols.append(b_n)
elif test_id.strip() == "threshold":
if nf is 0 and mf is 0:
o = stats.threshold(map(float, sample_one), newval=args.new)
else:
o = stats.threshold(map(float, sample_one), mf, nf, newval=args.new)
for list in o:
cols.append(list)
elif test_id.strip() == "trimboth":
o = stats.trimboth(map(float, sample_one), proportiontocut=args.proportiontocut)
for list in o:
cols.append(list)
elif test_id.strip() == "trim1":
t1 = stats.trim1(map(float, sample_one), proportiontocut=args.proportiontocut, tail=args.tail)
for list in t1:
cols.append(list)
elif test_id.strip() == "histogram":
if nf is 0 and mf is 0:
hi, low_range, binsize, ex = stats.histogram(map(float, sample_one), args.b)
else:
hi, low_range, binsize, ex = stats.histogram(map(float, sample_one), args.b, (mf, nf))
cols.append(hi)
cols.append(low_range)
cols.append(binsize)
cols.append(ex)
elif test_id.strip() == "cumfreq":
if nf is 0 and mf is 0:
cum, low_range, binsize, ex = stats.cumfreq(map(float, sample_one), args.b)
else:
cum, low_range, binsize, ex = stats.cumfreq(map(float, sample_one), args.b, (mf, nf))
cols.append(cum)
cols.append(low_range)
cols.append(binsize)
cols.append(ex)
elif test_id.strip() == "boxcox_normmax":
if nf is 0 and mf is 0:
ma = stats.boxcox_normmax(map(float, sample_one))
else:
ma = stats.boxcox_normmax(map(float, sample_one), (mf, nf), method=args.method)
cols.append(ma)
elif test_id.strip() == "boxcox":
if imbda is 0:
box, ma, ci = stats.boxcox(map(float, sample_one), alpha=args.alpha)
cols.append(box)
cols.append(ma)
cols.append(ci)
else:
box = stats.boxcox(map(float, sample_one), imbda, alpha=args.alpha)
cols.append(box)
elif test_id.strip() == "histogram2":
h2 = stats.histogram2(map(float, sample_one), map(float, sample_two))
for list in h2:
cols.append(list)
elif test_id.strip() == "ranksums":
z_statistic, p_value = stats.ranksums(map(float, sample_one), map(float, sample_two))
cols.append(z_statistic)
cols.append(p_value)
elif test_id.strip() == "ttest_1samp":
t, prob = stats.ttest_1samp(map(float, sample_one), map(float, sample_two))
for list in t:
cols.append(list)
for list in prob:
cols.append(list)
elif test_id.strip() == "ansari":
AB, p_value = stats.ansari(map(float, sample_one), map(float, sample_two))
cols.append(AB)
cols.append(p_value)
elif test_id.strip() == "linregress":
slope, intercept, r_value, p_value, stderr = stats.linregress(
map(float, sample_one), map(float, sample_two)
)
cols.append(slope)
cols.append(intercept)
cols.append(r_value)
cols.append(p_value)
cols.append(stderr)
elif test_id.strip() == "pearsonr":
cor, p_value = stats.pearsonr(map(float, sample_one), map(float, sample_two))
cols.append(cor)
cols.append(p_value)
elif test_id.strip() == "pointbiserialr":
r, p_value = stats.pointbiserialr(map(float, sample_one), map(float, sample_two))
cols.append(r)
cols.append(p_value)
elif test_id.strip() == "ks_2samp":
d, p_value = stats.ks_2samp(map(float, sample_one), map(float, sample_two))
cols.append(d)
cols.append(p_value)
elif test_id.strip() == "mannwhitneyu":
mw_stats_u, p_value = stats.mannwhitneyu(
map(float, sample_one), map(float, sample_two), use_continuity=args.mwu_use_continuity
)
cols.append(mw_stats_u)
cols.append(p_value)
elif test_id.strip() == "zmap":
z = stats.zmap(map(float, sample_one), map(float, sample_two), ddof=args.ddof)
for list in z:
cols.append(list)
elif test_id.strip() == "ttest_ind":
mw_stats_u, p_value = stats.ttest_ind(
map(float, sample_one), map(float, sample_two), equal_var=args.equal_var
)
cols.append(mw_stats_u)
cols.append(p_value)
elif test_id.strip() == "ttest_rel":
t, prob = stats.ttest_rel(map(float, sample_one), map(float, sample_two), axis=args.axis)
cols.append(t)
cols.append(prob)
elif test_id.strip() == "mood":
z, p_value = stats.mood(map(float, sample_one), map(float, sample_two), axis=args.axis)
cols.append(z)
cols.append(p_value)
elif test_id.strip() == "shapiro":
W, p_value, a = stats.shapiro(map(float, sample_one), map(float, sample_two), args.reta)
cols.append(W)
cols.append(p_value)
for list in a:
cols.append(list)
elif test_id.strip() == "kendalltau":
k, p_value = stats.kendalltau(
map(float, sample_one), map(float, sample_two), initial_lexsort=args.initial_lexsort
)
cols.append(k)
cols.append(p_value)
elif test_id.strip() == "entropy":
s = stats.entropy(map(float, sample_one), map(float, sample_two), base=args.base)
cols.append(s)
elif test_id.strip() == "spearmanr":
if sample2 == 1:
rho, p_value = stats.spearmanr(map(float, sample_one), map(float, sample_two))
else:
rho, p_value = stats.spearmanr(map(float, sample_one))
cols.append(rho)
cols.append(p_value)
elif test_id.strip() == "wilcoxon":
if sample2 == 1:
T, p_value = stats.wilcoxon(
map(float, sample_one),
map(float, sample_two),
zero_method=args.zero_method,
correction=args.correction,
)
else:
T, p_value = stats.wilcoxon(
map(float, sample_one), zero_method=args.zero_method, correction=args.correction
)
cols.append(T)
cols.append(p_value)
elif test_id.strip() == "chisquare":
if sample2 == 1:
rho, p_value = stats.chisquare(map(float, sample_one), map(float, sample_two), ddof=args.ddof)
else:
rho, p_value = stats.chisquare(map(float, sample_one), ddof=args.ddof)
cols.append(rho)
cols.append(p_value)
elif test_id.strip() == "power_divergence":
if sample2 == 1:
stat, p_value = stats.power_divergence(
map(float, sample_one), map(float, sample_two), ddof=args.ddof, lambda_=args.lambda_
)
else:
stat, p_value = stats.power_divergence(map(float, sample_one), ddof=args.ddof, lambda_=args.lambda_)
cols.append(stat)
cols.append(p_value)
elif test_id.strip() == "theilslopes":
if sample2 == 1:
mpe, met, lo, up = stats.theilslopes(map(float, sample_one), map(float, sample_two), alpha=args.alpha)
else:
mpe, met, lo, up = stats.theilslopes(map(float, sample_one), alpha=args.alpha)
cols.append(mpe)
cols.append(met)
cols.append(lo)
cols.append(up)
elif test_id.strip() == "combine_pvalues":
if sample2 == 1:
stat, p_value = stats.combine_pvalues(
map(float, sample_one), method=args.med, weights=map(float, sample_two)
)
else:
stat, p_value = stats.combine_pvalues(map(float, sample_one), method=args.med)
cols.append(stat)
cols.append(p_value)
elif test_id.strip() == "obrientransform":
ob = stats.obrientransform(*b_samples)
for list in ob:
elements = ",".join(map(str, list))
cols.append(elements)
elif test_id.strip() == "f_oneway":
f_value, p_value = stats.f_oneway(*b_samples)
cols.append(f_value)
cols.append(p_value)
elif test_id.strip() == "kruskal":
h, p_value = stats.kruskal(*b_samples)
cols.append(h)
cols.append(p_value)
elif test_id.strip() == "friedmanchisquare":
fr, p_value = stats.friedmanchisquare(*b_samples)
cols.append(fr)
cols.append(p_value)
elif test_id.strip() == "fligner":
xsq, p_value = stats.fligner(center=args.center, proportiontocut=args.proportiontocut, *b_samples)
cols.append(xsq)
cols.append(p_value)
elif test_id.strip() == "bartlett":
T, p_value = stats.bartlett(*b_samples)
cols.append(T)
cols.append(p_value)
elif test_id.strip() == "levene":
w, p_value = stats.levene(center=args.center, proportiontocut=args.proportiontocut, *b_samples)
cols.append(w)
cols.append(p_value)
elif test_id.strip() == "median_test":
stat, p_value, m, table = stats.median_test(
ties=args.ties, correction=args.correction, lambda_=args.lambda_, *b_samples
)
cols.append(stat)
cols.append(p_value)
cols.append(m)
cols.append(table)
for list in table:
elements = ",".join(map(str, list))
cols.append(elements)
outfile.write("%s\n" % "\t".join(map(str, cols)))
outfile.close()
def friction_stats(df, effect_variable='mu'):
"""
Builds a statistical report for a data frame
"""
pass
if __name__ == '__main__':
# par_dir = fh.get_dir()
par_dir = '/home/michael/Dropbox/Eyelid_edge_dev/Test_Data/BiomomentumData/TestData/MS1604'
df = analyze_multi(parent_dir=par_dir, save=False)
print(df)
# repeated measures variables
# Figure out n and get descriptive statistics
# Friedman chi square test
measurements = []
num_lenses = None
fried_stats, fried_ps = stats.friedmanchisquare()
