def main(directed): logfile = 'results-permutations/timigs-{}.txt'.format( 'directed' if directed else 'undirected') memory = Profiling( 'Permutations {}'.format('directed' if directed else 'undirected'), 'results-permutations/python-profiling-nperm-nodes{}-{}.png'.format( NCOUNTRIES, 'directed' if directed else 'undirected'), True) memory.check_memory('init-{}'.format('d' if directed else 'i')) ####################################################################### # Data Matrices ####################################################################### X1 = getMatrix('data-permutations/country_trade_index.txt', directed, True) memory.check_memory('X1-{}'.format('d' if directed else 'i')) X2 = getMatrix('data-permutations/country_distance_index.txt', directed, True) memory.check_memory('X2-{}'.format('d' if directed else 'i')) X3 = getMatrix('data-permutations/country_colonial_index.txt', directed) Y = getMatrix('data-permutations/country_lang_index.txt', directed) memory.check_memory('Y-{}'.format('d' if directed else 'i')) X = {'TRADE': X1, 'DISTANCE': X2, 'COLONIAL': X3} Y = {'LANG': Y} np.random.seed(1) ####################################################################### # QAP ####################################################################### perms = np.logspace(1, 7, num=7 - 1, endpoint=False) for nperm in perms: start_time = time.time() mrqap = MRQAP(Y=Y, X=X, npermutations=int(nperm), diagonal=False, directed=directed, logfile=logfile, memory=memory) mrqap.mrqap() utils.printf( "--- {}, nperm {}: {} seconds ---".format( 'directed' if directed else 'undirected', nperm, time.time() - start_time), logfile) mrqap.summary() fn = 'results-permutations/python-nperm{}-{}-<coef>.png'.format( nperm, 'directed' if directed else 'undirected') mrqap.plot('betas', fn.replace('<coef>', 'betas')) mrqap.plot('tvalues', fn.replace('<coef>', 'tvalues')) utils.printf( '******************************************************************************\n\n', logfile) del (mrqap) return
def _summary_ols(self): ''' Print the OLS summary :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary OLS (original) ===\n{}'.format(self.model.summary()), self.logfile) utils.printf('', self.logfile) utils.printf('# of Permutations: {}'.format(self.npermutations), self.logfile)
def _summary_ols(self): ''' Print the OLS summary :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary OLS (original) ===\n{}'.format(self.model.summary()), self.logfile) utils.printf('', self.logfile) utils.printf('# of Permutations: {}'.format(self.npermutations), self.logfile)
def init(self): ''' Generating the original OLS model. Y and Xs are flattened. Also, the betas and tvalues dictionaries are initialized (key:independent variables, value:[]) :return: ''' self.v[list(self.Y.keys())[0]] = self._getFlatten(list(self.Y.values())[0]) self._initCoefficients(INTERCEPT) for k,x in self.X.items(): if k == list(self.Y.keys())[0]: utils.printf('ERROR: Idependent variable cannot be named \'[}\''.format(list(self.Y.keys())[0]), self.logfile) sys.exit(0) self.v[k] = self._getFlatten(x) self._initCoefficients(k) self.data = pandas.DataFrame(self.v) self.model = self._fit(self.v.keys(), self.data) del(self.X)
def init(self): ''' Generating the original OLS model. Y and Xs are flattened. Also, the betas and tvalues dictionaries are initialized (key:independent variables, value:[]) :return: ''' self.v[self.Y.keys()[0]] = self._getFlatten(self.Y.values()[0]) self._initCoefficients(INTERCEPT) for k,x in self.X.items(): if k == self.Y.keys()[0]: utils.printf('ERROR: Idependent variable cannot be named \'[}\''.format(self.Y.keys()[0]), self.logfile) sys.exit(0) self.v[k] = self._getFlatten(x) self._initCoefficients(k) self.data = pandas.DataFrame(self.v) self.model = self._fit(self.v.keys(), self.data) del(self.X)
def correlation(self, x, y, show=True): ''' Computes Pearson's correlation value of variables x and y. Diagonal values are removed. :param x: numpy array independent variable :param y: numpu array dependent variable :param show: if True then shows pearson's correlation and p-value. :return: ''' if not self.diagonal: xflatten = np.delete(x, [i*(x.shape[0]+1)for i in range(x.shape[0])]) yflatten = np.delete(y, [i*(y.shape[0]+1)for i in range(y.shape[0])]) pc = pearsonr(xflatten, yflatten) else: pc = pearsonr(x.flatten(), y.flatten()) if show: utils.printf('Pearson Correlation: {}'.format(pc[0])) utils.printf('p-value: {}'.format(pc[1])) return pc
def main(directed): logfile = 'results-synthetic-ernos-renyi/timigs-{}.txt'.format('directed' if directed else 'undirected') memory = Profiling('Nodes {}'.format('directed' if directed else 'undirected'), 'results-synthetic-ernos-renyi/python-profiling-netsize-edgeprob{}-nperm{}-{}.png'.format(EDGEPROB, NPERMUTATIONS,'directed' if directed else 'undirected'), False) memory.check_memory('init-{}'.format('d' if directed else 'i')) ####################################################################### # Data Matrices ####################################################################### #nnodes = np.logspace(1,7,num=7-1, endpoint=False) nnodes = np.logspace(1,5,num=5-1, endpoint=False) for n in nnodes: n = int(n) fn = 'data-synthetic-ernos-renyi/nodes{}_edgeprob{}_<var>.dat'.format(n,EDGEPROB) memory.check_memory('nodes-{}'.format(n)) X1 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X1')) memory.check_memory('X1-{}'.format(n)) X2 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X2')) memory.check_memory('X2-{}'.format(n)) X3 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X3')) memory.check_memory('X3-{}'.format(n)) Y = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','Y')) memory.check_memory('Y-{}'.format(n)) X = {'X1':X1, 'X2':X2, 'X3':X3} Y = {'Y':Y} ####################################################################### # QAP ####################################################################### start_time = time.time() mrqap = MRQAP(Y=Y, X=X, npermutations=int(NPERMUTATIONS), diagonal=False, directed=directed, logfile=logfile, memory=memory) mrqap.mrqap() utils.printf("\n--- {}, nodes {}: {} seconds ---".format('directed' if directed else 'undirected', n, time.time() - start_time), logfile) mrqap.summary() fn = 'results-synthetic-ernos-renyi/python-nodes{}-edgeprob{}-nperm{}-{}-<coef>.png'.format(n, EDGEPROB, NPERMUTATIONS,'directed' if directed else 'undirected') mrqap.plot('betas',fn.replace('<coef>','betas')) mrqap.plot('tvalues',fn.replace('<coef>','tvalues')) utils.printf('******************************************************************************\n\n', logfile) del(mrqap) return
def _summary_tvalues(self): ''' Summary t-values :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary T-Values ===', self.logfile) utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','T-TEST','As Large', 'As Small'), self.logfile) for k,v in self.tvalues.items(): tstats = self.model.tvalues[k] aslarge = sum([1 for c in v if c >= tstats]) / float(len(v)) assmall = sum([1 for c in v if c <= tstats]) / float(len(v)) utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[int(len(v)/2)],sum(v)/len(v),max(v),round(np.std(v),6),round(float(tstats),2),aslarge,assmall), self.logfile)
def _summary_tvalues(self): ''' Summary t-values :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary T-Values ===', self.logfile) utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','T-TEST','As Large', 'As Small'), self.logfile) for k,v in self.tvalues.items(): tstats = self.model.tvalues[k] aslarge = sum([1 for c in v if c >= tstats]) / float(len(v)) assmall = sum([1 for c in v if c <= tstats]) / float(len(v)) utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[len(v)/2],sum(v)/len(v),max(v),round(np.std(v),6),round(float(tstats),2),aslarge,assmall), self.logfile)
def _summary_betas(self): ''' Summary of beta coefficients :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary beta coefficients ===', self.logfile) utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','B.COEFF.','As Large', 'As Small', 'P-VALUE'), self.logfile) for k,v in self.betas.items(): beta = self.model.params[k] pstats = self.model.pvalues[k] aslarge = sum([1 for c in v if c >= beta]) / float(len(v)) assmall = sum([1 for c in v if c <= beta]) / float(len(v)) utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[len(v)/2],sum(v)/len(v),max(v),round(np.std(v),6),beta,aslarge,assmall,round(float(pstats),2)), self.logfile)
def _summary_betas(self): ''' Summary of beta coefficients :return: ''' utils.printf('', self.logfile) utils.printf('=== Summary beta coefficients ===', self.logfile) utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','B.COEFF.','As Large', 'As Small', 'P-VALUE'), self.logfile) for k,v in self.betas.items(): beta = self.model.params[k] pstats = self.model.pvalues[k] aslarge = sum([1 for c in v if c >= beta]) / float(len(v)) assmall = sum([1 for c in v if c <= beta]) / float(len(v)) utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[int(len(v)/2)],sum(v)/len(v),max(v),round(np.std(v),6),beta,aslarge,assmall,round(float(pstats),2)), self.logfile)
def main(directed): logfile = 'results-permutations/timigs-{}.txt'.format('directed' if directed else 'undirected') memory = Profiling('Permutations {}'.format('directed' if directed else 'undirected'), 'results-permutations/python-profiling-nperm-nodes{}-{}.png'.format(NCOUNTRIES,'directed' if directed else 'undirected'), True) memory.check_memory('init-{}'.format('d' if directed else 'i')) ####################################################################### # Data Matrices ####################################################################### X1 = getMatrix('data-permutations/country_trade_index.txt',directed,True) memory.check_memory('X1-{}'.format('d' if directed else 'i')) X2 = getMatrix('data-permutations/country_distance_index.txt',directed,True) memory.check_memory('X2-{}'.format('d' if directed else 'i')) X3 = getMatrix('data-permutations/country_colonial_index.txt',directed) Y = getMatrix('data-permutations/country_lang_index.txt',directed) memory.check_memory('Y-{}'.format('d' if directed else 'i')) X = {'TRADE':X1, 'DISTANCE':X2, 'COLONIAL':X3} Y = {'LANG':Y} np.random.seed(1) ####################################################################### # QAP ####################################################################### perms = np.logspace(1,7,num=7-1, endpoint=False) for nperm in perms: start_time = time.time() mrqap = MRQAP(Y=Y, X=X, npermutations=int(nperm), diagonal=False, directed=directed, logfile=logfile, memory=memory) mrqap.mrqap() utils.printf("--- {}, nperm {}: {} seconds ---".format('directed' if directed else 'undirected', nperm, time.time() - start_time), logfile) mrqap.summary() fn = 'results-permutations/python-nperm{}-{}-<coef>.png'.format(nperm,'directed' if directed else 'undirected') mrqap.plot('betas', fn.replace('<coef>','betas')) mrqap.plot('tvalues', fn.replace('<coef>','tvalues')) utils.printf('******************************************************************************\n\n', logfile) del(mrqap) return
def _ttest(self): utils.printf('') utils.printf('========== T-TEST ==========') utils.printf('{:25s} {:25s} {:25s} {:25s}'.format('IND. VAR.','COEF.','T-STAT','P-VALUE')) ts = {} lines = {} for k,vlist in self.betas.items(): t = stats.ttest_1samp(vlist,self.model.params[k]) ts[k] = abs(round(float(t[0]),6)) lines[k] = '{:20s} {:25f} {:25f} {:25f}'.format(k,self.model.params[k],round(float(t[0]),6),round(float(t[1]),6)) ts = utils.sortDictByValue(ts,True) for t in ts: utils.printf(lines[t[0]])
def _ttest(self): utils.printf('') utils.printf('========== T-TEST ==========') utils.printf('{:25s} {:25s} {:25s} {:25s}'.format('IND. VAR.','COEF.','T-STAT','P-VALUE')) ts = {} lines = {} for k,vlist in self.betas.items(): t = stats.ttest_1samp(vlist,self.model.params[k]) ts[k] = abs(round(float(t[0]),6)) lines[k] = '{:20s} {:25f} {:25f} {:25f}'.format(k,self.model.params[k],round(float(t[0]),6),round(float(t[1]),6)) ts = utils.sortDictByValue(ts,True) for t in ts: utils.printf(lines[t[0]])
####################################################################### # References # - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf ####################################################################### ####################################################################### # Dependencies ####################################################################### import numpy as np from libs import utils from libs.qap import QAP ####################################################################### # Data # Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm ####################################################################### X = minfo = np.loadtxt('data/friendship.dat') Y = np.loadtxt('data/advice.dat') utils.printf('Friendship: \n{}'.format(X)) utils.printf('Advise: \n{}'.format(Y)) np.random.seed(831) ####################################################################### # QAP ####################################################################### qap = QAP(Y, X, 2000) qap.qap() qap.summary() qap.plot()
__author__ = 'lisette.espin' ####################################################################### # References # - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf ####################################################################### ####################################################################### # Dependencies ####################################################################### import numpy as np from libs import utils from libs.qap import QAP ####################################################################### # Data # Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm ####################################################################### minfo = minfo = np.loadtxt('data/info.dat') mmoney = np.loadtxt('data/money.dat') utils.printf('Information: \n{}'.format(minfo)) utils.printf('Money Exchange: \n{}'.format(mmoney)) ####################################################################### # QAP ####################################################################### qap = QAP(minfo, mmoney, 5000) qap.qap() qap.summary()
__author__ = 'lisette.espin' ####################################################################### # References # - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf ####################################################################### ####################################################################### # Dependencies ####################################################################### import numpy as np from libs import utils from libs.qap import QAP ####################################################################### # Data # Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm ####################################################################### minfo = minfo = np.loadtxt('data/info.dat') mmoney = np.loadtxt('data/money.dat') utils.printf('Information: \n{}'.format(minfo)) utils.printf('Money Exchange: \n{}'.format(mmoney)) ####################################################################### # QAP ####################################################################### qap = QAP(minfo, mmoney, 5000) qap.qap() qap.summary()
def stats(self, x, y): if not self.diagonal: xflatten = np.delete(x, [i*(x.shape[0]+1)for i in range(x.shape[0])]) yflatten = np.delete(y, [i*(y.shape[0]+1)for i in range(y.shape[0])]) p = np.corrcoef(xflatten,yflatten) utils.printf('Pearson\'s correlation:\n{}'.format(p)) utils.printf('Z-Test:{}'.format(ztest(xflatten, yflatten))) utils.printf('T-Test:{}'.format(ttest_ind(xflatten, yflatten))) else: p = np.corrcoef(x, y) utils.printf('Pearson\'s correlation:\n{}'.format(p)) utils.printf('Z-Test:{}'.format(ztest(x, y))) utils.printf('T-Test:{}'.format(ttest_ind(x, y)))
def summary(self): utils.printf('') utils.printf('# Permutations: {}'.format(self.npermutations)) utils.printf('Correlation coefficients: Obs. Value({}), Significance({})'.format(self.beta[0], self.beta[1])) utils.printf('') utils.printf('- Sum all betas: {}'.format(sum(self.betas))) utils.printf('- Min betas: {}'.format(min(self.betas))) utils.printf('- Max betas: {}'.format(max(self.betas))) utils.printf('- Average betas: {}'.format(np.average(self.betas))) utils.printf('- Std. Dev. betas: {}'.format(np.std(self.betas))) utils.printf('') utils.printf('prop >= {}: {}'.format(self.beta[0], sum([1 for b in self.betas if b >= self.beta[0] ])/float(len(self.betas)))) utils.printf('prop <= {}: {} (proportion of randomly generated correlations that were as {} as the observed)'.format(self.beta[0], sum([1 for b in self.betas if b <= self.beta[0] ])/float(len(self.betas)), 'large' if self.beta[0] >= 0 else 'small')) utils.printf('')
def main(directed): logfile = 'results-synthetic-ernos-renyi/timigs-{}.txt'.format( 'directed' if directed else 'undirected') memory = Profiling( 'Nodes {}'.format('directed' if directed else 'undirected'), 'results-synthetic-ernos-renyi/python-profiling-netsize-edgeprob{}-nperm{}-{}.png' .format(EDGEPROB, NPERMUTATIONS, 'directed' if directed else 'undirected'), False) memory.check_memory('init-{}'.format('d' if directed else 'i')) ####################################################################### # Data Matrices ####################################################################### #nnodes = np.logspace(1,7,num=7-1, endpoint=False) nnodes = np.logspace(1, 5, num=5 - 1, endpoint=False) for n in nnodes: n = int(n) fn = 'data-synthetic-ernos-renyi/nodes{}_edgeprob{}_<var>.dat'.format( n, EDGEPROB) memory.check_memory('nodes-{}'.format(n)) X1 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X1')) memory.check_memory('X1-{}'.format(n)) X2 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X2')) memory.check_memory('X2-{}'.format(n)) X3 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X3')) memory.check_memory('X3-{}'.format(n)) Y = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'Y')) memory.check_memory('Y-{}'.format(n)) X = {'X1': X1, 'X2': X2, 'X3': X3} Y = {'Y': Y} ####################################################################### # QAP ####################################################################### start_time = time.time() mrqap = MRQAP(Y=Y, X=X, npermutations=int(NPERMUTATIONS), diagonal=False, directed=directed, logfile=logfile, memory=memory) mrqap.mrqap() utils.printf( "\n--- {}, nodes {}: {} seconds ---".format( 'directed' if directed else 'undirected', n, time.time() - start_time), logfile) mrqap.summary() fn = 'results-synthetic-ernos-renyi/python-nodes{}-edgeprob{}-nperm{}-{}-<coef>.png'.format( n, EDGEPROB, NPERMUTATIONS, 'directed' if directed else 'undirected') mrqap.plot('betas', fn.replace('<coef>', 'betas')) mrqap.plot('tvalues', fn.replace('<coef>', 'tvalues')) utils.printf( '******************************************************************************\n\n', logfile) del (mrqap) return
__author__ = 'lisette.espin' ####################################################################### # References # - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf ####################################################################### ####################################################################### # Dependencies ####################################################################### import numpy as np from libs import utils from libs.qap import QAP ####################################################################### # Data # Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm ####################################################################### X = np.loadtxt('data/crudematerials.dat') Y = np.loadtxt('data/manufacturedgoods.dat') utils.printf('Crude Materials: \n{}'.format(X)) utils.printf('Manufactured Goods: \n{}'.format(Y)) np.random.seed(15843) ####################################################################### # QAP ####################################################################### qap = QAP(Y, X, 5000) qap.qap() qap.summary() qap.plot()