def main(directed):
logfile = 'results-permutations/timigs-{}.txt'.format(
'directed' if directed else 'undirected')
memory = Profiling(
'Permutations {}'.format('directed' if directed else 'undirected'),
'results-permutations/python-profiling-nperm-nodes{}-{}.png'.format(
NCOUNTRIES, 'directed' if directed else 'undirected'), True)
memory.check_memory('init-{}'.format('d' if directed else 'i'))
#######################################################################
# Data Matrices
#######################################################################
X1 = getMatrix('data-permutations/country_trade_index.txt', directed, True)
memory.check_memory('X1-{}'.format('d' if directed else 'i'))
X2 = getMatrix('data-permutations/country_distance_index.txt', directed,
True)
memory.check_memory('X2-{}'.format('d' if directed else 'i'))
X3 = getMatrix('data-permutations/country_colonial_index.txt', directed)
Y = getMatrix('data-permutations/country_lang_index.txt', directed)
memory.check_memory('Y-{}'.format('d' if directed else 'i'))
X = {'TRADE': X1, 'DISTANCE': X2, 'COLONIAL': X3}
Y = {'LANG': Y}
np.random.seed(1)
#######################################################################
# QAP
#######################################################################
perms = np.logspace(1, 7, num=7 - 1, endpoint=False)
for nperm in perms:
start_time = time.time()
mrqap = MRQAP(Y=Y,
X=X,
npermutations=int(nperm),
diagonal=False,
directed=directed,
logfile=logfile,
memory=memory)
mrqap.mrqap()
utils.printf(
"--- {}, nperm {}: {} seconds ---".format(
'directed' if directed else 'undirected', nperm,
time.time() - start_time), logfile)
mrqap.summary()
fn = 'results-permutations/python-nperm{}-{}-<coef>.png'.format(
nperm, 'directed' if directed else 'undirected')
mrqap.plot('betas', fn.replace('<coef>', 'betas'))
mrqap.plot('tvalues', fn.replace('<coef>', 'tvalues'))
utils.printf(
'******************************************************************************\n\n',
logfile)
del (mrqap)
return
def _summary_ols(self):
'''
Print the OLS summary
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary OLS (original) ===\n{}'.format(self.model.summary()), self.logfile)
utils.printf('', self.logfile)
utils.printf('# of Permutations: {}'.format(self.npermutations), self.logfile)
def _summary_ols(self):
'''
Print the OLS summary
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary OLS (original) ===\n{}'.format(self.model.summary()), self.logfile)
utils.printf('', self.logfile)
utils.printf('# of Permutations: {}'.format(self.npermutations), self.logfile)
def init(self):
'''
Generating the original OLS model. Y and Xs are flattened.
Also, the betas and tvalues dictionaries are initialized (key:independent variables, value:[])
:return:
'''
self.v[list(self.Y.keys())[0]] = self._getFlatten(list(self.Y.values())[0])
self._initCoefficients(INTERCEPT)
for k,x in self.X.items():
if k == list(self.Y.keys())[0]:
utils.printf('ERROR: Idependent variable cannot be named \'[}\''.format(list(self.Y.keys())[0]), self.logfile)
sys.exit(0)
self.v[k] = self._getFlatten(x)
self._initCoefficients(k)
self.data = pandas.DataFrame(self.v)
self.model = self._fit(self.v.keys(), self.data)
del(self.X)
def init(self):
'''
Generating the original OLS model. Y and Xs are flattened.
Also, the betas and tvalues dictionaries are initialized (key:independent variables, value:[])
:return:
'''
self.v[self.Y.keys()[0]] = self._getFlatten(self.Y.values()[0])
self._initCoefficients(INTERCEPT)
for k,x in self.X.items():
if k == self.Y.keys()[0]:
utils.printf('ERROR: Idependent variable cannot be named \'[}\''.format(self.Y.keys()[0]), self.logfile)
sys.exit(0)
self.v[k] = self._getFlatten(x)
self._initCoefficients(k)
self.data = pandas.DataFrame(self.v)
self.model = self._fit(self.v.keys(), self.data)
del(self.X)
def correlation(self, x, y, show=True):
'''
Computes Pearson's correlation value of variables x and y.
Diagonal values are removed.
:param x: numpy array independent variable
:param y: numpu array dependent variable
:param show: if True then shows pearson's correlation and p-value.
:return:
'''
if not self.diagonal:
xflatten = np.delete(x, [i*(x.shape[0]+1)for i in range(x.shape[0])])
yflatten = np.delete(y, [i*(y.shape[0]+1)for i in range(y.shape[0])])
pc = pearsonr(xflatten, yflatten)
else:
pc = pearsonr(x.flatten(), y.flatten())
if show:
utils.printf('Pearson Correlation: {}'.format(pc[0]))
utils.printf('p-value: {}'.format(pc[1]))
return pc
def main(directed):
logfile = 'results-synthetic-ernos-renyi/timigs-{}.txt'.format('directed' if directed else 'undirected')
memory = Profiling('Nodes {}'.format('directed' if directed else 'undirected'), 'results-synthetic-ernos-renyi/python-profiling-netsize-edgeprob{}-nperm{}-{}.png'.format(EDGEPROB, NPERMUTATIONS,'directed' if directed else 'undirected'), False)
memory.check_memory('init-{}'.format('d' if directed else 'i'))
#######################################################################
# Data Matrices
#######################################################################
#nnodes = np.logspace(1,7,num=7-1, endpoint=False)
nnodes = np.logspace(1,5,num=5-1, endpoint=False)
for n in nnodes:
n = int(n)
fn = 'data-synthetic-ernos-renyi/nodes{}_edgeprob{}_<var>.dat'.format(n,EDGEPROB)
memory.check_memory('nodes-{}'.format(n))
X1 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X1'))
memory.check_memory('X1-{}'.format(n))
X2 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X2'))
memory.check_memory('X2-{}'.format(n))
X3 = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','X3'))
memory.check_memory('X3-{}'.format(n))
Y = generateGraph(n,EDGEPROB,directed, fn.replace('<var>','Y'))
memory.check_memory('Y-{}'.format(n))
X = {'X1':X1, 'X2':X2, 'X3':X3}
Y = {'Y':Y}
#######################################################################
# QAP
#######################################################################
start_time = time.time()
mrqap = MRQAP(Y=Y, X=X, npermutations=int(NPERMUTATIONS), diagonal=False, directed=directed, logfile=logfile, memory=memory)
mrqap.mrqap()
utils.printf("\n--- {}, nodes {}: {} seconds ---".format('directed' if directed else 'undirected', n, time.time() - start_time), logfile)
mrqap.summary()
fn = 'results-synthetic-ernos-renyi/python-nodes{}-edgeprob{}-nperm{}-{}-<coef>.png'.format(n, EDGEPROB, NPERMUTATIONS,'directed' if directed else 'undirected')
mrqap.plot('betas',fn.replace('<coef>','betas'))
mrqap.plot('tvalues',fn.replace('<coef>','tvalues'))
utils.printf('******************************************************************************\n\n', logfile)
del(mrqap)
return
def _summary_tvalues(self):
'''
Summary t-values
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary T-Values ===', self.logfile)
utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','T-TEST','As Large', 'As Small'), self.logfile)
for k,v in self.tvalues.items():
tstats = self.model.tvalues[k]
aslarge = sum([1 for c in v if c >= tstats]) / float(len(v))
assmall = sum([1 for c in v if c <= tstats]) / float(len(v))
utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[int(len(v)/2)],sum(v)/len(v),max(v),round(np.std(v),6),round(float(tstats),2),aslarge,assmall), self.logfile)
def _summary_tvalues(self):
'''
Summary t-values
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary T-Values ===', self.logfile)
utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','T-TEST','As Large', 'As Small'), self.logfile)
for k,v in self.tvalues.items():
tstats = self.model.tvalues[k]
aslarge = sum([1 for c in v if c >= tstats]) / float(len(v))
assmall = sum([1 for c in v if c <= tstats]) / float(len(v))
utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[len(v)/2],sum(v)/len(v),max(v),round(np.std(v),6),round(float(tstats),2),aslarge,assmall), self.logfile)
def _summary_betas(self):
'''
Summary of beta coefficients
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary beta coefficients ===', self.logfile)
utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','B.COEFF.','As Large', 'As Small', 'P-VALUE'), self.logfile)
for k,v in self.betas.items():
beta = self.model.params[k]
pstats = self.model.pvalues[k]
aslarge = sum([1 for c in v if c >= beta]) / float(len(v))
assmall = sum([1 for c in v if c <= beta]) / float(len(v))
utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[len(v)/2],sum(v)/len(v),max(v),round(np.std(v),6),beta,aslarge,assmall,round(float(pstats),2)), self.logfile)
def _summary_betas(self):
'''
Summary of beta coefficients
:return:
'''
utils.printf('', self.logfile)
utils.printf('=== Summary beta coefficients ===', self.logfile)
utils.printf('{:20s}{:>10s}{:>10s}{:>10s}{:>10s}{:>12s}{:>12s}{:>12s}{:>12s}{:>12s}'.format('INDEPENDENT VAR.','MIN','MEDIAN','MEAN','MAX','STD. DEV.','B.COEFF.','As Large', 'As Small', 'P-VALUE'), self.logfile)
for k,v in self.betas.items():
beta = self.model.params[k]
pstats = self.model.pvalues[k]
aslarge = sum([1 for c in v if c >= beta]) / float(len(v))
assmall = sum([1 for c in v if c <= beta]) / float(len(v))
utils.printf('{:20s}{:10f}{:10f}{:10f}{:10f}{:12f}{:12f}{:12f}{:12f}{:12f}'.format(k,min(v),sorted(v)[int(len(v)/2)],sum(v)/len(v),max(v),round(np.std(v),6),beta,aslarge,assmall,round(float(pstats),2)), self.logfile)
def main(directed):
logfile = 'results-permutations/timigs-{}.txt'.format('directed' if directed else 'undirected')
memory = Profiling('Permutations {}'.format('directed' if directed else 'undirected'), 'results-permutations/python-profiling-nperm-nodes{}-{}.png'.format(NCOUNTRIES,'directed' if directed else 'undirected'), True)
memory.check_memory('init-{}'.format('d' if directed else 'i'))
#######################################################################
# Data Matrices
#######################################################################
X1 = getMatrix('data-permutations/country_trade_index.txt',directed,True)
memory.check_memory('X1-{}'.format('d' if directed else 'i'))
X2 = getMatrix('data-permutations/country_distance_index.txt',directed,True)
memory.check_memory('X2-{}'.format('d' if directed else 'i'))
X3 = getMatrix('data-permutations/country_colonial_index.txt',directed)
Y = getMatrix('data-permutations/country_lang_index.txt',directed)
memory.check_memory('Y-{}'.format('d' if directed else 'i'))
X = {'TRADE':X1, 'DISTANCE':X2, 'COLONIAL':X3}
Y = {'LANG':Y}
np.random.seed(1)
#######################################################################
# QAP
#######################################################################
perms = np.logspace(1,7,num=7-1, endpoint=False)
for nperm in perms:
start_time = time.time()
mrqap = MRQAP(Y=Y, X=X, npermutations=int(nperm), diagonal=False, directed=directed, logfile=logfile, memory=memory)
mrqap.mrqap()
utils.printf("--- {}, nperm {}: {} seconds ---".format('directed' if directed else 'undirected', nperm, time.time() - start_time), logfile)
mrqap.summary()
fn = 'results-permutations/python-nperm{}-{}-<coef>.png'.format(nperm,'directed' if directed else 'undirected')
mrqap.plot('betas', fn.replace('<coef>','betas'))
mrqap.plot('tvalues', fn.replace('<coef>','tvalues'))
utils.printf('******************************************************************************\n\n', logfile)
del(mrqap)
return
def _ttest(self):
utils.printf('')
utils.printf('========== T-TEST ==========')
utils.printf('{:25s} {:25s} {:25s} {:25s}'.format('IND. VAR.','COEF.','T-STAT','P-VALUE'))
ts = {}
lines = {}
for k,vlist in self.betas.items():
t = stats.ttest_1samp(vlist,self.model.params[k])
ts[k] = abs(round(float(t[0]),6))
lines[k] = '{:20s} {:25f} {:25f} {:25f}'.format(k,self.model.params[k],round(float(t[0]),6),round(float(t[1]),6))
ts = utils.sortDictByValue(ts,True)
for t in ts:
utils.printf(lines[t[0]])
def _ttest(self):
utils.printf('')
utils.printf('========== T-TEST ==========')
utils.printf('{:25s} {:25s} {:25s} {:25s}'.format('IND. VAR.','COEF.','T-STAT','P-VALUE'))
ts = {}
lines = {}
for k,vlist in self.betas.items():
t = stats.ttest_1samp(vlist,self.model.params[k])
ts[k] = abs(round(float(t[0]),6))
lines[k] = '{:20s} {:25f} {:25f} {:25f}'.format(k,self.model.params[k],round(float(t[0]),6),round(float(t[1]),6))
ts = utils.sortDictByValue(ts,True)
for t in ts:
utils.printf(lines[t[0]])
#######################################################################
# References
# - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf
#######################################################################
#######################################################################
# Dependencies
#######################################################################
import numpy as np
from libs import utils
from libs.qap import QAP
#######################################################################
# Data
# Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm
#######################################################################
X = minfo = np.loadtxt('data/friendship.dat')
Y = np.loadtxt('data/advice.dat')
utils.printf('Friendship: \n{}'.format(X))
utils.printf('Advise: \n{}'.format(Y))
np.random.seed(831)
#######################################################################
# QAP
#######################################################################
qap = QAP(Y, X, 2000)
qap.qap()
qap.summary()
qap.plot()
__author__ = 'lisette.espin'
#######################################################################
# References
# - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf
#######################################################################
#######################################################################
# Dependencies
#######################################################################
import numpy as np
from libs import utils
from libs.qap import QAP
#######################################################################
# Data
# Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm
#######################################################################
minfo = minfo = np.loadtxt('data/info.dat')
mmoney = np.loadtxt('data/money.dat')
utils.printf('Information: \n{}'.format(minfo))
utils.printf('Money Exchange: \n{}'.format(mmoney))
#######################################################################
# QAP
#######################################################################
qap = QAP(minfo, mmoney, 5000)
qap.qap()
qap.summary()
__author__ = 'lisette.espin'
#######################################################################
# References
# - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf
#######################################################################
#######################################################################
# Dependencies
#######################################################################
import numpy as np
from libs import utils
from libs.qap import QAP
#######################################################################
# Data
# Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm
#######################################################################
minfo = minfo = np.loadtxt('data/info.dat')
mmoney = np.loadtxt('data/money.dat')
utils.printf('Information: \n{}'.format(minfo))
utils.printf('Money Exchange: \n{}'.format(mmoney))
#######################################################################
# QAP
#######################################################################
qap = QAP(minfo, mmoney, 5000)
qap.qap()
qap.summary()
def stats(self, x, y):
if not self.diagonal:
xflatten = np.delete(x, [i*(x.shape[0]+1)for i in range(x.shape[0])])
yflatten = np.delete(y, [i*(y.shape[0]+1)for i in range(y.shape[0])])
p = np.corrcoef(xflatten,yflatten)
utils.printf('Pearson\'s correlation:\n{}'.format(p))
utils.printf('Z-Test:{}'.format(ztest(xflatten, yflatten)))
utils.printf('T-Test:{}'.format(ttest_ind(xflatten, yflatten)))
else:
p = np.corrcoef(x, y)
utils.printf('Pearson\'s correlation:\n{}'.format(p))
utils.printf('Z-Test:{}'.format(ztest(x, y)))
utils.printf('T-Test:{}'.format(ttest_ind(x, y)))
def summary(self):
utils.printf('')
utils.printf('# Permutations: {}'.format(self.npermutations))
utils.printf('Correlation coefficients: Obs. Value({}), Significance({})'.format(self.beta[0], self.beta[1]))
utils.printf('')
utils.printf('- Sum all betas: {}'.format(sum(self.betas)))
utils.printf('- Min betas: {}'.format(min(self.betas)))
utils.printf('- Max betas: {}'.format(max(self.betas)))
utils.printf('- Average betas: {}'.format(np.average(self.betas)))
utils.printf('- Std. Dev. betas: {}'.format(np.std(self.betas)))
utils.printf('')
utils.printf('prop >= {}: {}'.format(self.beta[0], sum([1 for b in self.betas if b >= self.beta[0] ])/float(len(self.betas))))
utils.printf('prop <= {}: {} (proportion of randomly generated correlations that were as {} as the observed)'.format(self.beta[0], sum([1 for b in self.betas if b <= self.beta[0] ])/float(len(self.betas)), 'large' if self.beta[0] >= 0 else 'small'))
utils.printf('')
def main(directed):
logfile = 'results-synthetic-ernos-renyi/timigs-{}.txt'.format(
'directed' if directed else 'undirected')
memory = Profiling(
'Nodes {}'.format('directed' if directed else 'undirected'),
'results-synthetic-ernos-renyi/python-profiling-netsize-edgeprob{}-nperm{}-{}.png'
.format(EDGEPROB, NPERMUTATIONS,
'directed' if directed else 'undirected'), False)
memory.check_memory('init-{}'.format('d' if directed else 'i'))
#######################################################################
# Data Matrices
#######################################################################
#nnodes = np.logspace(1,7,num=7-1, endpoint=False)
nnodes = np.logspace(1, 5, num=5 - 1, endpoint=False)
for n in nnodes:
n = int(n)
fn = 'data-synthetic-ernos-renyi/nodes{}_edgeprob{}_<var>.dat'.format(
n, EDGEPROB)
memory.check_memory('nodes-{}'.format(n))
X1 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X1'))
memory.check_memory('X1-{}'.format(n))
X2 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X2'))
memory.check_memory('X2-{}'.format(n))
X3 = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'X3'))
memory.check_memory('X3-{}'.format(n))
Y = generateGraph(n, EDGEPROB, directed, fn.replace('<var>', 'Y'))
memory.check_memory('Y-{}'.format(n))
X = {'X1': X1, 'X2': X2, 'X3': X3}
Y = {'Y': Y}
#######################################################################
# QAP
#######################################################################
start_time = time.time()
mrqap = MRQAP(Y=Y,
X=X,
npermutations=int(NPERMUTATIONS),
diagonal=False,
directed=directed,
logfile=logfile,
memory=memory)
mrqap.mrqap()
utils.printf(
"\n--- {}, nodes {}: {} seconds ---".format(
'directed' if directed else 'undirected', n,
time.time() - start_time), logfile)
mrqap.summary()
fn = 'results-synthetic-ernos-renyi/python-nodes{}-edgeprob{}-nperm{}-{}-<coef>.png'.format(
n, EDGEPROB, NPERMUTATIONS,
'directed' if directed else 'undirected')
mrqap.plot('betas', fn.replace('<coef>', 'betas'))
mrqap.plot('tvalues', fn.replace('<coef>', 'tvalues'))
utils.printf(
'******************************************************************************\n\n',
logfile)
del (mrqap)
return
__author__ = 'lisette.espin'
#######################################################################
# References
# - http://www.albany.edu/faculty/kretheme/PAD637/ClassNotes/Spring%202013/Lab8.pdf
#######################################################################
#######################################################################
# Dependencies
#######################################################################
import numpy as np
from libs import utils
from libs.qap import QAP
#######################################################################
# Data
# Source: http://vlado.fmf.uni-lj.si/pub/networks/data/ucinet/ucidata.htm
#######################################################################
X = np.loadtxt('data/crudematerials.dat')
Y = np.loadtxt('data/manufacturedgoods.dat')
utils.printf('Crude Materials: \n{}'.format(X))
utils.printf('Manufactured Goods: \n{}'.format(Y))
np.random.seed(15843)
#######################################################################
# QAP
#######################################################################
qap = QAP(Y, X, 5000)
qap.qap()
qap.summary()
qap.plot()