def test_build_dependencies_graph_from_bioc(cleanup):
print('Building dependencies graph...')
lsargs = {'bioc_release': '3.3', 'view': 'software'}
packinfoargs = lsargs
dt = DepTree(bioc, lsargs, packinfoargs)
dt.build(packagenames=['RGraph2js'])
assert (len(dt._g.nodes()) > 0)
assert (len(dt._g.edges()) > 0)
print('Generating graph...')
write_gml(dt._g, GML, stringizer=literal_stringizer)
assert (os.path.exists(GML))
def gmlize_graph():
from networkx.readwrite.gml import write_gml
d = 's2902a'
path = os.path.join(fetch_buckeye_resource('Graphs'),'%s.txt'% d)
g = pickle.load(open(path,'rb'))
newg = nx.Graph()
for n in g.nodes_iter():
newg.add_node(n.pk,name=n.WordType.Label,category=n.Category.Label,time=n.Begin)
for e in g.edges_iter(data=True):
newg.add_edge(e[0].pk,e[1].pk,weight=e[2]['weight'])
f = '/home/michael/dev/Linguistics/interactive_network/data/%s.gml' % d
write_gml(newg,f)
def test_build_dependencies_graph_from_cran(cleanup):
version2dates = cran.ls_snapshots(rversion='3.2.5')
print('Building dependencies graph...')
# select the first snapshot date in the list for R-3.2.5
lsargs = {'snapshot_date': version2dates['3.2.5'][0]}
packinfoargs = lsargs
dt = DepTree(cran, lsargs, packinfoargs)
dt.build(packagenames=['chromoR'])
assert (len(dt._g.nodes()) > 0)
assert (len(dt._g.edges()) > 0)
print('Generating graph...')
write_gml(dt._g, GML, stringizer=literal_stringizer)
assert (os.path.exists(GML))
def test_build_dependencies_graph_from_renvironment(cleanup):
RHOME = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'resources/R-fake-env')
LIBRARYPATH = 'packages'
renv = REnvironment(RHOME, LIBRARYPATH)
assert (renv.baseurl == RHOME)
assert (renv.repos == [LIBRARYPATH])
print('Building dependencies graph...')
lsargs = {'packagenamesonly': True}
dt = DepTree(renv, lsargs)
dt.build()
assert (len(dt._g.nodes()) > 0)
assert (len(dt._g.edges()) > 0)
print('Generating graph...')
write_gml(dt._g, GML, stringizer=literal_stringizer)
assert (os.path.exists(GML))
def test_build_dependencies_graph_from_local_packages_repository(cleanup):
# step 1: download packages from MRAN, a particular snapshot of CRAN
packagenames = [
"ggplot2", "data.table", "plyr", "knitr", "shiny", "xts", "lattice",
"digest", "gtable", "reshape2", "scales", "proto", "MASS", "Rcpp",
"stringr", "RColorBrewer", "dichromat", "munsell", "labeling",
"colorspace", "evaluate", "formatR", "highr", "markdown", "mime",
"httpuv", "caTools", "RJSONIO", "xtable", "htmltools", "bitops", "zoo",
"SparseM", "survival", "Formula", "latticeExtra", "cluster", "maps",
"sp", "foreign", "mvtnorm", "TH.data", "sandwich", "nlme", "Matrix",
"bit", "codetools", "iterators", "timeDate", "quadprog", "Hmisc", "BH",
"quantreg", "mapproj", "hexbin", "maptools", "multcomp", "testthat",
"mgcv", "chron", "reshape", "fastmatch", "bit64", "abind", "foreach",
"doMC", "itertools", "testit", "rgl", "XML", "RCurl", "Cairo",
"timeSeries", "tseries", "fts", "tis", "KernSmooth", "R6", "acepack",
"stringi", "praise", "cli", "glue", "htmlwidgets", "crayon", "yaml",
"quantmod", "magrittr", "MatrixModels", "jsonlite"
]
cran.download_multiple('2018-02-27',
packagenames,
dest=REPOFULLPATH,
procs=10)
# step 2: create a local repository for these downloaded packages
localRepo = LocalRepository(LOCALREPO_BASE, [LOCALREPO_LIBS])
assert (localRepo.baseurl == LOCALREPO_BASE)
assert (localRepo.repos == [LOCALREPO_LIBS])
# step 3: create a dependencies graph
print('Building dependencies graph...')
lsargs = {'packagenamesonly': True}
dt = DepTree(localRepo, lsargs)
dt.build()
assert (len(dt._g.nodes()) > 0)
assert (len(dt._g.edges()) > 0)
print('Generating graph...')
write_gml(dt._g, GML, stringizer=literal_stringizer)
assert (os.path.exists(GML))
def write_gml(G, path):
"""NetworkX write_gml method.
"""
rwgml.write_gml(G, path)
import graph_tool
import graph_tool.all as gt
from networkx.generators.random_graphs import barabasi_albert_graph
from networkx.algorithms.shortest_paths.generic import average_shortest_path_length
from networkx.readwrite.gml import write_gml
import numpy as np
import time
if __name__ == '__main__':
g = barabasi_albert_graph(int(1e4), 1)
start_t = time.process_time()
average_shortest_path_length(g)
print(time.process_time() - start_t)
write_gml(g, './graph.gml')
g = gt.load_graph('./graph.gml')
start_t = time.process_time()
all_sp = gt.shortest_distance(g)
vertex_avgs = graph_tool.stats.vertex_average(g, all_sp)
avg_path = np.sum(vertex_avgs[0]) / (g.num_vertices() - 1)
print(time.process_time() - start_t)
start_t = time.process_time()
sum([sum(i) for i in gt.shortest_distance(g)
]) / (g.num_vertices()**2 - g.num_vertices())
print(time.process_time() - start_t)
def rpacks_deps_graph():
parser = argparse.ArgumentParser(
description='Generate a dependencies graph')
parser.add_argument(
'--repo',
dest='repo',
action='store',
default=None,
required=True,
help=('The repository to work with. Identified by '
'its name in the configuration file. Use \"cran\" or '
'\"bioc\" to use CRAN or Bioconductor respectively'),
) and None
parser.add_argument(
'--repoparam',
dest='repoparam',
action='store',
default=None,
required=False,
help=('Additional repository parameter. '
'For Artifactory: \"repo name\"; all defined repositories '
'will be used otherwise. '
'Bioconductor: \"release numer, view\" '
'where \"view\" can be 1 of \"software\", '
'\"experimentData\", \"annotationData\". '
'CRAN: \"snapshot date\".')) and None
parser.add_argument(
'--packages',
dest='packages',
action='store',
default=None,
required=False,
help=('Comma separated list of root packages'
' to create the graph, by default all will be included'),
) and None
parser.add_argument(
'--traverse',
dest='traverse',
action='store',
default='imports,depends,linkingto',
required=False,
help=('By default \"imports,depends,linkingto\", to traverse all '
'required packages to build the dependency graph. '
'\"suggests\" is ignored by default.'),
) and None
parser.add_argument(
'--config',
dest='config',
action='store',
default=None,
required=False,
help=('RPackUtils configuration file, required unless '
'you use CRAN or Bioconductor as repository'),
) and None
parser.add_argument(
'--out',
dest='out',
action='store',
default=None,
required=True,
help='Output file where to write the GML',
) and None
lsargs = None
packinfoargs = None
args = parser.parse_args()
packages = None
if args.packages is not None:
packages = [x.strip() for x in args.packages.split(',')]
repo = args.repo
repoparam = args.repoparam
traverse = args.traverse
out = args.out
starttime = time.time()
if repo == 'cran':
repository = CRAN()
if args.config is not None:
logger.warning('Ignoring the --conf argument '
'since CRAN will be used.')
elif repo == 'bioc':
repository = Bioconductor()
if args.config is not None:
logger.warning('Ignoring the --conf argument '
'since Bioconductor will be used.')
else:
if args.config is None:
logger.error('Please specify the configuration '
'file to use with --config')
exit(-1)
configFile = args.config
# read the configuration file
config = Config(configFile)
# create the repositories defined there
reposConfig = ReposConfig(config)
repository = reposConfig.instance(repo)
if repository is None:
logger.error('Exiting due to previous error.')
exit(-1)
# for Artifactory we need the repo name
if isinstance(repository, Artifactory):
if repoparam is None:
# logger.error(
# 'Please specify the Artifactory ' \
# 'repo/folder to use with --repoparam')
# exit(-1)
logger.info('No repository specified for Artifactory, '
'using all defined in the configuration file.')
lsargs = {'repo': None}
packinfoargs = lsargs
else:
lsargs = {'repo': repoparam}
packinfoargs = lsargs
# for Bioconductor we need the release number and the view name
if isinstance(repository, Bioconductor):
if repoparam is None:
logger.error('Please specify the Bioconductor '
'release and view to use with --repoparam')
exit(-1)
else:
repoparams = [x.strip() for x in repoparam.split(',')]
if not len(repoparams) == 2:
logger.error('Please specify the Bioconductor '
'release and view to use with '
'--repoparam=\"release,view\"')
exit(-1)
lsargs = {'bioc_release': repoparams[0], 'view': repoparams[1]}
packinfoargs = lsargs
# for CRAN we need the snapshot date
if isinstance(repository, CRAN):
if repoparam is None:
logger.error('Please specify the CRAN '
'snapshot date to use with --repoparam')
exit(-1)
else:
lsargs = {'snapshot_date': repoparam}
packinfoargs = lsargs
# for REnvironment or LocalRepository it's fine
if isinstance(repository, REnvironment) \
or isinstance(repository, LocalRepository):
if repoparam is not None:
logger.warning('Ignoring the --repoparam argument')
# traverse options (imports, depends, suggests, linkingto)
traverse_imports = ('imports' in traverse)
traverse_depends = ('depends' in traverse)
traverse_suggests = ('suggests' in traverse)
traverse_linkingto = ('linkingto' in traverse)
# construct the dependencies tree
dt = DepTree(repository, lsargs, packinfoargs, traverse_imports,
traverse_depends, traverse_suggests, traverse_linkingto)
logger.info('Building the dependencies graph ...')
dt.build(packagenames=packages)
if len(dt._g.nodes()) < 2:
logger.info('The result graph is empty!')
logger.info('No output file generated')
exit(1)
logger.info('The result graph has {} nodes and {} edges'.format(
len(dt._g.nodes()), len(dt._g.edges())))
logger.info('Writting output GML file to \"{}\" ...'.format(out))
write_gml(dt._g, out, stringizer=literal_stringizer)
endtime = time.time()
logger.info('Time elapsed: {0:.3f} seconds.'.format(endtime - starttime))
def write_gml(G, path):
"""NetworkX write_gml method.
"""
rwgml.write_gml(G, path)
SEM_adult.add_nodes_from(white_adult.nodes())
white_adult.add_nodes_from(SEM_adult.nodes())
# Compare datasets.
print 'Edges in both datasets:', binary.intersection(SEM_adult, white_adult).number_of_edges()
print 'Edges not annotated by John White:', binary.difference(SEM_adult, white_adult).number_of_edges()
print 'Edges only annotated by John White:', binary.difference(white_adult, SEM_adult).number_of_edges()
for (pre, post) in binary.difference(SEM_adult, white_adult).edges():
print pre, post, SEM_adult[pre][post]['weight']
# Export for Cytoscape.
gml.write_gml(SEM_adult, 'data/SEM_adult.gml')
combined_graph = binary.compose(SEM_adult, white_adult)
specific_to_SEM_adult = binary.difference(SEM_adult, white_adult)
specific_to_white_adult = binary.difference(white_adult, SEM_adult)
for (pre, post) in combined_graph.edges():
if specific_to_SEM_adult.has_edge(pre, post):
combined_graph[pre][post]['dataset'] = 'only_SEM_adult'
elif specific_to_white_adult.has_edge(pre, post):
combined_graph[pre][post]['dataset'] = 'only_white_adult'
else:
combined_graph[pre][post]['dataset'] = 'shared'
gml.write_gml(combined_graph, 'data/combined_graph.gml')
).cumsum(),0)
nx_dc = np.array(
[it[1] for it in sorted(nx_degree_centrality(nx_g).items())]
)
nx_pr = np.array([val for val in pagerank(nx_g).values()])
nx_hc = np.array(
[val for val in nx_harmonic_centrality(nx_g).values()]
) / nx_g.number_of_nodes()
nx_bc = np.array(
[val for val in nx_betweenness_centrality(nx_g).values()]
)
write_gml(nx_g, './graph.gml')
gt_g = gt.load_graph('./graph.gml')
gt_apl = avg_path_length(gt_g)
gt_ad = avg_degree(gt_g)
gt_gcc = gb_clus_coef(gt_g)
gt_lcc = lcl_clus_coef(gt_g)
gt_md = max_degree(gt_g)
gt_drogc = degree_ratio_of_giant_comp(gt_g)
gt_v = variance(gt_g)
gt_ap = assortativity(gt_g)
gt_aknn = avg_neighbor_corr(gt_g)
gt_dh = degree_dist(gt_g)
gt_cdh = cum_degree_dist(gt_g)
gt_dc = degree_centrality(gt_g)
gt_hc = harmonic_centrality(gt_g)
"size": 1000
}, {
"random": 0.5,
"size": 1000
}]
# Clean old files.
dir_name = "/home/amir/Studies/Computer-Science-Classes/Parameterized Algorithms"
deleted = 0
items = os.listdir(dir_name)
for item in items:
if item.endswith(".gml"):
os.remove(os.path.join(dir_name, item))
deleted += 1
if (deleted > 0):
print("Deleted " + str(deleted) + " old graph files.")
print("Generating " + str(len(graphs)) + " graphs:")
for graph in graphs:
name = "" + str(graph["size"]) + "_" + str(graph["random"])
if ("name" in graph):
name = graph["name"]
print("Generating graph " + name + "...")
G = nx.fast_gnp_random_graph(graph["size"], graph["random"])
gml.write_gml(G, name + ".gml")
print("Graph generation done, saved as " + name + ".gml")
print "Done!"
exit()
def save_graph(graph, name):
if not name.endswith(".gml"):
name += ".gml"
gml.write_gml(graph, name)
