def create_graphs(options):
tags = tags_with_hellos(options, cur=options['topo_conn'].cursor())
cursor = options['topo_conn'].cursor()
cursor.execute('SELECT host FROM addr')
hosts = pylab.flatten(cursor.fetchall())
hosts = [
h for h in hosts if h in options['nodes'] or options['nodes'] == 'all'
]
digraphx = nx.DiGraph()
digraphx.add_nodes_from(hosts)
graphs = list()
for i, (tag_key, tag_id, helloSize) in enumerate(tags):
logging.info('tag_id=\"%s\" (%d/%d)', tag_id, i + 1, len(tags))
if helloSize not in options['gossip_params'][
'pkg_size'] and options['gossip_params']['pkg_size'][0] != 0:
logging.debug('\t skipping size = %d', helloSize)
continue
links = cursor.execute(
'''
SELECT src, host, pdr
FROM eval_helloPDR
WHERE tag_key=?
''', (tag_key, )).fetchall()
digraph = digraphx.copy()
for src, host, pdr in links:
if src in hosts and host in hosts:
digraph.add_edge(src, host, weight=pdr)
graphs.append((helloSize, digraph))
return graphs
def create_graphs(options):
tags = tags_with_hellos(options, cur=options['topo_conn'].cursor())
cursor = options['topo_conn'].cursor()
cursor.execute('SELECT host FROM addr')
hosts = pylab.flatten(cursor.fetchall())
hosts = [h for h in hosts if h in options['nodes'] or options['nodes'] == 'all']
digraphx = nx.DiGraph()
digraphx.add_nodes_from(hosts)
graphs = list()
for i, (tag_key, tag_id, helloSize) in enumerate(tags):
logging.info('tag_id=\"%s\" (%d/%d)', tag_id, i+1, len(tags))
if helloSize not in options['gossip_params']['pkg_size'] and options['gossip_params']['pkg_size'][0] != 0:
logging.debug('\t skipping size = %d', helloSize)
continue
links = cursor.execute('''
SELECT src, host, pdr
FROM eval_helloPDR
WHERE tag_key=?
''', (tag_key,)).fetchall()
digraph = digraphx.copy()
for src, host, pdr in links:
if src in hosts and host in hosts:
digraph.add_edge(src, host, weight=pdr)
graphs.append((helloSize, digraph))
return graphs
def create_graph_db(options):
logging.info('creating graph')
mode = options['mode']
cursor = options['db_conn'].cursor()
hosts = list(
pylab.flatten(
cursor.execute(
'SELECT DISTINCT(host) FROM addr ORDER BY host').fetchall()))
if mode in ['du', 'dw']:
protoGraph = nx.DiGraph()
else:
protoGraph = nx.Graph()
protoGraph.add_nodes_from(hosts)
tag_ids = cursor.execute(
'SELECT key, helloSize FROM tag where helloSize = ? ORDER BY helloSize',
(options['size'], )).fetchall()
if len(tag_ids) <= 0:
sizes = cursor.execute(
'SELECT DISTINCT(helloSize) FROM tag ORDER BY helloSize').fetchall(
)
raise Exception('Packet size not found in db; available: [%s]' %
', '.join([str(s[0]) for s in sizes]))
graphs = list()
for key, helloSize in tag_ids:
G = protoGraph.copy()
cursor.execute(
'''SELECT src,host,pdr FROM eval_helloPDR WHERE tag_key=?''',
(key, ))
for src, host, pdr in cursor.fetchall():
G.add_edge(host, src, weight=pdr)
graphs.append(G)
if len(graphs) == 0:
logging.critical('no graph created')
return graphs
def fuzzy_block_conductance(sg, off1, off2):
"""Computes the effective conductance of the fuzzy-block in the direction
off1-off2.
INPUT: sg: Fuzzy-block subgraph in iGraph format (see fuzzy_block_subgraph).
off1: First offset as a tuple / list , e.g. (-1,0,0) which would be
the neighboring block in -x direction.
off2: Second offset.
OUTPUT: Effective conductance in off1-off2 direction
"""
if 'pBC' in sg.vs.attribute_names():
del sg.vs['pBC']
vIn = sg.vs(xoffset_eq=off1[0], yoffset_eq=off1[1],
zoffset_eq=off1[2]).indices
vOut = sg.vs(xoffset_eq=off2[0], yoffset_eq=off2[1],
zoffset_eq=off2[2]).indices
if min(len(vIn), len(vOut)) == 0:
log.error("Cannot compute effective conductance")
return
if 'conductance' not in sg.es.attribute_names():
log.warning("Adding uniform conductance to all edges of the subgraph.")
sg.es['conductance'] = sp.ones(sg.ecount())
sg.vs(vIn)['pBC'] = sp.ones(len(vIn)) * 2.0
sg.vs(vOut)['pBC'] = sp.ones(len(vOut)) * 1.0
LS = linearSystem.LinearSystem(sg)
LS.solve('direct')
flow = sum(pl.flatten([sg.es(sg.adjacent(x, 'all'))['flow'] for x in vIn]))
return flow # G = F / dp, here dp == 1.0
def divide_into_slabs(G, slabThickness, overlap):
"""Divides a VascularGraph into subgraphs that are slabs along the z-axis.
INPUT: G: VascularGraph to be cut into slabs.
slabThickness: The thickness (in z-direction) of the resulting
slabs.
overlap: The overlap of the slabs.
OUTPUT: A list of slab subgraphs.
"""
minima, maxima, lengths = G.dimension_extrema()
zLength = lengths[2]
nSlabs = int(np.ceil(zLength / (slabThickness-overlap)))
G.vs['z'] = [r[2] for r in G.vs['r']]
zMin = minima[2]
zMax = zMin + slabThickness
slabSubgraphs = []
for slab in xrange(nSlabs):
slabVertices = G.vs(z_ge=zMin, z_lt=zMax)
SG = G.subgraph(np.unique(pylab.flatten([[G.neighbors(v.index)
for v in slabVertices],
slabVertices.indices])).tolist())
del SG.vs['z']
slabSubgraphs.append(SG)
zMin = zMax - overlap
zMax = zMin + slabThickness
del G.vs['z']
return slabSubgraphs
def create_graph_db(options):
logging.info('creating graph')
mode = options['mode']
cursor = options['db_conn'].cursor()
hosts = list(pylab.flatten(cursor.execute('SELECT DISTINCT(host) FROM addr ORDER BY host').fetchall()))
if mode in ['du', 'dw']:
protoGraph = nx.DiGraph()
else:
protoGraph = nx.Graph()
protoGraph.add_nodes_from(hosts)
tag_ids = cursor.execute('SELECT key, helloSize FROM tag where helloSize = ? ORDER BY helloSize', (options['size'],)).fetchall()
if len(tag_ids) <= 0:
sizes = cursor.execute('SELECT DISTINCT(helloSize) FROM tag ORDER BY helloSize').fetchall()
raise Exception('Packet size not found in db; available: [%s]' % ', '.join([str(s[0]) for s in sizes]))
graphs = list()
for key, helloSize in tag_ids:
G = protoGraph.copy()
cursor.execute('''SELECT src,host,pdr FROM eval_helloPDR WHERE tag_key=?''', (key,))
for src, host, pdr in cursor.fetchall():
G.add_edge(host, src, weight=pdr)
graphs.append(G)
if len(graphs) == 0:
logging.critical('no graph created')
return graphs
def exportDBToNED(options):
def write_nodes(f):
for i, h in enumerate(sorted(hosts)):
f.write(' %s: Gossip { @display (\"t=%s\"); id=%d; } \n' % (h.replace('-', '_'), h, i))
f.write(' connections:\n')
logging.info('exporting as NED')
c = options['db_conn'].cursor()
hosts = list(pylab.flatten(c.execute('SELECT DISTINCT(host) FROM addr ORDER BY host').fetchall()))
tag_ids = c.execute('SELECT key, helloSize FROM tag ORDER BY helloSize').fetchall()
for tag, helloSize in tag_ids:
undirected_unweighted = open('./%s/uu-%d.ned' % (options['outdir'], helloSize), 'w')
directed_unweighted = open('./%s/du-%d.ned' % (options['outdir'], helloSize), 'w')
directed_weighted = open('./%s/dw-%d.ned' % (options['outdir'], helloSize), 'w')
files = [undirected_unweighted, directed_unweighted, directed_weighted]
for f in files:
f.write('%s' % ned_head)
write_nodes(f)
c.execute('''SELECT src, host, pdr FROM eval_helloPDR WHERE tag_key=?''', (tag,))
added = list()
for src, host, pdr in c.fetchall():
src = src.replace('-', '_')
host = host.replace('-', '_')
directed_weighted.write(' %s.out++ --> Channel { per = %.3f; } --> %s.in++;\n' % (src, 1-pdr, host))
directed_unweighted.write(' %s.out++ --> Channel --> %s.in++;\n' % (src, host))
if (src, host) not in added and (host, src) not in added:
undirected_unweighted.write(' %s.out++ --> Channel --> %s.in++;\n' % (src, host))
undirected_unweighted.write(' %s.in++ <-- Channel <-- %s.out++;\n' % (src, host))
added.append((src, host))
for f in files:
f.write('}\n')
def get_html_tissue(self):
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools")
+ "/gdsctools/data/templates/")
template = env.get_template("boxplot_tissue.html")
data = self._get_boxplot_data("tissue")
if data is None:
return ""
# Show from bottom to top
labels = data[1][::-1]
data = data[0][::-1]
jinja = {}
N = len(self.odof.negatives) + len(self.odof.positives)
jinja["title"] = "FEATURE/MS-instability interactions"
jinja["subtitle"] = "%s versus %s" % (self.drug, self.feature)
factor = []
for i, thisdata in enumerate(data):
factor.extend( [labels[i]] * len(thisdata))
jinja['sign'] = [x.split()[1] for x in factor]
jinja['status'] = [x.split()[0] for x in factor]
jinja["data"] = list(pylab.flatten([list(this) for this in data]))
jinja['xlabel'] = '"logIC50"'
if len(labels)/2 >= 10:
jinja["minTextSize"] = 10
html = template.render(jinja)
return html
def fuzzy_block_conductance(sg,off1,off2):
"""Computes the effective conductance of the fuzzy-block in the direction
off1-off2.
INPUT: sg: Fuzzy-block subgraph in iGraph format (see fuzzy_block_subgraph).
off1: First offset as a tuple / list , e.g. (-1,0,0) which would be
the neighboring block in -x direction.
off2: Second offset.
OUTPUT: Effective conductance in off1-off2 direction
"""
if 'pBC' in sg.vs.attribute_names():
del sg.vs['pBC']
vIn = sg.vs(xoffset_eq=off1[0],
yoffset_eq=off1[1],
zoffset_eq=off1[2]).indices
vOut = sg.vs(xoffset_eq=off2[0],
yoffset_eq=off2[1],
zoffset_eq=off2[2]).indices
if min(len(vIn),len(vOut)) == 0:
log.error("Cannot compute effective conductance")
return
if 'conductance' not in sg.es.attribute_names():
log.warning("Adding uniform conductance to all edges of the subgraph.")
sg.es['conductance'] = sp.ones(sg.ecount())
sg.vs(vIn)['pBC'] = sp.ones(len(vIn)) * 2.0
sg.vs(vOut)['pBC'] = sp.ones(len(vOut)) * 1.0
LS = linearSystem.LinearSystem(sg)
LS.solve('direct')
flow = sum(pl.flatten([sg.es(sg.adjacent(x,'all'))['flow'] for x in vIn]))
return flow # G = F / dp, here dp == 1.0
def vertices_from_coordinates(G, coordinates, diameter_ll=0.0,
isEndpoint=False):
"""Given a list of x,y,z coordinates, locate the most closely matching set
of verties in a vascular graph of iGraph format.
INPUT: G: Vascular graph in iGraph format.
coordinates: List of lists specifying the coordinates (i.e.:
[[x1,y1,z1],[x2,y2,z2],...]).
diameter_ll: (Optional) lower limit of edge diameter, to select only
those vertices bordering a sufficiently large diameter
edge. Default is 0.0, i.e. all vertices are considered.
isEndpoint: Boolean whether or not the vertex searched for is
required to be an endpoint. Default is 'False'.
OUTPUT: vertex_indices: Array of vertex indices that represent the best
matches.
distances: Array of distances that the best matching vertices are
separated from the supplied coordinates. Units match
those of the graph vertex coordinates.
"""
# Select vertex indices based on diameter of adjacent edges:
si = unique(flatten([G.es[x].tuple for x in
G.es(diameter_ge=diameter_ll).indices])).tolist()
# Optionally filter for end-points:
if isEndpoint:
si = [i for i in si if G.degree(i) == 1]
# Construct k-dimensional seach-tree:
kdt = kdtree.KDTree(G.vs[si]['r'], leafsize=10)
search_result = kdt.query(coordinates)
sr_v = np.ravel([search_result[1]]).tolist()
vertex_indices = [si[x] for x in sr_v]
distances = np.ravel([search_result[0]]).tolist()
return vertex_indices, distances
def build_data_in(dm3, data_type, model_num):
# find standard error and use it for standard deviation
dm3 = mu.create_uncertainty(dm3, "log_normal")
# create data file
data_in = empty_data_in(dm3.input_data.index)
# add covariates
cov = dm3.input_data.filter(like="x_")
data_in = data_in.join(pandas.DataFrame(cov, columns=[""]))
cov_z = dm3.input_data.filter(like="z_")
if len(cov_z.columns) != 0:
data_in = data_in.join(pandas.DataFrame(cov_z, columns=[""]))
# add data
data_in["integrand"] = convert_data_type(data_type)
data_in["meas_value"] = dm3.input_data["value"]
data_in["meas_stdev"] = dm3.input_data["standard_error"]
data_in["sex"] = dm3.input_data["sex"]
data_in["age_lower"] = dm3.input_data["age_start"]
data_in["age_upper"] = dm3.input_data["age_end"] + 1.0
data_in["time_lower"] = dm3.input_data["year_start"]
data_in["time_upper"] = dm3.input_data["year_end"] + 1.0
data_in["x_sex"] = dm3.input_data["sex"].map(dict(male=0.5, female=-0.5, total=0))
# create data hierarchy
model = mu.load_new_model(model_num, "all", data_type)
superregion = set(model.hierarchy.neighbors("all"))
region = set(pl.flatten([model.hierarchy.neighbors(sr) for sr in model.hierarchy.neighbors("all")]))
country = set(
pl.flatten(
[
[model.hierarchy.neighbors(r) for r in model.hierarchy.neighbors(sr)]
for sr in model.hierarchy.neighbors("all")
]
)
)
# create data area levels
for i in dm3.input_data.index:
if dm3.input_data.ix[i, "area"] in country:
data_in.ix[i, "m_sub"] = dm3.input_data.ix[i, "area"]
data_in.ix[i, "m_region"] = model.hierarchy.in_edges(dm3.input_data.ix[i, "area"])[0][0]
data_in.ix[i, "m_super"] = model.hierarchy.in_edges(
model.hierarchy.in_edges(dm3.input_data.ix[i, "area"])[0][0]
)[0][0]
elif dm3.input_data.ix[i, "area"] in region:
data_in.ix[i, "m_region"] = dm3.input_data.ix[i, "area"]
data_in.ix[i, "m_super"] = model.hierarchy.in_edges(dm3.input_data.ix[i, "area"])[0][0]
elif dm3.input_data.ix[i, "area"] in superregion:
data_in.ix[i, "m_super"] = dm3.input_data.ix[i, "area"]
return data_in
def fuzzy_block_subgraph(G, sideLengths, origin, **kwargs):
"""Extracts a cuboid subgraph from the input graph. This also includes the
neighboring nodes that connect to nodes inside the cuboid, hence the name
fuzzy-block.
INPUT: G: Vascular graph in iGraph format.
sideLengths: Dimensions of the block (cuboid) as a tuple/list.
origin: The minimum x,y,z of the cuboid as tuple/list.
**kwargs:
dRange: Range of diameters to be included (edges with diameters
outside of this range are neglected). Provide as a
tuple/list, i.e. [minDiameter, maxDiameter].
OUTPUT: sg: Fuzzy-block subgraph. Additional vertex properties are added,
namely offsets in x, y, and z direction compared to the cuboid.
I.e. xoffset can take the values -1, 0, 1 corresponding to
positions in x-direction smaller than, equal to, or larger than
the cuboid x-dimensions. Analogous for yoffset, and zoffset.
"""
if len(sideLengths) == 1:
sideLengths = [sideLengths[0] for i in xrange(3)]
xMin = origin[0]
xMax = xMin + sideLengths[0]
yMin = origin[1]
yMax = yMin + sideLengths[1]
zMin = origin[2]
zMax = zMin + sideLengths[2]
block = G.vs(lambda x: xMin <= x['r'][0] <= xMax,
lambda y: yMin <= y['r'][1] <= yMax,
lambda z: zMin <= z['r'][2] <= zMax)
fbi = sp.unique(flatten([G.neighbors(x) for x in block.indices])).tolist()
log.info('Vertex indices of the subgraph: ')
log.info(fbi)
sg = G.subgraph(fbi)
if kwargs.has_key('dRange'):
sg.delete_edges(sg.es(diameter_lt=kwargs['dRange'][0]).indices)
sg.delete_edges(sg.es(diameter_gt=kwargs['dRange'][1]).indices)
log.info("Subgraph has %i components" % len(sg.components()))
sg.vs['xoffset'] = sp.zeros(sg.vcount())
ltx = sg.vs(lambda x: x['r'][0] < xMin).indices
sg.vs(ltx)['xoffset'] = sp.ones(len(ltx)) * -1
gtx = sg.vs(lambda x: x['r'][0] > xMax).indices
sg.vs(gtx)['xoffset'] = sp.ones(len(gtx))
sg.vs['yoffset'] = sp.zeros(sg.vcount())
lty = sg.vs(lambda y: y['r'][1] < yMin).indices
sg.vs(lty)['yoffset'] = sp.ones(len(lty)) * -1
gty = sg.vs(lambda y: y['r'][1] > yMax).indices
sg.vs(gty)['yoffset'] = sp.ones(len(gty))
sg.vs['zoffset'] = sp.zeros(sg.vcount())
ltz = sg.vs(lambda z: z['r'][2] < zMin).indices
sg.vs(ltz)['zoffset'] = sp.ones(len(ltz)) * -1
gtz = sg.vs(lambda z: z['r'][2] > zMax).indices
sg.vs(gtz)['zoffset'] = sp.ones(len(gtz))
return sg
def fuzzy_block_subgraph(G, sideLengths, origin, **kwargs):
"""Extracts a cuboid subgraph from the input graph. This also includes the
neighboring nodes that connect to nodes inside the cuboid, hence the name
fuzzy-block.
INPUT: G: Vascular graph in iGraph format.
sideLengths: Dimensions of the block (cuboid) as a tuple/list.
origin: The minimum x,y,z of the cuboid as tuple/list.
**kwargs:
dRange: Range of diameters to be included (edges with diameters
outside of this range are neglected). Provide as a
tuple/list, i.e. [minDiameter, maxDiameter].
OUTPUT: sg: Fuzzy-block subgraph. Additional vertex properties are added,
namely offsets in x, y, and z direction compared to the cuboid.
I.e. xoffset can take the values -1, 0, 1 corresponding to
positions in x-direction smaller than, equal to, or larger than
the cuboid x-dimensions. Analogous for yoffset, and zoffset.
"""
if len(sideLengths) == 1:
sideLengths = [sideLengths[0] for i in xrange(3)]
xMin = origin[0]; xMax = xMin + sideLengths[0]
yMin = origin[1]; yMax = yMin + sideLengths[1]
zMin = origin[2]; zMax = zMin + sideLengths[2]
block = G.vs(lambda x: xMin <= x['r'][0] <= xMax,
lambda y: yMin <= y['r'][1] <= yMax,
lambda z: zMin <= z['r'][2] <= zMax)
fbi = sp.unique(flatten([G.neighbors(x) for x in block.indices])).tolist()
log.info('Vertex indices of the subgraph: ')
log.info(fbi)
sg = G.subgraph(fbi)
if kwargs.has_key('dRange'):
sg.delete_edges(sg.es(diameter_lt=kwargs['dRange'][0]).indices)
sg.delete_edges(sg.es(diameter_gt=kwargs['dRange'][1]).indices)
log.info("Subgraph has %i components" % len(sg.components()))
sg.vs['xoffset'] = sp.zeros(sg.vcount())
ltx = sg.vs(lambda x: x['r'][0] < xMin).indices
sg.vs(ltx)['xoffset'] = sp.ones(len(ltx)) * -1
gtx = sg.vs(lambda x: x['r'][0] > xMax).indices
sg.vs(gtx)['xoffset'] = sp.ones(len(gtx))
sg.vs['yoffset'] = sp.zeros(sg.vcount())
lty = sg.vs(lambda y: y['r'][1] < yMin).indices
sg.vs(lty)['yoffset'] = sp.ones(len(lty)) * -1
gty = sg.vs(lambda y: y['r'][1] > yMax).indices
sg.vs(gty)['yoffset'] = sp.ones(len(gty))
sg.vs['zoffset'] = sp.zeros(sg.vcount())
ltz = sg.vs(lambda z: z['r'][2] < zMin).indices
sg.vs(ltz)['zoffset'] = sp.ones(len(ltz)) * -1
gtz = sg.vs(lambda z: z['r'][2] > zMax).indices
sg.vs(gtz)['zoffset'] = sp.ones(len(gtz))
return sg
def write_header(f, helloSize, options):
c = options['db_conn'].cursor()
hello_interval = list(set(pylab.flatten(c.execute('SELECT DISTINCT(hello_interval) FROM tag').fetchall())))
channel = list(set(pylab.flatten(c.execute('SELECT DISTINCT(channel) FROM tag').fetchall())))
assert(len(hello_interval) == 1)
assert(len(channel) == 1)
times = list(pylab.flatten(c.execute('SELECT time FROM tag').fetchall()))
durations = [times[i] - times[i-1] for i in range(1, len(times))]
avr_duration = scipy.mean(durations)
f.write('# DES-Testbed topology as probed with broadcast packets\n\n')
f.write('# %-15s: %s\n' % ('Database', options['db'].split('/')[-1]))
f.write('# %-15s: %d\n' % ('Channel', channel[0]))
f.write('# %-15s: %d byte\n' % ('Packet size', helloSize))
f.write('# %-15s: %d ms\n' % ('Packet interval', hello_interval[0]))
f.write('# %-15s: %d s\n' % ('Duration', (int(avr_duration)/60)*60))
f.write('\n# column 0: Name of transmitting node\n')
f.write('# column 1: Name of receiving node\n')
f.write('# column 2: Calculated packet delivery ration (PDR) after duration\n')
f.write('\n#%9s, %10s, %10s\n' % ('SRC', 'DEST', 'PDR'))
def plot_box(self):
"""
Plots the fraction of nodes that received a particular packet from the source
as a box-and-whisker with the probability p on the x-axis.
"""
logging.debug('')
configurations_per_variant = self.configurations_per_variant
gossip_variants_count = len(self.configurations['gossip'])
colors = self.options['color'](pylab.linspace(0, 0.8, configurations_per_variant))
labels = []
for li_of_frac in self.label_info:
s = str()
for i, (param, value) in enumerate(li_of_frac):
if i > 0:
s += '\n'
s += '%s=%s' % (_cname_to_latex(param), value)
labels.append(s)
labels *= len(self.configurations['p'])
ps = list(pylab.flatten(self.configurations_per_p*[p] for p in self.configurations['p']))
#################################################################
# box plot
#################################################################
array = numpy.zeros([len(self.fraction_of_nodes[0]), self.length()])
for i, fracs in enumerate(self.fraction_of_nodes):
array[:, i] = fracs
fig = MyFig(self.options, rect=[0.1, 0.2, 0.8, 0.75], figsize=(max(self.length(), 10), 10), xlabel='Probability p', ylabel='Fraction of Nodes', aspect='auto')
fig.ax.set_ylim(0, 1)
box_dict = fig.ax.boxplot(array, notch=1, sym='rx', vert=1)
#box_dict = fig.ax.boxplot(array, notch=1, sym='rx', vert=1, patch_artist=False)
for j, box in enumerate(box_dict['boxes']):
j = (j % self.configurations_per_p)
box.set_color(colors[j])
for _flier in box_dict['fliers']:
_flier.set_color('lightgrey')
fig.ax.set_xticklabels(ps, fontsize=self.options['fontsize']*0.6)
# draw vertical line to visually mark different probabilities
for x in range(0, self.length(), self.configurations_per_p):
fig.ax.plot([x+0.5, x+0.5], [0.0, 1.0], linestyle='dotted', color='red', alpha=0.8)
#################################################################
# create some dummy elements for the legend
#################################################################
if configurations_per_variant > 1:
proxies = []
for i in range(0, configurations_per_variant):
r = Rectangle((0, 0), 1, 1, edgecolor=colors[i % configurations_per_variant], facecolor='white')
proxies.append((r, labels[i]))
fig.ax.legend([proxy for proxy, label in proxies], [label for proxy, label in proxies], loc='lower right')
self.figures['boxplot'] = fig.save('boxplot_' + str(self.data_filter))
def prior_m_area(dm3, model_num, data_type):
# create 'm_sub'/'m_region' from unique input_data['area']
prior_in = empty_prior_in(pl.unique(dm3.input_data['area']).index)
prior_in['name'] = pl.unique(dm3.input_data['area'])
prior_in['mean'] = 0.
prior_in['std'] = 1.
prior_in['lower'] = '-inf'
prior_in['upper'] = 'inf'
# create hierarchy
model = mu.load_new_model(model_num, 'all', data_type)
superregion = set(model.hierarchy.neighbors('all'))
region = set(pl.flatten([model.hierarchy.neighbors(sr) for sr in model.hierarchy.neighbors('all')]))
country = set(pl.flatten([[model.hierarchy.neighbors(r) for r in model.hierarchy.neighbors(sr)] for sr in model.hierarchy.neighbors('all')]))
# create data area levels
for i in pl.unique(dm3.input_data['area']).index:
if dm3.input_data.ix[i,'area'] in country:
prior_in.ix[i,'type'] = 'm_sub'
elif dm3.input_data.ix[i,'area'] in region:
prior_in.ix[i,'type'] = 'm_region'
elif dm3.input_data.ix[i,'area'] in superregion:
prior_in.ix[i,'type'] = 'm_super'
return prior_in
def exportDBToNED(options):
def write_nodes(f):
for i, h in enumerate(sorted(hosts)):
f.write(' %s: Gossip { @display (\"t=%s\"); id=%d; } \n' %
(h.replace('-', '_'), h, i))
f.write(' connections:\n')
logging.info('exporting as NED')
c = options['db_conn'].cursor()
hosts = list(
pylab.flatten(
c.execute(
'SELECT DISTINCT(host) FROM addr ORDER BY host').fetchall()))
tag_ids = c.execute(
'SELECT key, helloSize FROM tag ORDER BY helloSize').fetchall()
for tag, helloSize in tag_ids:
undirected_unweighted = open(
'./%s/uu-%d.ned' % (options['outdir'], helloSize), 'w')
directed_unweighted = open(
'./%s/du-%d.ned' % (options['outdir'], helloSize), 'w')
directed_weighted = open(
'./%s/dw-%d.ned' % (options['outdir'], helloSize), 'w')
files = [undirected_unweighted, directed_unweighted, directed_weighted]
for f in files:
f.write('%s' % ned_head)
write_nodes(f)
c.execute(
'''SELECT src, host, pdr FROM eval_helloPDR WHERE tag_key=?''',
(tag, ))
added = list()
for src, host, pdr in c.fetchall():
src = src.replace('-', '_')
host = host.replace('-', '_')
directed_weighted.write(
' %s.out++ --> Channel { per = %.3f; } --> %s.in++;\n' %
(src, 1 - pdr, host))
directed_unweighted.write(
' %s.out++ --> Channel --> %s.in++;\n' % (src, host))
if (src, host) not in added and (host, src) not in added:
undirected_unweighted.write(
' %s.out++ --> Channel --> %s.in++;\n' %
(src, host))
undirected_unweighted.write(
' %s.in++ <-- Channel <-- %s.out++;\n' %
(src, host))
added.append((src, host))
for f in files:
f.write('}\n')
def plot_histogram(self):
configurations_per_variant = self.configurations_per_variant
colors = self.options['color'](pylab.linspace(0, 0.8, configurations_per_variant))
labels = []
for li_of_frac in self.label_info:
s = str()
for i, (param, value) in enumerate(li_of_frac):
if i > 0:
s += '\n'
s += '%s=%s' % (_cname_to_latex(param), value)
labels.append(s)
labels *= len(self.configurations['p'])
ps = list(pylab.flatten(self.configurations_per_p*[p] for p in self.configurations['p']))
#################################################################
# histogram plot
#################################################################
fig2 = MyFig(self.options, rect=[0.1, 0.2, 0.8, 0.75], figsize=(max(self.length(), 10), 10), xlabel='Probability p', ylabel='Fraction of Nodes', aspect='auto')
patches = []
for i, fracs in enumerate(self.fraction_of_nodes):
hist, bin_edges = numpy.histogram(fracs, bins=pylab.linspace(0, 1, self.options['bins']))
hist_norm = hist/float(len(fracs))*0.4
yvals = list(bin_edges.repeat(2))[1:-1]
yvals.extend(list(bin_edges.repeat(2))[1:-1][::-1])
xvals = list((i+1+hist_norm).repeat(2))
xvals.extend(list((i+1-hist_norm).repeat(2))[::-1])
i = (i % self.configurations_per_p)
poly = Polygon(zip(xvals, yvals), edgecolor='black', facecolor=colors[i], closed=True)
patches.append(poly)
patch_collection = PatchCollection(patches, match_original=True)
fig2.ax.add_collection(patch_collection)
fig2.ax.set_xticks(range(1, self.length() + 1))
fig2.ax.set_xticklabels(ps, fontsize=self.options['fontsize']*0.6)
for x in range(0, self.length(), self.configurations_per_p):
fig2.ax.plot([x+0.5, x+0.5], [0.0, 1.0], linestyle='dotted', color='red', alpha=0.8)
fig2.ax.set_ylim(0, 1)
#################################################################
# create some dummy elements for the legend
#################################################################
if configurations_per_variant > 1:
proxies = []
#for i in range(0, len(self.configurations['gossip'])):
for i in range(0, configurations_per_variant):
r = Rectangle((0, 0), 1, 1, facecolor=colors[i % configurations_per_variant], edgecolor='black')
proxies.append((r, labels[i]))
fig2.ax.legend([proxy for proxy, label in proxies], [label for proxy, label in proxies], loc='lower right')
self.figures['histogram'] = fig2.save('histogram_' + str(self.data_filter))
def plot_feedback_loops_species(self, cmap="heat", **kargs):
"""Returns and plots species part of feedback loops
:param str cmap: a color map
:return: dictionary with key (species) and values (number of feedback loop
containing the species) pairs.
.. plot::
:include-source:
:width: 50%
from cno import XCNOGraph, cnodata
c = XCNOGraph(cnodata("PKN-ToyPB.sif"), cnodata("MD-ToyPB.csv"))
c.plot_feedback_loops_species(cmap='heat', colorbar=True)
"""
if len(self) == 0:
self.logging.warning("Empty graph")
return
data = nx.simple_cycles(self)
data = list(pylab.flatten(data))
# FIXME: may not be robust to have "and": could be a valid name
counting = [(x, data.count(x)) for x in self.nodes()
if data.count(x)!=0 and unicode(x).startswith('and') is False
and self.isand(x) is False]
for node in self.nodes():
self.node[node]['loops'] = 0
if len(counting):
M = float(max([count[1] for count in counting]))
# set a default
#for node in self.nodes():
# self.node[node]['loops'] = "#FFFFFF"
for count in counting:
#ratio_count = sm.to_rgba(count[1]/M)
ratio_count = count[1]/M
colorHex = ratio_count
#self.node[count[0]]['loops'] = colorHex
self.node[count[0]]['loops'] = ratio_count
self.node[count[0]]['style'] = 'filled,bold'
self.plot(node_attribute="loops", cmap=cmap, **kargs)
return counting
def _get_data(self):
cursor = self.options['db_conn'].cursor()
#######################################################################
# create dynamic query based on the parameters that have changed
# in different tags
#######################################################################
parameters = []
for param in self.options['configurations']:
if param not in [
'p', 'gossip'
] and len(self.options['configurations'][param]) > 1:
parameters.append(param)
parameters_str = str()
if len(parameters):
parameters_str = ', T.' + ', T.'.join(str(s) for s in parameters)
query = '''
SELECT DISTINCT(E.tag_key), T.p, T.gossip %s
FROM eval_rx_fracs_for_tag AS E JOIN tag as T
ON E.tag_key = T.key
WHERE src=\'%s\' %s
ORDER BY T.p, T.gossip %s
''' % (parameters_str, self.src, self.data_filter.sql_filter(),
parameters_str)
tags = cursor.execute(query).fetchall()
assert (len(tags))
#######################################################################
# get the fraction of nodes the received the packets from the source
# for each tag
#######################################################################
self.fraction_of_nodes = []
self.label_info = []
for tag in tags:
tag_key = tag[0]
fracs_for_p = cursor.execute(
'''
SELECT frac
FROM eval_rx_fracs_for_tag
WHERE src=? AND tag_key=?
''', (self.src, tag_key)).fetchall()
label_info = zip(parameters, tag[3::])
self.fraction_of_nodes.append(list(pylab.flatten(fracs_for_p)))
self.label_info.append(label_info)
self.tag_keys = [tag[0] for tag in tags]
def get_guess(self):
"""Random guess to initialise optimisation"""
params = {}
m = self.data.min()
M = self.data.max()
range_ = M - m
mus = [m + range_ / (self.k+1.) * i for i in range(1, self.k+1)]
params['mus'] = mus
sigma = range_ / float(self.k+1) / 2.
params['sigmas'] = [sigma] * self.k
params['pis'] = [1./self.k] * self.k
params = [ [mu,sigma,pi] for mu,sigma,pi in
zip(params['mus'], params['sigmas'], params['pis'])]
params = list(pylab.flatten(params))
return params
def get_guess(self):
"""Random guess to initialise optimisation"""
params = {}
m = self.data.min()
M = self.data.max()
range_ = M - m
mus = [m + range_ / (self.k + 1.) * i for i in range(1, self.k + 1)]
params['mus'] = mus
sigma = range_ / float(self.k + 1) / 2.
params['sigmas'] = [sigma] * self.k
params['pis'] = [1. / self.k] * self.k
params = [[mu, sigma, pi] for mu, sigma, pi in zip(
params['mus'], params['sigmas'], params['pis'])]
params = list(pylab.flatten(params))
return params
def px_combine(i_list, e_list, x_list, level, qm_levels):
""" Combine differents photoionization cross-sections if necessary and adjust the statistical weight
i_list: index list of qm_levels for the level considered
e_list: smoothed and undersampled photoionization energies
x_list: smoothed and undersampled photoionization cross-sections
"""
if len(i_list) == 1:
g_theo = qm_levels[i_list[0]].g
if MPI:
g = g_theo
else:
g = level.g
e = e_list[0]
x = g / g_theo * x_list[0]
else:
e_set = set(pl.flatten(e_list))
eilg = pl.linspace(min(pl.log10(list(e_set))), max(pl.log10(list(e_set))), 50000)
ei = 10 ** eilg
xi_list = []
for i, x in enumerate(x_list):
xi = pl.interp(ei, e_list[i], x)
xi_list.append(xi[::N_US])
g_theo = pl.sum([qm_levels[i].g for i in i_list])
if MPI:
g = g_theo
else:
g = level.g
e = ei[::N_US]
x = g / g_theo * pl.sum([xi for xi in xi_list], axis=0)
return e, x
def _get_data(self):
cursor = self.options['db_conn'].cursor()
#######################################################################
# create dynamic query based on the parameters that have changed
# in different tags
#######################################################################
parameters = []
for param in self.options['configurations']:
if param not in ['p', 'gossip'] and len(self.options['configurations'][param]) > 1:
parameters.append(param)
parameters_str = str()
if len(parameters):
parameters_str = ', T.' + ', T.'.join(str(s) for s in parameters)
query = '''
SELECT DISTINCT(E.tag_key), T.p, T.gossip %s
FROM eval_rx_fracs_for_tag AS E JOIN tag as T
ON E.tag_key = T.key
WHERE src=\'%s\' %s
ORDER BY T.p, T.gossip %s
''' % (parameters_str, self.src, self.data_filter.sql_filter(), parameters_str)
tags = cursor.execute(query).fetchall()
assert(len(tags))
#######################################################################
# get the fraction of nodes the received the packets from the source
# for each tag
#######################################################################
self.fraction_of_nodes = []
self.label_info = []
for tag in tags:
tag_key = tag[0]
fracs_for_p = cursor.execute('''
SELECT frac
FROM eval_rx_fracs_for_tag
WHERE src=? AND tag_key=?
''', (self.src, tag_key)).fetchall()
label_info = zip(parameters, tag[3::])
self.fraction_of_nodes.append(list(pylab.flatten(fracs_for_p)))
self.label_info.append(label_info)
self.tag_keys = [tag[0] for tag in tags]
def paths_to_subgraph(G,paths,filename=None):
"""Constructs a subgraph of the input-graph, based on the given paths. That
is, only nodes that are listed in the variable 'paths' will be included in
the subgraph. Optionally, the subgraph can be saved to disk in form of a
vtp-file.
INPUT: G: Vascular graph in iGraph format.
paths: Paths in the vascular graph in form of a list of vertex-
lists. Note that this list can be produced with
'all_paths_between_two vertices' or
'all_paths_of_a_given_length'.
filename: (Optional.) Name of the vtp-file to which the subgraph
should be saved.
OUTPUT: sg: Subgraph containing only those nodes in the paths variable.
"""
sg = g.subgraph(pl.flatten(paths))
if filename != None:
g_output.write_vgm(sg,filename)
return sg
def paths_to_subgraph(G,paths,filename=None):
"""Constructs a subgraph of the input-graph, based on the given paths. That
is, only nodes that are listed in the variable 'paths' will be included in
the subgraph. Optionally, the subgraph can be saved to disk in form of a
vtp-file.
INPUT: G: Vascular graph in iGraph format.
paths: Paths in the vascular graph in form of a list of vertex-
lists. Note that this list can be produced with
'all_paths_between_two vertices' or
'all_paths_of_a_given_length'.
filename: (Optional.) Name of the vtp-file to which the subgraph
should be saved.
OUTPUT: sg: Subgraph containing only those nodes in the paths variable.
"""
sg = g.subgraph(pl.flatten(paths))
if filename != None:
g_output.write_vgm(sg,filename)
return sg
def vertices_from_coordinates(G,
coordinates,
diameter_ll=0.0,
isEndpoint=False):
"""Given a list of x,y,z coordinates, locate the most closely matching set
of verties in a vascular graph of iGraph format.
INPUT: G: Vascular graph in iGraph format.
coordinates: List of lists specifying the coordinates (i.e.:
[[x1,y1,z1],[x2,y2,z2],...]).
diameter_ll: (Optional) lower limit of edge diameter, to select only
those vertices bordering a sufficiently large diameter
edge. Default is 0.0, i.e. all vertices are considered.
isEndpoint: Boolean whether or not the vertex searched for is
required to be an endpoint. Default is 'False'.
OUTPUT: vertex_indices: Array of vertex indices that represent the best
matches.
distances: Array of distances that the best matching vertices are
separated from the supplied coordinates. Units match
those of the graph vertex coordinates.
"""
# Select vertex indices based on diameter of adjacent edges:
si = unique(
flatten([G.es[x].tuple
for x in G.es(diameter_ge=diameter_ll).indices])).tolist()
# Optionally filter for end-points:
if isEndpoint:
si = [i for i in si if G.degree(i) == 1]
# Construct k-dimensional seach-tree:
kdt = kdtree.KDTree(G.vs[si]['r'], leafsize=10)
search_result = kdt.query(coordinates)
sr_v = np.ravel([search_result[1]]).tolist()
vertex_indices = [si[x] for x in sr_v]
distances = np.ravel([search_result[0]]).tolist()
return vertex_indices, distances
def modified_strahler_order_v2(G, startingOrder=1, enforceMonotony=True):
"""Assigns order numbers to the edges of a tree-structured VascularGraph.
The order is determined using the diameter-modified Strahler method, as
described in the work of Kassab et al. 'Morphometry of pig coronary arterial
trees' (Am J Physiol, 1993).
This function reassigns orders based primarily on the mean and std of the
order bins. The enforce-monotony correction, which takes the topological
ordering into account is added as a final reassingment step. Without this
correction, the diameter-ranges of the different orders (> startingOrder)
are non-overlapping.
INPUT: G: VascularGraph of tree-structure.
startingOrder: The lowest order that should be included in the
sorting process.
enforceMonotony: Whether or not to enforce that orders will never
decrease going downstream.
OUTPUT: None, the edges of G are given a new property 'order'
"""
# Begin by assigning the classical Strahler order:
classical_strahler_order(G)
# Iteratively assign modified Strahler orders until convergence:
diameters = np.array(G.es['diameter'])
preserve = G.es(order_lt=startingOrder).indices
while True:
oldOrder = G.es['order']
maxOrder = max(oldOrder)
mean = []
std = []
for order in range(maxOrder + 1):
mean.append(np.mean(G.es(order_eq=order)['diameter']))
std.append(np.std(G.es(order_eq=order)['diameter']))
bounds = {}
for order in range(1, maxOrder):
bounds[order] = ([
(mean[order - 1] + std[order - 1] + mean[order] - std[order]) /
2.0,
(mean[order] + std[order] + mean[order + 1] - std[order + 1]) /
2.0
])
bounds[0] = [0.0, bounds[1][0]]
bounds[maxOrder] = [bounds[maxOrder - 1][1], 1e100]
for order in range(startingOrder, maxOrder + 1):
eIndices = np.nonzero((diameters >= bounds[order][0]) *
(diameters < bounds[order][1]))[0].tolist()
eIndices = [e for e in eIndices if e not in preserve]
G.es[eIndices]['order'] = [order for e in eIndices]
if G.es['order'] == oldOrder:
break
# Ensure that orders do not decrease in downstream direction:
if enforceMonotony:
while True:
oldOrder = G.es['order']
for edge in G.es:
edge['order'] = max(
pylab.flatten([
edge['order'],
G.es[upstream_edges(G, edge.index)]['order']
]))
if G.es['order'] == oldOrder:
break
def estimate(self, guess=None, k=2):
"""
:param list guess: a list to provide the initial guess. Order is mu1, sigma1,
pi1, mu2, ...
:param int k: number of models to be used.
"""
#print("EM estimation")
self.k = k
# Initial guess of parameters and initializations
if guess is None:
# estimate the mu/sigma/pis from the data
guess = self.get_guess()
mu = np.array(guess[0::3])
sig = np.array(guess[1::3])
pi_ = np.array(guess[2::3])
N_ = len(pi_)
gamma = np.zeros((N_, self.size))
N_ = np.zeros(N_)
p_new = guess
# EM loop
counter = 0
converged = False
self.mus = []
while not converged:
# Compute the responsibility func. and new parameters
for k in range(0, self.k):
# unstable if eslf.model.pdf is made of zeros
if self.model.pdf(self.data, p_new,normalise=False).sum()!=0:
gamma[k, :] = pi_[k]*pylab.normpdf(self.data, mu[k],
sig[k])/(self.model.pdf(self.data, p_new, normalise=False))
else:
gamma[k, :] = pi_[k]*pylab.normpdf(self.data, mu[k],
sig[k])/(self.model.pdf(self.data, p_new,
normalise=False)+1e-6)
N_[k] = 1.*gamma[k].sum()
mu[k] = sum(gamma[k]*self.data)/N_[k]
sig[k] = pylab.sqrt( sum(gamma[k]*(self.data - mu[k])**2)/N_[k] )
#N_[k] += 1
pi_[k] = N_[k] / self.size
self.results = {'x': p_new, 'nfev':counter, 'success': converged}
p_new = [(mu[x], sig[x], pi_[x]) for x in range(0, self.k)]
p_new = list(pylab.flatten(p_new))
self.status = True
try:
assert abs(N_.sum() - self.size)/self.size < 1e-6
assert abs(pi_.sum() - 1) < 1e-6
except:
print("issue arised at iteration %s" % counter)
self.debug = {'N':N_, 'pis':pi_}
self.status = False
break
self.mus.append(mu)
# Convergence check
counter += 1
converged = counter >= self.max_iter
self.gamma = gamma
if self.status is True:
self.results = {'x': p_new, 'nfev':counter, 'success': converged}
self.results = AttrDict(**self.results)
self.results.mus = self.results.x[0::3]
self.results.sigmas = self.results.x[1::3]
self.results.pis = self.results.x[2::3]
log_likelihood = self.model.log_likelihood(self.results.x, self.data)
self.results.AIC = criteria.AIC(log_likelihood, k, logL=True)
self.results.log_likelihood = log_likelihood
self.results.AIC = criteria.AIC(log_likelihood, self.k, logL=True)
self.results.AICc = criteria.AICc(log_likelihood, self.k, self.data.size, logL=True)
self.results.BIC = criteria.BIC(log_likelihood, self.k, self.data.size, logL=True)
def makeGroupsFigure(groupsList,nameDict,fontsize=12, \
cmap=pylab.cm.Oranges,fontColor='bw',sortByMag=True,edgecolor=None, \
linewidth=0.75,numBases=None,offset=[0.,0.],newFigure=True, \
includeTimesUsed=False,maxAbsMag=None,eraseNegatives=True,sp=None, \
numNonzero=None,threshold=None,swapxy=False,relatedMatrix=None):
"""
eraseNegatives : Gets rid of any minus sign at the beginning of
a name. (This doesn't matter if each basis
vector has a single sign, which is true of the
bases I've been looking at.)
numBases : Number of bases that includes each group, in the
original (unsorted) order. The order is taken
from the indices listed in the given rules.
sp : If given sparsenessProblem, use given
groupsList as basis and order by number of
times used to reconstruct the data.
relatedMatrix : If given a relatedMatrix
(see biologicallyMeaningfulGroups.py), resort
and draw rectangles around related groups
"""
padHorz = 0.1 #0.1
padVert = 0.3
relatedLineColor = 'k'
relatedLinewidth = 4. * linewidth
padRelated = 0. #padHorz/2.
if newFigure: pylab.figure()
axes = pylab.subplot(111)
groupsThresh = thresholdMatrix(groupsList,thold=threshold, \
numNonzero=numNonzero)
xVals, yVals = [], []
if maxAbsMag is None:
#maxAbsMag = max(pylab.flatten([ rule[-1] for rule in rules ]))
maxAbsMag = max(pylab.flatten(abs(groupsThresh)))
if numBases is not None or includeTimesUsed:
offset[0] += 1 + padHorz
if sp is not None: # 4.25.2011
rules = sp.sortedRules(groupsList,numNonzero=numNonzero, \
includeIndices=True,includeMagnitudes=True,nameDict=nameDict)
indices = [rule[2] for rule in rules]
groupsThresh = groupsThresh[indices]
#groupsThresh = filteredBasis(groupsThresh[indices],0)
#print "len(groupsThresh) =",len(groupsThresh)
if eraseNegatives:
for group in groupsThresh:
filteredGroup = scipy.sort(filter(lambda x: abs(x) > 0., group))
if len(filteredGroup) > 0.:
allSameSign = 0.5*(1.+ \
scipy.sign(filteredGroup[0])*scipy.sign(filteredGroup[-1]))
if not allSameSign:
print("makeGroupsFigure: WARNING: using eraseNegatives, " \
"but group doesn't have all equal sign: group =", \
filteredGroup)
groupsThresh = abs(groupsThresh)
i = 0
#for rule in rules:
for groupIndex, group in enumerate(groupsThresh):
#mags = abs(scipy.array(rule[-1]))
mags = filter(lambda x: abs(x) > 0, group)
if len(mags) > 0:
magsR = mags / maxAbsMag * cmap.N
#names = scipy.array(rule[0][0])
nonzeroIndices = scipy.array( filter(lambda x: x>0, \
(scipy.arange(len(group))+1)*(abs(group)>0)) ) - 1
names = scipy.array([nameDict[ind] for ind in nonzeroIndices])
srt = scipy.argsort(-magsR)
if sortByMag:
magsR = magsR[srt]
names = names[srt]
nonzeroIndices = nonzeroIndices[srt]
# 4.20.2012 corral related individuals
if relatedMatrix is not None:
srt2 = []
relatedRects = []
currentPos = 0
for k in range(len(nonzeroIndices)):
numRelated = 0
if k not in srt2: # we're starting a new group
srt2.append(k)
numRelated = 1
for ell in range(k + 1, len(nonzeroIndices)):
indiv1, indiv2 = nonzeroIndices[k], nonzeroIndices[
ell]
if relatedMatrix[indiv1][indiv2]:
srt2.append(ell)
numRelated += 1
# draw rectangle around group if bigger than one
if numRelated > 1:
xloc = currentPos * (
1 + padHorz) + offset[0] - padRelated
yloc = -(i + 1) * (
1 + padVert) + offset[1] - padRelated
width = numRelated * (
padHorz + 1.) - padHorz + 2. * padRelated
height = 1 + 2. * padRelated
rect = pylab.Rectangle((xloc,yloc), \
width,height,fill=False,lw=relatedLinewidth, \
ec=relatedLineColor)
#axes.add_artist(rect)
relatedRects.append(rect)
currentPos += numRelated
# sort
magsR = magsR[srt2]
names = names[srt2]
# draw the individual squares
for j, mag in enumerate(magsR):
xloc = j * (1 + padHorz) + offset[0]
yloc = -(i + 1) * (1 + padVert) + offset[1]
if swapxy:
tmpYloc = copy.copy(yloc)
yloc = copy.copy(xloc)
xloc = -copy.copy(tmpYloc)
rect = pylab.Rectangle((xloc,yloc), \
1,1,color=cmap(int(mag)),ec=edgecolor,lw=linewidth)
axes.add_artist(rect)
if fontColor is 'auto':
fc = cmap((cmap.N / 2 + int(mag)) % cmap.N)
elif fontColor is 'bw':
blackThresh = 1.2 #1.5
if sum(cmap(int(mag))[:-1]) > blackThresh: fc = 'black'
else: fc = 'white'
else: fc = fontColor
#if eraseNegatives: n = names[j][-2:]
#else: n = names[j]
pylab.annotate("$\mathrm{"+names[j]+"}$", \
(xloc+0.5,yloc+0.5), \
color=fc,fontsize=fontsize, \
ha='center',va='center')
xVals.append(xloc)
yVals.append(yloc)
if numBases is not None:
#if scipy.iterable(rule[2]):
# raise Exception, "makeGroupsFigure: it appears the" \
# +" given rules include no indices. Cannot" \
# +" associate numBases with given groups."
pylab.annotate("$\mathrm{"+str(numBases[groupIndex])+"}$", \
(-1*(1+padHorz)+offset[0]+0.5,yloc+0.5),color='black', \
fontsize=fontsize,ha='center',va='center')
xVals.append(-1 * (1 + padHorz) + offset[0])
if includeTimesUsed:
pylab.annotate(str(rule[1]), \
(-1*(1+padHorz)+offset[0]+0.5,yloc+0.5),color='black', \
fontsize=fontsize,ha='center',va='center')
xVals.append(-1 * (1 + padHorz) + offset[0])
i += 1
# draw the related rectangles last
if relatedMatrix is not None:
for rect in relatedRects:
axes.add_artist(rect)
pylab.axis('scaled')
if not swapxy:
pylab.axis(xmin=min(xVals)-padHorz-1,xmax=max(xVals)+1+padHorz+1, \
ymin=min(yVals)-padVert-1,ymax=+1)
else:
pylab.axis(xmin=0,xmax=max(xVals)+padVert+1+1, \
ymin=min(yVals)-padHorz-1,ymax=max(yVals)+1+padHorz+1)
pylab.xticks([])
pylab.yticks([])
pylab.show()
return pylab.axis()
x = (X[:,2] - T0)/P
xx.append(x - int(x[0]))
aa.append(X[:,0])
# let phase at 0.8 -> 1.0
tpp = X[:,1]
tpp -= pl.average(tpp[0])
#tpp += 1.0
pp.append(tpp)
sa.append(X[:,3])
sp.append(X[:,4])
# now sort observations in terms of orbital phase
xx = pl.array([i for i in pl.flatten(xx)])
pp = pl.array([i for i in pl.flatten(pp)])
aa = pl.array([i for i in pl.flatten(aa)])
sa = pl.array([i for i in pl.flatten(sa)])
sp = pl.array([i for i in pl.flatten(sp)])
arg = xx.argsort()
xx = xx[arg]
pp = pp[arg]
aa = aa[arg]
sa = sa[arg]
sp = sp[arg]
# limit orb phase to [0.8,1.2]
lt = xx < 1.2
def covariate_level_constraints(name, model, vars, ages):
""" Generate PyMC objects implementing priors on the value of the covariate adjusted rate function
:Parameters:
- `name` : str
- `parameters` : dict
- `unconstrained_mu_age` : pymc.Node with values of PCGP
- `ages` : array
:Results:
- Returns dict of PyMC objects, including 'unconstrained_mu_age' and 'mu_age'
"""
if name not in model.parameters or 'level_value' not in model.parameters[name] or 'level_bounds' not in model.parameters[name]:
return {}
# X_out = model.output_template
# X_out['x_sex'] = .5
# for x_i in vars['X_shift'].index:
# X_out[x_i] = pl.array(X_out[x_i], dtype=float) - vars['X_shift'][x_i] # shift covariates so that the root node has X_ar,sr,yr == 0
# X_all = vars['X'].append(X_out.select(lambda c: c in vars['X'].columns, 1), ignore_index=True)
# X_all['x_sex'] = .5 - vars['X_shift']['x_sex']
# X_max = X_all.max()
# X_min = X_all.min()
# X_min['x_sex'] = -.5 - vars['X_shift']['x_sex'] # make sure that the range of sex covariates is included
X_sex_max = .5 - vars['X_shift']['x_sex']
X_sex_min = -.5 - vars['X_shift']['x_sex'] # make sure that the range of sex covariates is included
index_map = dict([[key, i] for i,key in enumerate(vars['X_shift'].index)])
sex_index = index_map['x_sex']
U_all = []
nodes = ['all']
for l in range(1,4):
nodes = [n for n in pl.flatten([model.hierarchy.successors(n) for n in nodes])]
U_i = pl.array([col in nodes for col in vars['U'].columns])
if U_i.sum() > 0:
U_all.append(U_i)
@mc.potential(name='covariate_constraint_%s'%name)
def covariate_constraint(mu=vars['mu_age'], alpha=vars['alpha'], beta=vars['beta'],
U_all=U_all,
X_sex_max=X_sex_max,
X_sex_min=X_sex_min,
lower=pl.log(model.parameters[name]['level_bounds']['lower']),
upper=pl.log(model.parameters[name]['level_bounds']['upper'])):
log_mu_max = pl.log(mu.max())
log_mu_min = pl.log(mu.min())
alpha = pl.array([float(x) for x in alpha])
if len(alpha) > 0:
for U_i in U_all:
log_mu_max += max(0, alpha[U_i].max())
log_mu_min += min(0, alpha[U_i].min())
# this estimate is too crude, and is causing problems
#if len(beta) > 0:
# log_mu_max += pl.sum(pl.maximum(X_max*beta, X_min*beta))
# log_mu_min += pl.sum(pl.minimum(X_max*beta, X_min*beta))
# but leaving out the sex effect results in strange problems, too
log_mu_max += X_sex_max*float(beta[sex_index])
log_mu_min += X_sex_min*float(beta[sex_index])
lower_violation = min(0., log_mu_min - lower)
upper_violation = max(0., log_mu_max - upper)
return mc.normal_like([lower_violation, upper_violation], 0., 1.e-6**-2)
return dict(covariate_constraint=covariate_constraint)
def estimate(self, guess=None, k=2):
"""
:param list guess: a list to provide the initial guess. Order is mu1, sigma1,
pi1, mu2, ...
:param int k: number of models to be used.
"""
#print("EM estimation")
self.k = k
# Initial guess of parameters and initializations
if guess is None:
# estimate the mu/sigma/pis from the data
guess = self.get_guess()
mu = np.array(guess[0::3])
sig = np.array(guess[1::3])
pi_ = np.array(guess[2::3])
N_ = len(pi_)
gamma = np.zeros((N_, self.size))
N_ = np.zeros(N_)
p_new = guess
# EM loop
counter = 0
converged = False
self.mus = []
while not converged:
# Compute the responsibility func. and new parameters
for k in range(0, self.k):
# unstable if eslf.model.pdf is made of zeros
if self.model.pdf(self.data, p_new,
normalise=False).sum() != 0:
gamma[k, :] = pi_[k] * pylab.normpdf(
self.data, mu[k], sig[k]) / (self.model.pdf(
self.data, p_new, normalise=False))
else:
gamma[k, :] = pi_[k] * pylab.normpdf(
self.data, mu[k], sig[k]) / (self.model.pdf(
self.data, p_new, normalise=False) + 1e-6)
N_[k] = 1. * gamma[k].sum()
mu[k] = sum(gamma[k] * self.data) / N_[k]
sig[k] = pylab.sqrt(
sum(gamma[k] * (self.data - mu[k])**2) / N_[k])
#N_[k] += 1
pi_[k] = N_[k] / self.size
self.results = {'x': p_new, 'nfev': counter, 'success': converged}
p_new = [(mu[x], sig[x], pi_[x]) for x in range(0, self.k)]
p_new = list(pylab.flatten(p_new))
self.status = True
try:
assert abs(N_.sum() - self.size) / self.size < 1e-6
assert abs(pi_.sum() - 1) < 1e-6
except:
print("issue arised at iteration %s" % counter)
self.debug = {'N': N_, 'pis': pi_}
self.status = False
break
self.mus.append(mu)
# Convergence check
counter += 1
converged = counter >= self.max_iter
self.gamma = gamma
if self.status is True:
self.results = {'x': p_new, 'nfev': counter, 'success': converged}
self.results = AttrDict(**self.results)
self.results.mus = self.results.x[0::3]
self.results.sigmas = self.results.x[1::3]
self.results.pis = self.results.x[2::3]
log_likelihood = self.model.log_likelihood(self.results.x, self.data)
self.results.AIC = criteria.AIC(log_likelihood, k, logL=True)
self.results.log_likelihood = log_likelihood
self.results.AIC = criteria.AIC(log_likelihood, self.k, logL=True)
self.results.AICc = criteria.AICc(log_likelihood,
self.k,
self.data.size,
logL=True)
self.results.BIC = criteria.BIC(log_likelihood,
self.k,
self.data.size,
logL=True)
def main():
"""
The main function of the plotting module
"""
logging.basicConfig(level=logging.INFO,
format='%(levelname)s [%(funcName)s] %(message)s')
try:
import psyco
psyco.full()
psyco.profile(0.0)
except ImportError:
print "INFO: psyco module not available"
parser = argparse.ArgumentParser(
description='Plot data stored in an sqlite database')
parser.add_argument(
'--topo',
required=True,
help='name of the database where the network topology is available')
parser.add_argument(
'--db',
required=True,
help='name of the database where the results shall be stored')
parser.add_argument('--clone',
default=None,
help='clone the scenarios from another experiment')
parser.add_argument('--src',
nargs='+',
default=['all'],
help='source node sending packets')
parser.add_argument('--gossip',
nargs='+',
default=['0'],
help='gossip variants')
parser.add_argument('-m', nargs='+', default=['1'], help='m')
parser.add_argument('-k',
nargs='+',
default=['1'],
help='flood on first k hops')
parser.add_argument('--probability',
'-p',
nargs='+',
default=list(pylab.linspace(0.1, 1, 10)),
help='gossip variants')
parser.add_argument('--outdir',
default='',
help='output directory for the plots')
parser.add_argument('--pkg_num',
default=100,
type=int,
help='number of packets')
parser.add_argument('--pkg_size',
nargs='+',
default=[0],
type=int,
help='select a particular graph')
parser.add_argument('--no_loss',
nargs='?',
const=True,
default=False,
help='List all callable plotting functions')
args = parser.parse_args()
options = {}
options['topo'] = args.topo
options['outdir'] = args.outdir
options['loss'] = not args.no_loss
options['db'] = args.db
options['nodes'] = 'all'
gossip_params = dict()
options['gossip_params'] = gossip_params
gossip_params['src'] = args.src
gossip_params['gossip'] = args.gossip
gossip_params['m'] = args.m
gossip_params['k'] = args.k
gossip_params['p'] = args.probability
gossip_params['pkg_num'] = args.pkg_num
gossip_params['pkg_size'] = args.pkg_size
db = options['topo']
conn = sqlite3.connect(db)
options['topo_conn'] = conn
cursor = conn.cursor()
if 'all' in args.src and not args.clone:
logging.info('all routers selected as sources')
cursor.execute('''SELECT host FROM addr''')
hosts = list(pylab.flatten(cursor.fetchall()))
options['src'] = hosts
if args.clone:
clone_conn = sqlite3.connect(args.clone)
clone_cursor = clone_conn.cursor()
clone(options, clone_cursor)
cursor = prepare_db(options)
graphs = create_graphs(options)
#for helloSize, G in graphs:
#for gossip in gossip_params['gossip']:
#for p in gossip_params['p']:
#t = (None, '', '', p, p2, k, n, m, timeout, gossip, 0, 0, 0, 0, T_MAX, p_min, p_max, helloSize)
#cursor.execute('INSERT INTO tag VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', t)
#curr_tag_key = c.lastrowid
for node in graphs[0][1]:
cursor.execute('INSERT INTO addr VALUES (?)', (node, ))
options['db_conn'].commit()
for i, pkg_size in enumerate(gossip_params['pkg_size']):
G = None
for helloSize, graph in graphs:
if helloSize == pkg_size:
G = (helloSize, graph)
break
if not G:
logging.critical('no graph found for pkg_size=%d', pkg_size)
assert (False)
logging.info('helloSize=\"%d\" (%d/%d)', G[0], i + 1, len(graphs))
simulate(G, options)
def main():
"""
The main function of the plotting module
"""
parser = argparse.ArgumentParser(description='Plot data stored in an sqlite database')
parser.add_argument('--db', default='./sqlite.db', help='name of the database where the parsed and evaluated data is stored')
parser.add_argument('--src', nargs='+', default=['all'], help='source node sending packets')
parser.add_argument('--bins', default=10, type=int, help='granularity of the 3d box plot; bin size')
parser.add_argument('--fix', nargs='+', default=[0], type=int, help='fix the number of parallel sources when routers failed [steps]')
parser.add_argument('--outdir', default='', help='output directory for the plots')
parser.add_argument('--fontsize', default=matplotlib.rcParams['font.size'], type=int, help='base font size of plots')
parser.add_argument('--username', help='username for the database access to get the nodes\' positions')
parser.add_argument('--password', help='password for the database access to get the nodes\' positions')
parser.add_argument('--dbhost', default='uhu.imp.fu-berlin.de', help='hostname for the database access to get the nodes\' positions')
parser.add_argument('--execute', '-e', nargs='+', default=['all'], help='Execute the specified functions')
parser.add_argument('--list', nargs='?', const=True, default=False, help='List all callable plotting functions')
parser.add_argument('--update', nargs='?', const=True, default=False, help='Download or update data from the testbed database')
parser.add_argument('--statistics', nargs='?', const=True, default=False, help='Calculate and plot distance statistics')
parser.add_argument('--deadline', default=40, type=float, help='Maximum time for the retransmit plot')
parser.add_argument('--restarts', default=numpy.infty, type=float, help='Maximum number of retransmsmissions to plot')
parser.add_argument('--repetitions', default=13, type=int, help='for MPR phi plotting')
parser.add_argument('--intervals', nargs='+', default=[], type=float, help='Selected intervals for the evaluation')
parser.add_argument('--special', nargs='+', default=[''], help='Special options for the plots')
parser.add_argument('--mark', nargs='+', default=[], type=int, help='Mark p_s in plots for sources')
parser.add_argument('--grayscale', nargs='?', const=True, default=False, help='Create plots with grayscale colormap')
args = parser.parse_args()
if args.list:
print('Callable plot functions:')
for func in sorted([key for key in globals().keys() if key.startswith('plot_')]):
print('\t'+func)
sys.exit(0)
logging.basicConfig(level=logging.INFO, format='%(levelname)s [%(funcName)s] %(message)s')
options = get_options()
options['db'] = args.db
options['src'] = args.src
options['bins'] = args.bins
options['outdir'] = args.outdir
options['fontsize'] = args.fontsize
options['username'] = args.username
options['dbpassword'] = args.password
options['fix'] = args.fix
options['dbhost'] = args.dbhost
options['show3d'] = False
options['statistics'] = args.statistics
options['restarts'] = args.restarts
options['deadline'] = args.deadline
options['intervals'] = args.intervals
options['special'] = args.special
options['mark'] = args.mark
options['repetitions'] = args.repetitions
options['grayscale'] = args.grayscale
db = options["db"]
conn = sqlite3.connect(db)
options['db_conn'] = conn
cursor = conn.cursor()
options['neg_rssi'] = False
logging.info('old or new rssi format?')
#max_rssi = max(cursor.execute('''SELECT rssi FROM rx WHERE NOT rssi=0 LIMIT 1''').fetchone()[0], cursor.execute('''SELECT rssi FROM rx WHERE NOT rssi=0 LIMIT 1''').fetchone()[0])
#if max_rssi>0:
#options['neg_rssi'] = False
logging.basicConfig(level=logging.INFO, format='%(levelname)s [%(funcName)s] %(message)s')
logging.info('connecting to database')
if args.update:
logging.info('updating positions from %s', options['dbhost'])
update_data(options)
sys.exit(0)
if 'all' in args.src:
logging.info('all sources selected for plotting')
try:
cursor.execute('''
CREATE TABLE eval_sources (
host TEXT NOT NULL,
FOREIGN KEY(host) REFERENCES addr(host)
)
''')
logging.info('extracting all source node names')
cursor.execute('''
INSERT INTO eval_sources
SELECT DISTINCT(host)
FROM tx
ORDER BY host
''')
conn.commit()
except sqlite3.OperationalError:
pass
sources = cursor.execute('''
SELECT DISTINCT(host)
FROM eval_sources
ORDER BY host
''').fetchall()
options['src'] = list(pylab.flatten(sources))
prepare_outdir(options)
open_latex_file(options)
eval_scenarios(options)
write_scenarios(options)
#try:
prepare_coordinates(options)
#except ValueError:
#logging.warning('no router positions found in DB')
if 'all' in args.execute:
for func in [globals()[key] for key in globals().keys() if key.startswith('plot_')]:
func(options)
else:
for func in args.execute:
try:
globals()[func](options)
except KeyError:
logging.critical('function not found: %s' % func)
raise
cursor.close()
def covariate_level_constraints(name, model, vars, ages):
""" Generate PyMC objects implementing priors on the value of the covariate adjusted rate function
:Parameters:
- `name` : str
- `parameters` : dict
- `unconstrained_mu_age` : pymc.Node with values of PCGP
- `ages` : array
:Results:
- Returns dict of PyMC objects, including 'unconstrained_mu_age' and 'mu_age'
"""
if name not in model.parameters or 'level_value' not in model.parameters[
name] or 'level_bounds' not in model.parameters[name]:
return {}
# X_out = model.output_template
# X_out['x_sex'] = .5
# for x_i in vars['X_shift'].index:
# X_out[x_i] = pl.array(X_out[x_i], dtype=float) - vars['X_shift'][x_i] # shift covariates so that the root node has X_ar,sr,yr == 0
# X_all = vars['X'].append(X_out.select(lambda c: c in vars['X'].columns, 1), ignore_index=True)
# X_all['x_sex'] = .5 - vars['X_shift']['x_sex']
# X_max = X_all.max()
# X_min = X_all.min()
# X_min['x_sex'] = -.5 - vars['X_shift']['x_sex'] # make sure that the range of sex covariates is included
X_sex_max = .5 - vars['X_shift']['x_sex']
X_sex_min = -.5 - vars['X_shift'][
'x_sex'] # make sure that the range of sex covariates is included
index_map = dict([[key, i] for i, key in enumerate(vars['X_shift'].index)])
sex_index = index_map['x_sex']
U_all = []
nodes = ['all']
for l in range(1, 4):
nodes = [
n
for n in pl.flatten([model.hierarchy.successors(n) for n in nodes])
]
U_i = pl.array([col in nodes for col in vars['U'].columns])
if U_i.sum() > 0:
U_all.append(U_i)
@mc.potential(name='covariate_constraint_%s' % name)
def covariate_constraint(
mu=vars['mu_age'],
alpha=vars['alpha'],
beta=vars['beta'],
U_all=U_all,
X_sex_max=X_sex_max,
X_sex_min=X_sex_min,
lower=pl.log(model.parameters[name]['level_bounds']['lower']),
upper=pl.log(model.parameters[name]['level_bounds']['upper'])):
log_mu_max = pl.log(mu.max())
log_mu_min = pl.log(mu.min())
alpha = pl.array([float(x) for x in alpha])
if len(alpha) > 0:
for U_i in U_all:
log_mu_max += max(0, alpha[U_i].max())
log_mu_min += min(0, alpha[U_i].min())
# this estimate is too crude, and is causing problems
#if len(beta) > 0:
# log_mu_max += pl.sum(pl.maximum(X_max*beta, X_min*beta))
# log_mu_min += pl.sum(pl.minimum(X_max*beta, X_min*beta))
# but leaving out the sex effect results in strange problems, too
log_mu_max += X_sex_max * float(beta[sex_index])
log_mu_min += X_sex_min * float(beta[sex_index])
lower_violation = min(0., log_mu_min - lower)
upper_violation = max(0., log_mu_max - upper)
return mc.normal_like([lower_violation, upper_violation], 0.,
1.e-6**-2)
return dict(covariate_constraint=covariate_constraint)
def spline(name, ages, knots, smoothing, interpolation_method='linear'):
""" Generate PyMC objects for a spline model of age-specific rate
Parameters
----------
name : str
knots : array
ages : array, points to interpolate to
smoothing : pymc.Node, smoothness parameter for smoothing spline
interpolation_method : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, 'cubic'
Results
-------
Returns dict of PyMC objects, including 'gamma' (log of rate at
knots) and 'mu_age' (age-specific rate interpolated at all age
points)
"""
assert pl.all(
pl.diff(knots) > 0), 'Spline knots must be strictly increasing'
# TODO: consider changing this prior distribution to be something more familiar in linear space
gamma = [
mc.Normal('gamma_%s_%d' % (name, k), 0., 10.**-2, value=-10.)
for k in knots
]
#gamma = [mc.Uniform('gamma_%s_%d'%(name,k), -20., 20., value=-10.) for k in knots]
# TODO: fix AdaptiveMetropolis so that this is not necessary
flat_gamma = mc.Lambda(
'flat_gamma_%s' % name,
lambda gamma=gamma: pl.array([x for x in pl.flatten(gamma)]))
import scipy.interpolate
@mc.deterministic(name='mu_age_%s' % name)
def mu_age(gamma=flat_gamma, knots=knots, ages=ages):
mu = scipy.interpolate.interp1d(knots,
pl.exp(gamma),
kind=interpolation_method,
bounds_error=False,
fill_value=0.)
return mu(ages)
vars = dict(gamma=gamma, mu_age=mu_age, ages=ages, knots=knots)
if (smoothing > 0) and (not pl.isinf(smoothing)):
#print 'adding smoothing of', smoothing
@mc.potential(name='smooth_mu_%s' % name)
def smooth_gamma(gamma=flat_gamma, knots=knots, tau=smoothing**-2):
# the following is to include a "noise floor" so that level value
# zero prior does not exert undue influence on age pattern
# smoothing
# TODO: consider changing this to an offset log normal
gamma = gamma.clip(
pl.log(pl.exp(gamma).mean() / 10.),
pl.inf) # only include smoothing on values within 10x of mean
return mc.normal_like(
pl.sqrt(pl.sum(pl.diff(gamma)**2 / pl.diff(knots))), 0, tau)
vars['smooth_gamma'] = smooth_gamma
return vars
def optimise(self, NAFac=1, pmutation=0.5, selpress=1.2, popsize=50,
reltol=0.1, elitism=5, maxtime=60, sizefactor=0.0001,
time_index_1=1, maxgens=500, maxstallgens=100, ga_verbose=True):
"""Perform the optimisation and save results
* Results are stored in :attr:`results`
* Models with the tolerance are stored in :attr:`results.models`
Parameters are those of a Genetic Algorithm used to perform
the analysis.
If you run again, it uses the previous best bitstirng.
Set self.session.best_bitstring = None to start from the
full network.
"""
self.logging.info("Running the optimisation. Can take a very long"
"time. To see the progression, set verboseR "
"attribute to True")
# update config GA section with user parameters
self._update_config('GA', self.optimise.actual_kwargs)
# keep track of the GA parameters, which may have been update above
gad = self.config.GA.as_dict()
bs = self.session.get('best_bitstring')
if bs is not None:
bs = "c(" + ",".join([str(x) for x in list(bs)]) + ")"
else:
bs = 'NULL'
# do we want to pre process the data ?
script_template = """
library(CellNOptR)
pknmodel = readSIF("%(pkn)s")
cnolist = CNOlist("%(midas)s")
model = preprocessing(cnolist, pknmodel, compression=%(compression)s,
expansion=%(expansion)s, cutNONC=%(cutnonc)s,
maxInputsPerGate=%(maxInputsPerGate)s)
res = gaBinaryT1(cnolist, model, popSize=%(popsize)s, maxGens=%(maxgens)s,
maxTime=%(maxtime)s, elitism=%(elitism)s, pMutation=%(pmutation)s,
NAFac=%(NAFac)s, selPress=%(selpress)s, relTol=%(reltol)s, sizeFac=%(sizefactor)s,
stallGenMax=%(maxstallgens)s, initBstring=%(bs)s)
sim_results = cutAndPlot(cnolist, model, list(res$bString),
plotParams=list(maxrow = 80, cex=0.5),
plotPDF=F)
sim_results2 = NULL
# output are not the same... as in T1
signals = colnames(cnolist@signals$`0`)
colnames(sim_results$mse) = signals
for (i in seq_along(sim_results$simResults[[1]])){
colnames(sim_results$simResults[[1]][[i]]) = signals
}
# to be retrieved inside Python code
best_bitstring = res$bString
best_score = res$bScore
all_scores = res$stringsTolScores
all_bitstrings = res$stringsTol
reactions = model$reacID
results = as.data.frame(res$results)
stimuli = as.data.frame(cnolist@stimuli)
inhibitors = as.data.frame(cnolist@inhibitors)
species = colnames(cnolist@signals[[1]])
optim1 = T
"""
params = {
'pkn': self.pknmodel.filename,
'midas': self.data.filename,
'compression': bool2R(self._compression),
'expansion': bool2R(self._expansion),
'cutnonc': bool2R(self._cutnonc),
'maxInputsPerGate': self._max_inputs_per_gate,
'bs':bs
}
params.update(gad)
script = script_template % params
self.session.run(script)
self.reactions_r = self.session.reactions
# need to change type of some columns, which are all string
results = self.session.results
results.columns = [x.strip() for x in results.columns]
columns_int = ['Generation', 'Stall_Generation']
columns_float = ['Best_score', 'Avg_Score_Gen', 'Best_score_Gen', 'Iter_time']
results[columns_int] = results[columns_int].astype(int)
results[columns_float] = results[columns_float].astype(float)
# cnograph created automatically from the reactions
try:
N = len(self.session.best_bitstring)
all_bs = self.session.all_bitstrings
df = pd.DataFrame(all_bs, columns=self.reactions_r)
models = BooleanModels(df)
# flatten to handle exhaustive
import numpy as np
models.scores = np.array(list(pylab.flatten(self.session.all_scores)))
try:
models.cnograph.midas = self.data.copy()
except Exception as err:
# does not work with ExtLiverPCB
# CNOError: 'The cues IFNg was found in the MIDAS file but is not present in the model. Change your model or MIDAS file.'
print("something wrong in the copying of the midas into cnograph(models)")
print(err.message)
except:
N = len(self.session.best_bitstring)
all_bs = self.session.all_bitstrings
if N == len(self.reactions_r) and len(self.session.all_bitstrings)>0:
df = pd.DataFrame([self.session.all_bitstrings],
columns=self.reactions_r)
models = BooleanModels(df)
models.scores = easydev.to_list(self.session.all_scores)
self._models = models
else:
df = pd.DataFrame(columns=self.reactions_r)
models = BooleanModels(df)
models.scores = easydev.to_list(self.session.all_scores)
self._models = models
models.cnograph.midas = self.data.copy()
results = {
'best_score': self.session.best_score,
'best_bitstring': self.session.best_bitstring,
'all_scores': self.session.all_scores,
'all_bitstrings': self.session.all_bitstrings,
'reactions': self._reac_cnor2cno(self.reactions_r),
'sim_results': self.session.sim_results, # contains mse and sim at t0,t1,
'results': results,
'models': models,
'stimuli': self.session.stimuli.copy(),
'inhibitors': self.session.inhibitors.copy(),
'species': self.session.species,
}
results['pkn'] = self.pknmodel
results['midas'] = self.data
self.results.results = results
self.results.models = models
self._called.append('optimise')
T.append(vs)
f.close()
T = np.array(T)
cols = [2, 3]
F__ = T[:, cols]
# normalizamos
M = np.mean(F__, axis=0)
D = np.std(F__, axis=0)
F = np.nan_to_num((F__ - M) / D)
# LDA
X = F
y = np.array([i for i in py.flatten([[i] * 20 for i in range(12)])])
target_names = np.array(conf.artistas)
#pca = PCA(n_components=2)
#X_r2 = pca.fit(X).transform(X)
lda = LDA(n_components=2)
X_r2 = lda.fit(X, y).transform(X)
print lda.coef_
print X_r2
# separamos as pinturas em classes de 20 pinturas, uma para cada pintor
# e calculamos os protótipos (pontos médios)
Fs = []
Prots = []
for i in range(12):
def plot_box(self):
"""
Plots the fraction of nodes that received a particular packet from the source
as a box-and-whisker with the probability p on the x-axis.
"""
logging.debug('')
configurations_per_variant = self.configurations_per_variant
gossip_variants_count = len(self.configurations['gossip'])
colors = self.options['color'](pylab.linspace(
0, 0.8, configurations_per_variant))
labels = []
for li_of_frac in self.label_info:
s = str()
for i, (param, value) in enumerate(li_of_frac):
if i > 0:
s += '\n'
s += '%s=%s' % (_cname_to_latex(param), value)
labels.append(s)
labels *= len(self.configurations['p'])
ps = list(
pylab.flatten(self.configurations_per_p * [p]
for p in self.configurations['p']))
#################################################################
# box plot
#################################################################
array = numpy.zeros([len(self.fraction_of_nodes[0]), self.length()])
for i, fracs in enumerate(self.fraction_of_nodes):
array[:, i] = fracs
fig = MyFig(self.options,
rect=[0.1, 0.2, 0.8, 0.75],
figsize=(max(self.length(), 10), 10),
xlabel='Probability p',
ylabel='Fraction of Nodes',
aspect='auto')
fig.ax.set_ylim(0, 1)
box_dict = fig.ax.boxplot(array, notch=1, sym='rx', vert=1)
#box_dict = fig.ax.boxplot(array, notch=1, sym='rx', vert=1, patch_artist=False)
for j, box in enumerate(box_dict['boxes']):
j = (j % self.configurations_per_p)
box.set_color(colors[j])
for _flier in box_dict['fliers']:
_flier.set_color('lightgrey')
fig.ax.set_xticklabels(ps, fontsize=self.options['fontsize'] * 0.6)
# draw vertical line to visually mark different probabilities
for x in range(0, self.length(), self.configurations_per_p):
fig.ax.plot([x + 0.5, x + 0.5], [0.0, 1.0],
linestyle='dotted',
color='red',
alpha=0.8)
#################################################################
# create some dummy elements for the legend
#################################################################
if configurations_per_variant > 1:
proxies = []
for i in range(0, configurations_per_variant):
r = Rectangle((0, 0),
1,
1,
edgecolor=colors[i % configurations_per_variant],
facecolor='white')
proxies.append((r, labels[i]))
fig.ax.legend([proxy for proxy, label in proxies],
[label for proxy, label in proxies],
loc='lower right')
self.figures['boxplot'] = fig.save('boxplot_' + str(self.data_filter))
def all_routers(options):
cursor = options['db_conn'].cursor()
cursor.execute('''SELECT host FROM addr''')
return list(pylab.flatten(cursor.fetchall()))
def get_hosts(options):
cursor = options['db_conn'].cursor()
hosts = list(pylab.flatten(cursor.execute('''SELECT host FROM addr''').fetchall()))
return hosts
return (np.array(P_max,
dtype=np.float_), np.array(w_at_max, dtype=np.float_))
if __name__ == '__main__':
L = [10, 20, 50] # , 100]
A_m = [10.0 * distance for distance in [10]] # ., 20., 30., 40]]
A_f = [0.5, 1.0, 1.5] # , 2.0]
tau = [3.0, 4.0, 5.0] # , 6.0] # , 7.0, 8.0]
s = [0.1] # , 0.2], #0.3, 0.4, 0.5, 0.6]
params = tau, s, A_f, A_m, L
test_grid = np.meshgrid(*params)
test_grid_shape = test_grid[0].shape
test_program = np.vstack([p.flatten() for p in test_grid]).T
print('inputs')
print(test_program)
output_vars = run_test_program(test_program)
print('outputs')
print(output_vars)
out_grids = [out.reshape(*test_grid_shape) for out in output_vars]
print(out_grids[0])
print(test_grid[0].shape)
print(out_grids[0].shape)
# mlab.contour3d(test_grid[0], test_grid[2], test_grid[4], out_grids[0])
mlab.contour3d(out_grids[0][0, :, :, 0, :])
mlab.show()
def optimise2(self, NAFac=1, pmutation=0.5, selpress=1.2, popsize=50,
reltol=0.1, elitism=5, maxtime=60, sizefactor=0.0001,
maxgens=500, maxstallgens=100, ga_verbose=True,
time_index_2=3):
"""
:param NAFac:
:param pmutation:
:param selpress:
:param popsize:
:param reltol:
:param elitism:
:param maxtime:
:param sizefactor:
:param maxgens:
:param maxstallgens:
:param ga_verbose:
:param time_index_2: indices as R expects it. 3, means third time points.
time_index 1 is time zero. time_index 2 is T1, time_index 3 is T2
TODO decrement by 1 to be Python like syntax
:return:
"""
# TODO assert there are 2 time indices
# something interesting to do is to run steady stte not only
# for the best bitstring but all those within the tolerance !
self._update_config('GA2', self.optimise2.actual_kwargs)
#if best_bitstring is None:
best_bitstring = self.results.results.best_bitstring
reactions = [r for r,b in zip(self.reactions_r, best_bitstring) if b==0]
script_template = """
# model, cnolist and best_bitstring are created when calling
# optimise()
res2 = gaBinaryTN(cnolist, model, popSize=%(popsize)s, maxGens=%(maxgens)s,
maxTime=%(maxtime)s, elitism=%(elitism)s, pMutation=%(pmutation)s,
NAFac=%(NAFac)s, selPress=%(selpress)s, relTol=%(reltol)s, sizeFac=%(sizefactor)s,
stallGenMax=%(maxstallgens)s, timeIndex=%(timeindex)s, bStrings=list(best_bitstring))
# bitstring is provided when calling optimise()
sim_results2 = cutAndPlot(cnolist, model, list(res$bString, res2$bString),
plotParams=list(maxrow = 80, cex=0.5),
plotPDF=F)
signals = colnames(cnolist@signals$`0`)
colnames(sim_results2$mse) = signals
for (i in seq_along(sim_results2$simResults)){
colnames(sim_results2$simResults[[i]]) = signals
}
results2 = as.data.frame(res2$results)
best_bitstring2 = res2$bString
best_score2 = res2$bScore
all_scores2 = res2$stringsTolScores
all_bitstrings2 = res2$stringsTol
optim2 = T
"""
params = dict([(k, self.config.GA[k].value)
for k in self.config.GA._get_names()])
params['timeindex'] = time_index_2
params['reltol'] = reltol
script = script_template % params
self.session.run(script)
results = self.session.results2
results.columns = [x.strip() for x in results.columns]
columns_int = ['Generation', 'Stall_Generation']
columns_float = ['Best_score', 'Avg_Score_Gen', 'Best_score_Gen',
'Iter_time']
results[columns_int] = results[columns_int].astype(int)
results[columns_float] = results[columns_float].astype(float)
# cnograph created automatically from the reactions
# TODneed to gure outat are reactio names
try:
df = pd.DataFrame(self.session.all_bitstrings2,
columns=reactions)
self.df = df
except:
df = pd.DataFrame(self.session.all_bitstrings2)
df = df.transpose()
self.reactions2 = reactions
df.columns = reactions
self.df = df
models = BooleanModels(df)
# flatten to handle exhaustive
try:
# if only 1 item, it is not a list...
models.scores = np.array(list(pylab.flatten(self.session.all_scores2)))
except:
models.scores = np.array([self.session.all_scores2])
# models.cnograph.midas = self.data.copy()
# reactions = self._reac_cnor2cno(self.session.reactions)
results = {
'best_score': self.session.best_score2,
'best_bitstring': self.session.best_bitstring2,
'all_scores': self.session.all_scores2,
'all_bitstrings': self.session.all_bitstrings2,
'reactions': self._reac_cnor2cno(reactions),
'sim_results': self.session.sim_results2, # contains mse and sim at t0,t1,
'results': results,
'models': models,
'stimuli': self.session.stimuli.copy(),
'inhibitors': self.session.inhibitors.copy(),
'species': self.session.species,
}
# work around to have sme results as in T1
this = results['sim_results']['simResults']
this = [dict([("t"+str(i), x) for i,x in enumerate(this)])]
results['sim_results']['simResults'] = this
results['pkn'] = self.pknmodel
results['midas'] = self.data
self.results2.results = results
self.results2.models = models
self._called.append('optimise2')
def plot_histogram(self):
configurations_per_variant = self.configurations_per_variant
colors = self.options['color'](pylab.linspace(
0, 0.8, configurations_per_variant))
labels = []
for li_of_frac in self.label_info:
s = str()
for i, (param, value) in enumerate(li_of_frac):
if i > 0:
s += '\n'
s += '%s=%s' % (_cname_to_latex(param), value)
labels.append(s)
labels *= len(self.configurations['p'])
ps = list(
pylab.flatten(self.configurations_per_p * [p]
for p in self.configurations['p']))
#################################################################
# histogram plot
#################################################################
fig2 = MyFig(self.options,
rect=[0.1, 0.2, 0.8, 0.75],
figsize=(max(self.length(), 10), 10),
xlabel='Probability p',
ylabel='Fraction of Nodes',
aspect='auto')
patches = []
for i, fracs in enumerate(self.fraction_of_nodes):
hist, bin_edges = numpy.histogram(fracs,
bins=pylab.linspace(
0, 1, self.options['bins']))
hist_norm = hist / float(len(fracs)) * 0.4
yvals = list(bin_edges.repeat(2))[1:-1]
yvals.extend(list(bin_edges.repeat(2))[1:-1][::-1])
xvals = list((i + 1 + hist_norm).repeat(2))
xvals.extend(list((i + 1 - hist_norm).repeat(2))[::-1])
i = (i % self.configurations_per_p)
poly = Polygon(zip(xvals, yvals),
edgecolor='black',
facecolor=colors[i],
closed=True)
patches.append(poly)
patch_collection = PatchCollection(patches, match_original=True)
fig2.ax.add_collection(patch_collection)
fig2.ax.set_xticks(range(1, self.length() + 1))
fig2.ax.set_xticklabels(ps, fontsize=self.options['fontsize'] * 0.6)
for x in range(0, self.length(), self.configurations_per_p):
fig2.ax.plot([x + 0.5, x + 0.5], [0.0, 1.0],
linestyle='dotted',
color='red',
alpha=0.8)
fig2.ax.set_ylim(0, 1)
#################################################################
# create some dummy elements for the legend
#################################################################
if configurations_per_variant > 1:
proxies = []
#for i in range(0, len(self.configurations['gossip'])):
for i in range(0, configurations_per_variant):
r = Rectangle((0, 0),
1,
1,
facecolor=colors[i % configurations_per_variant],
edgecolor='black')
proxies.append((r, labels[i]))
fig2.ax.legend([proxy for proxy, label in proxies],
[label for proxy, label in proxies],
loc='lower right')
self.figures['histogram'] = fig2.save('histogram_' +
str(self.data_filter))
yy = []
for f in files:
print 'Reading %s ' % f
name, eph = string.split(f)
T0 = ephemeris[eph]
P = 0.154525
X = pl.load(name)
x = (X[:,0] - T0)/P
xx.append(x - int(x[0]))
yt = X[:,1] + X[:,2]
yt -= pl.average(yt)
yy.append(yt)
# now sort observations in terms of orbital phase
xx = pl.array([i for i in pl.flatten(xx)])
yy = pl.array([i for i in pl.flatten(yy)])
arg = xx.argsort()
xx = xx[arg]
yy = yy[arg]
# limit orb phase to [0.8,1.2]
lt = xx < 1.2
gt = xx > 0.8
xx = xx[lt*gt]
yy = yy[lt*gt]
# average lightcurve in N bins
N = 40
# try-except block takes away points at the end of the array if array cannot be split in N equal parts
def spline(name, ages, knots, smoothing, interpolation_method='linear'):
""" Generate PyMC objects for a piecewise constant Gaussian process (PCGP) model
Parameters
----------
name : str
knots : array, locations of the discontinuities in the piecewise constant function
ages : array, points to interpolate to
smoothing : pymc.Node, smoothness parameter for smoothing spline
interpolation_method : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, 'cubic'
Results
-------
Returns dict of PyMC objects, including 'gamma' and 'mu_age'
the observed stochastic likelihood and data predicted stochastic
"""
assert pl.all(
pl.diff(knots) > 0), 'Spline knots must be strictly increasing'
gamma = [
mc.Normal('gamma_%s_%d' % (name, k), 0., 10.**-2, value=-10.)
for k in knots
]
#gamma = [mc.Uniform('gamma_%s_%d'%(name,k), -20., 20., value=-10.) for k in knots]
# TODO: fix AdaptiveMetropolis so that this is not necessary
flat_gamma = mc.Lambda(
'flat_gamma_%s' % name,
lambda gamma=gamma: pl.array([x for x in pl.flatten(gamma)]))
import scipy.interpolate
@mc.deterministic(name='mu_age_%s' % name)
def mu_age(gamma=flat_gamma, knots=knots, ages=ages):
mu = scipy.interpolate.interp1d(knots,
pl.exp(gamma),
kind=interpolation_method,
bounds_error=False,
fill_value=0.)
return mu(ages)
vars = dict(gamma=gamma, mu_age=mu_age, ages=ages, knots=knots)
if (smoothing > 0) and (not pl.isinf(smoothing)):
print 'adding smoothing of', smoothing
@mc.potential(name='smooth_mu_%s' % name)
def smooth_gamma(gamma=flat_gamma, knots=knots, tau=smoothing**-2):
# the following is to include a "noise floor" so that level value
# zero prior does not exert undue influence on age pattern
# smoothing
gamma = gamma.clip(
pl.log(pl.exp(gamma).mean() / 10.),
pl.inf) # only include smoothing on values within 10x of mean
return mc.normal_like(
pl.sqrt(pl.sum(pl.diff(gamma)**2 / pl.diff(knots))), 0, tau)
vars['smooth_gamma'] = smooth_gamma
return vars
def spline(name, ages, knots, smoothing, interpolation_method='linear'):
""" Generate PyMC objects for a spline model of age-specific rate
Parameters
----------
name : str
knots : array
ages : array, points to interpolate to
smoothing : pymc.Node, smoothness parameter for smoothing spline
interpolation_method : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, 'cubic'
Results
-------
Returns dict of PyMC objects, including 'gamma' (log of rate at
knots) and 'mu_age' (age-specific rate interpolated at all age
points)
"""
assert pl.all(pl.diff(knots) > 0), 'Spline knots must be strictly increasing'
# TODO: consider changing this prior distribution to be something more familiar in linear space
gamma = [mc.Normal('gamma_%s_%d'%(name,k), 0., 10.**-2, value=-10.) for k in knots]
#gamma = [mc.Uniform('gamma_%s_%d'%(name,k), -20., 20., value=-10.) for k in knots]
# TODO: fix AdaptiveMetropolis so that this is not necessary
flat_gamma = mc.Lambda('flat_gamma_%s'%name, lambda gamma=gamma: pl.array([x for x in pl.flatten(gamma)]))
import scipy.interpolate
@mc.deterministic(name='mu_age_%s'%name)
def mu_age(gamma=flat_gamma, knots=knots, ages=ages):
mu = scipy.interpolate.interp1d(knots, pl.exp(gamma), kind=interpolation_method, bounds_error=False, fill_value=0.)
return mu(ages)
vars = dict(gamma=gamma, mu_age=mu_age, ages=ages, knots=knots)
if (smoothing > 0) and (not pl.isinf(smoothing)):
#print 'adding smoothing of', smoothing
@mc.potential(name='smooth_mu_%s'%name)
def smooth_gamma(gamma=flat_gamma, knots=knots, tau=smoothing**-2):
# the following is to include a "noise floor" so that level value
# zero prior does not exert undue influence on age pattern
# smoothing
# TODO: consider changing this to an offset log normal
gamma = gamma.clip(pl.log(pl.exp(gamma).mean()/10.), pl.inf) # only include smoothing on values within 10x of mean
return mc.normal_like(pl.sqrt(pl.sum(pl.diff(gamma)**2 / pl.diff(knots))), 0, tau)
vars['smooth_gamma'] = smooth_gamma
return vars
# e um conjunto de teste, com 10 últimas pinturas de cada artista
Fteste = np.vstack([F__ord[i:i+20][rind[10:]] for i in range(0, 12*20, 20)])
# normalizamos todos
Mtreino = np.mean(Ftreino, axis=0)
Dtreino = np.std(Ftreino, axis=0)
Ftreino = np.nan_to_num((Ftreino-Mtreino) / Dtreino)
Mteste = np.mean(Fteste, axis=0)
Dteste = np.std(Fteste, axis=0)
Fteste = np.nan_to_num((Fteste-Mteste) / Dteste)
# LDA
Xtreino = Ftreino
Xteste = Fteste
y = np.array([i for i in py.flatten([[i]*10 for i in range(12)])])
target_names = np.array(conf.artistas)
# aplicamos LDA no conjunto de treino e teste (após fitar... treinar com o
# conjunto de treino)
lda = LDA(n_components=2)
# lda.fit(Xtreino, y, store_covariance=True)
Xtreino_r2 = lda.fit(Xtreino, y, store_covariance=True).transform(Xtreino)
y_pred = lda.predict(Xteste)
print y_pred
cm = confusion_matrix(y, y_pred)
cms.append(cm)
print 'cm', cm
cm_media = sum([np.array(cm, dtype=float) for cm in cms]) / N
levels.remove(ION)
else:
print("Warning: in ionization Level of", SPE, DEG, "the State component")
state.print()
print("not found in", IFILE1)
quit(1)
else:
if ION in levels:
levels.remove(ION)
else:
print("Warning: ionization State of", SPE, DEG, "not found in", IFILE1)
IL = False
terms = [level.remove_label_term(level.term) for level in levels if (level.term or type(level.term) != list)]
terms = list(set(pl.flatten(terms)))
terms = sorted(terms, key=term2num)
terms = sorted(terms, key=term2mult)
print('Number of terms:', len(terms))
print(terms)
# Extract levels by class
fl = []
ml = []
sl = []
hl = []
for level in levels:
if isinstance(level, HyperLevel) and hasattr(level, 'superlevels'):
hl.append(level)
elif isinstance(level, SuperLevel) and hasattr(level, 'levels'):
def spline(name, ages, knots, smoothing, interpolation_method='linear'):
""" Generate PyMC objects for a piecewise constant Gaussian process (PCGP) model
Parameters
----------
name : str
knots : array, locations of the discontinuities in the piecewise constant function
ages : array, points to interpolate to
smoothing : pymc.Node, smoothness parameter for smoothing spline
interpolation_method : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, 'cubic'
Results
-------
Returns dict of PyMC objects, including 'gamma' and 'mu_age'
the observed stochastic likelihood and data predicted stochastic
"""
assert pl.all(pl.diff(knots) > 0), 'Spline knots must be strictly increasing'
gamma = [mc.Normal('gamma_%s_%d'%(name,k), 0., 10.**-2, value=-10.) for k in knots]
#gamma = [mc.Uniform('gamma_%s_%d'%(name,k), -20., 20., value=-10.) for k in knots]
# TODO: fix AdaptiveMetropolis so that this is not necessary
flat_gamma = mc.Lambda('flat_gamma_%s'%name, lambda gamma=gamma: pl.array([x for x in pl.flatten(gamma)]))
import scipy.interpolate
@mc.deterministic(name='mu_age_%s'%name)
def mu_age(gamma=flat_gamma, knots=knots, ages=ages):
mu = scipy.interpolate.interp1d(knots, pl.exp(gamma), kind=interpolation_method, bounds_error=False, fill_value=0.)
return mu(ages)
vars = dict(gamma=gamma, mu_age=mu_age, ages=ages, knots=knots)
if (smoothing > 0) and (not pl.isinf(smoothing)):
print 'adding smoothing of', smoothing
@mc.potential(name='smooth_mu_%s'%name)
def smooth_gamma(gamma=flat_gamma, knots=knots, tau=smoothing**-2):
# the following is to include a "noise floor" so that level value
# zero prior does not exert undue influence on age pattern
# smoothing
gamma = gamma.clip(pl.log(pl.exp(gamma).mean()/10.), pl.inf) # only include smoothing on values within 10x of mean
return mc.normal_like(pl.sqrt(pl.sum(pl.diff(gamma)**2 / pl.diff(knots))), 0, tau)
vars['smooth_gamma'] = smooth_gamma
return vars
def main():
"""
The main function of the plotting module
"""
logging.basicConfig(level=logging.INFO, format='%(levelname)s [%(funcName)s] %(message)s')
try:
import psyco
psyco.full()
psyco.profile(0.0)
except ImportError:
print "INFO: psyco module not available"
parser = argparse.ArgumentParser(description='Plot data stored in an sqlite database')
parser.add_argument('--topo', required=True, help='name of the database where the network topology is available')
parser.add_argument('--db', required=True, help='name of the database where the results shall be stored')
parser.add_argument('--clone', default=None, help='clone the scenarios from another experiment')
parser.add_argument('--src', nargs='+', default=['all'], help='source node sending packets')
parser.add_argument('--gossip', nargs='+', default=['0'], help='gossip variants')
parser.add_argument('-m', nargs='+', default=['1'], help='m')
parser.add_argument('-k', nargs='+', default=['1'], help='flood on first k hops')
parser.add_argument('--probability', '-p', nargs='+', default=list(pylab.linspace(0.1, 1, 10)), help='gossip variants')
parser.add_argument('--outdir', default='', help='output directory for the plots')
parser.add_argument('--pkg_num', default=100, type=int, help='number of packets')
parser.add_argument('--pkg_size', nargs='+', default=[0], type=int, help='select a particular graph')
parser.add_argument('--no_loss', nargs='?', const=True, default=False, help='List all callable plotting functions')
args = parser.parse_args()
options = {}
options['topo'] = args.topo
options['outdir'] = args.outdir
options['loss'] = not args.no_loss
options['db'] = args.db
options['nodes'] = 'all'
gossip_params = dict()
options['gossip_params'] = gossip_params
gossip_params['src'] = args.src
gossip_params['gossip'] = args.gossip
gossip_params['m'] = args.m
gossip_params['k'] = args.k
gossip_params['p'] = args.probability
gossip_params['pkg_num'] = args.pkg_num
gossip_params['pkg_size'] = args.pkg_size
db = options['topo']
conn = sqlite3.connect(db)
options['topo_conn'] = conn
cursor = conn.cursor()
if 'all' in args.src and not args.clone:
logging.info('all routers selected as sources')
cursor.execute('''SELECT host FROM addr''')
hosts = list(pylab.flatten(cursor.fetchall()))
options['src'] = hosts
if args.clone:
clone_conn = sqlite3.connect(args.clone)
clone_cursor = clone_conn.cursor()
clone(options, clone_cursor)
cursor = prepare_db(options)
graphs = create_graphs(options)
#for helloSize, G in graphs:
#for gossip in gossip_params['gossip']:
#for p in gossip_params['p']:
#t = (None, '', '', p, p2, k, n, m, timeout, gossip, 0, 0, 0, 0, T_MAX, p_min, p_max, helloSize)
#cursor.execute('INSERT INTO tag VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', t)
#curr_tag_key = c.lastrowid
for node in graphs[0][1]:
cursor.execute('INSERT INTO addr VALUES (?)', (node, ))
options['db_conn'].commit()
for i, pkg_size in enumerate(gossip_params['pkg_size']):
G = None
for helloSize, graph in graphs:
if helloSize == pkg_size:
G = (helloSize, graph)
break
if not G:
logging.critical('no graph found for pkg_size=%d', pkg_size)
assert(False)
logging.info('helloSize=\"%d\" (%d/%d)', G[0], i+1, len(graphs))
simulate(G, options)
