def test_multigraph(self):
G = nx.MultiGraph()
G.add_edge(1, 2, key="first")
G.add_edge(1, 2, key="second", color="blue")
H = adjacency_graph(adjacency_data(G))
nx.is_isomorphic(G, H)
assert H[1][2]["second"]["color"] == "blue"
def with_empty_graph(cls, structure, name="bonds",
edge_weight_name=None,
edge_weight_units=None):
"""
Constructor for StructureGraph, returns a StructureGraph
object with an empty graph (no edges, only nodes defined
that correspond to Sites in Structure).
:param structure (Structure):
:param name (str): name of graph, e.g. "bonds"
:param edge_weight_name (str): name of edge weights,
e.g. "bond_length" or "exchange_constant"
:param edge_weight_units (str): name of edge weight units
e.g. "Å" or "eV"
:return (StructureGraph):
"""
if edge_weight_name and (edge_weight_units is None):
raise ValueError("Please specify units associated "
"with your edge weights. Can be "
"empty string if arbitrary or "
"dimensionless.")
# construct graph with one node per site
# graph attributes don't change behavior of graph,
# they're just for book-keeping
graph = nx.MultiDiGraph(edge_weight_name=edge_weight_name,
edge_weight_units=edge_weight_units,
name=name)
graph.add_nodes_from(range(len(structure)))
graph_data = json_graph.adjacency_data(graph)
return cls(structure, graph_data=graph_data)
def to_json(net, gformat):
if "list" in gformat:
jsn = jsg.node_link_data(net)
else:
jsn = jsg.adjacency_data(net)
jsn = json.dumps(jsn, indent=2, sort_keys=True)
return jsn
def test_multigraph(self):
G = nx.MultiGraph()
G.add_edge(1, 2, key='first')
G.add_edge(1, 2, key='second', color='blue')
H = adjacency_graph(adjacency_data(G))
nx.is_isomorphic(G, H)
assert H[1][2]['second']['color'] == 'blue'
def with_empty_graph(cls, structure, name="bonds",
edge_weight_name=None,
edge_weight_units=None):
"""
Constructor for StructureGraph, returns a StructureGraph
object with an empty graph (no edges, only nodes defined
that correspond to Sites in Structure).
:param structure (Structure):
:param name (str): name of graph, e.g. "bonds"
:param edge_weight_name (str): name of edge weights,
e.g. "bond_length" or "exchange_constant"
:param edge_weight_units (str): name of edge weight units
e.g. "Å" or "eV"
:return (StructureGraph):
"""
if edge_weight_name and (edge_weight_units is None):
raise ValueError("Please specify units associated "
"with your edge weights. Can be "
"empty string if arbitrary or "
"dimensionless.")
# construct graph with one node per site
# graph attributes don't change behavior of graph,
# they're just for book-keeping
graph = nx.MultiDiGraph(edge_weight_name=edge_weight_name,
edge_weight_units=edge_weight_units,
name=name)
graph.add_nodes_from(range(len(structure)))
graph_data = json_graph.adjacency_data(graph)
return cls(structure, graph_data=graph_data)
def create_relations_graph():
client = MongoClient('localhost', 27017)
db = client["cinema-of-israel-db"]["movies"]
movies_list = db.find({})
all_casts = []
for rec in movies_list:
all_casts.append(get_movie_cast(rec))
#every movie represents by clique sub graph
all_subgraph = [nx.complete_graph(x[1]) for x in all_casts]
#set edges attribute to movie's name for each sub graph
for x in range(0, len(all_casts)):
nx.set_edge_attributes(all_subgraph[x], all_casts[x][0], "שם הסרט")
#union graph includes all movies
israel_cinema_relations = nx.compose_all(all_subgraph)
#save graph in json format
data = json_graph.adjacency_data(israel_cinema_relations)
with open('json_graph.json', 'w') as outfile:
json.dump(data, outfile)
return
def gsource_gdata(config, graphSource, graphData):
"""Create a graph from the config file GRAPH_SOURCE and GRAPH_DATA sections"""
# make sure the config file has graph information in it
graph_source_field = "gSource"
save_graph_field = "save_json"
required_graph_data_fields = ['id', 'pop', 'area', 'cd']
if not config.has_section(graphData):
raise configparser.NoSectionError(graphData)
if not config.has_section(graphSource):
raise configparser.NoSectionError(graphSource)
configGraphData = config[graphData]
configGraphSource = config[graphSource]
missing = [
x for x in required_graph_data_fields if x not in configGraphData
]
if missing:
missing_str = " ".join(missing)
raise configparser.NoOptionError(missing_str, graphData)
if graph_source_field not in configGraphSource:
raise configparser.NoOptionError(graph_source_field, graphSource)
ID = configGraphData['id']
POP = configGraphData['pop']
AREA = configGraphData['area']
CD = configGraphData['cd']
# create graph from data and load required data
path = configGraphSource[graph_source_field]
save_graph = False
load_graph = False
if save_graph_field in configGraphSource:
save_graph = True
if os.path.isfile(configGraphSource[save_graph_field]):
print("trying to load graph from", path)
path = configGraphSource[save_graph_field]
save_graph = False
load_graph = True
type = "json" if load_graph else "fiona"
graph = mgs.construct_graph(path,
ID,
pop_col=POP,
area_col=AREA,
district_col=CD,
cols_to_add=[POP, AREA, CD],
data_source_type=type)
if save_graph:
print("saving graph to", configGraphSource[save_graph_field])
with open(configGraphSource[save_graph_field], "w") as f:
json.dump(json_graph.adjacency_data(graph), f)
return graph, POP, AREA, CD
def write_json(G, i):
data = json_graph.adjacency_data(G)
s = json.dumps(data)
f = open(outFileName.format("-{0}".format(i), "json"), "w")
f.write(s)
f.close()
if not outputOnlyReducedGraphs:
export_as_coord_list(G, "nodes-{0}.json".format(i))
def graph2json(graph,filename):
# convert to json
jdata = json_graph.adjacency_data(graph)
# write to file
f = open(filename+'.json','w')
json.dump(jdata,f)
f.close()
def test_graph_attributes(self):
G = nx.path_graph(4)
G.add_node(1, color="red")
G.add_edge(1, 2, width=7)
G.graph["foo"] = "bar"
G.graph[1] = "one"
H = adjacency_graph(adjacency_data(G))
assert H.graph["foo"] == "bar"
assert H.nodes[1]["color"] == "red"
assert H[1][2]["width"] == 7
d = json.dumps(adjacency_data(G))
H = adjacency_graph(json.loads(d))
assert H.graph["foo"] == "bar"
assert H.graph[1] == "one"
assert H.nodes[1]["color"] == "red"
assert H[1][2]["width"] == 7
def test_graph_attributes(self):
G = nx.path_graph(4)
G.add_node(1, color='red')
G.add_edge(1, 2, width=7)
G.graph['foo'] = 'bar'
G.graph[1] = 'one'
H = adjacency_graph(adjacency_data(G))
assert H.graph['foo'] == 'bar'
assert H.nodes[1]['color'] == 'red'
assert H[1][2]['width'] == 7
d = json.dumps(adjacency_data(G))
H = adjacency_graph(json.loads(d))
assert H.graph['foo'] == 'bar'
assert H.graph[1] == 'one'
assert H.nodes[1]['color'] == 'red'
assert H[1][2]['width'] == 7
def serialize(self):
data = json_graph.adjacency_data(self._graph)
data["adjacency"] = [
sorted(outbounds, key=lambda x: x["id"])
for outbounds in data["adjacency"]
]
return data
def saveGraph(G, file_name):
data = json_graph.adjacency_data(G)
try:
with open(file_name, 'w') as outfile:
json.dump(json.dumps(data), outfile)
return(True)
except IOError:
return(False)
def serialize(self):
data = json_graph.adjacency_data(self._graph)
data['adjacency'] = [
sorted(outbounds, key=lambda x: x['id'])
for outbounds in data['adjacency']
]
return data
def json_networkx(obj):
logger.debug("nx graph")
json_string = json_graph.adjacency_data(obj,
attrs={
'id': 'json_id',
'key': 'json_key'
})
d = with_signature(obj, json_string, obj_module="networkx")
return d
def create_subgraph(poi_input_path: str, graph_input_path: str, subgraph_output_path: str):
"""
Read csv in poi_input_path containing point of interests (POIs) {name, lat, lon, id}
Read json file in graph_input_path containing the complete graph (where the POIs are some of the nodes)
Create subgraph for given POIs and save it as JSON in subgraph_output_path
Args:
poi_input_path (str): path of the csv containing POI having rows {name, lat, lon, id}
graph_input_path (str): path to networkx json_graph.adjacency_data
subgraph_output_path (str): path to write networkx subgraph as json_graph.adjacency_data
"""
# read input
node_ids = []
print(f'reading POI file {poi_input_path}')
with open(poi_input_path) as f:
reader = csv.reader(f)
for row in reader:
assert len(row) >= 5
node_id = row[4]
node_ids.append(int(node_id))
node_ids = list(set(node_ids))
node_ids.sort()
data = {}
print(f'reading graph file {graph_input_path}')
with open(graph_input_path) as f:
data = json.load(f)
# create subgraph
print(f'creating subgraph')
G = json_graph.adjacency_graph(data)
len_node_ids = len(node_ids)
# H = nx.to_undirected(G)
# num_combinations = math.factorial(len_node_ids) / (2*math.factorial((len_node_ids-2)))
# print(f'* getting the shortest paths of {num_combinations} combinations')
# futures = []
# with concurrent.futures.ThreadPoolExecutor(max_workers=mp.cpu_count()) as executor:
# for nodes in combinations(node_ids, r=2):
# futures.append(executor.submit(nx.shortest_path, H, *nodes))
num_permutations = math.factorial(len_node_ids) / math.factorial((len_node_ids-2))
print(f'* getting the shortest paths of {num_permutations} permutations')
futures = []
with concurrent.futures.ThreadPoolExecutor(max_workers=mp.cpu_count()) as executor:
for nodes in permutations(node_ids, r=2):
futures.append(executor.submit(nx.shortest_path, G, *nodes))
min_nodes = set()
for future in futures:
nodes = future.result()
min_nodes.update(nodes)
print('* create subgraph from the longest induced path')
I = nx.subgraph(G, min_nodes)
subgraph_data = json_graph.adjacency_data(I)
with open(subgraph_output_path, 'w+') as f:
json.dump(subgraph_data, f)
print(f'subgraph saved in {subgraph_output_path}')
def save_json_file(graph):
data = json_graph.adjacency_data(graph)
data['graph'] = []
bad_node = []
for node in data['nodes']:
# node['pos'] = list(node['pos'])
if 'pos' in node:
del node['pos']
if 'rotation' in node:
# node['rotation'] = list(node['rotation'])
del node['rotation']
# I guess here the frozenset and pos is the information for face, should we keep this information?
for i in range(len(node)):
if isinstance(list(node.values())[i], set):
key = list(node.keys())[i]
del node[key]
if len(node) == 0:
bad_node.append(node)
elif type(list(node.values())[i]) is frozenset:
key = list(node.keys())[i]
del node[key]
if len(node) == 0:
bad_node.append(node)
for i in bad_node:
data['nodes'].remove(i)
bad_node = []
break_bool = False
for adj_node in data['adjacency']:
bad_key = []
for node in adj_node:
for i in range(len(node)):
if isinstance(list(node.values())[i], set):
bad_key.append(node)
elif type(list(node.values())[i]) is frozenset:
bad_key.append(node)
for i in bad_key:
adj_node.remove(i)
for node in adj_node:
for i in range(len(node)):
if i == 0 and list(node.keys())[0] == 'id':
bad_node.append(adj_node)
break_bool = True
break
if break_bool:
break_bool = False
break
for i in bad_node:
data['adjacency'].remove(i)
remove_geometries(data)
with open("some.json", "w") as f:
json.dump(data, f)
def _serialize_graph(graph):
graph = json_graph.adjacency_data(graph)
graph["nodes"] = [{
"id":
n["id"] if not isinstance(n["id"], Step) else repr(n["id"])
} for n in graph["nodes"]]
graph["adjacency"] = [[{
"id":
a["id"] if not isinstance(a["id"], Step) else repr(a["id"])
} for a in adj] for adj in graph["adjacency"]]
return graph
def find_crowds(weak_comps):
"""
break up big connected components using crowd detection algorithm, add
details to crowds
"""
crowds = []
for crowd,weak_comp in enumerate(weak_comps):
g = json_graph.adjacency_graph(weak_comp)
dendo = community.generate_dendogram(nx.Graph(g))
if len(dendo)>=2:
partition = community.partition_at_level(dendo, 1 )
crowd_ids = collections.defaultdict(list)
for uid,subcrowd in partition.iteritems():
crowd_ids[subcrowd].append(uid)
for subcrowd,uids in sorted(crowd_ids.iteritems()):
subg = nx.DiGraph(nx.subgraph(g,uids))
if len(subg)>2:
crowds.append(subg)
else:
crowds.append(g)
def _blur(angle):
return random.triangular(angle-.02,angle+.02)
big_spots = collections.defaultdict(list)
lil_spots = collections.defaultdict(list)
for index,g in enumerate(crowds):
# location is location of user with greatest degree
uid,degree = max(g.degree_iter(),key=operator.itemgetter(1))
lng,lat = g.node[uid]['loc']
big_spots[int(lng/2),int(lat/2)].append(g)
lil_spots[int(lng*5),int(lat*5)].append(g)
g.graph['loc'] = lng,lat
g.graph['id'] = index
# add noise to each location based on how popular that area is.
for lng_lat,graphs in lil_spots.iteritems():
graphs.sort(key=len,reverse=True)
for index,g in enumerate(graphs):
lng,lat = g.graph['loc']
ang = random.random()*2*np.pi
dist = .001 * math.sqrt(index)
g.graph['loc'] = lng+1.2*dist*math.cos(ang), lat+dist*math.sin(ang)
# pick crowds to show on map based on size
for lng_lat,graphs in big_spots.iteritems():
graphs.sort(key=len,reverse=True)
for index,g in enumerate(graphs):
g.graph['zoom'] = int(math.floor(1+math.log(index,3))) if index else 0
return (json_graph.adjacency_data(g) for g in crowds)
def as_dict(self):
"""
As in :Class: `pymatgen.core.Structure` except
with using `to_dict_of_dicts` from NetworkX
to store graph information.
"""
d = {"@module": self.__class__.__module__,
"@class": self.__class__.__name__,
"structure": self.structure.as_dict(),
"graphs": json_graph.adjacency_data(self.graph)}
return d
def as_dict(self):
"""
As in :Class: `pymatgen.core.Structure` except
with using `to_dict_of_dicts` from NetworkX
to store graph information.
"""
d = {"@module": self.__class__.__module__,
"@class": self.__class__.__name__,
"structure": self.structure.as_dict(),
"graphs": json_graph.adjacency_data(self.graph)}
return d
def to_json(glist, fname):
""" Accepts a list of networkx graphs and dumps them to file for later
reading """
count = 1
master = {}
for plan in glist:
data = json_graph.adjacency_data(plan)
master["graph-{}".format(count)] = data
count += 1
with open("{}".format(fname), "w") as f:
json.dump(master, f, indent=2)
def grids(horizontal_file, vertical_file):
"""Generates two 10x10 grids with vertically striped districts.
One grid (the A/B grid) has a 40%/60% binary partisan split along
a horizontal line, perpendicular to the districts. Another grid
(the A'/B' grid) has a 40%/60% binary partisan split along a vertical
line, parallel to the districts.
:param horizontal_file: The JSON file to dump the grid with horizontal
partisan split to.
:param vertical_file: The JSON file to dump the grid with horizontal
partisan split to.
"""
graph = nx.grid_graph(dim=[N, N])
for node in graph.nodes:
graph.nodes[node]['population'] = 1
graph.nodes[node]['district'] = (node[0] // 6) + 1
graph.nodes[node]['x'] = node[0] + 1
graph.nodes[node]['y'] = node[1] + 1
horizontal_graph = graph.copy()
vertical_graph = graph.copy()
for node in graph.nodes:
a_share = int(node[1] < SPLIT)
horizontal_graph.nodes[node]['a_share'] = a_share
horizontal_graph.nodes[node]['b_share'] = 1 - a_share
for node in vertical_graph.nodes:
a_share = int(node[0] < SPLIT)
vertical_graph.nodes[node]['a_share'] = a_share
vertical_graph.nodes[node]['b_share'] = 1 - a_share
mapping = {(x, y): (x * N) + y for x, y in horizontal_graph.nodes}
horizontal_graph = nx.relabel_nodes(horizontal_graph, mapping)
vertical_graph = nx.relabel_nodes(vertical_graph, mapping)
with open(horizontal_file, 'w') as adj_file:
json.dump(json_graph.adjacency_data(horizontal_graph), adj_file)
with open(vertical_file, 'w') as adj_file:
json.dump(json_graph.adjacency_data(vertical_graph), adj_file)
def __del__(self):
new = []
for doc in Block.ADJ_GRAPHS.values():
if 'new' in doc:
del doc['new']
data = json_graph.adjacency_data(doc['graph'])
doc['graph'] = bson.Binary(bz2.compress(pickletools.optimize(
pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL))))
new.append(doc)
if doc:
mdb = pymongo.MongoClient('mongodb://127.0.0.1:27017')
mdb.config['graph'].insert_many(new)
mdb.close()
Log.info('New scene graphs uploaded')
Block.ADJ_GRAPHS = {}
def gml2json(infile):
graph = nx.read_gml(infile)
# delete all nodes
for n,d in graph.nodes(data=True):
del d['id']
# convert to json
jdata = json_graph.adjacency_data(graph)
# write to file
f = open(infile.replace('gml','json'),'w')
json.dump(jdata,f)
f.close()
print("[i] Conversion successful...")
def save_following_network_graph(user_list, file_name):
graph = nx.DiGraph()
for user in user_list:
print user
user_followings = get_user_following(user)
graph_edges = [(user, u) for u in user_followings]
graph.add_edges_from(graph_edges)
print 'added {1} edges for user @{0}'.format(user, len(graph_edges))
graph_data = json_graph.adjacency_data(graph)
with open(file_name, 'w') as f:
json.dump(graph_data, f, indent=4)
def __init__(self,
graph: Graph,
initial_state: Partition,
total_steps: int,
pop_col: str,
pop_tol: float,
compressed: bool = True,
reversible: bool = True,
seed: int = None):
"""
:param graph: the GerryChain graph to compile to a Julia graph.
:param initial_state: The state to start from.
:param total_steps: The total steps to take in the chain.
:param pop_col: The column of the graph with population data.
:param pop_tol: The population tolerance to use.
:param compressed: If `True`, plans are represented in `RecomStep`
format. Otherwise, behavior mimics GerryChain.
:param reversible: Determines if the reversible version of the chain
should be used.
:param seed: The random seed to use for plan generation.
"""
for node, assignment in initial_state.assignment.items():
graph.nodes[node]['__district'] = assignment
graph_data = json_graph.adjacency_data(graph)
self.graph = Flips.IndexedGraph(graph_data, pop_col)
# TODO: verify initial state is valid w.r.t. population constraints
# (+ more?)
self.plan = Flips.Plan(self.graph, '__district')
total_pop = sum(graph.nodes[node][pop_col] for node in graph.nodes)
districts = list(initial_state.assignment.values())
n_districts = max(districts) - min(districts) + 1
district_pop = total_pop / n_districts
self.min_pop = int(ceil((1 - pop_tol) * district_pop))
self.max_pop = int(floor((1 + pop_tol) * district_pop))
self.reversible = reversible
self.compressed = compressed
self._curr_parent = initial_state
self.total_steps = total_steps
self._curr_step = 0
self._buf = deque() # https://stackoverflow.com/a/4426727/8610749
self.data = None
if seed is None:
self._twister = Random.MersenneTwister()
else:
self._twister = Random.MersenneTwister(seed)
def to_json(self, json_file, *, include_geometries_as_geojson=False):
"""Save a graph to a JSON file in the NetworkX json_graph format.
:param json_file: Path to target JSON file.
:param bool include_geometry_as_geojson: (optional) Whether to include any
:mod:`shapely` geometry objects encountered in the graph's node attributes
as GeoJSON. The default (``False``) behavior is to remove all geometry
objects because they are not serializable. Including the GeoJSON will result
in a much larger JSON file.
"""
data = json_graph.adjacency_data(self)
if include_geometries_as_geojson:
convert_geometries_to_geojson(data)
else:
remove_geometries(data)
with open(json_file, "w") as f:
json.dump(data, f)
def add_flow(self, name, input_flow):
new_flow = [[y[0:2] for y in x] for x in input_flow]
for flow in input_flow:
for app in flow:
if not self.graph.has_node(app[0:2]):
raise Exception('Please define application {0} first.'.format(app))
elif self.graph.node[app[0:2]]['full_name'] != app:
raise Exception('Names of app abbreviations {0} and '
'{1} don\'t match.'.format(self.graph.node[app[0:2]]['full_name'], app))
flow = nx.DiGraph()
for path in new_flow:
flow.add_path(path, weight=1)
# copy node attributes
for node in flow.nodes():
flow.node[node] = self.graph.node[node]
self.flows[name] = flow
print "graph {0} created: {1}".format(name, json_graph.adjacency_data(flow))
self.save_flows()
self.save_graph(name, flow)
self.merge_graphs(flow)
def simulate(p=0, alpha=25e-6, resolution=40000, fname='results/data.json'):
""" Simulate Bornholdt/Sneppen model
"""
if os.path.exists(fname):
print(f'File {fname} exists, skipping')
return
N = 128
tmax = resolution * 2 #80000
freq = int(tmax / resolution) #1/N**2
print(f'Simulating with p={p}, alpha={alpha} ({fname})')
print(
f'Saving data every {freq*N**2} time steps, resulting in {int(tmax/(freq*N**2))} ({int(tmax/freq)}) data points'
)
node_list = [(i // N, i % N) for i in range(N**2)]
graph = generate_graph(N, p)
graph_mat = nx.to_numpy_matrix(graph, nodelist=node_list)
graph_repr = convert_matrix(graph_mat)
sim = Simulator(N, p, alpha, tmax, freq, fname, graph_repr)
start = time.time()
snapshot_num = sim.run()
end = time.time()
assert snapshot_num == int(tmax / freq), snapshot_num
print(f' > Runtime: {end-start:.0f}s')
# save graph
with open(fname) as fd:
data = json.load(fd)
data['graph'] = json_graph.adjacency_data(graph)
with open(fname, 'w') as fd:
json.dump(data, fd)
def weak_comps(edges,user_locs):
"""convert near_edges into networkx format, keep weakly-connected users"""
g = nx.DiGraph()
for edge in edges:
ne = NearEdge(*edge)
conv = (ne.day,ne.at,ne.rt)
if g.has_edge(ne.frm,ne.to):
g[ne.frm][ne.to]['conv'].append(conv)
else:
g.add_edge(ne.frm, ne.to, dist=ne.dist, conv=[conv])
# remove edges that are only retweets
for frm,to,data in g.edges(data=True):
if not any(at for day,at,rt in data['conv']):
g.remove_edge(frm,to)
# remove nodes with a degree greater than 50
popular = [uid for uid,degree in g.degree_iter() if degree>50]
g.remove_nodes_from(popular)
# add locations
for node in g.nodes_iter():
g.node[node]['loc'] = user_locs[node]
# yield weakly connected crowds
for subg in nx.weakly_connected_component_subgraphs(g):
if len(subg)>2:
yield json_graph.adjacency_data(subg)
def network_x_block_together(self,
population,
location=None,
additional=None):
"""
:param location: The location parameters which the blocks are checked for (scene_type for example).
:param population: These are the parameters which the blocks are checked for.
:param additional: Additional attributes for the blocks calculation:
1. calculate_all_scenes: whether to check all the relevant scenes for a block or stop once a block was found.
If True, the function will search for blocks in all scenes (which will affect the performance!!).
If False, the function will stop running once a scene was found with the relevant block.
2. horizontal_bucket_size: Used to calculate the number of products in order to determine whether a block
is vertical or horizontal. Usually 1ft - the length of the bay
3. outliers_threshold: The threshold for removing the outliers for the horizontal/vertical calculation.
4. include_stacking: Whether the products can be stacked on the shelf or not
5. minimum_facing_for_block: Minimum number of facings that needs to be in a block to consider it as one.
6. adjacency_overlap_ratio: Minimal threshold the overlap between the products must exceeds to be considered
as adjacent.
7. allowed_products_filters: These are the parameters which are allowed to corrupt the block without failing it.
8. minimum_block_ratio: The minimum (block number of facings / total number of relevant facings) ratio
in order for KPI to pass (if ratio=1, then only one block is allowed).
9. check_vertical_horizontal: True if the orientation (vertical or horizontal) of the block should be checked for.
10. allowed_edge_type: This determines what allowed products will actually be allowed. That is, allowed products
in scene can be filtered based on their connections (or lack of) to relevant group products. Format is a
Mix and Match list of -> [unconnected, connected, encapsulated].
unconnected-> allowed nodes with no connections to relevant products.
connected -> allowed nodes that connect to both relevant group and irrelevant group nodes
encapsulated -> allowed nodes that only connect to relevant group products
:return: df with the following fields;
block (Graph), scene_fk, orientation (string - vertical or horizontal), facing_percentage.
"""
block_parameters = {
AdditionalAttr.MINIMUM_BLOCK_RATIO: Default.minimum_block_ratio,
AdditionalAttr.CHECK_VERTICAL_HORIZONTAL:
Default.check_vertical_horizontal,
AdditionalAttr.ALLOWED_PRODUCTS_FILTERS: None,
AdditionalAttr.ADJACENCY_OVERLAP_RATIO:
Default.adjacency_overlap_ratio,
AdditionalAttr.MINIMUM_FACING_FOR_BLOCK:
Default.minimum_facing_for_block,
AdditionalAttr.INCLUDE_STACKING: Default.include_stacking,
AdditionalAttr.IGNORE_EMPTY: Default.ignore_empty,
AdditionalAttr.OUTLIERS_THRESHOLD: Default.outliers_threshold,
AdditionalAttr.CALCULATE_ALL_SCENES: Default.calculate_all_scenes,
AdditionalAttr.ALLOWED_EDGE_TYPE: Default.allowed_edge_type,
AdditionalAttr.FILTER_OPERATOR: Default.filter_operator
}
if additional is not None:
block_parameters.update(additional)
self.outliers_threshold = block_parameters[
AdditionalAttr.OUTLIERS_THRESHOLD]
self.check_vertical_horizontal = block_parameters[
AdditionalAttr.CHECK_VERTICAL_HORIZONTAL]
# self.include_stacking = block_parameters[AdditionalAttr.INCLUDE_STACKING]
self.include_stacking = True
self.ignore_empty = block_parameters[AdditionalAttr.IGNORE_EMPTY]
self.allowed_edge_type = block_parameters[
AdditionalAttr.ALLOWED_EDGE_TYPE]
operator = block_parameters[AdditionalAttr.FILTER_OPERATOR]
# Constructing the result_df that will be returned if calculate_all_scenes = True
results_df = pd.DataFrame(columns=[
ColumnNames.CLUSTER, ColumnNames.SCENE_FK, ColumnNames.ORIENTATION,
ColumnNames.FACING_PERCENTAGE, ColumnNames.IS_BLOCK
])
# Separate the filters on products from the filters on scene and get the relevant scenes
if location is not None:
conditions = {'location': location}
relevant_scenes = self.input_parser.filter_df(
conditions, self.scif).scene_id.unique()
else:
relevant_scenes = self.scif.scene_id.unique()
if not relevant_scenes.any():
Log.info('No scenes with the requested location filter.')
return results_df
# Identify relevant block values
block_value = [
'_'.join(str(x) for x in elem)
for elem in list(itertools.product(*population.values()))
]
# Creating unified filters from the block filters and the allowed filters
allowed_product_filter = []
if block_parameters[AdditionalAttr.ALLOWED_PRODUCTS_FILTERS]:
allowed_product_filter = block_parameters[
AdditionalAttr.ALLOWED_PRODUCTS_FILTERS].keys()
# For each relevant scene check if a block is exist
for scene in relevant_scenes:
try:
# filter masking and matches data by the scene
scene_mask = self.masking_data[self.masking_data['scene_fk'] ==
scene].drop('scene_fk', axis=1)
scene_matches = self.matches_df[self.matches_df['scene_fk'] ==
scene]
relevant_matches_for_block = self.filter_graph_data(
scene_matches,
population,
operator=operator,
is_blocks_graph=False)
scene_matches = scene_matches.drop(
'pk', axis=1).rename(columns={'scene_match_fk': 'pk'})
if relevant_matches_for_block.empty:
continue
import time
times = []
start = time.time()
graph = AdjacencyGraphBuilder.initiate_graph_by_dataframe(
scene_matches,
scene_mask,
additional_attributes=['rect_x', 'rect_y'] +
allowed_product_filter + list(scene_matches.columns))
start = time.time() - start
times.append(['build graph', start])
print('{} to build graph'.format(start))
start = time.time()
graph = AdjacencyGraphBuilder.condense_graph_by_level(
CalcConst.PRODUCT_FK, graph)
start = time.time() - start
times.append(['simplify graph', start])
print('{} to build simplify'.format(start))
start = time.time()
import pymongo
from networkx.readwrite import json_graph
import json
import cPickle as pickle
import pickletools
import bz2
import bson
mdb = pymongo.MongoClient('mongodb://127.0.0.1:27017')
col = mdb.config['graph']
start = time.time() - start
times.append(['mongo connect', start])
print('{} to connect to mongo'.format(start))
start = time.time()
data = json_graph.adjacency_data(graph)
start = time.time() - start
times.append(['extract graph data', start])
print('{} extract graph data'.format(start))
start = time.time()
print('~~~~~~~~~~~~')
g = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
start = time.time() - start
print('{} pickle'.format(start))
start = time.time()
g = pickletools.optimize(g)
start = time.time() - start
print('{} optimize'.format(start))
start = time.time()
g = bz2.compress(g)
start = time.time() - start
print('{} compress'.format(start))
start = time.time()
g = bson.Binary(g)
start = time.time() - start
print('{} bsonify'.format(start))
start = time.time()
print('~~~~~~~~~~~~')
# bson.Binary(bz2.compress(pickletools.optimize(pickle.dumps(data))))
pkl = {
'project': self.data_provider.project_name,
'scene': scene,
'session': self.data_provider.session_uid,
'graph': g
}
start = time.time() - start
times.append(['pickle graph', start])
print('{} to pickle graph'.format(start))
start = time.time()
col.insert_one(pkl)
start = time.time() - start
times.append(['insert to mongo', start])
print('{} to insert into mongo'.format(start))
start = time.time()
doc = col.find({
'project': self.data_provider.project_name,
'scene': scene
})[0]
start = time.time() - start
times.append(['load from mongo', start])
print('{} to load graph from mongo'.format(start))
start = time.time()
upkl = pickle.loads(bz2.decompress(doc['graph']))
start = time.time() - start
times.append(['unpickle graph', start])
print('{} to unpickle'.format(start))
start = time.time()
if not self.include_stacking:
scene_matches = self._get_no_stack_data(scene_matches)
# check if the adj_g already exists for this scene, if not create it and save it
if scene not in self.adj_graphs_by_scene:
# Create the adjacency graph on the data we filtered above
graph = AdjacencyGraphBuilder.initiate_graph_by_dataframe(
scene_matches,
scene_mask,
additional_attributes=['rect_x', 'rect_y'] +
allowed_product_filter + list(scene_matches.columns))
self.adj_graphs_by_scene[scene] = graph
adj_g = self.adj_graphs_by_scene[scene].copy()
# Update the block_key node attribute based on the population fields
for i, n in adj_g.nodes(data=True):
n[Block.BLOCK_KEY].stored_values = set(
['_'.join([str(*n[x]) for x in population.keys()])])
if allowed_product_filter:
adj_g = self._set_allowed_nodes(
adj_g, block_parameters[
AdditionalAttr.ALLOWED_PRODUCTS_FILTERS],
block_value)
# TODO: filter nodes to subgraph
# Filter only relevant nodes for block out of the graph
filtered_nodes = list(
n for n, d in adj_g.nodes(data=True)
if list(d[Block.BLOCK_KEY])[0] in block_value +
self.allowed_edge_type)
# Create a sub graph based on the adj_g containing only the filtered nodes
adj_g = adj_g.subgraph(filtered_nodes)
# Creating the condensed graph based on the adjacency graph created above
if len(population.keys()
) == 1 and population.keys()[0] != CalcConst.PRODUCT_FK:
# adj_g = AdjacencyGraphBuilder.build_adjacency_graph_by_level(Block.BLOCK_KEY, adj_g)
adj_g = AdjacencyGraphBuilder.condense_graph_by_level(
Block.BLOCK_KEY, adj_g)
# Transferring the graph to be undirected
condensed_graph_sku = adj_g.to_undirected()
# Calculating all the components in the graph
components = list(
nx.connected_component_subgraphs(condensed_graph_sku))
# Constructing the block_res df that will contain each block's data
blocks_res = pd.DataFrame(
columns=['component', 'sum_of_facings', 'num_of_shelves'])
# For each component in the graph calculate the blocks data
for component in components:
component_data = self.calculate_block_data(
component, relevant_matches_for_block)
blocks_res = blocks_res.append(component_data)
total_facings = blocks_res['sum_of_facings'].sum()
# For each block check if its valid or not
for row in blocks_res.itertuples():
if not row.sum_of_facings:
continue
orientation = None
# Calculate facing percentage
facing_percentage = np.divide(float(row.sum_of_facings),
float(total_facings))
if (facing_percentage >= block_parameters[AdditionalAttr.MINIMUM_BLOCK_RATIO]) and \
(row.sum_of_facings >= block_parameters[AdditionalAttr.MINIMUM_FACING_FOR_BLOCK]):
# If needed, calculate the block's orientation
if block_parameters[
AdditionalAttr.CHECK_VERTICAL_HORIZONTAL]:
orientation = self.handle_horizontal_and_vertical(
row.component, row.num_of_shelves)
results_df = results_df.append(
pd.DataFrame(columns=[
ColumnNames.CLUSTER, ColumnNames.SCENE_FK,
ColumnNames.ORIENTATION,
ColumnNames.FACING_PERCENTAGE,
ColumnNames.IS_BLOCK
],
data=[[
row.component, scene, orientation,
facing_percentage, True
]]))
if not block_parameters[
AdditionalAttr.CALCULATE_ALL_SCENES]:
return results_df
else:
results_df = results_df.append(
pd.DataFrame(columns=[
ColumnNames.CLUSTER, ColumnNames.SCENE_FK,
ColumnNames.ORIENTATION,
ColumnNames.FACING_PERCENTAGE,
ColumnNames.IS_BLOCK
],
data=[[
row.component, scene, orientation,
facing_percentage, False
]]))
except AttributeError as err:
Log.error('{}'.format(err))
continue
except Exception as err:
Log.error('{}'.format(err))
return results_df
return results_df
def __mul__(self, scaling_matrix):
"""
Replicates the graph, creating a supercell,
intelligently joining together
edges that lie on periodic boundaries.
In principle, any operations on the expanded
graph could also be done on the original
graph, but a larger graph can be easier to
visualize and reason about.
:param scaling_matrix: same as Structure.__mul__
:return:
"""
# Developer note: a different approach was also trialed, using
# a simple Graph (instead of MultiDiGraph), with node indices
# representing both site index and periodic image. Here, the
# number of nodes != number of sites in the Structure. This
# approach has many benefits, but made it more difficult to
# keep the graph in sync with its corresponding Structure.
# Broadly, it would be easier to multiply the Structure
# *before* generating the StructureGraph, but this isn't
# possible when generating the graph using critic2 from
# charge density.
# Multiplication works by looking for the expected position
# of an image node, and seeing if that node exists in the
# supercell. If it does, the edge is updated. This is more
# computationally expensive than just keeping track of the
# which new lattice images present, but should hopefully be
# easier to extend to a general 3x3 scaling matrix.
# code adapted from Structure.__mul__
scale_matrix = np.array(scaling_matrix, np.int16)
if scale_matrix.shape != (3, 3):
scale_matrix = np.array(scale_matrix * np.eye(3), np.int16)
else:
# TODO: test __mul__ with full 3x3 scaling matrices
raise NotImplementedError('Not tested with 3x3 scaling matrices yet.')
new_lattice = Lattice(np.dot(scale_matrix, self.structure.lattice.matrix))
f_lat = lattice_points_in_supercell(scale_matrix)
c_lat = new_lattice.get_cartesian_coords(f_lat)
new_sites = []
new_graphs = []
for v in c_lat:
# create a map of nodes from original graph to its image
mapping = {n: n + len(new_sites) for n in range(len(self.structure))}
for idx, site in enumerate(self.structure):
s = PeriodicSite(site.species_and_occu, site.coords + v,
new_lattice, properties=site.properties,
coords_are_cartesian=True, to_unit_cell=False)
new_sites.append(s)
new_graphs.append(nx.relabel_nodes(self.graph, mapping, copy=True))
new_structure = Structure.from_sites(new_sites)
# merge all graphs into one big graph
new_g = nx.MultiDiGraph()
for new_graph in new_graphs:
new_g = nx.union(new_g, new_graph)
edges_to_remove = [] # tuple of (u, v, k)
edges_to_add = [] # tuple of (u, v, attr_dict)
# list of new edges inside supercell
# for duplicate checking
edges_inside_supercell = [{u, v} for u, v, d in new_g.edges(data=True)
if d['to_jimage'] == (0, 0, 0)]
new_periodic_images = []
orig_lattice = self.structure.lattice
# use k-d tree to match given position to an
# existing Site in Structure
kd_tree = KDTree(new_structure.cart_coords)
# tolerance in Å for sites to be considered equal
# this could probably be a lot smaller
tol = 0.05
for u, v, k, d in new_g.edges(keys=True, data=True):
to_jimage = d['to_jimage'] # for node v
# reduce unnecessary checking
if to_jimage != (0, 0, 0):
# get index in original site
n_u = u % len(self.structure)
n_v = v % len(self.structure)
# get fractional co-ordinates of where atoms defined
# by edge are expected to be, relative to original
# lattice (keeping original lattice has
# significant benefits)
v_image_frac = np.add(self.structure[n_v].frac_coords, to_jimage)
u_frac = self.structure[n_u].frac_coords
# using the position of node u as a reference,
# get relative Cartesian co-ordinates of where
# atoms defined by edge are expected to be
v_image_cart = orig_lattice.get_cartesian_coords(v_image_frac)
u_cart = orig_lattice.get_cartesian_coords(u_frac)
v_rel = np.subtract(v_image_cart, u_cart)
# now retrieve position of node v in
# new supercell, and get absolute Cartesian
# co-ordinates of where atoms defined by edge
# are expected to be
v_expec = new_structure[u].coords + v_rel
# now search in new structure for these atoms
# query returns (distance, index)
v_present = kd_tree.query(v_expec)
v_present = v_present[1] if v_present[0] <= tol else None
# check if image sites now present in supercell
# and if so, delete old edge that went through
# periodic boundary
if v_present is not None:
new_u = u
new_v = v_present
new_d = d.copy()
# node now inside supercell
new_d['to_jimage'] = (0, 0, 0)
edges_to_remove.append((u, v, k))
# make sure we don't try to add duplicate edges
# will remove two edges for everyone one we add
if {new_u, new_v} not in edges_inside_supercell:
# normalize direction
if new_v < new_u:
new_u, new_v = new_v, new_u
edges_inside_supercell.append({new_u, new_v})
edges_to_add.append((new_u, new_v, new_d))
else:
# want to find new_v such that we have
# full periodic boundary conditions
# so that nodes on one side of supercell
# are connected to nodes on opposite side
v_expec_frac = new_structure.lattice.get_fractional_coords(v_expec)
# find new to_jimage
# use np.around to fix issues with finite precision leading to incorrect image
v_expec_image = np.around(v_expec_frac, decimals=3)
v_expec_image = v_expec_image - v_expec_image%1
v_expec_frac = np.subtract(v_expec_frac, v_expec_image)
v_expec = new_structure.lattice.get_cartesian_coords(v_expec_frac)
v_present = kd_tree.query(v_expec)
v_present = v_present[1] if v_present[0] <= tol else None
if v_present is not None:
new_u = u
new_v = v_present
new_d = d.copy()
new_to_jimage = tuple(map(int, v_expec_image))
# normalize direction
if new_v < new_u:
new_u, new_v = new_v, new_u
new_to_jimage = tuple(np.multiply(-1, d['to_jimage']).astype(int))
new_d['to_jimage'] = new_to_jimage
edges_to_remove.append((u, v, k))
if (new_u, new_v, new_to_jimage) not in new_periodic_images:
edges_to_add.append((new_u, new_v, new_d))
new_periodic_images.append((new_u, new_v, new_to_jimage))
logger.debug("Removing {} edges, adding {} new edges.".format(len(edges_to_remove),
len(edges_to_add)))
# add/delete marked edges
for edges_to_remove in edges_to_remove:
new_g.remove_edge(*edges_to_remove)
for (u, v, d) in edges_to_add:
new_g.add_edge(u, v, **d)
# return new instance of StructureGraph with supercell
d = {"@module": self.__class__.__module__,
"@class": self.__class__.__name__,
"structure": new_structure.as_dict(),
"graphs": json_graph.adjacency_data(new_g)}
sg = StructureGraph.from_dict(d)
return sg
def serialize(self):
return json_graph.adjacency_data(self._graph)
def get_prereq_graph(course_id, format=None):
"""
Generate a graph of prerequisites within a course. If format is not
requested, simply return a NetworkX graph object.
couse_id: the ID of the requested course
format: what format to return in (optional)
node: json formatted as node-link style
adjacency: json formatted as adjacency style
tree: json formatted as tree style
"""
from trajectory.models import Department, Course
from trajectory.models.meta import session
from trajectory.utils.common import row2dict
from networkx.readwrite import json_graph
import networkx as nx
import json
if format not in [None, "node", "adjacency", "tree"]:
raise RuntimeError("Unknown requested data format %s" % format)
# Initialize a new NetworkX graph.
G = nx.DiGraph()
# Attempt to look up the requested course.
course = session.query(Course).get(course_id)
if course is None:
return None
# Recursively add course ids in a subtree to the graph.
def add_tree(G, tree, parent=None):
cid = tree[0] # unpack information
prereqs = tree[1] # unpack information
course = session.query(Course).get(cid)
# Insert all known data, including department abbreviation.
node_data = row2dict(course)
node_data['dept'] = course.department.abbreviation
# Identify the primary course in the graph (the requested).
if str(cid) == str(course_id):
node_data['prime'] = True
else:
node_data['prime'] = False
# If the course has already been added, generate a unique ID for it
# based on its parent, and add it anyway. But don't recurse into
# its list of prereqs.
seen = False
if cid in G.nodes():
cid = str(parent) + "-" + str(cid)
seen = True
# Add course and an edge from its parent, if relevant.
G.add_node(cid, node_data)
if parent is not None:
G.add_edge(parent, cid)
# Recurse through the prerequisite tree and add in subtrees.
if not seen:
for prereq in prereqs:
add_tree(G, prereq, cid)
# Navigate the prerequisite tree and add the course ids as nodes, and
# prerequisite relationships as unweighted edges.
prereq_tree = get_prereq_tree(course_id)
add_tree(G, prereq_tree)
if G is None:
return G
# Calculate and apply a basic layout.
pos = nx.spring_layout(G)
for node in G.nodes():
G.node[node]["viz"] = {
'position': {
'x': pos[node][0],
'y': pos[node][1]
}
}
# Apply any requested data output formatting.
if format == "node":
return json.dumps(json_graph.node_link_data(G))
elif format == "adjacency":
return json.dumps(json_graph.adjacency_data(G))
elif format == "tree":
return json.dumps(json_graph.tree_data(G, int(course_id)))
else:
return G
import json
import glob
# Blockgroups 20
bg_dgs = glob.glob("bg20_dgs/*")
for dg in bg_dgs:
st_name = dg.split("/")[-1].split(".")[0]
with open(dg) as fin:
data = json.load(fin)
graph = json_graph.adjacency_graph(data)
mapping = {n: graph.nodes()[n]["GEOID20"] for n in graph.nodes()}
nx.relabel.relabel_nodes(graph, mapping=mapping, copy=False)
data_out = json_graph.adjacency_data(graph)
node_data = [{"id": n["id"]} for n in data_out["nodes"]]
data_out["nodes"] = node_data
with open("graphs/{}_blockgroups20.json".format(st_name), "w") as fout:
json.dump(data_out, fout)
# VTDs 20
vtd_dgs = glob.glob("vtd_dg/*")
for dg in vtd_dgs:
st_name = dg.split("/")[-1].split(".")[0]
with open(dg) as fin:
data = json.load(fin)
graph = json_graph.adjacency_graph(data)
if __name__ == '__main__':
graph = nx.read_edgelist('input/example_graph.edgelist', nodetype=int, data=(('weight', float),))
assert isinstance(graph, nx.Graph)
print 'edges:', graph.edges()
# raw
nx.write_adjlist(graph, 'output_raw/example_graph.adjlist')
nx.write_multiline_adjlist(graph, 'output_raw/example_graph.multiline_adjlist')
nx.write_edgelist(graph, 'output_raw/example_graph.edgelist')
# better serialization
nx.write_gpickle(graph, 'output_serialization/example_graph.pickle')
nx.write_yaml(graph, 'output_serialization/example_graph.yaml')
nx.write_graph6(graph, 'output_serialization/example_graph.graph6')
# xml
nx.write_gexf(graph, 'output_xml/example_graph.gexf')
nx.write_graphml(graph, 'output_xml/example_graph.graphml')
# json
with open('output_json/node_link.json', 'w') as outfile:
json.dump(json_graph.node_link_data(graph), outfile, indent=2)
with open('output_json/adjacency.json', 'w') as outfile:
json.dump(json_graph.adjacency_data(graph), outfile, indent=2)
# other
nx.write_gml(graph, 'output_other/example_graph.gml')
nx.write_pajek(graph, 'output_other/example_graph.pajek')
def json_graph_list_dump(graph_list,file):
with open_filename(file,'w') as f:
json.dump([json_graph.adjacency_data(graph) for graph in graph_list],f)
pass
def __mul__(self, scaling_matrix):
"""
Replicates the graph, creating a supercell,
intelligently joining together
edges that lie on periodic boundaries.
In principle, any operations on the expanded
graph could also be done on the original
graph, but a larger graph can be easier to
visualize and reason about.
:param scaling_matrix: same as Structure.__mul__
:return:
"""
# Developer note: a different approach was also trialed, using
# a simple Graph (instead of MultiDiGraph), with node indices
# representing both site index and periodic image. Here, the
# number of nodes != number of sites in the Structure. This
# approach has many benefits, but made it more difficult to
# keep the graph in sync with its corresponding Structure.
# Broadly, it would be easier to multiply the Structure
# *before* generating the StructureGraph, but this isn't
# possible when generating the graph using critic2 from
# charge density.
# Multiplication works by looking for the expected position
# of an image node, and seeing if that node exists in the
# supercell. If it does, the edge is updated. This is more
# computationally expensive than just keeping track of the
# which new lattice images present, but should hopefully be
# easier to extend to a general 3x3 scaling matrix.
# code adapted from Structure.__mul__
scale_matrix = np.array(scaling_matrix, np.int16)
if scale_matrix.shape != (3, 3):
scale_matrix = np.array(scale_matrix * np.eye(3), np.int16)
else:
# TODO: test __mul__ with full 3x3 scaling matrices
raise NotImplementedError('Not tested with 3x3 scaling matrices yet.')
new_lattice = Lattice(np.dot(scale_matrix, self.structure.lattice.matrix))
f_lat = lattice_points_in_supercell(scale_matrix)
c_lat = new_lattice.get_cartesian_coords(f_lat)
new_sites = []
new_graphs = []
for v in c_lat:
# create a map of nodes from original graph to its image
mapping = {n: n + len(new_sites) for n in range(len(self.structure))}
for idx, site in enumerate(self.structure):
s = PeriodicSite(site.species_and_occu, site.coords + v,
new_lattice, properties=site.properties,
coords_are_cartesian=True, to_unit_cell=False)
new_sites.append(s)
new_graphs.append(nx.relabel_nodes(self.graph, mapping, copy=True))
new_structure = Structure.from_sites(new_sites)
# merge all graphs into one big graph
new_g = nx.MultiDiGraph()
for new_graph in new_graphs:
new_g = nx.union(new_g, new_graph)
edges_to_remove = [] # tuple of (u, v, k)
edges_to_add = [] # tuple of (u, v, attr_dict)
# list of new edges inside supercell
# for duplicate checking
edges_inside_supercell = [{u, v} for u, v, d in new_g.edges(data=True)
if d['to_jimage'] == (0, 0, 0)]
new_periodic_images = []
orig_lattice = self.structure.lattice
# use k-d tree to match given position to an
# existing Site in Structure
kd_tree = KDTree(new_structure.cart_coords)
# tolerance in Å for sites to be considered equal
# this could probably be a lot smaller
tol = 0.05
for u, v, k, d in new_g.edges(keys=True, data=True):
to_jimage = d['to_jimage'] # for node v
# reduce unnecessary checking
if to_jimage != (0, 0, 0):
# get index in original site
n_u = u % len(self.structure)
n_v = v % len(self.structure)
# get fractional co-ordinates of where atoms defined
# by edge are expected to be, relative to original
# lattice (keeping original lattice has
# significant benefits)
v_image_frac = np.add(self.structure[n_v].frac_coords, to_jimage)
u_frac = self.structure[n_u].frac_coords
# using the position of node u as a reference,
# get relative Cartesian co-ordinates of where
# atoms defined by edge are expected to be
v_image_cart = orig_lattice.get_cartesian_coords(v_image_frac)
u_cart = orig_lattice.get_cartesian_coords(u_frac)
v_rel = np.subtract(v_image_cart, u_cart)
# now retrieve position of node v in
# new supercell, and get absolute Cartesian
# co-ordinates of where atoms defined by edge
# are expected to be
v_expec = new_structure[u].coords + v_rel
# now search in new structure for these atoms
# query returns (distance, index)
v_present = kd_tree.query(v_expec)
v_present = v_present[1] if v_present[0] <= tol else None
# check if image sites now present in supercell
# and if so, delete old edge that went through
# periodic boundary
if v_present is not None:
new_u = u
new_v = v_present
new_d = d.copy()
# node now inside supercell
new_d['to_jimage'] = (0, 0, 0)
edges_to_remove.append((u, v, k))
# make sure we don't try to add duplicate edges
# will remove two edges for everyone one we add
if {new_u, new_v} not in edges_inside_supercell:
# normalize direction
if new_v < new_u:
new_u, new_v = new_v, new_u
edges_inside_supercell.append({new_u, new_v})
edges_to_add.append((new_u, new_v, new_d))
else:
# want to find new_v such that we have
# full periodic boundary conditions
# so that nodes on one side of supercell
# are connected to nodes on opposite side
v_expec_frac = new_structure.lattice.get_fractional_coords(v_expec)
# find new to_jimage
# use np.around to fix issues with finite precision leading to incorrect image
v_expec_image = np.around(v_expec_frac, decimals=3)
v_expec_image = v_expec_image - v_expec_image%1
v_expec_frac = np.subtract(v_expec_frac, v_expec_image)
v_expec = new_structure.lattice.get_cartesian_coords(v_expec_frac)
v_present = kd_tree.query(v_expec)
v_present = v_present[1] if v_present[0] <= tol else None
if v_present is not None:
new_u = u
new_v = v_present
new_d = d.copy()
new_to_jimage = tuple(map(int, v_expec_image))
# normalize direction
if new_v < new_u:
new_u, new_v = new_v, new_u
new_to_jimage = tuple(np.multiply(-1, d['to_jimage']).astype(int))
new_d['to_jimage'] = new_to_jimage
edges_to_remove.append((u, v, k))
if (new_u, new_v, new_to_jimage) not in new_periodic_images:
edges_to_add.append((new_u, new_v, new_d))
new_periodic_images.append((new_u, new_v, new_to_jimage))
logger.debug("Removing {} edges, adding {} new edges.".format(len(edges_to_remove),
len(edges_to_add)))
# add/delete marked edges
for edges_to_remove in edges_to_remove:
new_g.remove_edge(*edges_to_remove)
for (u, v, d) in edges_to_add:
new_g.add_edge(u, v, **d)
# return new instance of StructureGraph with supercell
d = {"@module": self.__class__.__module__,
"@class": self.__class__.__name__,
"structure": new_structure.as_dict(),
"graphs": json_graph.adjacency_data(new_g)}
sg = StructureGraph.from_dict(d)
return sg
def to_json(self): return json_graph.adjacency_data(self.network)
def to_json(self):
return json_graph.adjacency_data(self.network)
def saveGraph(self):
data = json_graph.adjacency_data(self.G)
with open('graph.json', 'w') as outfile:
json.dump(data, outfile, cls=SetEncoder)
def communities(args, neighbor_weight=None):
graphname = args.graphname or 'network'
edge_weights = args.weight
vertex_weights = str('FamilyFrequency')
normalize = args.normalize
edgefilter = args.edgefilter
threshold = args.threshold or 1
neighbor_weight = neighbor_weight or 5
_graph = args.api.load_graph(graphname, threshold, edgefilter)
args.log.info('loaded graph')
for n, d in tqdm(_graph.nodes(data=True), desc='vertex-weights', leave=False):
d[vertex_weights] = int(d[vertex_weights])
if normalize:
for edgeA, edgeB, data in tqdm(_graph.edges(data=True), desc='normalizing', leave=False):
data[str('weight')] = data[edge_weights] ** 2 / (
_graph.node[edgeA][vertex_weights] +
_graph.node[edgeB][vertex_weights] -
data[edge_weights])
vertex_weights = None
edge_weights = 'weight'
args.log.info('computed weights')
graph = networkx2igraph(_graph)
args.log.info('starting infomap')
args.log.info('converted graph...')
comps = graph.community_infomap(
edge_weights=str(edge_weights), vertex_weights=vertex_weights)
args.log.info('finished infomap')
D, Com = {}, defaultdict(list)
for i, comp in enumerate(sorted(comps.subgraphs(), key=lambda x: len(x.vs), reverse=True)):
for vertex in [v['name'] for v in comp.vs]:
D[graph.vs[vertex]['ConcepticonId']] = str(i + 1)
Com[i + 1].append(graph.vs[vertex]['ConcepticonId'])
for node, data in _graph.nodes(data=True):
data['infomap'] = D[node]
data['ClusterName'] = ''
data['CentralConcept'] = ''
# get the articulation points etc. immediately
for idx, nodes in sorted(Com.items()):
sg = _graph.subgraph(nodes)
if len(sg) > 1:
d_ = sorted(sg.degree(), key=lambda x: x[1], reverse=True)
d = [_graph.node[a]['Gloss'] for a, b in d_][0]
cluster_name = 'infomap_{0}_{1}'.format(idx, d)
else:
d = _graph.node[nodes[0]]['Gloss']
cluster_name = 'infomap_{0}_{1}'.format(idx, _graph.node[nodes[0]]['Gloss'])
args.log.debug(cluster_name, d)
for node in nodes:
_graph.node[node]['ClusterName'] = cluster_name
_graph.node[node]['CentralConcept'] = d
args.log.info('computed cluster names')
cluster_dir = args.api.existing_dir('app', 'cluster', clean=True)
cluster_names = {}
removed = []
for idx, nodes in tqdm(sorted(Com.items()), desc='export to app', leave=False):
sg = _graph.subgraph(nodes)
for node, data in sg.nodes(data=True):
data['OutEdge'] = []
neighbors = [
n for n in _graph if
n in _graph[node] and
_graph[node][n]['FamilyWeight'] >= neighbor_weight and
n not in sg]
if neighbors:
sg.node[node]['OutEdge'] = []
for n in neighbors:
sg.node[node]['OutEdge'].append([
_graph.node[n]['ClusterName'],
_graph.node[n]['CentralConcept'],
_graph.node[n]['Gloss'],
_graph[node][n]['WordWeight'],
n
])
if len(sg) > 1:
jsonlib.dump(
json_graph.adjacency_data(sg),
cluster_dir / (_graph.node[nodes[0]]['ClusterName'] + '.json'),
sort_keys=True)
for node in nodes:
cluster_names[_graph.node[node]['Gloss']] = _graph.node[node]['ClusterName']
else:
removed += [list(nodes)[0]]
_graph.remove_nodes_from(removed)
for node, data in _graph.nodes(data=True):
if 'OutEdge' in data:
data['OutEdge'] = '//'.join(['/'.join([str(y) for y in x]) for x in data['OutEdge']])
removed = []
for nA, nB, data in tqdm(_graph.edges(data=True), desc='remove edges', leave=False):
if _graph.node[nA]['infomap'] != _graph.node[nB]['infomap'] and data['FamilyWeight'] < 5:
removed += [(nA, nB)]
_graph.remove_edges_from(removed)
args.api.save_graph(_graph, 'infomap', threshold, edgefilter)
args.api.write_js_var('INFO', cluster_names, 'app', 'source', 'infomap-names.js')
def adjacency_data_json():
H = generate_complete_graph()
return json.dumps(json_graph.adjacency_data(H))
sys.stdout.flush()
count += 1
trace = get_trace(f)
if not trace: break
last_trace = trace
add_hops_to_graph(G, trace)
print "\nSetting source"
global source
source = last_trace['source']
print_stats(G)
return G
G = getGraph()
print "Pruning"
min_weight = 5000 # If you want to prune to a very high number (basically no pruning), set this to 1 instead.
while G.number_of_nodes() > N:
min_weight += 1
prune(G, min_weight, source)
print "Pruned the graph!"
print_stats(G)
import geodb
geo_data = {n: geodb.get_geo(n) for n in G.nodes()}
data = {'graph': json_graph.adjacency_data(G), 'geo': geo_data}
with open("data/backbone.json", "w") as f:
json.dump(data, f)
def to_jsongraph(graph):
return json.dumps(json_graph.adjacency_data(graph))
def tree_ready_json(asn):
H = flare_tree_as_json_for_asn(asn)
return json.dumps(json_graph.adjacency_data(H))
def subgraph(args, neighbor_weight=None):
args.api._log = args.log
graphname = args.graphname or 'network'
threshold = args.threshold or 1
edgefilter = args.edgefilter
neighbor_weight = neighbor_weight or 5
_graph = args.api.load_graph(graphname, threshold, edgefilter)
for node, data in _graph.nodes(data=True):
generations = [{node}]
while generations[-1] and len(set.union(*generations)) < 30 and len(generations) < 3:
nextgen = set.union(*[set(_graph[n].keys()) for n in generations[-1]])
if len(nextgen) > 50:
break # pragma: no cover
else:
generations.append(set.union(*[set(_graph[n].keys()) for n in generations[-1]]))
data['subgraph'] = list(set.union(*generations))
args.api.save_graph(_graph, 'subgraph', threshold, edgefilter)
outdir = args.api.existing_dir('app', 'subgraph', clean=True)
cluster_names = {}
nodes2cluster = {}
nidx = 1
for node, data in tqdm(
sorted(_graph.nodes(data=True), key=lambda x: len(x[1]['subgraph']), reverse=True),
leave=False):
nodes = tuple(sorted(data['subgraph']))
sg = _graph.subgraph(data['subgraph'])
if nodes not in nodes2cluster:
d_ = sorted(sg.degree(), key=lambda x: x[1], reverse=True)
d = [_graph.node[a]['Gloss'] for a, b in d_][0]
nodes2cluster[nodes] = 'subgraph_{0}_{1}'.format(nidx, d)
nidx += 1
cluster_name = nodes2cluster[nodes]
data['ClusterName'] = cluster_name
for n, d in sg.nodes(data=True):
d['OutEdge'] = []
neighbors = [
n_ for n_ in _graph if
n_ in _graph[node] and
_graph[node][n_]['FamilyWeight'] >= neighbor_weight and
n_ not in sg]
if neighbors:
sg.node[node]['OutEdge'] = []
for n_ in neighbors:
sg.node[node]['OutEdge'].append([
'subgraph_' + n_ + '_' + _graph.node[n]['Gloss'],
_graph.node[n_]['Gloss'],
_graph.node[n_]['Gloss'],
_graph[node][n_]['FamilyWeight'],
n_
])
sg.node[node]['OutEdge'].append([
_graph.node[n]['ClusterName'],
_graph.node[n]['CentralConcept'],
_graph.node[n]['Gloss'],
_graph[node][n]['WordWeight'],
n
])
if len(sg) > 1:
jsonlib.dump(
json_graph.adjacency_data(sg), outdir / (cluster_name + '.json'), sort_keys=True)
cluster_names[data['Gloss']] = cluster_name
for node, data in _graph.nodes(data=True):
if 'OutEdge' in data:
data['OutEdge'] = '//'.join([str(x) for x in data['OutEdge']])
args.api.write_js_var('SUBG', cluster_names, 'app', 'source', 'subgraph-names.js')
def get_structure_components(bonded_structure, inc_orientation=False,
inc_site_ids=False, inc_molecule_graph=False):
"""
Gets information on the components in a bonded structure.
Correctly determines the dimensionality of all structures, regardless of
structure type or improper connections due to periodic boundary conditions.
Requires a StructureGraph object as input. This can be generated using one
of the NearNeighbor classes. For example, using the CrystalNN class::
bonded_structure = CrystalNN().get_bonded_structure(structure)
Based on the modified breadth-first-search algorithm described in:
P. Larsem, M. Pandey, M. Strange, K. W. Jacobsen, 2018, arXiv:1808.02114
Args:
bonded_structure (StructureGraph): A structure with bonds, represented
as a pymatgen structure graph. For example, generated using the
CrystalNN.get_bonded_structure() method.
inc_orientation (bool, optional): Whether to include the orientation
of the structure component. For surfaces, the miller index is given,
for one-dimensional structures, the direction of the chain is given.
inc_site_ids (bool, optional): Whether to include the site indices
of the sites in the structure component.
inc_molecule_graph (bool, optional): Whether to include MoleculeGraph
objects for zero-dimensional components.
Returns:
(list of dict): Information on the components in a structure as a list
of dictionaries with the keys:
- "structure_graph": A pymatgen StructureGraph object for the
component.
- "dimensionality": The dimensionality of the structure component as an
int.
- "orientation": If inc_orientation is `True`, the orientation of the
component as a tuple. E.g. (1, 1, 1)
- "site_ids": If inc_site_ids is `True`, the site indices of the
sites in the component as a tuple.
- "molecule_graph": If inc_molecule_graph is `True`, the site a
MoleculeGraph object for zero-dimensional components.
"""
import networkx as nx # optional dependency therefore not top level import
comp_graphs = (bonded_structure.graph.subgraph(c) for c in
nx.weakly_connected_components(bonded_structure.graph))
components = []
for graph in comp_graphs:
dimensionality, vertices = calculate_dimensionality_of_site(
bonded_structure, list(graph.nodes())[0], inc_vertices=True)
component = {'dimensionality': dimensionality}
if inc_orientation:
if dimensionality in [1, 2]:
vertices = np.array(vertices)
g = vertices.sum(axis=0) / vertices.shape[0]
# run singular value decomposition
_, _, vh = np.linalg.svd(vertices - g)
# get direction (first column is best fit line,
# 3rd column is unitary norm)
index = 2 if dimensionality == 2 else 0
orientation = get_integer_index(vh[index, :])
else:
orientation = None
component['orientation'] = orientation
if inc_site_ids:
component['site_ids'] = tuple(graph.nodes())
if inc_molecule_graph and dimensionality == 0:
component['molecule_graph'] = zero_d_graph_to_molecule_graph(
bonded_structure, graph)
component_structure = Structure.from_sites(
[bonded_structure.structure[n] for n in sorted(graph.nodes())])
sorted_graph = nx.convert_node_labels_to_integers(
graph, ordering="sorted")
component_graph = StructureGraph(
component_structure,
graph_data=json_graph.adjacency_data(sorted_graph))
component['structure_graph'] = component_graph
components.append(component)
return components
def json_graph_list_dumps(graph_list):
return json.dumps([json_graph.adjacency_data(graph) for graph in graph_list])
def save_graphs(graphs): for g,d in graphs.items(): f = open(g, 'w') data = json_graph.adjacency_data(d) s = json.dump(data, f) f.close()
