def lambda_handler(event, context):
if 'body' in event.keys():
event = json.loads(event["body"])
bucket = "districtr"
state = event["state"].lower().replace(" ", "_")
units = event["units"].lower().replace(" ", "")
# district = event["dist_id"]
plan_assignment = event["assignment"]
keys = set(plan_assignment.keys())
key = "dual_graphs/{}_{}.json".format(state, units)
try:
data = s3.get_object(Bucket=bucket, Key=key)
g = json_graph.adjacency_graph(json.load(data['Body']))
graph = Graph(g)
assignment = {
n: plan_assignment[n] if n in keys else -1
for n in graph.nodes()
}
part = GeographicPartition(graph, assignment)
return {'statusCode': 200, 'body': json.dumps(plan_evaluation(part))}
except Exception as e:
print(e)
return {
"error":
"This state/units ({}, {}) is not supported".format(
event["state"], event["units"])
}
def set_up_graph(link):
"""
The function used to convert online json file to adjacency graph
Parameters:
link (string): the link for online json file
Returns:
Graph: graph with the information
mean: the mean value used for partition
"""
link = "https://people.csail.mit.edu/ddeford//COUNTY/COUNTY_13.json"
r = requests.get(link)
data = json.loads(r.content)
g = json_graph.adjacency_graph(data)
graph = Graph(g)
graph.issue_warnings()
horizontal = []
node_degree = []
# find the node with degree 1 or 2 and remove it
for node in graph.nodes():
graph.nodes[node]["pos"] = [
graph.node[node]['C_X'], graph.node[node]['C_Y']
]
horizontal.append(graph.node[node]['C_X'])
if graph.degree(node) == 1 or graph.degree(node) == 2:
node_degree.append(node)
# remove node with degree 1 or 2 since it will impact the outcome of graph
for i in node_degree:
graph.remove_node(i)
# calculate mean value for partition
mean = sum(horizontal) / len(horizontal)
return graph, mean
def verify_gerrychain(df):
try:
Graph.from_geodataframe(fix_buffer(df))
print("GerryChain graph created")
return True
except Exception as error:
print("Unable to create GerryChain graph: ", error)
return False
def graph_from_url():
link = input("Put graph link: ")
r = requests.get(link)
data = json.loads(r.content)
g = json_graph.adjacency_graph(data)
graph = Graph(g)
graph.issue_warnings()
return graph
def graph_from_url(link):
r = requests.get(url=link)
data = json.loads(r.content)
g = json_graph.adjacency_graph(data)
graph = Graph(g)
graph.issue_warnings()
pos = {}
for node in graph.nodes():
pos[node] = [graph.node[node]['C_X'], graph.node[node]['C_Y']]
return graph
def graph_from_url():
# link = input("Put graph link: ")
link = "https://people.csail.mit.edu/ddeford//COUNTY/COUNTY_37.json" # County
# link = "https://people.csail.mit.edu/ddeford//COUSUB/COUSUB_37.json" # COUNTY SUB
r = requests.get(link)
data = json.loads(r.content)
g = json_graph.adjacency_graph(data)
graph = Graph(g)
graph.issue_warnings()
return graph
def save_graph(place, filename):
print("Creating rook graph")
rook = Graph.from_file(filename,
adjacency="rook",
columns=[col.key for col in place.columns])
rook.to_json("./graphs/{}_rook.json".format(place.id))
print("Creating queen graph")
queen = Graph.from_file(filename,
adjacency="queen",
columns=[col.key for col in place.columns])
queen.to_json("./graphs/{}_queen.json".format(place.id))
def test_works_when_no_flips_occur():
graph = Graph([(0, 1), (1, 2), (2, 3), (3, 0)])
for node in graph:
graph.nodes[node]["pop"] = node + 1
partition = Partition(graph, {0: 0, 1: 0, 2: 1, 3: 1}, {"pop": Tally("pop")})
chain = MarkovChain(lambda p: p.flip({}), [], always_accept, partition, 10)
expected = {0: 3, 1: 7}
for partition in chain:
assert partition["pop"] == expected
def preprocessing(path_to_json):
"""Takes file path to JSON graph, and returns the appropriate
Args:
path_to_json (String): path to graph in JSON format
Returns:
graph (Gerrychain Graph): graph in JSON file following cleaning
dual (Gerrychain Graph): planar dual of graph
"""
graph = Graph.from_json(path_to_json)
# For each node in graph, set 'pos' parameter to position
for node in graph.nodes():
graph.nodes[node]['pos'] = (graph.nodes[node][config['X_POSITION']],
graph.nodes[node][config['Y_POSITION']])
save_fig(graph, config['UNDERLYING_GRAPH_FILE'], config['WIDTH'])
# Cleans graph to be able to find planar dual
# TODO: why is this necessary?
#graph = duality_cleaning(graph)
print('Making Dual')
dual = facefinder.restricted_planar_dual(graph)
print('Made Dual')
return graph, dual
def main(config_data, id):
try:
timeBeg = time.time()
print('Experiment', id, 'has begun')
# Save configuration into global variable
global config
config = config_data
graph = Graph.from_json(config['INPUT_GRAPH_FILENAME'])
# List of districts in original graph
parts = list(
set([
graph.nodes[node][config['ASSIGN_COL']]
for node in graph.nodes()
]))
# Ideal population of districts
ideal_pop = sum(
[graph.nodes[node][config['POP_COL']]
for node in graph.nodes()]) / len(parts)
election = Election(config['ELECTION_NAME'], {
'PartyA': config['PARTY_A_COL'],
'PartyB': config['PARTY_B_COL']
})
updaters = {
'population': Tally(config['POP_COL']),
'cut_edges': cut_edges,
config['ELECTION_NAME']: election
}
partDict = recursive_tree_part(graph, parts, ideal_pop,
config['POP_COL'], config['EPSILON'],
config['NODE_REPEATS'])
for node in graph.nodes():
graph.nodes[node][config['ASSIGN_COL']] = partDict[node]
part = Partition(graph=graph,
assignment=config['ASSIGN_COL'],
updaters=updaters)
for len_ in config['RUN_LENGTHS']:
for num in range(config['RUNS_PER_LEN']):
run_chain(part, config['CHAIN_TYPE'], len_, ideal_pop,
'{}_{}_{}_{}'.format(config['TAG'], id, len_, num))
print('Experiment {} completed in {} seconds'.format(
id,
time.time() - timeBeg))
except Exception as e:
# Print notification if any experiment fails to complete
track = traceback.format_exc()
print(track)
print('Experiment {} failed to complete after {:.2f} seconds'.format(
id,
time.time() - timeBeg))
def graph_from_url_processing(link):
r = requests.get(url=link)
data = json.loads(r.content)
g = json_graph.adjacency_graph(data)
graph = Graph(g)
graph.issue_warnings()
for node in graph.nodes():
graph.nodes[node]["pos"] = [graph.nodes[node]['C_X'], graph.nodes[node]['C_Y'] ]
deg_one_nodes = []
for v in graph:
if graph.degree(v) == 1:
deg_one_nodes.append(v)
for node in deg_one_nodes:
graph.remove_node(node)
return graph
def test_uses_graph_geometries_by_default(self, geodataframe):
mock_plot = MagicMock()
gp.GeoDataFrame.plot = mock_plot
graph = Graph.from_geodataframe(geodataframe)
partition = Partition(graph=graph,
assignment={node: 0
for node in graph})
partition.plot()
assert mock_plot.call_count == 1
def test():
graph_path = "./Data/PA_VTDALL.json"
graph = Graph.from_json(graph_path)
k = 18
ep = 0.05
unit_id = "GEOID10"
pop_col = "TOT_POP"
plot_path = "./Data/VTD_FINAL"
unit_df = gpd.read_file(plot_path)
division_col = "COUNTYFP10"
divisions = unit_df[[division_col, 'geometry']].dissolve(by=division_col,
aggfunc='sum')
updaters = {"population": updaters.Tally(pop_col, alias="population")}
cddict = recursive_tree_part(graph,
range(k),
df[pop_col].sum() / k,
pop_col,
.01,
node_repeats=1)
initial_partition = Partition(graph, cddict, updaters)
ideal_population = sum(
initial_partition["population"].values()) / len(initial_partition)
division_proposal = partial(recom,
pop_col=pop_col,
pop_target=ideal_population,
epsilon=0.05,
method=partial(division_bipartition_tree,
division_col=division_col),
node_repeats=2)
chain = MarkovChain(proposal=division_proposal,
constraints=[
constraints.within_percent_of_ideal_population(
initial_partition, 0.05),
],
accept=accept.always_accept,
initial_state=initial_partition,
total_steps=1000)
t = 0
snapshot = 100
for part in chain:
if t % snapshot == 0:
draw_graph(graph,
part.assignment,
unit_df,
divisions,
'./chain_' + str(t) + '.png',
geo_id=unit_id)
t += 1
def three_by_three_grid():
"""Returns a graph that looks like this:
0 1 2
3 4 5
6 7 8
"""
graph = Graph()
graph.add_edges_from([
(0, 1),
(0, 3),
(1, 2),
(1, 4),
(2, 5),
(3, 4),
(3, 6),
(4, 5),
(4, 7),
(5, 8),
(6, 7),
(7, 8),
])
return graph
def test_pa_freeze():
from gerrychain import (
GeographicPartition,
Graph,
MarkovChain,
proposals,
updaters,
constraints,
accept,
)
import hashlib
from gerrychain.proposals import recom
from functools import partial
graph = Graph.from_json("docs/user/PA_VTDs.json")
my_updaters = {"population": updaters.Tally("TOTPOP", alias="population")}
initial_partition = GeographicPartition(graph,
assignment="CD_2011",
updaters=my_updaters)
ideal_population = sum(
initial_partition["population"].values()) / len(initial_partition)
# We use functools.partial to bind the extra parameters (pop_col, pop_target, epsilon, node_repeats)
# of the recom proposal.
proposal = partial(recom,
pop_col="TOTPOP",
pop_target=ideal_population,
epsilon=0.02,
node_repeats=2)
pop_constraint = constraints.within_percent_of_ideal_population(
initial_partition, 0.02)
chain = MarkovChain(
proposal=proposal,
constraints=[pop_constraint],
accept=accept.always_accept,
initial_state=initial_partition,
total_steps=100,
)
result = ""
for count, partition in enumerate(chain):
result += str(list(sorted(partition.population.values())))
result += str(len(partition.cut_edges))
result += str(count) + "\n"
assert hashlib.sha256(result.encode()).hexdigest(
) == "3bef9ac8c0bfa025fb75e32aea3847757a8fba56b2b2be6f9b3b952088ae3b3c"
def generate_reports(geometries, population=None):
adj = maup.adjacencies(geometries,
warn_for_overlaps=False,
warn_for_islands=False)
adj.crs = geometries.crs
graph = Graph(list(adj.index))
reports = [
graph_report(geometries, adj),
degree_report(geometries, adj),
connectivity_report(graph, geometries),
topology_report(geometries, adj),
]
if population is not None:
reports.append(population_report(geometries, population))
return reports
def preprocessing(path_to_json):
graph = Graph.from_json(path_to_json)
for node in graph.nodes():
graph.nodes[node]['pos'] = [ graph.nodes[node]["C_X"], graph.nodes[node]["C_Y"] ]
graph = duality_cleaning(graph)
print("making dual")
dual = restricted_planar_dual(graph)
print("made dual")
save_fig(graph,"./plots/UnderlyingGraph.png",1)
for node in graph.nodes():
graph.nodes[node]["population"] = graph.nodes[node]["TOTPOP"]
return graph, dual
def __init__(self,
graph: Graph,
initial_state: Partition,
total_steps: int,
pop_col: str,
pop_tol: float,
compressed: bool = True,
reversible: bool = True,
seed: int = None):
"""
:param graph: the GerryChain graph to compile to a Julia graph.
:param initial_state: The state to start from.
:param total_steps: The total steps to take in the chain.
:param pop_col: The column of the graph with population data.
:param pop_tol: The population tolerance to use.
:param compressed: If `True`, plans are represented in `RecomStep`
format. Otherwise, behavior mimics GerryChain.
:param reversible: Determines if the reversible version of the chain
should be used.
:param seed: The random seed to use for plan generation.
"""
for node, assignment in initial_state.assignment.items():
graph.nodes[node]['__district'] = assignment
graph_data = json_graph.adjacency_data(graph)
self.graph = Flips.IndexedGraph(graph_data, pop_col)
# TODO: verify initial state is valid w.r.t. population constraints
# (+ more?)
self.plan = Flips.Plan(self.graph, '__district')
total_pop = sum(graph.nodes[node][pop_col] for node in graph.nodes)
districts = list(initial_state.assignment.values())
n_districts = max(districts) - min(districts) + 1
district_pop = total_pop / n_districts
self.min_pop = int(ceil((1 - pop_tol) * district_pop))
self.max_pop = int(floor((1 + pop_tol) * district_pop))
self.reversible = reversible
self.compressed = compressed
self._curr_parent = initial_state
self.total_steps = total_steps
self._curr_step = 0
self._buf = deque() # https://stackoverflow.com/a/4426727/8610749
self.data = None
if seed is None:
self._twister = Random.MersenneTwister()
else:
self._twister = Random.MersenneTwister(seed)
def make_dual_graph(st, year):
"""
Takes a 2-letter state postal code abbreviation (e.g. GA for Georgia), makes a dual graph of its shapefile, and writes the dual graph to a JSON
Arguments:
st -- 2 letter state postal code
"""
# Fold state postal code to lowercase:
st = st.lower()
##2016 Census Tract- TigerLine File for the appropriate state, follow link to download
geo = gpd.read_file("./data/" + st + "_" + str(year) + "_tract.shp")
graph = Graph.from_geodataframe(
geo
) #if graph is successfully generated, we should be able to run chain
graph.add_data(geo, columns=geo.columns)
#nx.is_connected(graph) # if returns true, graph is connected
graph.to_json("./data/" + st + "_tract.json")
def preprocessing(path_to_json, output_directory):
"""Takes file path to JSON graph, and returns the appropriate
Args:
path_to_json ([String]): path to graph in JSON format
output_directory: path of directory to save output in
Returns:
graph (Gerrychain Graph): graph in JSON file following cleaning
dual (Gerrychain Graph): planar dual of graph
"""
graph = Graph.from_json(path_to_json)
# For each node in graph, set 'pos' keyword to position
for node in graph.nodes():
graph.nodes[node]['pos'] = (graph.nodes[node][config['X_POSITION']],
graph.nodes[node][config['Y_POSITION']])
save_fig(graph, output_directory + '/UnderlyingGraph.png', config['WIDTH'])
#restricted planar dual does not include unbounded face
dual = facefinder.restricted_planar_dual(graph)
return graph, dual
def preprocessing(path_to_json):
"""Takes file path to JSON graph, and returns the appropriate
Args:
path_to_json ([String]): path to graph in JSON format
Returns:
graph (Gerrychain Graph): graph in JSON file following cleaning
dual (Gerrychain Graph): planar dual of graph
"""
graph = Graph.from_json(path_to_json)
# For each node in graph, set 'pos' keyword to position
for node in graph.nodes():
graph.nodes[node]['pos'] = (graph.nodes[node][config['X_POSITION']],
graph.nodes[node][config['Y_POSITION']])
save_fig(graph, config['UNDERLYING_GRAPH_FILE'], config['WIDTH'])
dual = facefinder.restricted_planar_dual(graph)
return graph, dual
def load_graph_from_shp(file_path, write_path, id_key, adjacency):
global graph
print("Loading graph...")
start = time.perf_counter()
graph = Graph.from_file(file_path, adjacency=adjacency)
end = time.perf_counter()
print("Took " + str((end - start)) + " seconds to create the graph.")
print("This state has " + str(len(graph.nodes)) + " blocks")
# write matrix
adj_mat = np.zeros((len(graph.nodes), len(graph.nodes)))
if id_key not in graph.nodes[0].keys():
print(id_key, "not found in node attributes.")
print("Attributes: ", graph.nodes[0].keys())
ids = [graph.nodes[i][id_key] for i in range(len(graph.nodes))]
for e in graph.edges:
adj_mat[e[0], e[1]] = 1
adj_mat[e[1], e[0]] = 1
print("Writing adjacency matrix and IDs to file...")
with open(write_path, 'w') as outfile:
json.dump({"order": ids, "adj": adj_mat.tolist()}, outfile)
print("Done! JSON written to " + write_path)
def slow_reversible_propose(partition):
flip = random.choice(list(partition["b_nodes"]))
return partition.flip({flip[0]: flip[1]})
#graph = Graph.from_file("./State_Data/VA_Trimmed.shp")
#with open("./State_Data/VA_Trimmedj.json", 'w') as outfile:
# json.dump(json_graph.adjacency_data(graph), outfile)
#graph.to_json("./State_Data/VA_Trimmed.json")
graph = Graph.from_json("./State_Data/VA_Trimmed.json")
df = gpd.read_file("./State_Data/VA_Trimmed.shp")
print("loaded data")
#print("nodes",len(graph.nodes()))
#print("edges",len(graph.edges()))
for node in graph.nodes():
graph.nodes[node]["nBVAP"] = graph.nodes[node]["VAPERSONS"] - graph.nodes[node]["VAPBLACK"]
"""
updater = { "population": updaters.Tally("TAPERSONS", alias="population"),
"cut_edges",cut_edges,
"BVAP":Election("BVAP",{"BVAP":"VAPBLACK","nBVAP":"nBVAP"}),
import json
from gerrychain import (
Election,
Graph,
MarkovChain,
Partition,
accept,
constraints,
updaters,
)
from gerrychain.tree import recursive_tree_part
import numpy as np
import pickle
import wasserplan
graph = Graph.from_json("NC_VTD.json")
def get_partition(partition_file, fid_to_geoid_file, geoidcol):
fid_to_node = get_fid_to_node(fid_to_geoid_file, geoidcol)
fid_to_district = {}
f = open(partition_file, 'r')
for line in f:
lineitems = line.rstrip("\n").split("\t")
fid, district = int(lineitems[0]) + 1, int(lineitems[1])
fid_to_district[fid] = district
assignment = {}
for fid in fid_to_district:
assignment[
fid_to_node[fid]] = fid_to_district[fid] - 1 #make 0-indexed
return assignment
from numpy import linalg as LA
import matplotlib.pyplot as plt
from sklearn.cluster import SpectralClustering
from sklearn import metrics
from networkx.algorithms.community import greedy_modularity_communities
from networkx.algorithms.community import label_propagation_communities
from gerrychain import Graph
import geopandas as gpd
G = nx.karate_club_graph()
n = 34
G = nx.grid_graph([5, 5])
n = 25
G = Graph.from_json("./County05.json")
n = len(list(G.nodes()))
df = gpd.read_file("./County05.shp")
AM = nx.adjacency_matrix(G)
NLM = (nx.normalized_laplacian_matrix(G)).todense()
LM = (nx.laplacian_matrix(G)).todense()
'''
Labels = [G.nodes[i]['club'] != 'Mr. Hi' for i in G.nodes()]
plt.figure()
plt.title("Data Labels")
nx.draw(G, node_color=Labels )
plt.show()
'''
def example_partition():
graph = Graph.from_networkx(networkx.complete_graph(3))
assignment = {0: 1, 1: 1, 2: 2}
partition = Partition(graph, assignment, {"cut_edges": cut_edges})
return partition
import matplotlib.pyplot as plt
import random
import numpy as np
from gerrychain import Graph
import geopandas as gpd
from gerrychain.tree import recursive_tree_part
from gerrychain.partition import Partition
from gerrychain.updaters import Tally, cut_edges
g = nx.grid_graph([10, 10])
g = nx.karate_club_graph()
ns = 50
g = Graph.from_json("./Data/BG05.json")
df = gpd.read_file("./Data/BG05.shp")
centroids = df.centroid
c_x = centroids.x
c_y = centroids.y
shape = True
nlist = list(g.nodes())
n = len(nlist)
totpop = 0
for node in g.nodes():
g.node[node]["TOTPOP"] = int(g.node[node]["TOTPOP"])
totpop += g.node[node]["TOTPOP"]
# fill-in unspecified configs using default values
for ckey in available_config.keys():
if ckey not in config.keys():
print("Using default value", ckey, "=", default_config[ckey],
"since no option was selected.")
config[ckey] = default_config[ckey]
# initialize dictionary to store this run's results
result = config
result['run'] = key
# read input data
state = config['state']
code = state_codes[state]
level = config['level']
G = Graph.from_json("../data/" + level + "/dual_graphs/" + level + code +
".json")
DG = nx.DiGraph(G) # bidirected version of G
df = gpd.read_file("../data/" + level + "/shape_files/" + state + "_" +
level + ".shp")
# set parameters
k = number_of_congressional_districts[state]
population = [G.nodes[i]['TOTPOP'] for i in G.nodes()]
deviation = 0.01
U = math.floor((1 + deviation / 2) * (sum(population) / k))
L = math.ceil((1 - deviation / 2) * (sum(population) / k))
ideal = math.floor(sum(population) / k)
print("k =", k)
result['k'] = k
result['n'] = G.number_of_nodes()
"SEN16R",
"SEN16L",
]
for x in df.columns:
if x in variables:
df[x] = df[x].astype(int)
#county_col = "COUNTYFP10"
pop_col = "TOTPOP"
df["CPOP"] = df["TOTPOP"] - df["NCPOP"]
ccol = "CPOP"
uid = "ID"
num_districts = 14
graph = Graph.from_geodataframe(df, ignore_errors=True)
graph.add_data(df, list(df))
graph = nx.relabel_nodes(graph, df[uid])
elections = [
Election("PRES16", {
"Democratic": "PRES16D",
"Republican": "PRES16R"
}),
Election("SEN16", {
"Democratic": "SEN16D",
"Republican": "SEN16R"
})
]
#my_updaters = {"population" : updaters.Tally("TOTPOP", alias="population")}
test_05["polsbypopper"]
plt.hist(test_05["eg"], bins=50)
# load actual gerrychain results
sen_05 = np.load(
"/Users/hopecj/projects/gerryspam/MO/res_0817/MO_state_senate_300000_0.05.p"
)
sen_01 = np.load(
"/Users/hopecj/projects/gerryspam/MO/res_0817/MO_state_senate_100000_0.01.p"
)
sen_05.keys() # shows "columns" available
print(sen_05["mean_median_ussen16"]) # results from chain
# open enacted map
graph = Graph.from_file(
"/Users/hopecj/projects/gerryspam/MO/dat/final_prec/prec_labeled.shp")
elections = [
Election("USSEN16", {
"Dem": "G16USSDKAN",
"Rep": "G16USSRBLU"
}),
Election("PRES16", {
"Dem": "G16PREDCLI",
"Rep": "G16PRERTRU"
})
]
mo_updaters = {
"population": Tally("POP10", alias="population"),
"cut_edges": cut_edges
}
