def build_from_blkequiv_csvbak(self, blk_equivfile):
arr = np.loadtxt(blk_equivfile, delimiter=',')
print arr.shape
# if <blockid is a not single string.
if arr.shape[1] > 2:
arr = arr.astype(str)
dists = SortedDict()
dict = SortedDict()
if arr.shape[1] > 2:
for blkvals in arr:
pkey = util.hash_cblocks_dist(tuple(
blkvals[0:-1])) #key:blockid
dict[pkey] = tuple(blkvals)
dists[tuple(blkvals)[-1]] = 0
#
self.__block_to_dist_map = dict
self.__nblocks = len(list(dict.keys()))
self.__merkleroot = util.merkle_root(self.__block_to_dist_map)
self.__problemacc = DEFAULT_PROBLEM_ACC
self.__solacc = DEFAULT_SOLUTION_ACC
self.__planid = self.get_planid()
self.__keyschainhash = self.get_keyschainhash()
self.__timestamp = time.time()
self.__dists = dists
def dproblem_from_shape_files(self,
block_shapefile,
census_tract_file=False):
# open block shape file.
if census_tract_file:
idfieldname = "GEOID10"
popfieldname = "P0010001"
else:
idfieldname = "BLOCKID10"
popfieldname = "POP10"
blk_to_attr_map = SortedDict()
shapef = ogr.Open(block_shapefile)
layer = shapef.GetLayer()
#print schema
schema = []
ldefn = layer.GetLayerDefn()
for n in range(ldefn.GetFieldCount()):
fdefn = ldefn.GetFieldDefn(n)
schema.append(fdefn.name)
print schema
#create maping.
sourceSpatialRef = layer.GetSpatialRef()
Nfeats = layer.GetFeatureCount()
ctract = {}
for fid in xrange(Nfeats):
fit = layer.GetFeature(fid)
blkgeo = fit.GetGeometryRef()
d = fit.GetField(idfieldname) #id
pop = fit.GetField(popfieldname) #population
blockkey = util.hash_cblocks_dist([d])
# block attrs
blk_to_attr_map[blockkey] = {"FEAT_KEY": d, "POPULATION": pop}
blk_to_attr_map[blockkey]["AREA"] = blkgeo.GetArea()
blk_to_attr_map[blockkey]["PERI"] = blkgeo.GetBoundary().Length()
blk_to_attr_map[blockkey]["ENV"] = blkgeo.GetEnvelope()
blk_to_attr_map[blockkey]["CENTROID"] = blkgeo.Centroid(
).GetPoints()[0]
#set property
self.__block_to_attr_map = blk_to_attr_map
self.__keyschainhash = self.get_keyschainhash()
self.__nblocks = len(self.__block_to_attr_map.keys())
if self.__problemacc is None:
self.__problemacc = DEFAULT_PROBLEM_ACC
self.__releasetime = time.time()
self.__closetime = self.__releasetime + PLAYTIME
def blk_feat_to_dist_mapping(self,
block_shapefile,
blk_to_dist_map,
census_tract_file=False):
# open block shape file.
home = os.path.dirname(block_shapefile)
if census_tract_file:
idfieldname = "GEOID10"
popfieldname = "P0010001"
else:
idfieldname = "BLOCKID10"
popfieldname = "POP10"
blk_to_attr_map = SortedDict()
shapef = ogr.Open(block_shapefile)
layer = shapef.GetLayer()
#print schema
ordered_dist = []
ldefn = layer.GetLayerDefn()
#create maping.
sourceSpatialRef = layer.GetSpatialRef()
Nfeats = layer.GetFeatureCount()
ctract = {}
for fid in xrange(Nfeats):
fit = layer.GetFeature(fid)
blkgeo = fit.GetGeometryRef()
d = fit.GetField(idfieldname) #id
pop = fit.GetField(popfieldname) #population
blockkey = util.hash_cblocks_dist([d])
#get dist value for this 'd'
ordered_dist += [blk_to_dist_map[blockkey][1:]]
#set property
#save as csv.
print ordered_dist[0]
with open(home + '/ordered_blk_eqiv.csv', 'wb') as writeFile:
writer = csv.writer(writeFile)
writer.writerows(ordered_dist)
writeFile.close()
def dproblem_from_files(self, block_shapefile):
# open shape file.
blk_to_attr_map = SortedDict()
shapef = ogr.Open(block_shapefile)
layer = shapef.GetLayer()
#print schema
schema = []
ldefn = layer.GetLayerDefn()
for n in range(ldefn.GetFieldCount()):
fdefn = ldefn.GetFieldDefn(n)
schema.append(fdefn.name)
print schema
#create maping.
sourceSpatialRef = layer.GetSpatialRef()
Nfeats = layer.GetFeatureCount()
ctract = {}
for fid in xrange(Nfeats):
fit = layer.GetFeature(fid)
geom = fit.GetGeometryRef()
a = fit.GetField('STATEFP10')
b = fit.GetField('COUNTYFP10')
c = fit.GetField('TRACTCE10')
d = fit.GetField('BLOCKID10')
ctract[c] = 0
pkey = util.hash_cblocks_dist([d])
blk_to_attr_map[pkey] = (fit.GetField('POP10'))
print("n(census-tracts)"), len(ctract)
#set property
self.__block_to_attr_map = blk_to_attr_map
self.__keyschainhash = self.get_keyschainhash()
self.__nblocks = len(self.__block_to_attr_map.keys())
self.__problemacc = DEFAULT_PROBLEM_ACC
self.__releasetime = time.time()
self.__closetime = self.__releasetime + PLAYTIME
def build_from_blkequiv_csv(self, blk_equivfile):
dists = SortedDict()
dict = SortedDict()
with open(blk_equivfile) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for blkvals in csv_reader:
pkey = util.hash_cblocks_dist([blkvals[0]]) #key:blockid
dict[pkey] = tuple(blkvals)
dists[blkvals[-1]] = 0
#
self.__block_to_dist_map = dict
self.__nblocks = len(list(dict.keys()))
self.__merkleroot = util.merkle_root(self.__block_to_dist_map)
self.__problemacc = DEFAULT_PROBLEM_ACC
self.__solacc = DEFAULT_SOLUTION_ACC
self.__planid = self.get_planid()
self.__keyschainhash = self.get_keyschainhash()
self.__timestamp = time.time()
self.__dists = dists
def test_metric_block_integration(self, state_id="28", filter_dists=[]):
timing = {}
state_prop = {}
data_info = {}
home_cong = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/cd116/"
home_blkpop = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/census_blocks_by_states/" + "tabblock2010_" + state_id + "_pophu/"
ctract_shapefile = home_blkpop + "prjtabblock2010_" + state_id + "_pophu.shp"
blk_equivfile = home_cong + "blk_eqiv" + state_id + ".csv" #with 2 cols #tract equiv file
dist_idsfile = home_cong + "dist_ids" + state_id + ".csv" #with 2 cols #tract equiv file
census_tract_file = False
if census_tract_file:
idfieldname = "GEOID10"
dist_id_field = "STID3" #dist-id
else:
idfieldname = "BLOCKID10"
dist_id_field = "STID3"
prefix = "blk" #integration-type
dplandatafile = prefix + "dplan" + state_id
dplanproblemfile = prefix + "dpmap" + state_id
dplan_block_propfile = prefix + "dpblock_prop" + state_id
data_info["size_shp"] = util.get_size(
os.path.dirname(ctract_shapefile))
data_info["dists"] = {}
#get a dist problem
t0 = time.clock()
dpmap = DPmap()
dpmap.load_json_dprob(home_blkpop + dplanproblemfile)
timing["tAPEC_BY_BLK"] = round(time.clock() - t0, 6)
timing["PROB_ACC"] = dpmap.get_problemacc()
#get a plan
dplan = Dplan()
dplan.load_json_dplan(home_cong + dplandatafile)
timing["SOL_ACC"] = dplan.get_solacc()
#for which prob, which solution
state_prop["PROB_ACC"] = dpmap.get_problemacc()
state_prop["SOL_ACC"] = dplan.get_solacc()
state_prop["DIST_KEYS"] = sorted(dpmap.get_dist_keys())
#filter by district ids.
for k in filter_dists:
try:
state_prop["DIST_KEYS"].remove(k)
except:
continue
# fill up block's prop.
block_prop = dpmap.get_block_to_attr_map() #transient properties
#IO().insert_block_prop(dpmap) #TODO: isssue with unicode keys in dict
del dpmap
t0 = time.clock()
# collection of blocks by dist.
districts_blocks = {}
for ctractkey, idvalues in dplan.get_block_to_attr_map().items():
#TODO:
#this blockkey must be in dpmap.
distid = idvalues[-1]
try:
districts_blocks[distid] += [ctractkey]
except:
districts_blocks[distid] = [ctractkey]
timing["tCOLL_BLKS_BY_DIST"] = round(time.clock() - t0, 6)
print "tCOLL_BLKS_BY_DIST", timing["tCOLL_BLKS_BY_DIST"]
###
#Validate:
##dist_keys in plan must be in problem-maps's dist keys.
##all of the input blocks must be mapped to one of the district.
print("# of Districts :"), len(state_prop["DIST_KEYS"])
for distkey in state_prop["DIST_KEYS"]:
t0 = time.clock()
state_prop[distkey] = {}
print("# of blocks in this dist:"), distkey, len(
districts_blocks[distkey])
udist_geom = ogr.Geometry(ogr.wkbMultiPolygon)
cd_container = {distkey: udist_geom}
#get each block's shape to create a district's map.
fcount = 0
shapef = ogr.Open(ctract_shapefile)
layer = shapef.GetLayer()
#
t0 = time.clock()
t2 = time.time()
for feature in layer:
bkey = util.hash_cblocks_dist([feature.GetField(idfieldname)])
blkgeo = feature.GetGeometryRef()
#distid = feature.GetField(dist_id_field) #or
distid = dplan.get_block_to_attr_map()[bkey][1]
try:
cd_container[distid].AddGeometry(blkgeo)
except:
continue
#end-for
layer = None
print("time:"), time.clock() - t0, time.time() - t2
print("-udist_geom count-"), udist_geom.GetGeometryCount()
#take union of blocks.
t0 = time.clock()
udist_geomc = udist_geom.UnionCascaded()
timing["tBLKS_UCASCADE_" + str(distkey)] = round(
time.clock() - t0, 6)
count = 0
#for each ring geom, search for a largest ring.
for geom in udist_geomc:
if geom.GetGeometryName() == 'LINEARRING':
state_prop[distkey]["BOUNDARY_PTS"] = geom.GetPoints()
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(geom)
util.geom_toshp(poly,
home_cong + "dists/" + str(state_id) +
"-" + "dist-" + str(distkey) + "-" +
str(count),
save_as_multipt=False)
break
elif geom.GetGeometryName() == "POLYGON":
for geo in geom:
state_prop[distkey]["BOUNDARY_PTS"] = geo.GetPoints()
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(geo)
util.geom_toshp(poly,
home_cong + "dists/" + str(state_id) +
"-" + "dist-" + str(distkey) + "-" +
str(count),
save_as_multipt=False)
break
count += 1
print("\t\t dist-boundary completed"), len(
state_prop[distkey]["BOUNDARY_PTS"])
#TODO: cascadedunion of these linearring must be performed.
##
#util.save(state_prop, home_blkpop + "temp_distprop28-")
print("Completed block integration to construct a district")
#compute state's APEC properties
t0 = time.clock()
for distkey in state_prop["DIST_KEYS"][:]:
polypts = state_prop[distkey]["BOUNDARY_PTS"]
poly = ogr.Geometry(ogr.wkbPolygon)
ring = ogr.Geometry(ogr.wkbLinearRing)
for x, y in polypts:
x, y = float(x), float(y)
ring.AddPoint(x, y)
poly.AddGeometry(ring)
state_prop[distkey]["AREA"] = poly.GetArea()
state_prop[distkey]["PERI"] = poly.GetBoundary().Length()
state_prop[distkey]["ENV"] = poly.GetEnvelope()
state_prop[distkey]["CENTROID"] = poly.Centroid().GetPoints()[0]
timing["tAPEC_" + str(distkey)] = round(time.clock() - t0, 6)
state_prop[distkey]["FAILED"] = []
print("Completed APEC for district "), distkey
#for each block in district, update block's properties relative to comprising district.
for distkey in state_prop["DIST_KEYS"][:]:
st_centroid = state_prop[distkey]["CENTROID"]
data_info["dists"]["#" + prefix + "-" + distkey] = len(
districts_blocks[distkey])
t0 = time.clock()
try:
Metri.bctr_to_dctr(st_centroid, districts_blocks[distkey],
block_prop)
timing["tBLKCTR_TO_DIST_CTR_" + distkey] = round(
time.clock() - t0, 6)
except Exception, e:
state_prop[distkey]["FAILED"] = ["BLKCTR_TO_DIST_CTR_"]
print("Exception:"), str(e)
print("1")
t0 = time.clock()
try:
Metri.blkctr_to_dist_peri_np(
state_prop[distkey]["BOUNDARY_PTS"],
districts_blocks[distkey], block_prop)
timing["tDBLKCTR_TO_DIST_PERI_" + distkey] = round(
time.clock() - t0, 6)
pass
except Exception, e:
state_prop[distkey]["FAILED"] += ["DBLKCTR_TO_DIST_PERI_"]
print("Exception--"), str(e)
def test_metric_ctract_integration(self):
timing = {}
state_prop = {}
home = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/"
ctract_shapefile = home + "Tracts10PopnHou28/Tracts10PopnHou.shp"
blk_equivfile = "tr_equiv28.csv" #with 2 cols #tract equiv file
#save and get a plan
dplan = Dplan()
dplan.build_from_blkequiv_csv(home + blk_equivfile)
dplan.to_json(home + "ctdplan28")
del dplan
dplan = Dplan()
dplan.load_json_dplan(home + "ctdplan28.json")
IO().insert_plan(dplan)
timing["SOL_ACC"] = dplan.get_solacc()
#save and get a dist problem
t0 = time.clock()
dpmap = DPmap()
dpmap.dproblem_from_shape_files(ctract_shapefile,
census_tract_file=True)
dpmap.to_json(home + "ctdpmap28.json")
del dpmap
dpmap = DPmap()
dpmap.load_json_dprob(home + "ctdpmap28.json")
timing["tAPEC_BY_BLK"] = round(time.clock() - t0, 6)
timing["PROB_ACC"] = dpmap.get_problemacc()
#for which prob, which solution
state_prop["PROB_ACC"] = dpmap.get_problemacc()
state_prop["SOL_ACC"] = dplan.get_solacc()
state_prop["DIST_KEYS"] = dpmap.get_dist_keys()
# fill up block's prop.
block_prop = dpmap.get_block_to_attr_map() #transient properties
util.save(block_prop, home + "blkprop28-")
IO().insert_block_prop(dpmap)
del dpmap
t0 = time.clock()
# collection of blocks by dist.
districts_blocks = {}
for ctractkey, idvalues in dplan.get_block_to_attr_map().items():
#TODO:
#this blockkey must be in dpmap.
distid = idvalues[-1]
try:
districts_blocks[distid] += [ctractkey]
except:
districts_blocks[distid] = [ctractkey]
del dplan
timing["tCOLL_BLKS_BY_DIST"] = round(time.clock() - t0, 6)
###
#Validate:
##dist_keys in plan must be in problem-maps's dist keys.
##all of the input blocks must be mapped to one of the district.
for distkey in sorted(state_prop["DIST_KEYS"])[:]:
t0 = time.clock()
state_prop[distkey] = {}
ctractkeys = districts_blocks[distkey][:]
print("# of blocks in this dist:"), len(ctractkeys)
udist_geom = ogr.Geometry(ogr.wkbMultiPolygon)
#get each block's shape to create a district's map.
fcount = 0
shapef = ogr.Open(ctract_shapefile)
layer = shapef.GetLayer()
#
for feature in layer:
bkey = util.hash_cblocks_dist([feature.GetField('GEOID10')])
blkgeo = feature.GetGeometryRef()
if bkey in ctractkeys:
if blkgeo.GetGeometryName() == "LINEARRING":
print("\t-.-"), "linearring"
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(blkgeo)
if blkgeo.GetGeometryName() == 'MULTIPOLYGON':
print("\t-.-"), "multipolygon"
for geom in blkgeo:
udist_geom.AddGeometry(geom)
elif blkgeo.GetGeometryName() == 'POLYGON':
udist_geom.AddGeometry(blkgeo)
else:
print("\t\t-.-"), blkgeo.GetGeometryName()
udist_geom.AddGeometry(blkgeo)
#end-for
layer = None
#take union of blocks.
print("--"), udist_geom.GetGeometryName(
), udist_geom.GetGeometryCount(), fcount
udist_geom = udist_geom.UnionCascaded()
count = 0
#for each ring geom
for geom in udist_geom:
print("\t"), count, geom.GetGeometryName()
if geom.GetGeometryName() == 'LINEARRING':
state_prop[distkey]["BOUNDARY_PTS"] = geom.GetPoints()
#poly = ogr.Geometry(ogr.wkbPolygon)
#poly.AddGeometry(geom)
#util.geom_toshp(poly, home + "dists/st-" + str(distkey) + "-" + str(count), save_as_multipt=False)
else:
#util.geom_toshp(geom,home + "dists/st-"+str(distkey)+"-"+str(count), save_as_multipt=False)
pass
count += 1
print("Completed union of blks for a district")
timing["tUNION_CASCADE_BLOCKS_BY_DIST_" + str(distkey)] = round(
time.clock() - t0, 6)
util.save(state_prop, home + "temp_distprop28-")
print("Completed all district construction")
#compute state's APEC properties
t0 = time.clock()
for distkey in state_prop["DIST_KEYS"]:
polypts = state_prop[distkey]["BOUNDARY_PTS"]
poly = ogr.Geometry(ogr.wkbPolygon)
ring = ogr.Geometry(ogr.wkbLinearRing)
for x, y in polypts:
x, y = float(x), float(y)
ring.AddPoint(x, y)
poly.AddGeometry(ring)
state_prop[distkey]["AREA"] = poly.GetArea()
state_prop[distkey]["PERI"] = poly.GetBoundary().Length()
state_prop[distkey]["ENV"] = poly.GetEnvelope()
state_prop[distkey]["CENTROID"] = poly.Centroid().GetPoints()[0]
timing["tAPEC_BY_DIST" + str(distkey)] = round(
time.clock() - t0, 6)
#for each block in district, update block's properties relative to comprising district.
for distkey in state_prop["DIST_KEYS"]:
st_centroid = state_prop[distkey]["CENTROID"]
t0 = time.clock()
Metri.bctr_to_dctr(st_centroid, districts_blocks[distkey],
block_prop)
timing["tBLKCTR_TO_DIST_CTR_" + distkey] = round(
time.clock() - t0, 6)
t0 = time.clock()
Metri.blkctr_to_dist_peri(state_prop[distkey]["BOUNDARY_PTS"],
districts_blocks[distkey], block_prop)
timing["tDBLKCTR_TO_DIST_PERI_" + distkey] = round(
time.clock() - t0, 6)
t0 = time.clock()
#compute distr population
state_prop[distkey]["POPULATION"] = Metri.dist_population(
districts_blocks[distkey], block_prop)
timing["tPOPULATION_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# moments for district
Ia = Metri.moment_area(districts_blocks[distkey], block_prop)
state_prop[distkey]["MOMENT_AREA"] = Ia
timing["tMOMENT_AREA_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
Ip = Metri.moment_popu(districts_blocks[distkey], block_prop)
state_prop[distkey]["MOMENT_POPU"] = Ip
timing["tMOMENT_POPU_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# ISO-perimetric ratio A/P
Metri.dpolsby_popper2(distkey, state_prop)
timing["tISO_PERIMETRIC_IDX_" + distkey] = round(
time.clock() - t0, 6)
t0 = time.clock()
# Equal-perimeter-circle
Metri.darea_by_ac_eqpd2(distkey, state_prop)
timing["tDAREA_EQPC_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# Mean block-ctrroid-to-district-centroid
state_prop[distkey][
"MEAN_RADIUS_TO_DISTCTR"] = Metri.mean_radius_to_distctr(
districts_blocks[distkey], block_prop)
state_prop[distkey][
"DYN_RADIUS_TO_DISTCTR"] = Metri.dynamic_radius_to_distctr(
districts_blocks[distkey], block_prop)
state_prop[distkey][
"HRM_RADIUS_TO_DISTCTR"] = Metri.harmonic_radius_to_distctr(
districts_blocks[distkey], block_prop)
timing["t3R_TO_DISTCTR_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# Interpersonal-distance
state_prop[distkey][
"INTERPERSONAL_DIST"] = Metri.interpersonal_distance(
districts_blocks[distkey], block_prop)
timing["tINTERPERSONAL_DIST_" + distkey] = round(
time.clock() - t0, 6)
t0 = time.clock()
# HULL
state_hull = Metri.get_convexhull(
state_prop[distkey]["BOUNDARY_PTS"])
state_prop[distkey]["HULL_AREA"] = state_hull.GetArea()
timing["tHULL_AREA_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
state_prop[distkey][
"AREA_HULL_RATIO"] = Metri.convex_hull_area_ratio(
distkey, state_prop)
timing["tAREA_HULL_RATIO_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
state_prop[distkey]["POP_HULL_RATIO"] = Metri.population_polygon(
state_hull, state_prop[distkey]["POPULATION"],
districts_blocks[distkey], block_prop)
timing["tPOP_HULL_RATIO_" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# Exchange-idx
eqarea_circle = Metri.comp_equal_area_circle(
state_prop[distkey]["AREA"], state_prop[distkey]["CENTROID"],
None)
overlap_area_eqcircle = Metri.overlaping_area(
eqarea_circle, state_prop[distkey]["BOUNDARY_PTS"])
state_prop[distkey][
"OVERLAP_AREA_EQCIRCLE"] = overlap_area_eqcircle
state_prop[distkey]["EXCHG_IDX"] = Metri.exchange_index(
distkey, state_prop)
timing["tEXCHG_IDX" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
# Rohrbach-index
state_prop[distkey]["ROHRBACH_IDX"] = Metri.rohrbach_index(
districts_blocks[distkey], block_prop)
timing["tROHRBACH_IDX" + distkey] = round(time.clock() - t0, 6)
t0 = time.clock()
print("Completed prop for "), distkey
#update
IO().insert_timings(timing)
del timing
IO().insert_dist_metric(state_prop)
del state_prop