def build_from_blkequiv_csvbak(self, blk_equivfile): arr = np.loadtxt(blk_equivfile, delimiter=',') print arr.shape # if <blockid is a not single string. if arr.shape[1] > 2: arr = arr.astype(str) dists = SortedDict() dict = SortedDict() if arr.shape[1] > 2: for blkvals in arr: pkey = util.hash_cblocks_dist(tuple( blkvals[0:-1])) #key:blockid dict[pkey] = tuple(blkvals) dists[tuple(blkvals)[-1]] = 0 # self.__block_to_dist_map = dict self.__nblocks = len(list(dict.keys())) self.__merkleroot = util.merkle_root(self.__block_to_dist_map) self.__problemacc = DEFAULT_PROBLEM_ACC self.__solacc = DEFAULT_SOLUTION_ACC self.__planid = self.get_planid() self.__keyschainhash = self.get_keyschainhash() self.__timestamp = time.time() self.__dists = dists
def dproblem_from_shape_files(self, block_shapefile, census_tract_file=False): # open block shape file. if census_tract_file: idfieldname = "GEOID10" popfieldname = "P0010001" else: idfieldname = "BLOCKID10" popfieldname = "POP10" blk_to_attr_map = SortedDict() shapef = ogr.Open(block_shapefile) layer = shapef.GetLayer() #print schema schema = [] ldefn = layer.GetLayerDefn() for n in range(ldefn.GetFieldCount()): fdefn = ldefn.GetFieldDefn(n) schema.append(fdefn.name) print schema #create maping. sourceSpatialRef = layer.GetSpatialRef() Nfeats = layer.GetFeatureCount() ctract = {} for fid in xrange(Nfeats): fit = layer.GetFeature(fid) blkgeo = fit.GetGeometryRef() d = fit.GetField(idfieldname) #id pop = fit.GetField(popfieldname) #population blockkey = util.hash_cblocks_dist([d]) # block attrs blk_to_attr_map[blockkey] = {"FEAT_KEY": d, "POPULATION": pop} blk_to_attr_map[blockkey]["AREA"] = blkgeo.GetArea() blk_to_attr_map[blockkey]["PERI"] = blkgeo.GetBoundary().Length() blk_to_attr_map[blockkey]["ENV"] = blkgeo.GetEnvelope() blk_to_attr_map[blockkey]["CENTROID"] = blkgeo.Centroid( ).GetPoints()[0] #set property self.__block_to_attr_map = blk_to_attr_map self.__keyschainhash = self.get_keyschainhash() self.__nblocks = len(self.__block_to_attr_map.keys()) if self.__problemacc is None: self.__problemacc = DEFAULT_PROBLEM_ACC self.__releasetime = time.time() self.__closetime = self.__releasetime + PLAYTIME
def blk_feat_to_dist_mapping(self, block_shapefile, blk_to_dist_map, census_tract_file=False): # open block shape file. home = os.path.dirname(block_shapefile) if census_tract_file: idfieldname = "GEOID10" popfieldname = "P0010001" else: idfieldname = "BLOCKID10" popfieldname = "POP10" blk_to_attr_map = SortedDict() shapef = ogr.Open(block_shapefile) layer = shapef.GetLayer() #print schema ordered_dist = [] ldefn = layer.GetLayerDefn() #create maping. sourceSpatialRef = layer.GetSpatialRef() Nfeats = layer.GetFeatureCount() ctract = {} for fid in xrange(Nfeats): fit = layer.GetFeature(fid) blkgeo = fit.GetGeometryRef() d = fit.GetField(idfieldname) #id pop = fit.GetField(popfieldname) #population blockkey = util.hash_cblocks_dist([d]) #get dist value for this 'd' ordered_dist += [blk_to_dist_map[blockkey][1:]] #set property #save as csv. print ordered_dist[0] with open(home + '/ordered_blk_eqiv.csv', 'wb') as writeFile: writer = csv.writer(writeFile) writer.writerows(ordered_dist) writeFile.close()
def dproblem_from_files(self, block_shapefile): # open shape file. blk_to_attr_map = SortedDict() shapef = ogr.Open(block_shapefile) layer = shapef.GetLayer() #print schema schema = [] ldefn = layer.GetLayerDefn() for n in range(ldefn.GetFieldCount()): fdefn = ldefn.GetFieldDefn(n) schema.append(fdefn.name) print schema #create maping. sourceSpatialRef = layer.GetSpatialRef() Nfeats = layer.GetFeatureCount() ctract = {} for fid in xrange(Nfeats): fit = layer.GetFeature(fid) geom = fit.GetGeometryRef() a = fit.GetField('STATEFP10') b = fit.GetField('COUNTYFP10') c = fit.GetField('TRACTCE10') d = fit.GetField('BLOCKID10') ctract[c] = 0 pkey = util.hash_cblocks_dist([d]) blk_to_attr_map[pkey] = (fit.GetField('POP10')) print("n(census-tracts)"), len(ctract) #set property self.__block_to_attr_map = blk_to_attr_map self.__keyschainhash = self.get_keyschainhash() self.__nblocks = len(self.__block_to_attr_map.keys()) self.__problemacc = DEFAULT_PROBLEM_ACC self.__releasetime = time.time() self.__closetime = self.__releasetime + PLAYTIME
def build_from_blkequiv_csv(self, blk_equivfile): dists = SortedDict() dict = SortedDict() with open(blk_equivfile) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') for blkvals in csv_reader: pkey = util.hash_cblocks_dist([blkvals[0]]) #key:blockid dict[pkey] = tuple(blkvals) dists[blkvals[-1]] = 0 # self.__block_to_dist_map = dict self.__nblocks = len(list(dict.keys())) self.__merkleroot = util.merkle_root(self.__block_to_dist_map) self.__problemacc = DEFAULT_PROBLEM_ACC self.__solacc = DEFAULT_SOLUTION_ACC self.__planid = self.get_planid() self.__keyschainhash = self.get_keyschainhash() self.__timestamp = time.time() self.__dists = dists
def test_metric_block_integration(self, state_id="28", filter_dists=[]): timing = {} state_prop = {} data_info = {} home_cong = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/cd116/" home_blkpop = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/census_blocks_by_states/" + "tabblock2010_" + state_id + "_pophu/" ctract_shapefile = home_blkpop + "prjtabblock2010_" + state_id + "_pophu.shp" blk_equivfile = home_cong + "blk_eqiv" + state_id + ".csv" #with 2 cols #tract equiv file dist_idsfile = home_cong + "dist_ids" + state_id + ".csv" #with 2 cols #tract equiv file census_tract_file = False if census_tract_file: idfieldname = "GEOID10" dist_id_field = "STID3" #dist-id else: idfieldname = "BLOCKID10" dist_id_field = "STID3" prefix = "blk" #integration-type dplandatafile = prefix + "dplan" + state_id dplanproblemfile = prefix + "dpmap" + state_id dplan_block_propfile = prefix + "dpblock_prop" + state_id data_info["size_shp"] = util.get_size( os.path.dirname(ctract_shapefile)) data_info["dists"] = {} #get a dist problem t0 = time.clock() dpmap = DPmap() dpmap.load_json_dprob(home_blkpop + dplanproblemfile) timing["tAPEC_BY_BLK"] = round(time.clock() - t0, 6) timing["PROB_ACC"] = dpmap.get_problemacc() #get a plan dplan = Dplan() dplan.load_json_dplan(home_cong + dplandatafile) timing["SOL_ACC"] = dplan.get_solacc() #for which prob, which solution state_prop["PROB_ACC"] = dpmap.get_problemacc() state_prop["SOL_ACC"] = dplan.get_solacc() state_prop["DIST_KEYS"] = sorted(dpmap.get_dist_keys()) #filter by district ids. for k in filter_dists: try: state_prop["DIST_KEYS"].remove(k) except: continue # fill up block's prop. block_prop = dpmap.get_block_to_attr_map() #transient properties #IO().insert_block_prop(dpmap) #TODO: isssue with unicode keys in dict del dpmap t0 = time.clock() # collection of blocks by dist. districts_blocks = {} for ctractkey, idvalues in dplan.get_block_to_attr_map().items(): #TODO: #this blockkey must be in dpmap. distid = idvalues[-1] try: districts_blocks[distid] += [ctractkey] except: districts_blocks[distid] = [ctractkey] timing["tCOLL_BLKS_BY_DIST"] = round(time.clock() - t0, 6) print "tCOLL_BLKS_BY_DIST", timing["tCOLL_BLKS_BY_DIST"] ### #Validate: ##dist_keys in plan must be in problem-maps's dist keys. ##all of the input blocks must be mapped to one of the district. print("# of Districts :"), len(state_prop["DIST_KEYS"]) for distkey in state_prop["DIST_KEYS"]: t0 = time.clock() state_prop[distkey] = {} print("# of blocks in this dist:"), distkey, len( districts_blocks[distkey]) udist_geom = ogr.Geometry(ogr.wkbMultiPolygon) cd_container = {distkey: udist_geom} #get each block's shape to create a district's map. fcount = 0 shapef = ogr.Open(ctract_shapefile) layer = shapef.GetLayer() # t0 = time.clock() t2 = time.time() for feature in layer: bkey = util.hash_cblocks_dist([feature.GetField(idfieldname)]) blkgeo = feature.GetGeometryRef() #distid = feature.GetField(dist_id_field) #or distid = dplan.get_block_to_attr_map()[bkey][1] try: cd_container[distid].AddGeometry(blkgeo) except: continue #end-for layer = None print("time:"), time.clock() - t0, time.time() - t2 print("-udist_geom count-"), udist_geom.GetGeometryCount() #take union of blocks. t0 = time.clock() udist_geomc = udist_geom.UnionCascaded() timing["tBLKS_UCASCADE_" + str(distkey)] = round( time.clock() - t0, 6) count = 0 #for each ring geom, search for a largest ring. for geom in udist_geomc: if geom.GetGeometryName() == 'LINEARRING': state_prop[distkey]["BOUNDARY_PTS"] = geom.GetPoints() poly = ogr.Geometry(ogr.wkbPolygon) poly.AddGeometry(geom) util.geom_toshp(poly, home_cong + "dists/" + str(state_id) + "-" + "dist-" + str(distkey) + "-" + str(count), save_as_multipt=False) break elif geom.GetGeometryName() == "POLYGON": for geo in geom: state_prop[distkey]["BOUNDARY_PTS"] = geo.GetPoints() poly = ogr.Geometry(ogr.wkbPolygon) poly.AddGeometry(geo) util.geom_toshp(poly, home_cong + "dists/" + str(state_id) + "-" + "dist-" + str(distkey) + "-" + str(count), save_as_multipt=False) break count += 1 print("\t\t dist-boundary completed"), len( state_prop[distkey]["BOUNDARY_PTS"]) #TODO: cascadedunion of these linearring must be performed. ## #util.save(state_prop, home_blkpop + "temp_distprop28-") print("Completed block integration to construct a district") #compute state's APEC properties t0 = time.clock() for distkey in state_prop["DIST_KEYS"][:]: polypts = state_prop[distkey]["BOUNDARY_PTS"] poly = ogr.Geometry(ogr.wkbPolygon) ring = ogr.Geometry(ogr.wkbLinearRing) for x, y in polypts: x, y = float(x), float(y) ring.AddPoint(x, y) poly.AddGeometry(ring) state_prop[distkey]["AREA"] = poly.GetArea() state_prop[distkey]["PERI"] = poly.GetBoundary().Length() state_prop[distkey]["ENV"] = poly.GetEnvelope() state_prop[distkey]["CENTROID"] = poly.Centroid().GetPoints()[0] timing["tAPEC_" + str(distkey)] = round(time.clock() - t0, 6) state_prop[distkey]["FAILED"] = [] print("Completed APEC for district "), distkey #for each block in district, update block's properties relative to comprising district. for distkey in state_prop["DIST_KEYS"][:]: st_centroid = state_prop[distkey]["CENTROID"] data_info["dists"]["#" + prefix + "-" + distkey] = len( districts_blocks[distkey]) t0 = time.clock() try: Metri.bctr_to_dctr(st_centroid, districts_blocks[distkey], block_prop) timing["tBLKCTR_TO_DIST_CTR_" + distkey] = round( time.clock() - t0, 6) except Exception, e: state_prop[distkey]["FAILED"] = ["BLKCTR_TO_DIST_CTR_"] print("Exception:"), str(e) print("1") t0 = time.clock() try: Metri.blkctr_to_dist_peri_np( state_prop[distkey]["BOUNDARY_PTS"], districts_blocks[distkey], block_prop) timing["tDBLKCTR_TO_DIST_PERI_" + distkey] = round( time.clock() - t0, 6) pass except Exception, e: state_prop[distkey]["FAILED"] += ["DBLKCTR_TO_DIST_PERI_"] print("Exception--"), str(e)
def test_metric_ctract_integration(self): timing = {} state_prop = {} home = "D:/workspace/sqdm-repo/sqdm/out/tmp/redist/" ctract_shapefile = home + "Tracts10PopnHou28/Tracts10PopnHou.shp" blk_equivfile = "tr_equiv28.csv" #with 2 cols #tract equiv file #save and get a plan dplan = Dplan() dplan.build_from_blkequiv_csv(home + blk_equivfile) dplan.to_json(home + "ctdplan28") del dplan dplan = Dplan() dplan.load_json_dplan(home + "ctdplan28.json") IO().insert_plan(dplan) timing["SOL_ACC"] = dplan.get_solacc() #save and get a dist problem t0 = time.clock() dpmap = DPmap() dpmap.dproblem_from_shape_files(ctract_shapefile, census_tract_file=True) dpmap.to_json(home + "ctdpmap28.json") del dpmap dpmap = DPmap() dpmap.load_json_dprob(home + "ctdpmap28.json") timing["tAPEC_BY_BLK"] = round(time.clock() - t0, 6) timing["PROB_ACC"] = dpmap.get_problemacc() #for which prob, which solution state_prop["PROB_ACC"] = dpmap.get_problemacc() state_prop["SOL_ACC"] = dplan.get_solacc() state_prop["DIST_KEYS"] = dpmap.get_dist_keys() # fill up block's prop. block_prop = dpmap.get_block_to_attr_map() #transient properties util.save(block_prop, home + "blkprop28-") IO().insert_block_prop(dpmap) del dpmap t0 = time.clock() # collection of blocks by dist. districts_blocks = {} for ctractkey, idvalues in dplan.get_block_to_attr_map().items(): #TODO: #this blockkey must be in dpmap. distid = idvalues[-1] try: districts_blocks[distid] += [ctractkey] except: districts_blocks[distid] = [ctractkey] del dplan timing["tCOLL_BLKS_BY_DIST"] = round(time.clock() - t0, 6) ### #Validate: ##dist_keys in plan must be in problem-maps's dist keys. ##all of the input blocks must be mapped to one of the district. for distkey in sorted(state_prop["DIST_KEYS"])[:]: t0 = time.clock() state_prop[distkey] = {} ctractkeys = districts_blocks[distkey][:] print("# of blocks in this dist:"), len(ctractkeys) udist_geom = ogr.Geometry(ogr.wkbMultiPolygon) #get each block's shape to create a district's map. fcount = 0 shapef = ogr.Open(ctract_shapefile) layer = shapef.GetLayer() # for feature in layer: bkey = util.hash_cblocks_dist([feature.GetField('GEOID10')]) blkgeo = feature.GetGeometryRef() if bkey in ctractkeys: if blkgeo.GetGeometryName() == "LINEARRING": print("\t-.-"), "linearring" poly = ogr.Geometry(ogr.wkbPolygon) poly.AddGeometry(blkgeo) if blkgeo.GetGeometryName() == 'MULTIPOLYGON': print("\t-.-"), "multipolygon" for geom in blkgeo: udist_geom.AddGeometry(geom) elif blkgeo.GetGeometryName() == 'POLYGON': udist_geom.AddGeometry(blkgeo) else: print("\t\t-.-"), blkgeo.GetGeometryName() udist_geom.AddGeometry(blkgeo) #end-for layer = None #take union of blocks. print("--"), udist_geom.GetGeometryName( ), udist_geom.GetGeometryCount(), fcount udist_geom = udist_geom.UnionCascaded() count = 0 #for each ring geom for geom in udist_geom: print("\t"), count, geom.GetGeometryName() if geom.GetGeometryName() == 'LINEARRING': state_prop[distkey]["BOUNDARY_PTS"] = geom.GetPoints() #poly = ogr.Geometry(ogr.wkbPolygon) #poly.AddGeometry(geom) #util.geom_toshp(poly, home + "dists/st-" + str(distkey) + "-" + str(count), save_as_multipt=False) else: #util.geom_toshp(geom,home + "dists/st-"+str(distkey)+"-"+str(count), save_as_multipt=False) pass count += 1 print("Completed union of blks for a district") timing["tUNION_CASCADE_BLOCKS_BY_DIST_" + str(distkey)] = round( time.clock() - t0, 6) util.save(state_prop, home + "temp_distprop28-") print("Completed all district construction") #compute state's APEC properties t0 = time.clock() for distkey in state_prop["DIST_KEYS"]: polypts = state_prop[distkey]["BOUNDARY_PTS"] poly = ogr.Geometry(ogr.wkbPolygon) ring = ogr.Geometry(ogr.wkbLinearRing) for x, y in polypts: x, y = float(x), float(y) ring.AddPoint(x, y) poly.AddGeometry(ring) state_prop[distkey]["AREA"] = poly.GetArea() state_prop[distkey]["PERI"] = poly.GetBoundary().Length() state_prop[distkey]["ENV"] = poly.GetEnvelope() state_prop[distkey]["CENTROID"] = poly.Centroid().GetPoints()[0] timing["tAPEC_BY_DIST" + str(distkey)] = round( time.clock() - t0, 6) #for each block in district, update block's properties relative to comprising district. for distkey in state_prop["DIST_KEYS"]: st_centroid = state_prop[distkey]["CENTROID"] t0 = time.clock() Metri.bctr_to_dctr(st_centroid, districts_blocks[distkey], block_prop) timing["tBLKCTR_TO_DIST_CTR_" + distkey] = round( time.clock() - t0, 6) t0 = time.clock() Metri.blkctr_to_dist_peri(state_prop[distkey]["BOUNDARY_PTS"], districts_blocks[distkey], block_prop) timing["tDBLKCTR_TO_DIST_PERI_" + distkey] = round( time.clock() - t0, 6) t0 = time.clock() #compute distr population state_prop[distkey]["POPULATION"] = Metri.dist_population( districts_blocks[distkey], block_prop) timing["tPOPULATION_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # moments for district Ia = Metri.moment_area(districts_blocks[distkey], block_prop) state_prop[distkey]["MOMENT_AREA"] = Ia timing["tMOMENT_AREA_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() Ip = Metri.moment_popu(districts_blocks[distkey], block_prop) state_prop[distkey]["MOMENT_POPU"] = Ip timing["tMOMENT_POPU_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # ISO-perimetric ratio A/P Metri.dpolsby_popper2(distkey, state_prop) timing["tISO_PERIMETRIC_IDX_" + distkey] = round( time.clock() - t0, 6) t0 = time.clock() # Equal-perimeter-circle Metri.darea_by_ac_eqpd2(distkey, state_prop) timing["tDAREA_EQPC_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # Mean block-ctrroid-to-district-centroid state_prop[distkey][ "MEAN_RADIUS_TO_DISTCTR"] = Metri.mean_radius_to_distctr( districts_blocks[distkey], block_prop) state_prop[distkey][ "DYN_RADIUS_TO_DISTCTR"] = Metri.dynamic_radius_to_distctr( districts_blocks[distkey], block_prop) state_prop[distkey][ "HRM_RADIUS_TO_DISTCTR"] = Metri.harmonic_radius_to_distctr( districts_blocks[distkey], block_prop) timing["t3R_TO_DISTCTR_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # Interpersonal-distance state_prop[distkey][ "INTERPERSONAL_DIST"] = Metri.interpersonal_distance( districts_blocks[distkey], block_prop) timing["tINTERPERSONAL_DIST_" + distkey] = round( time.clock() - t0, 6) t0 = time.clock() # HULL state_hull = Metri.get_convexhull( state_prop[distkey]["BOUNDARY_PTS"]) state_prop[distkey]["HULL_AREA"] = state_hull.GetArea() timing["tHULL_AREA_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() state_prop[distkey][ "AREA_HULL_RATIO"] = Metri.convex_hull_area_ratio( distkey, state_prop) timing["tAREA_HULL_RATIO_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() state_prop[distkey]["POP_HULL_RATIO"] = Metri.population_polygon( state_hull, state_prop[distkey]["POPULATION"], districts_blocks[distkey], block_prop) timing["tPOP_HULL_RATIO_" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # Exchange-idx eqarea_circle = Metri.comp_equal_area_circle( state_prop[distkey]["AREA"], state_prop[distkey]["CENTROID"], None) overlap_area_eqcircle = Metri.overlaping_area( eqarea_circle, state_prop[distkey]["BOUNDARY_PTS"]) state_prop[distkey][ "OVERLAP_AREA_EQCIRCLE"] = overlap_area_eqcircle state_prop[distkey]["EXCHG_IDX"] = Metri.exchange_index( distkey, state_prop) timing["tEXCHG_IDX" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() # Rohrbach-index state_prop[distkey]["ROHRBACH_IDX"] = Metri.rohrbach_index( districts_blocks[distkey], block_prop) timing["tROHRBACH_IDX" + distkey] = round(time.clock() - t0, 6) t0 = time.clock() print("Completed prop for "), distkey #update IO().insert_timings(timing) del timing IO().insert_dist_metric(state_prop) del state_prop