def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool): if isinstance(buildings_to_be_demolished, list): buildings_to_be_demolished = array(buildings_to_be_demolished) if buildings_to_be_demolished.size <= 0: return id_index_in_buildings = building_dataset.get_id_index(buildings_to_be_demolished) parcels = dataset_pool.get_dataset('parcel') idx_pcl = parcels.get_id_index(unique(building_dataset['parcel_id'][id_index_in_buildings])) # remove occupants from buildings to be demolished JAMM = JoinAttributeModificationModel() for agent_name in ['household', 'job']: agents = dataset_pool.get_dataset(agent_name) JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1) building_dataset.remove_elements(id_index_in_buildings) logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size) # set land_use_type 'vacant' to parcels with demolished buildings land_types = dataset_pool.get_dataset('land_use_type') vac_idx = land_types["land_use_name"] == 'vacant' if vac_idx.sum() > 0: code = land_types.get_id_attribute()[vac_idx][0] nvac = (parcels['land_use_type_id'][idx_pcl] == code).sum() parcels['land_use_type_id'][idx_pcl] = code logger.log_status("%s parcels set to vacant." % (idx_pcl.size - nvac))
def test_demolition(self): demolish_buildings = array([10, 3, 180]) household_data = { 'household_id': arange(10)+1, 'building_id': array([10, 3, 6, 150, 10, 10, -1, 5, 3, 3]) # demolished [* * * * * *] } person_data = { 'person_id':arange(15)+1, 'household_id': array([1,1,2,3,3,5,4,4,4,6,7,8,9,10,10]), # in demolished bldgs [* * * * * * * * *] 'job_id': array([5,4,1,2,2,1,3,5,1,5,5,4,3, 3, 1]) # in demolished bldgs [ * * * * * * * * *] } job_data = { 'job_id': arange(5)+1, 'building_id': array([180, 145, 10, 180, 179]) } building_data = { 'building_id': arange(200)+1, 'parcel_id': arange(200)+1 } parcel_data = { 'parcel_id': arange(200)+1, 'land_use_type_id': array(150*[1]+50*[2]), } lut_data = { 'land_use_type_id': array([1,2]), 'land_use_name': array(['non_vacant', 'vacant']) } storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name = 'households', table_data = household_data) storage.write_table(table_name = 'buildings', table_data = building_data) storage.write_table(table_name = 'jobs', table_data = job_data) storage.write_table(table_name = 'persons', table_data = person_data) storage.write_table(table_name = 'parcels', table_data = parcel_data) storage.write_table(table_name = 'land_use_types', table_data = lut_data) dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim']) BCM = BuildingConstructionModel() BCM.demolish_buildings(demolish_buildings, dataset_pool.get_dataset('building'), dataset_pool) JAMM = JoinAttributeModificationModel() JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('household'), attribute_to_be_modified='job_id', value=-1, filter='household.building_id <=0') JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('job'), attribute_to_be_modified='job_id', value=-1, filter='job.building_id <=0') self.assertEqual(all(dataset_pool.get_dataset('household').get_attribute('building_id') == array([-1, -1, 6, 150, -1, -1, -1, 5, -1, -1])), True) self.assertEqual(all(dataset_pool.get_dataset('job').get_attribute('building_id') == array([-1, 145, -1, -1, 179])), True) self.assertEqual(dataset_pool.get_dataset('building').size()==197, True) self.assertEqual(all(dataset_pool.get_dataset('person').get_attribute('job_id') == array([-1,-1,-1,2,2,-1,-1,5,-1,-1,-1,-1,-1,-1,-1])), True) self.assertEqual(dataset_pool.get_dataset('parcel')['land_use_type_id'][9], 2) self.assertEqual(dataset_pool.get_dataset('parcel')['land_use_type_id'][2], 2)
def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool): if isinstance(buildings_to_be_demolished, list): buildings_to_be_demolished = array(buildings_to_be_demolished) if buildings_to_be_demolished.size <= 0: return id_index_in_buildings = building_dataset.get_id_index(buildings_to_be_demolished) # remove occupants from buildings to be demolished JAMM = JoinAttributeModificationModel() for agent_name in ['household', 'job']: agents = dataset_pool.get_dataset(agent_name) JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1) building_dataset.remove_elements(id_index_in_buildings) logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size)
def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool): if isinstance(buildings_to_be_demolished, list): buildings_to_be_demolished = array(buildings_to_be_demolished) if buildings_to_be_demolished.size <= 0: return id_index_in_buildings = building_dataset.get_id_index( buildings_to_be_demolished) # remove occupants from buildings to be demolished JAMM = JoinAttributeModificationModel() for agent_name in ['household', 'job']: agents = dataset_pool.get_dataset(agent_name) JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1) building_dataset.remove_elements(id_index_in_buildings) logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size)
def run(my, cache_dir=None, year=None): global parcel_set, z, node_set, submarket, esubmarket, isr, parcelfees, costdiscount ''' if 0: z = Zoning() p = Parcels() cPickle.dump((z,p),open('databaseinfo.jar','w')) else: print "Reading db info from jar..." z,p = cPickle.load(open(os.path.join(os.environ['OPUS_DATA'],'bay_area_parcel/databaseinfo.jar'))) ''' ## when developer_model is invoked alone from command line if cache_dir is not None and year is not None: #data_path = paths.get_opus_data_path_path() cache_dir = os.path.join(data_path, 'bay_area_parcel/runs/run_105.2012_05_03_09') #year = 2011 simulation_state = SimulationState() simulation_state.set_current_time(year) SimulationState().set_cache_directory(cache_dir) attribute_cache = AttributeCache() dataset_pool = SessionConfiguration(new_instance=True, package_order=['bayarea', 'urbansim_parcel', 'urbansim', 'opus_core'], in_storage=attribute_cache ).get_dataset_pool() dataset_pool = SessionConfiguration().get_dataset_pool() current_year = SimulationState().get_current_time() cache_dir = SimulationState().get_cache_directory() parcel_set = dataset_pool.get_dataset('parcel') building_set = dataset_pool.get_dataset('building') household_set = dataset_pool.get_dataset('household') node_set = dataset_pool.get_dataset('node') unit_set = dataset_pool.get_dataset('residential_unit') submarket = dataset_pool.get_dataset('submarket') esubmarket = dataset_pool.get_dataset('employment_submarket') #print numpy.array(unit_set['rent'] > 0).size #for i in range(unit_set.size()): # print unit_set['unit_price'][i], unit_set['unit_sqft'][i] #transit_set = dataset_pool.get_dataset('transit_station') #print dataset_pool.datasets_in_pool() ''' from bayarea.node import transit_type_DDD_within_DDD_meters for i in range(7): print i v = transit_type_DDD_within_DDD_meters.transit_type_DDD_within_DDD_meters(i,500) d = v.compute(dataset_pool) print d.size found = d[numpy.nonzero(d)] print found.size sys.exit() ''' compute_devmdl_accvars(node_set) ###################### ### CAREFUL - THIS IS WHERE SCNERARIO SPECIFIC INFO GOES ###################### current_year = SimulationState().get_current_time() z = Zoning(my.scenario,current_year) isr = None if my.scenario.startswith('Transit'): isr = ISR() parcelfees = None if my.scenario.startswith('Preferred'): parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_preferred')) #elif my.scenario.startswith('Transit'): # parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_transit')) elif my.scenario.startswith('Equity'): parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_equity')) elif my.scenario.startswith('Infill'): parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_infill')) costdiscount = 0.0 if not my.scenario.startswith('No Project') and not my.scenario.startswith('Equity'): costdiscount = .01 ################################# ################################# from numpy import logical_not empty_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)==0)*(parcel.node_id>0)*(parcel.shape_area>80)") res_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)>0)*(parcel.node_id>0)*(parcel.shape_area>80)") bart_parcels = parcel_set.compute_variables("(parcel.disaggregate(bayarea.node.transit_type_1_within_800_meters))") caltrain_parcels = parcel_set.compute_variables("(parcel.disaggregate(bayarea.node.transit_type_2_within_800_meters))") #pda_parcels = parcel_set.compute_variables("(parcel.pda_id > -1)*(numpy.logical_not(parcel.county_id==38))") pda_parcels = parcel_set.compute_variables("(parcel.pda_id > -1)") SAMPLE_RATE = 0.01 from opus_core.sampling_toolbox import sample_noreplace from numpy import concatenate, where sampled_res_parcels_index = sample_noreplace(where(res_parcels)[0], int(SAMPLE_RATE * parcel_set.size())) test_parcels = concatenate((where(empty_parcels==1)[0], sampled_res_parcels_index,where(bart_parcels==1)[0],where(caltrain_parcels==1)[0],where(pda_parcels==1)[0])) test_parcels = sample_noreplace(test_parcels, int(.08 * 154877)) numpy.random.shuffle(test_parcels) """ sample = [] for i in range(parcel_set.size()): if empty_parcels[i] == 1: sample.append(i+1) elif res_parcels[i] == 1 and numpy.random.ranf() < SAMPLE_RATE: sample.append(i+1) test_parcels = array(sample) """ #empty_parcels = parcel_set.compute_variables("(parcel.node_id>0)*(parcel.shape_area>80)") #test_parcels = numpy.where(empty_parcels==1)[0] global building_sqft, building_price building_sqft = parcel_set.compute_variables('parcel.aggregate(building.building_sqft)') building_price_owner_residential=parcel_set.compute_variables('building_price_owner_res=parcel.aggregate((residential_unit.sale_price)*(residential_unit.sale_price>0),intermediates=[building])') building_price_rental_residential=parcel_set.compute_variables('building_price_rental_res=parcel.aggregate((residential_unit.rent*12*17.9)*(residential_unit.rent>0),intermediates=[building])') building_price_nonresidential = parcel_set.compute_variables('building_price_nonres = parcel.aggregate((building.non_residential_rent*7*building.non_residential_sqft))') sum_building_p = parcel_set.compute_variables('sum_building_price = parcel.building_price_owner_res + parcel.building_price_rental_res + building_price_nonres') ##sum_building_price = building_price_owner_residential + building_price_rental_residential + building_price_nonresidential vacant_parcel = parcel_set.compute_variables('parcel.sum_building_price == 0') price_per_sqft_land = (parcel_set.compute_variables('parcel.disaggregate(safe_array_divide(zone.aggregate(parcel.sum_building_price),zone.aggregate(building.building_sqft)))'))/4 parcel_land_area = parcel_set.compute_variables('parcel.shape_area') vacant_land_price = vacant_parcel*price_per_sqft_land*parcel_land_area building_price = sum_building_p + vacant_land_price ##sum_building_price = building_price_owner_residential + building_price_rental_residential + building_price_nonresidential #land_price = (sum_building_p==0) * (parcel_set.compute_variables('parcel.disaggregate(safe_array_divide(zone.aggregate(parcel.sum_building_price),zone.aggregate(building.building_sqft)))'))* parcel_set.compute_variables('parcel.land_area') #info used to match from proposal_component to submarket parcel_set.compute_variables(["bayarea.parcel.within_half_mile_transit", "bayarea.parcel.schooldistrict", "bayarea.parcel.jurisdiction_id", ]) #test_parcels = array([i+1 for i in range(parcel_set.size())]) #test_parcels = test_parcels[:10000] #test_parcels = test_parcels[:150] #test_parcels = numpy.where(parcel_set['parcel_id'] == 1608920)[0] #print test_parcels logger.log_status("%s parcels to test" % (test_parcels.size)) print "Num of parcels:", test_parcels.size import time HOTSHOT = 0 if MP: from multiprocessing import Pool, Queue pool = Pool(processes=4) import hotshot, hotshot.stats#, test.pystone if HOTSHOT: prof = hotshot.Profile('devmdl.prof') prof.start() outf = open(os.path.join(cache_dir,'buildings-%d.csv' % current_year),'w') outf.write('pid,county,dev_btype,stories,sqft,res_sqft,nonres_sqft,tenure,year_built,res_units,npv,actualfee,btype\n') debugf = open(os.path.join(cache_dir,'proforma-debug-%d.csv' % current_year),'w') bformdbg = 'county_id,far,height,max_dua,bform.sf_builtarea(),bform.sfunitsizes,bform.mf_builtarea(),bform.mfunitsizes,bform.num_units,bform.nonres_sqft,bform.buildable_area' otherdbg = 'isr,parcelfees,existing_sqft,existing_price,lotsize,unitsize,unitsize2,bform.sales_absorption,bform.rent_absorption,bform.leases_absorption,bform.sales_vacancy_rates,bform.vacancy_rates' debugf.write('pid,btype,npv,actualfee,pricesf,pricemf,rentsf,rentmf,rentof,rentret,rentind,%s,%s\n' % (bformdbg,otherdbg)) t1 = time.time() aggd = {} def chunks(l, n): for i in xrange(0, len(l), n): yield l[i:i+n] for test_chunk in chunks(test_parcels,1000): print "Executing CHUNK" sales_absorption = submarket.compute_variables('bayarea.submarket.sales_absorption') rent_absorption = submarket.compute_variables('bayarea.submarket.rent_absorption') vacancy_rates = submarket.compute_variables('bayarea.submarket.vacancy_rates') leases_absorption = esubmarket.compute_variables('bayarea.employment_submarket.leases_absorption') nr_vacancy_rates = esubmarket.compute_variables('bayarea.employment_submarket.vacancy_rates') if HOTSHOT: results = [] for p in test_chunk: r = process_parcel(p) if r <> None and r <> -1: results.append(list(r)) else: if MP: results = pool.map(process_parcel,test_chunk) else: results = [process_parcel(p) for p in test_chunk] results_bldg = [list(x[0]) for x in results if x <> None and x[0] <> -1] #each row of units represents number of units of [1, 2, 3, 4] bedrooms units = array([x[1][0] for x in results if x <> None and x[0] <> -1]) sqft_per_unit = array([x[1][1] for x in results if x <> None and x[0] <> -1]) for x in results: if x <> None: debugf.write(x[2]) results = results_bldg for result in results: #print result out_btype = devmdltypes[int(result[2])-1] outf.write(string.join([str(x) for x in result]+[str(out_btype)],sep=',')+'\n') ##TODO: id of buildings to be demolished buildings_to_demolish = [] idx_buildings_to_demolish = building_set.get_id_index(buildings_to_demolish) JAMM = JoinAttributeModificationModel() JAMM.run(household_set, building_set, index=idx_buildings_to_demolish, value=-1) building_set.remove_elements(idx_buildings_to_demolish) column_names = ["parcel_id","county","building_type_id","stories", "building_sqft","residential_sqft","non_residential_sqft", "tenure","year_built","residential_units"] buildings_data = copy.deepcopy(results) for i in range(len(buildings_data)): buildings_data[i][2] = devmdltypes[int(buildings_data[i][2])-1] buildings_data = array(buildings_data) new_buildings = {} available_bldg_id = building_set['building_id'].max() + 1 new_bldg_ids = arange(available_bldg_id, available_bldg_id+buildings_data.shape[0], dtype=building_set['building_id'].dtype) if buildings_data.size > 0: for icol, col_name in enumerate(column_names): if col_name in building_set.get_known_attribute_names(): ddtype = building_set[col_name].dtype new_buildings[col_name] = (buildings_data[:, icol]).astype(ddtype) else: #if the col_name is not in dataset, it will be discarded anyway pass new_buildings['building_id'] = new_bldg_ids # recode tenure: 1 - rent, 2 - own from 0 - own, 1 - rent new_buildings['tenure'][new_buildings['tenure']==0] = 2 ## pid is the index to parcel_set; convert them to actual parcel_id #new_buildings['parcel_id'] = parcel_set['parcel_id'][new_buildings['parcel_id']] building_set.add_elements(new_buildings, require_all_attributes=False, change_ids_if_not_unique=True) building_set.flush_dataset() assert new_bldg_ids.size == units.shape[0] == sqft_per_unit.shape[0] units_bldg_ids = repeat(new_bldg_ids, 4) bedrooms = array([1, 2, 3, 4] * units.size) units = round(units.ravel()) sqft_per_unit = sqft_per_unit.ravel() new_units = {'building_id': array([], dtype='i4'), 'bedrooms': array([], dtype='i4'), 'sqft_per_unit': array([], dtype='i4') } for i_unit, unit in enumerate(units): if unit <= 0: continue new_units['building_id'] = concatenate((new_units['building_id'], repeat(units_bldg_ids[i_unit], unit)) ) new_units['bedrooms'] = concatenate((new_units['bedrooms'], repeat(bedrooms[i_unit], unit)) ) new_units['sqft_per_unit'] = concatenate((new_units['sqft_per_unit'], repeat(sqft_per_unit[i_unit], unit)) ) ##force dtype conversion to the same dtype as unit_set for col_name in ['building_id', 'bedrooms', 'sqft_per_unit']: if col_name in unit_set.get_known_attribute_names(): new_units[col_name] = new_units[col_name].astype(unit_set[col_name].dtype) unit_set.add_elements(new_units, require_all_attributes=False, change_ids_if_not_unique=True) unit_set.flush_dataset() for result in results: units = result[-1] nonres_sqft = 1 #result[6]/1000.0 county = result[1] btype = result[2] key = (county,btype) aggd.setdefault(key,0) if btype < 7: aggd[key] += units else: aggd[key] += nonres_sqft aggd.setdefault(county,0) aggd[county] += units aggf = open('county_aggregations-%d.csv' % current_year,'w') county_names = {49:'son',41:'smt',1:'ala',43:'scl',28:'nap',38:'sfr',7:'cnc',48:'sol',21:'mar',0:'n/a'} btype_names = {1:'SF',2:'SFBUILD',3:'MF',4:'MXMF',5:'CONDO',6:'MXC',7:'OF',8:'MXO',9:'CHOOD',10:'CAUTO',11:'CBOX',12:'MANU',13:'WHE'} aggf.write('county,total,'+string.join(btype_names.values(),sep=',')+'\n') for county in [38,41,43,1,7,48,28,49,21]: aggf.write(county_names[county]+','+str(aggd.get(county,0))) for btype in btype_names.keys(): key = (county,btype) val = aggd.get(key,0) aggf.write(','+str(val)) aggf.write('\n') t2 = time.time() print "Finished in %f seconds" % (t2-t1) print "Ran optimization %d times" % devmdl_optimize.OBJCNT global NOZONINGCNT, NOBUILDTYPES print "Did not find zoning for parcel %d times" % NOZONINGCNT print "Did not find building types for parcel %d times" % NOBUILDTYPES print "DONE" my.post_run() #remove price_shifter & cost_shifter to avoid them being cached if HOTSHOT: prof.stop() prof.close() stats = hotshot.stats.load('devmdl.prof') stats.strip_dirs() stats.sort_stats('cumulative') stats.print_stats(20)
def test_demolition(self): demolish_buildings = array([10, 3, 180]) household_data = { 'household_id': arange(10) + 1, 'building_id': array([10, 3, 6, 150, 10, 10, -1, 5, 3, 3]) # demolished [* * * * * *] } person_data = { 'person_id': arange(15) + 1, 'household_id': array([1, 1, 2, 3, 3, 5, 4, 4, 4, 6, 7, 8, 9, 10, 10]), # in demolished bldgs [* * * * * * * * *] 'job_id': array([5, 4, 1, 2, 2, 1, 3, 5, 1, 5, 5, 4, 3, 3, 1]) # in demolished bldgs [ * * * * * * * * *] } job_data = { 'job_id': arange(5) + 1, 'building_id': array([180, 145, 10, 180, 179]) } building_data = { 'building_id': arange(200) + 1, } storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='households', table_data=household_data) storage.write_table(table_name='buildings', table_data=building_data) storage.write_table(table_name='jobs', table_data=job_data) storage.write_table(table_name='persons', table_data=person_data) dataset_pool = DatasetPool( storage=storage, package_order=['urbansim_parcel', 'urbansim']) BCM = BuildingConstructionModel() BCM.demolish_buildings(demolish_buildings, dataset_pool.get_dataset('building'), dataset_pool) JAMM = JoinAttributeModificationModel() JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('household'), attribute_to_be_modified='job_id', value=-1, filter='household.building_id <=0') JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('job'), attribute_to_be_modified='job_id', value=-1, filter='job.building_id <=0') self.assertEqual( all( dataset_pool.get_dataset('household').get_attribute( 'building_id') == array( [-1, -1, 6, 150, -1, -1, -1, 5, -1, -1])), True) self.assertEqual( all( dataset_pool.get_dataset('job').get_attribute('building_id') == array([-1, 145, -1, -1, 179])), True) self.assertEqual( dataset_pool.get_dataset('building').size() == 197, True) self.assertEqual( all( dataset_pool.get_dataset('person').get_attribute('job_id') == array([ -1, -1, -1, 2, 2, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1 ])), True)
def run(my, cache_dir=None, year=None): global parcel_set, z, node_set, zone_set, submarket, esubmarket, isr, parcelfees, costdiscount, building_sqft, building_price dataset_pool = SessionConfiguration().get_dataset_pool() current_year = SimulationState().get_current_time() cache_dir = SimulationState().get_cache_directory() parcel_set = dataset_pool.get_dataset('parcel') building_set = dataset_pool.get_dataset('building') household_set = dataset_pool.get_dataset('household') unit_set = dataset_pool.get_dataset('residential_unit') submarket = dataset_pool.get_dataset('submarket') esubmarket = dataset_pool.get_dataset('employment_submarket') capped_rents = building_set.compute_variables('_adj_rent = building.non_residential_rent*(building.non_residential_rent<120) + 120*(building.non_residential_rent>=120)') building_set.modify_attribute('non_residential_rent', capped_rents) if NODES: node_set = dataset_pool.get_dataset('node') compute_devmdl_accvars_nodal(node_set) else: zone_set = dataset_pool.get_dataset('zone') compute_devmdl_accvars_zonal(zone_set) z = Zoning(my.scenario,current_year) costdiscount = 0.0 SAMPLE_RATE = 0.05 isr = None parcelfees = None empty_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)==0)*(parcel.parcel_sqft>800)") res_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)>0)*(parcel.parcel_sqft>800)") sampled_res_parcels_index = sample_noreplace(where(res_parcels)[0], int(SAMPLE_RATE * parcel_set.size())) test_parcels1 = concatenate((where(empty_parcels==1)[0], sampled_res_parcels_index)) test_parcels = sample_noreplace(test_parcels1, int(.05 * test_parcels1.size)) numpy.random.shuffle(test_parcels) building_sqft = parcel_set.compute_variables('parcel.aggregate(building.non_residential_sqft + building.residential_units*building.sqft_per_unit)') building_price_owner_residential=parcel_set.compute_variables('building_price_owner_res=parcel.aggregate((residential_unit.sale_price)*(residential_unit.sale_price>0),intermediates=[building])') building_price_rental_residential=parcel_set.compute_variables('building_price_rental_res=parcel.aggregate((residential_unit.rent*12*17.9)*(residential_unit.rent>0),intermediates=[building])') ##change 17.9 building_price_nonresidential = parcel_set.compute_variables('building_price_nonres = parcel.aggregate((building.non_residential_rent*7*building.non_residential_sqft))') ##change 7 sum_building_p = parcel_set.compute_variables('sum_building_price = parcel.building_price_owner_res + parcel.building_price_rental_res + building_price_nonres') vacant_parcel = parcel_set.compute_variables('parcel.sum_building_price == 0') price_per_sqft_land = parcel_set.compute_variables('safe_array_divide(parcel.land_value,parcel.parcel_sqft)') parcel_land_area = parcel_set.compute_variables('parcel.parcel_sqft') vacant_land_price = vacant_parcel*price_per_sqft_land*parcel_land_area building_price = sum_building_p + vacant_land_price #rename property price #info used to match from proposal_component to submarket parcel_set.compute_variables(["drcog.parcel.within_half_mile_transit", "drcog.parcel.zone", ]) logger.log_status("%s parcels to test" % (test_parcels.size)) if MP: from multiprocessing import Pool, Queue pool = Pool(processes=4) if HOTSHOT: import hotshot, hotshot.stats prof = hotshot.Profile('devmdl.prof') prof.start() outf = open(os.path.join(cache_dir,'buildings-%d.csv' % current_year),'w') outf.write('pid,county,dev_btype,stories,sqft,res_sqft,nonres_sqft,tenure,year_built,res_units,npv,actualfee,btype\n') debugf = open(os.path.join(cache_dir,'proforma-debug-%d.csv' % current_year),'w') bformdbg = 'county_id,far,height,max_dua,bform.sf_builtarea(),bform.sfunitsizes,bform.mf_builtarea(),bform.mfunitsizes,bform.num_units,bform.nonres_sqft,bform.buildable_area' otherdbg = 'isr,parcelfees,existing_sqft,existing_price,lotsize,unitsize,unitsize2,bform.sales_absorption,bform.rent_absorption,bform.leases_absorption,bform.sales_vacancy_rates,bform.vacancy_rates' debugf.write('pid,btype,npv,actualfee,pricesf,pricemf,rentsf,rentmf,rentof,rentret,rentind,%s,%s\n' % (bformdbg,otherdbg)) t1 = time.time() aggd = {} def chunks(l, n): for i in xrange(0, len(l), n): yield l[i:i+n] for test_chunk in chunks(test_parcels,1000): print "Executing CHUNK" sales_absorption = submarket.compute_variables('drcog.submarket.sales_absorption') ######################!! rent_absorption = submarket.compute_variables('drcog.submarket.rent_absorption') vacancy_rates = submarket.compute_variables('drcog.submarket.vacancy_rates') leases_absorption = esubmarket.compute_variables('drcog.employment_submarket.leases_absorption') nr_vacancy_rates = esubmarket.compute_variables('drcog.employment_submarket.vacancy_rates') if HOTSHOT: results = [] for p in test_chunk: r = process_parcel(p) if r <> None and r <> -1: results.append(list(r)) else: if MP: results = pool.map(process_parcel,test_chunk) else: results = [process_parcel(p) for p in test_chunk] results_bldg = [list(x[0]) for x in results if x <> None and x[0] <> -1] #each row of units represents number of units of [1, 2, 3, 4] bedrooms units = array([x[1][0] for x in results if x <> None and x[0] <> -1]) sqft_per_unit = array([x[1][1] for x in results if x <> None and x[0] <> -1]) for x in results: if x <> None: debugf.write(x[2]) results = results_bldg for result in results: #print result out_btype = int(result[2]) outf.write(string.join([str(x) for x in result]+[str(out_btype)],sep=',')+'\n') ##TODO: id of buildings to be demolished buildings_to_demolish = [] idx_buildings_to_demolish = building_set.get_id_index(buildings_to_demolish) JAMM = JoinAttributeModificationModel() JAMM.run(household_set, building_set, index=idx_buildings_to_demolish, value=-1) building_set.remove_elements(idx_buildings_to_demolish) column_names = ["parcel_id","county","building_type_id","stories", "building_sqft","residential_sqft","non_residential_sqft", "tenure","year_built","residential_units"] buildings_data = copy.deepcopy(results) #for i in range(len(buildings_data)): #buildings_data[i][2] = devmdltypes[int(buildings_data[i][2])-1] buildings_data = array(buildings_data) new_buildings = {} available_bldg_id = building_set['building_id'].max() + 1 new_bldg_ids = arange(available_bldg_id, available_bldg_id+buildings_data.shape[0], dtype=building_set['building_id'].dtype) if buildings_data.size > 0: for icol, col_name in enumerate(column_names): if col_name in building_set.get_known_attribute_names(): ddtype = building_set[col_name].dtype new_buildings[col_name] = (buildings_data[:, icol]).astype(ddtype) else: #if the col_name is not in dataset, it will be discarded anyway pass new_buildings['building_id'] = new_bldg_ids # recode tenure: 1 - rent, 2 - own from 0 - own, 1 - rent new_buildings['tenure'][new_buildings['tenure']==0] = 1 ## pid is the index to parcel_set; convert them to actual parcel_id #new_buildings['parcel_id'] = parcel_set['parcel_id'][new_buildings['parcel_id']] building_set.add_elements(new_buildings, require_all_attributes=False, change_ids_if_not_unique=True) building_set.flush_dataset() assert new_bldg_ids.size == units.shape[0] == sqft_per_unit.shape[0] units_bldg_ids = repeat(new_bldg_ids, 4) bedrooms = array([1, 2, 3, 4] * units.size) units = round(units.ravel()) sqft_per_unit = sqft_per_unit.ravel() new_units = {'building_id': array([], dtype='i4'), 'bedrooms': array([], dtype='i4'), 'unit_sqft': array([], dtype='i4') } for i_unit, unit in enumerate(units): if unit <= 0: continue new_units['building_id'] = concatenate((new_units['building_id'], repeat(units_bldg_ids[i_unit], unit)) ) new_units['bedrooms'] = concatenate((new_units['bedrooms'], repeat(bedrooms[i_unit], unit)) ) new_units['unit_sqft'] = concatenate((new_units['unit_sqft'], repeat(sqft_per_unit[i_unit], unit)) ) ##force dtype conversion to the same dtype as unit_set for col_name in ['building_id', 'bedrooms', 'unit_sqft']: if col_name in unit_set.get_known_attribute_names(): new_units[col_name] = new_units[col_name].astype(unit_set[col_name].dtype) unit_set.add_elements(new_units, require_all_attributes=False, change_ids_if_not_unique=True) unit_set.flush_dataset() for result in results: units = result[-1] nonres_sqft = 1 #result[6]/1000.0 county = result[1] btype = result[2] key = (county,btype) aggd.setdefault(key,0) if btype < 7: aggd[key] += units else: aggd[key] += nonres_sqft aggd.setdefault(county,0) aggd[county] += units ######################These hard-coded counties are problematic!!! # aggf = open('county_aggregations-%d.csv' % current_year,'w') # county_names = {49:'son',41:'smt',1:'ala',43:'scl',28:'nap',38:'sfr',7:'cnc',48:'sol',21:'mar',0:'n/a'} # btype_names = {1:'SF',2:'SFBUILD',3:'MF',4:'MXMF',5:'CONDO',6:'MXC',7:'OF',8:'MXO',9:'CHOOD',10:'CAUTO',11:'CBOX',12:'MANU',13:'WHE'} # aggf.write('county,total,'+string.join(btype_names.values(),sep=',')+'\n') # for county in [38,41,43,1,7,48,28,49,21]: # aggf.write(county_names[county]+','+str(aggd.get(county,0))) # for btype in btype_names.keys(): # key = (county,btype) # val = aggd.get(key,0) # aggf.write(','+str(val)) # aggf.write('\n') t2 = time.time() print "Finished in %f seconds" % (t2-t1) print "Ran optimization %d times" % devmdl_optimize.OBJCNT global NOZONINGCNT, NOBUILDTYPES print "Did not find zoning for parcel %d times" % NOZONINGCNT print "Did not find building types for parcel %d times" % NOBUILDTYPES print "DONE" my.post_run() #remove price_shifter & cost_shifter to avoid them being cached if HOTSHOT: prof.stop() prof.close() stats = hotshot.stats.load('devmdl.prof') stats.strip_dirs() stats.sort_stats('cumulative') stats.print_stats(20)