def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool):
if isinstance(buildings_to_be_demolished, list):
buildings_to_be_demolished = array(buildings_to_be_demolished)
if buildings_to_be_demolished.size <= 0:
return
id_index_in_buildings = building_dataset.get_id_index(buildings_to_be_demolished)
parcels = dataset_pool.get_dataset('parcel')
idx_pcl = parcels.get_id_index(unique(building_dataset['parcel_id'][id_index_in_buildings]))
# remove occupants from buildings to be demolished
JAMM = JoinAttributeModificationModel()
for agent_name in ['household', 'job']:
agents = dataset_pool.get_dataset(agent_name)
JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1)
building_dataset.remove_elements(id_index_in_buildings)
logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size)
# set land_use_type 'vacant' to parcels with demolished buildings
land_types = dataset_pool.get_dataset('land_use_type')
vac_idx = land_types["land_use_name"] == 'vacant'
if vac_idx.sum() > 0:
code = land_types.get_id_attribute()[vac_idx][0]
nvac = (parcels['land_use_type_id'][idx_pcl] == code).sum()
parcels['land_use_type_id'][idx_pcl] = code
logger.log_status("%s parcels set to vacant." % (idx_pcl.size - nvac))
def test_demolition(self):
demolish_buildings = array([10, 3, 180])
household_data = {
'household_id': arange(10)+1,
'building_id': array([10, 3, 6, 150, 10, 10, -1, 5, 3, 3])
# demolished [* * * * * *]
}
person_data = {
'person_id':arange(15)+1,
'household_id': array([1,1,2,3,3,5,4,4,4,6,7,8,9,10,10]),
# in demolished bldgs [* * * * * * * * *]
'job_id': array([5,4,1,2,2,1,3,5,1,5,5,4,3, 3, 1])
# in demolished bldgs [ * * * * * * * * *]
}
job_data = {
'job_id': arange(5)+1,
'building_id': array([180, 145, 10, 180, 179])
}
building_data = {
'building_id': arange(200)+1,
'parcel_id': arange(200)+1
}
parcel_data = {
'parcel_id': arange(200)+1,
'land_use_type_id': array(150*[1]+50*[2]),
}
lut_data = {
'land_use_type_id': array([1,2]),
'land_use_name': array(['non_vacant', 'vacant'])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'households', table_data = household_data)
storage.write_table(table_name = 'buildings', table_data = building_data)
storage.write_table(table_name = 'jobs', table_data = job_data)
storage.write_table(table_name = 'persons', table_data = person_data)
storage.write_table(table_name = 'parcels', table_data = parcel_data)
storage.write_table(table_name = 'land_use_types', table_data = lut_data)
dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim'])
BCM = BuildingConstructionModel()
BCM.demolish_buildings(demolish_buildings, dataset_pool.get_dataset('building'), dataset_pool)
JAMM = JoinAttributeModificationModel()
JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('household'),
attribute_to_be_modified='job_id', value=-1, filter='household.building_id <=0')
JAMM.run(dataset_pool.get_dataset('person'), dataset_pool.get_dataset('job'),
attribute_to_be_modified='job_id', value=-1, filter='job.building_id <=0')
self.assertEqual(all(dataset_pool.get_dataset('household').get_attribute('building_id') ==
array([-1, -1, 6, 150, -1, -1, -1, 5, -1, -1])), True)
self.assertEqual(all(dataset_pool.get_dataset('job').get_attribute('building_id') ==
array([-1, 145, -1, -1, 179])), True)
self.assertEqual(dataset_pool.get_dataset('building').size()==197, True)
self.assertEqual(all(dataset_pool.get_dataset('person').get_attribute('job_id') ==
array([-1,-1,-1,2,2,-1,-1,5,-1,-1,-1,-1,-1,-1,-1])), True)
self.assertEqual(dataset_pool.get_dataset('parcel')['land_use_type_id'][9], 2)
self.assertEqual(dataset_pool.get_dataset('parcel')['land_use_type_id'][2], 2)
def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool):
if isinstance(buildings_to_be_demolished, list):
buildings_to_be_demolished = array(buildings_to_be_demolished)
if buildings_to_be_demolished.size <= 0:
return
id_index_in_buildings = building_dataset.get_id_index(buildings_to_be_demolished)
# remove occupants from buildings to be demolished
JAMM = JoinAttributeModificationModel()
for agent_name in ['household', 'job']:
agents = dataset_pool.get_dataset(agent_name)
JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1)
building_dataset.remove_elements(id_index_in_buildings)
logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size)
def demolish_buildings(self, buildings_to_be_demolished, building_dataset,
dataset_pool):
if isinstance(buildings_to_be_demolished, list):
buildings_to_be_demolished = array(buildings_to_be_demolished)
if buildings_to_be_demolished.size <= 0:
return
id_index_in_buildings = building_dataset.get_id_index(
buildings_to_be_demolished)
# remove occupants from buildings to be demolished
JAMM = JoinAttributeModificationModel()
for agent_name in ['household', 'job']:
agents = dataset_pool.get_dataset(agent_name)
JAMM.run(agents,
building_dataset,
index=id_index_in_buildings,
value=-1)
building_dataset.remove_elements(id_index_in_buildings)
logger.log_status("%s buildings demolished." %
buildings_to_be_demolished.size)
def run(my, cache_dir=None, year=None):
global parcel_set, z, node_set, submarket, esubmarket, isr, parcelfees, costdiscount
'''
if 0:
z = Zoning()
p = Parcels()
cPickle.dump((z,p),open('databaseinfo.jar','w'))
else:
print "Reading db info from jar..."
z,p = cPickle.load(open(os.path.join(os.environ['OPUS_DATA'],'bay_area_parcel/databaseinfo.jar')))
'''
## when developer_model is invoked alone from command line
if cache_dir is not None and year is not None:
#data_path = paths.get_opus_data_path_path()
cache_dir = os.path.join(data_path, 'bay_area_parcel/runs/run_105.2012_05_03_09')
#year = 2011
simulation_state = SimulationState()
simulation_state.set_current_time(year)
SimulationState().set_cache_directory(cache_dir)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(new_instance=True,
package_order=['bayarea', 'urbansim_parcel',
'urbansim', 'opus_core'],
in_storage=attribute_cache
).get_dataset_pool()
dataset_pool = SessionConfiguration().get_dataset_pool()
current_year = SimulationState().get_current_time()
cache_dir = SimulationState().get_cache_directory()
parcel_set = dataset_pool.get_dataset('parcel')
building_set = dataset_pool.get_dataset('building')
household_set = dataset_pool.get_dataset('household')
node_set = dataset_pool.get_dataset('node')
unit_set = dataset_pool.get_dataset('residential_unit')
submarket = dataset_pool.get_dataset('submarket')
esubmarket = dataset_pool.get_dataset('employment_submarket')
#print numpy.array(unit_set['rent'] > 0).size
#for i in range(unit_set.size()):
# print unit_set['unit_price'][i], unit_set['unit_sqft'][i]
#transit_set = dataset_pool.get_dataset('transit_station')
#print dataset_pool.datasets_in_pool()
'''
from bayarea.node import transit_type_DDD_within_DDD_meters
for i in range(7):
print i
v = transit_type_DDD_within_DDD_meters.transit_type_DDD_within_DDD_meters(i,500)
d = v.compute(dataset_pool)
print d.size
found = d[numpy.nonzero(d)]
print found.size
sys.exit()
'''
compute_devmdl_accvars(node_set)
######################
### CAREFUL - THIS IS WHERE SCNERARIO SPECIFIC INFO GOES
######################
current_year = SimulationState().get_current_time()
z = Zoning(my.scenario,current_year)
isr = None
if my.scenario.startswith('Transit'): isr = ISR()
parcelfees = None
if my.scenario.startswith('Preferred'):
parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_preferred'))
#elif my.scenario.startswith('Transit'):
# parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_transit'))
elif my.scenario.startswith('Equity'):
parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_equity'))
elif my.scenario.startswith('Infill'):
parcelfees = ParcelFees(dataset_pool.get_dataset('parcelfees_infill'))
costdiscount = 0.0
if not my.scenario.startswith('No Project') and not my.scenario.startswith('Equity'):
costdiscount = .01
#################################
#################################
from numpy import logical_not
empty_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)==0)*(parcel.node_id>0)*(parcel.shape_area>80)")
res_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)>0)*(parcel.node_id>0)*(parcel.shape_area>80)")
bart_parcels = parcel_set.compute_variables("(parcel.disaggregate(bayarea.node.transit_type_1_within_800_meters))")
caltrain_parcels = parcel_set.compute_variables("(parcel.disaggregate(bayarea.node.transit_type_2_within_800_meters))")
#pda_parcels = parcel_set.compute_variables("(parcel.pda_id > -1)*(numpy.logical_not(parcel.county_id==38))")
pda_parcels = parcel_set.compute_variables("(parcel.pda_id > -1)")
SAMPLE_RATE = 0.01
from opus_core.sampling_toolbox import sample_noreplace
from numpy import concatenate, where
sampled_res_parcels_index = sample_noreplace(where(res_parcels)[0], int(SAMPLE_RATE * parcel_set.size()))
test_parcels = concatenate((where(empty_parcels==1)[0], sampled_res_parcels_index,where(bart_parcels==1)[0],where(caltrain_parcels==1)[0],where(pda_parcels==1)[0]))
test_parcels = sample_noreplace(test_parcels, int(.08 * 154877))
numpy.random.shuffle(test_parcels)
"""
sample = []
for i in range(parcel_set.size()):
if empty_parcels[i] == 1:
sample.append(i+1)
elif res_parcels[i] == 1 and numpy.random.ranf() < SAMPLE_RATE:
sample.append(i+1)
test_parcels = array(sample)
"""
#empty_parcels = parcel_set.compute_variables("(parcel.node_id>0)*(parcel.shape_area>80)")
#test_parcels = numpy.where(empty_parcels==1)[0]
global building_sqft, building_price
building_sqft = parcel_set.compute_variables('parcel.aggregate(building.building_sqft)')
building_price_owner_residential=parcel_set.compute_variables('building_price_owner_res=parcel.aggregate((residential_unit.sale_price)*(residential_unit.sale_price>0),intermediates=[building])')
building_price_rental_residential=parcel_set.compute_variables('building_price_rental_res=parcel.aggregate((residential_unit.rent*12*17.9)*(residential_unit.rent>0),intermediates=[building])')
building_price_nonresidential = parcel_set.compute_variables('building_price_nonres = parcel.aggregate((building.non_residential_rent*7*building.non_residential_sqft))')
sum_building_p = parcel_set.compute_variables('sum_building_price = parcel.building_price_owner_res + parcel.building_price_rental_res + building_price_nonres')
##sum_building_price = building_price_owner_residential + building_price_rental_residential + building_price_nonresidential
vacant_parcel = parcel_set.compute_variables('parcel.sum_building_price == 0')
price_per_sqft_land = (parcel_set.compute_variables('parcel.disaggregate(safe_array_divide(zone.aggregate(parcel.sum_building_price),zone.aggregate(building.building_sqft)))'))/4
parcel_land_area = parcel_set.compute_variables('parcel.shape_area')
vacant_land_price = vacant_parcel*price_per_sqft_land*parcel_land_area
building_price = sum_building_p + vacant_land_price
##sum_building_price = building_price_owner_residential + building_price_rental_residential + building_price_nonresidential
#land_price = (sum_building_p==0) * (parcel_set.compute_variables('parcel.disaggregate(safe_array_divide(zone.aggregate(parcel.sum_building_price),zone.aggregate(building.building_sqft)))'))* parcel_set.compute_variables('parcel.land_area')
#info used to match from proposal_component to submarket
parcel_set.compute_variables(["bayarea.parcel.within_half_mile_transit",
"bayarea.parcel.schooldistrict",
"bayarea.parcel.jurisdiction_id",
])
#test_parcels = array([i+1 for i in range(parcel_set.size())])
#test_parcels = test_parcels[:10000]
#test_parcels = test_parcels[:150]
#test_parcels = numpy.where(parcel_set['parcel_id'] == 1608920)[0]
#print test_parcels
logger.log_status("%s parcels to test" % (test_parcels.size))
print "Num of parcels:", test_parcels.size
import time
HOTSHOT = 0
if MP:
from multiprocessing import Pool, Queue
pool = Pool(processes=4)
import hotshot, hotshot.stats#, test.pystone
if HOTSHOT:
prof = hotshot.Profile('devmdl.prof')
prof.start()
outf = open(os.path.join(cache_dir,'buildings-%d.csv' % current_year),'w')
outf.write('pid,county,dev_btype,stories,sqft,res_sqft,nonres_sqft,tenure,year_built,res_units,npv,actualfee,btype\n')
debugf = open(os.path.join(cache_dir,'proforma-debug-%d.csv' % current_year),'w')
bformdbg = 'county_id,far,height,max_dua,bform.sf_builtarea(),bform.sfunitsizes,bform.mf_builtarea(),bform.mfunitsizes,bform.num_units,bform.nonres_sqft,bform.buildable_area'
otherdbg = 'isr,parcelfees,existing_sqft,existing_price,lotsize,unitsize,unitsize2,bform.sales_absorption,bform.rent_absorption,bform.leases_absorption,bform.sales_vacancy_rates,bform.vacancy_rates'
debugf.write('pid,btype,npv,actualfee,pricesf,pricemf,rentsf,rentmf,rentof,rentret,rentind,%s,%s\n' % (bformdbg,otherdbg))
t1 = time.time()
aggd = {}
def chunks(l, n):
for i in xrange(0, len(l), n):
yield l[i:i+n]
for test_chunk in chunks(test_parcels,1000):
print "Executing CHUNK"
sales_absorption = submarket.compute_variables('bayarea.submarket.sales_absorption')
rent_absorption = submarket.compute_variables('bayarea.submarket.rent_absorption')
vacancy_rates = submarket.compute_variables('bayarea.submarket.vacancy_rates')
leases_absorption = esubmarket.compute_variables('bayarea.employment_submarket.leases_absorption')
nr_vacancy_rates = esubmarket.compute_variables('bayarea.employment_submarket.vacancy_rates')
if HOTSHOT:
results = []
for p in test_chunk:
r = process_parcel(p)
if r <> None and r <> -1: results.append(list(r))
else:
if MP:
results = pool.map(process_parcel,test_chunk)
else:
results = [process_parcel(p) for p in test_chunk]
results_bldg = [list(x[0]) for x in results if x <> None and x[0] <> -1]
#each row of units represents number of units of [1, 2, 3, 4] bedrooms
units = array([x[1][0] for x in results if x <> None and x[0] <> -1])
sqft_per_unit = array([x[1][1] for x in results if x <> None and x[0] <> -1])
for x in results:
if x <> None:
debugf.write(x[2])
results = results_bldg
for result in results:
#print result
out_btype = devmdltypes[int(result[2])-1]
outf.write(string.join([str(x) for x in result]+[str(out_btype)],sep=',')+'\n')
##TODO: id of buildings to be demolished
buildings_to_demolish = []
idx_buildings_to_demolish = building_set.get_id_index(buildings_to_demolish)
JAMM = JoinAttributeModificationModel()
JAMM.run(household_set, building_set, index=idx_buildings_to_demolish, value=-1)
building_set.remove_elements(idx_buildings_to_demolish)
column_names = ["parcel_id","county","building_type_id","stories",
"building_sqft","residential_sqft","non_residential_sqft",
"tenure","year_built","residential_units"]
buildings_data = copy.deepcopy(results)
for i in range(len(buildings_data)):
buildings_data[i][2] = devmdltypes[int(buildings_data[i][2])-1]
buildings_data = array(buildings_data)
new_buildings = {}
available_bldg_id = building_set['building_id'].max() + 1
new_bldg_ids = arange(available_bldg_id, available_bldg_id+buildings_data.shape[0],
dtype=building_set['building_id'].dtype)
if buildings_data.size > 0:
for icol, col_name in enumerate(column_names):
if col_name in building_set.get_known_attribute_names():
ddtype = building_set[col_name].dtype
new_buildings[col_name] = (buildings_data[:, icol]).astype(ddtype)
else:
#if the col_name is not in dataset, it will be discarded anyway
pass
new_buildings['building_id'] = new_bldg_ids
# recode tenure: 1 - rent, 2 - own from 0 - own, 1 - rent
new_buildings['tenure'][new_buildings['tenure']==0] = 2
## pid is the index to parcel_set; convert them to actual parcel_id
#new_buildings['parcel_id'] = parcel_set['parcel_id'][new_buildings['parcel_id']]
building_set.add_elements(new_buildings, require_all_attributes=False,
change_ids_if_not_unique=True)
building_set.flush_dataset()
assert new_bldg_ids.size == units.shape[0] == sqft_per_unit.shape[0]
units_bldg_ids = repeat(new_bldg_ids, 4)
bedrooms = array([1, 2, 3, 4] * units.size)
units = round(units.ravel())
sqft_per_unit = sqft_per_unit.ravel()
new_units = {'building_id': array([], dtype='i4'),
'bedrooms': array([], dtype='i4'),
'sqft_per_unit': array([], dtype='i4')
}
for i_unit, unit in enumerate(units):
if unit <= 0:
continue
new_units['building_id'] = concatenate((new_units['building_id'],
repeat(units_bldg_ids[i_unit], unit))
)
new_units['bedrooms'] = concatenate((new_units['bedrooms'],
repeat(bedrooms[i_unit], unit))
)
new_units['sqft_per_unit'] = concatenate((new_units['sqft_per_unit'],
repeat(sqft_per_unit[i_unit], unit))
)
##force dtype conversion to the same dtype as unit_set
for col_name in ['building_id', 'bedrooms', 'sqft_per_unit']:
if col_name in unit_set.get_known_attribute_names():
new_units[col_name] = new_units[col_name].astype(unit_set[col_name].dtype)
unit_set.add_elements(new_units, require_all_attributes=False,
change_ids_if_not_unique=True)
unit_set.flush_dataset()
for result in results:
units = result[-1]
nonres_sqft = 1 #result[6]/1000.0
county = result[1]
btype = result[2]
key = (county,btype)
aggd.setdefault(key,0)
if btype < 7: aggd[key] += units
else: aggd[key] += nonres_sqft
aggd.setdefault(county,0)
aggd[county] += units
aggf = open('county_aggregations-%d.csv' % current_year,'w')
county_names = {49:'son',41:'smt',1:'ala',43:'scl',28:'nap',38:'sfr',7:'cnc',48:'sol',21:'mar',0:'n/a'}
btype_names = {1:'SF',2:'SFBUILD',3:'MF',4:'MXMF',5:'CONDO',6:'MXC',7:'OF',8:'MXO',9:'CHOOD',10:'CAUTO',11:'CBOX',12:'MANU',13:'WHE'}
aggf.write('county,total,'+string.join(btype_names.values(),sep=',')+'\n')
for county in [38,41,43,1,7,48,28,49,21]:
aggf.write(county_names[county]+','+str(aggd.get(county,0)))
for btype in btype_names.keys():
key = (county,btype)
val = aggd.get(key,0)
aggf.write(','+str(val))
aggf.write('\n')
t2 = time.time()
print "Finished in %f seconds" % (t2-t1)
print "Ran optimization %d times" % devmdl_optimize.OBJCNT
global NOZONINGCNT, NOBUILDTYPES
print "Did not find zoning for parcel %d times" % NOZONINGCNT
print "Did not find building types for parcel %d times" % NOBUILDTYPES
print "DONE"
my.post_run() #remove price_shifter & cost_shifter to avoid them being cached
if HOTSHOT:
prof.stop()
prof.close()
stats = hotshot.stats.load('devmdl.prof')
stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_stats(20)
def test_demolition(self):
demolish_buildings = array([10, 3, 180])
household_data = {
'household_id': arange(10) + 1,
'building_id': array([10, 3, 6, 150, 10, 10, -1, 5, 3, 3])
# demolished [* * * * * *]
}
person_data = {
'person_id': arange(15) + 1,
'household_id':
array([1, 1, 2, 3, 3, 5, 4, 4, 4, 6, 7, 8, 9, 10, 10]),
# in demolished bldgs [* * * * * * * * *]
'job_id': array([5, 4, 1, 2, 2, 1, 3, 5, 1, 5, 5, 4, 3, 3, 1])
# in demolished bldgs [ * * * * * * * * *]
}
job_data = {
'job_id': arange(5) + 1,
'building_id': array([180, 145, 10, 180, 179])
}
building_data = {
'building_id': arange(200) + 1,
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households', table_data=household_data)
storage.write_table(table_name='buildings', table_data=building_data)
storage.write_table(table_name='jobs', table_data=job_data)
storage.write_table(table_name='persons', table_data=person_data)
dataset_pool = DatasetPool(
storage=storage, package_order=['urbansim_parcel', 'urbansim'])
BCM = BuildingConstructionModel()
BCM.demolish_buildings(demolish_buildings,
dataset_pool.get_dataset('building'),
dataset_pool)
JAMM = JoinAttributeModificationModel()
JAMM.run(dataset_pool.get_dataset('person'),
dataset_pool.get_dataset('household'),
attribute_to_be_modified='job_id',
value=-1,
filter='household.building_id <=0')
JAMM.run(dataset_pool.get_dataset('person'),
dataset_pool.get_dataset('job'),
attribute_to_be_modified='job_id',
value=-1,
filter='job.building_id <=0')
self.assertEqual(
all(
dataset_pool.get_dataset('household').get_attribute(
'building_id') == array(
[-1, -1, 6, 150, -1, -1, -1, 5, -1, -1])), True)
self.assertEqual(
all(
dataset_pool.get_dataset('job').get_attribute('building_id') ==
array([-1, 145, -1, -1, 179])), True)
self.assertEqual(
dataset_pool.get_dataset('building').size() == 197, True)
self.assertEqual(
all(
dataset_pool.get_dataset('person').get_attribute('job_id') ==
array([
-1, -1, -1, 2, 2, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1
])), True)
def run(my, cache_dir=None, year=None):
global parcel_set, z, node_set, zone_set, submarket, esubmarket, isr, parcelfees, costdiscount, building_sqft, building_price
dataset_pool = SessionConfiguration().get_dataset_pool()
current_year = SimulationState().get_current_time()
cache_dir = SimulationState().get_cache_directory()
parcel_set = dataset_pool.get_dataset('parcel')
building_set = dataset_pool.get_dataset('building')
household_set = dataset_pool.get_dataset('household')
unit_set = dataset_pool.get_dataset('residential_unit')
submarket = dataset_pool.get_dataset('submarket')
esubmarket = dataset_pool.get_dataset('employment_submarket')
capped_rents = building_set.compute_variables('_adj_rent = building.non_residential_rent*(building.non_residential_rent<120) + 120*(building.non_residential_rent>=120)')
building_set.modify_attribute('non_residential_rent', capped_rents)
if NODES:
node_set = dataset_pool.get_dataset('node')
compute_devmdl_accvars_nodal(node_set)
else:
zone_set = dataset_pool.get_dataset('zone')
compute_devmdl_accvars_zonal(zone_set)
z = Zoning(my.scenario,current_year)
costdiscount = 0.0
SAMPLE_RATE = 0.05
isr = None
parcelfees = None
empty_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)==0)*(parcel.parcel_sqft>800)")
res_parcels = parcel_set.compute_variables("(parcel.number_of_agents(building)>0)*(parcel.parcel_sqft>800)")
sampled_res_parcels_index = sample_noreplace(where(res_parcels)[0], int(SAMPLE_RATE * parcel_set.size()))
test_parcels1 = concatenate((where(empty_parcels==1)[0], sampled_res_parcels_index))
test_parcels = sample_noreplace(test_parcels1, int(.05 * test_parcels1.size))
numpy.random.shuffle(test_parcels)
building_sqft = parcel_set.compute_variables('parcel.aggregate(building.non_residential_sqft + building.residential_units*building.sqft_per_unit)')
building_price_owner_residential=parcel_set.compute_variables('building_price_owner_res=parcel.aggregate((residential_unit.sale_price)*(residential_unit.sale_price>0),intermediates=[building])')
building_price_rental_residential=parcel_set.compute_variables('building_price_rental_res=parcel.aggregate((residential_unit.rent*12*17.9)*(residential_unit.rent>0),intermediates=[building])') ##change 17.9
building_price_nonresidential = parcel_set.compute_variables('building_price_nonres = parcel.aggregate((building.non_residential_rent*7*building.non_residential_sqft))') ##change 7
sum_building_p = parcel_set.compute_variables('sum_building_price = parcel.building_price_owner_res + parcel.building_price_rental_res + building_price_nonres')
vacant_parcel = parcel_set.compute_variables('parcel.sum_building_price == 0')
price_per_sqft_land = parcel_set.compute_variables('safe_array_divide(parcel.land_value,parcel.parcel_sqft)')
parcel_land_area = parcel_set.compute_variables('parcel.parcel_sqft')
vacant_land_price = vacant_parcel*price_per_sqft_land*parcel_land_area
building_price = sum_building_p + vacant_land_price #rename property price
#info used to match from proposal_component to submarket
parcel_set.compute_variables(["drcog.parcel.within_half_mile_transit",
"drcog.parcel.zone",
])
logger.log_status("%s parcels to test" % (test_parcels.size))
if MP:
from multiprocessing import Pool, Queue
pool = Pool(processes=4)
if HOTSHOT:
import hotshot, hotshot.stats
prof = hotshot.Profile('devmdl.prof')
prof.start()
outf = open(os.path.join(cache_dir,'buildings-%d.csv' % current_year),'w')
outf.write('pid,county,dev_btype,stories,sqft,res_sqft,nonres_sqft,tenure,year_built,res_units,npv,actualfee,btype\n')
debugf = open(os.path.join(cache_dir,'proforma-debug-%d.csv' % current_year),'w')
bformdbg = 'county_id,far,height,max_dua,bform.sf_builtarea(),bform.sfunitsizes,bform.mf_builtarea(),bform.mfunitsizes,bform.num_units,bform.nonres_sqft,bform.buildable_area'
otherdbg = 'isr,parcelfees,existing_sqft,existing_price,lotsize,unitsize,unitsize2,bform.sales_absorption,bform.rent_absorption,bform.leases_absorption,bform.sales_vacancy_rates,bform.vacancy_rates'
debugf.write('pid,btype,npv,actualfee,pricesf,pricemf,rentsf,rentmf,rentof,rentret,rentind,%s,%s\n' % (bformdbg,otherdbg))
t1 = time.time()
aggd = {}
def chunks(l, n):
for i in xrange(0, len(l), n):
yield l[i:i+n]
for test_chunk in chunks(test_parcels,1000):
print "Executing CHUNK"
sales_absorption = submarket.compute_variables('drcog.submarket.sales_absorption') ######################!!
rent_absorption = submarket.compute_variables('drcog.submarket.rent_absorption')
vacancy_rates = submarket.compute_variables('drcog.submarket.vacancy_rates')
leases_absorption = esubmarket.compute_variables('drcog.employment_submarket.leases_absorption')
nr_vacancy_rates = esubmarket.compute_variables('drcog.employment_submarket.vacancy_rates')
if HOTSHOT:
results = []
for p in test_chunk:
r = process_parcel(p)
if r <> None and r <> -1: results.append(list(r))
else:
if MP:
results = pool.map(process_parcel,test_chunk)
else:
results = [process_parcel(p) for p in test_chunk]
results_bldg = [list(x[0]) for x in results if x <> None and x[0] <> -1]
#each row of units represents number of units of [1, 2, 3, 4] bedrooms
units = array([x[1][0] for x in results if x <> None and x[0] <> -1])
sqft_per_unit = array([x[1][1] for x in results if x <> None and x[0] <> -1])
for x in results:
if x <> None:
debugf.write(x[2])
results = results_bldg
for result in results:
#print result
out_btype = int(result[2])
outf.write(string.join([str(x) for x in result]+[str(out_btype)],sep=',')+'\n')
##TODO: id of buildings to be demolished
buildings_to_demolish = []
idx_buildings_to_demolish = building_set.get_id_index(buildings_to_demolish)
JAMM = JoinAttributeModificationModel()
JAMM.run(household_set, building_set, index=idx_buildings_to_demolish, value=-1)
building_set.remove_elements(idx_buildings_to_demolish)
column_names = ["parcel_id","county","building_type_id","stories",
"building_sqft","residential_sqft","non_residential_sqft",
"tenure","year_built","residential_units"]
buildings_data = copy.deepcopy(results)
#for i in range(len(buildings_data)):
#buildings_data[i][2] = devmdltypes[int(buildings_data[i][2])-1]
buildings_data = array(buildings_data)
new_buildings = {}
available_bldg_id = building_set['building_id'].max() + 1
new_bldg_ids = arange(available_bldg_id, available_bldg_id+buildings_data.shape[0],
dtype=building_set['building_id'].dtype)
if buildings_data.size > 0:
for icol, col_name in enumerate(column_names):
if col_name in building_set.get_known_attribute_names():
ddtype = building_set[col_name].dtype
new_buildings[col_name] = (buildings_data[:, icol]).astype(ddtype)
else:
#if the col_name is not in dataset, it will be discarded anyway
pass
new_buildings['building_id'] = new_bldg_ids
# recode tenure: 1 - rent, 2 - own from 0 - own, 1 - rent
new_buildings['tenure'][new_buildings['tenure']==0] = 1
## pid is the index to parcel_set; convert them to actual parcel_id
#new_buildings['parcel_id'] = parcel_set['parcel_id'][new_buildings['parcel_id']]
building_set.add_elements(new_buildings, require_all_attributes=False,
change_ids_if_not_unique=True)
building_set.flush_dataset()
assert new_bldg_ids.size == units.shape[0] == sqft_per_unit.shape[0]
units_bldg_ids = repeat(new_bldg_ids, 4)
bedrooms = array([1, 2, 3, 4] * units.size)
units = round(units.ravel())
sqft_per_unit = sqft_per_unit.ravel()
new_units = {'building_id': array([], dtype='i4'),
'bedrooms': array([], dtype='i4'),
'unit_sqft': array([], dtype='i4')
}
for i_unit, unit in enumerate(units):
if unit <= 0:
continue
new_units['building_id'] = concatenate((new_units['building_id'],
repeat(units_bldg_ids[i_unit], unit))
)
new_units['bedrooms'] = concatenate((new_units['bedrooms'],
repeat(bedrooms[i_unit], unit))
)
new_units['unit_sqft'] = concatenate((new_units['unit_sqft'],
repeat(sqft_per_unit[i_unit], unit))
)
##force dtype conversion to the same dtype as unit_set
for col_name in ['building_id', 'bedrooms', 'unit_sqft']:
if col_name in unit_set.get_known_attribute_names():
new_units[col_name] = new_units[col_name].astype(unit_set[col_name].dtype)
unit_set.add_elements(new_units, require_all_attributes=False,
change_ids_if_not_unique=True)
unit_set.flush_dataset()
for result in results:
units = result[-1]
nonres_sqft = 1 #result[6]/1000.0
county = result[1]
btype = result[2]
key = (county,btype)
aggd.setdefault(key,0)
if btype < 7: aggd[key] += units
else: aggd[key] += nonres_sqft
aggd.setdefault(county,0)
aggd[county] += units
######################These hard-coded counties are problematic!!!
# aggf = open('county_aggregations-%d.csv' % current_year,'w')
# county_names = {49:'son',41:'smt',1:'ala',43:'scl',28:'nap',38:'sfr',7:'cnc',48:'sol',21:'mar',0:'n/a'}
# btype_names = {1:'SF',2:'SFBUILD',3:'MF',4:'MXMF',5:'CONDO',6:'MXC',7:'OF',8:'MXO',9:'CHOOD',10:'CAUTO',11:'CBOX',12:'MANU',13:'WHE'}
# aggf.write('county,total,'+string.join(btype_names.values(),sep=',')+'\n')
# for county in [38,41,43,1,7,48,28,49,21]:
# aggf.write(county_names[county]+','+str(aggd.get(county,0)))
# for btype in btype_names.keys():
# key = (county,btype)
# val = aggd.get(key,0)
# aggf.write(','+str(val))
# aggf.write('\n')
t2 = time.time()
print "Finished in %f seconds" % (t2-t1)
print "Ran optimization %d times" % devmdl_optimize.OBJCNT
global NOZONINGCNT, NOBUILDTYPES
print "Did not find zoning for parcel %d times" % NOZONINGCNT
print "Did not find building types for parcel %d times" % NOBUILDTYPES
print "DONE"
my.post_run() #remove price_shifter & cost_shifter to avoid them being cached
if HOTSHOT:
prof.stop()
prof.close()
stats = hotshot.stats.load('devmdl.prof')
stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_stats(20)
