def initialize_new_units(buildings, residential_units):
"""
This data maintenance step initializes units for buildings that have been
newly created, conforming to the data requirements of the
'residential_units' table.
Data expectations
-----------------
- 'buildings' table has the following columns:
- index that serves as its identifier
- 'residential_units' (int, count of units in building)
- 'residential_units' table has the following columns:
- index named 'unit_id' that serves as its identifier
- 'building_id' corresponding to the index of the 'buildings' table
Results
-------
- extends the 'residential_units' table, following the same schema as the
'initialize_residential_units' model step
"""
# Verify initial data characteristics
'''
ot.assert_orca_spec(OrcaSpec(
'',
TableSpec(
'buildings',
ColumnSpec('building_id', primary_key=True),
ColumnSpec('residential_units', min=0)),
TableSpec(
'residential_units',
ColumnSpec('unit_id', primary_key=True),
ColumnSpec('building_id', foreign_key='buildings.building_id'))))
'''
old_units = residential_units.to_frame(residential_units.local_columns)
bldgs = buildings.to_frame(['residential_units', 'deed_restricted_units', 'building_id'])
# Filter for residential buildings not currently represented in
# the units table
new_bldgs = bldgs[~bldgs.building_id.isin(old_units.building_id)]
new_bldgs = new_bldgs[new_bldgs.residential_units > 0]
if len(new_bldgs) == 0:
return
# Create new units, merge them, and update the table
new_units = _create_empty_units(new_bldgs)
all_units = dev.merge(old_units, new_units)
all_units.index.name = 'unit_id'
print "Creating %d residential units for %d new buildings" % \
(len(new_units), len(new_bldgs))
orca.add_table('residential_units', all_units)
# Verify final data characteristics
'''
def initialize_new_units(buildings, residential_units):
"""
This data maintenance step initializes units for buildings that have been
newly created, conforming to the data requirements of the
'residential_units' table.
Data expectations
-----------------
- 'buildings' table has the following columns:
- index that serves as its identifier
- 'residential_units' (int, count of units in building)
- 'residential_units' table has the following columns:
- index named 'unit_id' that serves as its identifier
- 'building_id' corresponding to the index of the 'buildings' table
Results
-------
- extends the 'residential_units' table, following the same schema as the
'initialize_residential_units' model step
"""
# Verify initial data characteristics
'''
ot.assert_orca_spec(OrcaSpec(
'',
TableSpec(
'buildings',
ColumnSpec('building_id', primary_key=True),
ColumnSpec('residential_units', min=0)),
TableSpec(
'residential_units',
ColumnSpec('unit_id', primary_key=True),
ColumnSpec('building_id', foreign_key='buildings.building_id'))))
'''
old_units = residential_units.to_frame(residential_units.local_columns)
bldgs = buildings.to_frame(['residential_units', 'deed_restricted_units'])
# Filter for residential buildings not currently represented in
# the units table
new_bldgs = bldgs[~bldgs.index.isin(old_units.building_id)]
new_bldgs = new_bldgs[new_bldgs.residential_units > 0]
# Create new units, merge them, and update the table
new_units = _create_empty_units(new_bldgs)
all_units = dev.merge(old_units, new_units)
all_units.index.name = 'unit_id'
print "Creating %d residential units for %d new buildings" % \
(len(new_units), len(new_bldgs))
orca.add_table('residential_units', all_units)
# Verify final data characteristics
'''
def add_buildings(buildings, new_buildings, remove_developed_buildings=True):
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
unplace_agents = ["households", "jobs"]
old_buildings = \
_remove_developed_buildings(old_buildings, new_buildings,
unplace_agents)
all_buildings = dev.merge(old_buildings, new_buildings)
orca.add_table("buildings", all_buildings)
def add_buildings(buildings, new_buildings,
remove_developed_buildings=True):
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
unplace_agents = ["households", "jobs"]
old_buildings = \
_remove_developed_buildings(old_buildings, new_buildings,
unplace_agents)
all_buildings = dev.merge(old_buildings, new_buildings)
orca.add_table("buildings", all_buildings)
def scheduled_development_events(buildings, scheduled_development_events):
year = get_year()
sched_dev = scheduled_development_events.to_frame()
sched_dev = sched_dev[sched_dev.year_built == year]
sched_dev[
'residential_sqft'] = sched_dev.sqft_per_unit * sched_dev.residential_units
sched_dev['job_spaces'] = sched_dev.non_residential_sqft / 400
if len(sched_dev) > 0:
max_bid = buildings.index.values.max()
idx = np.arange(max_bid + 1, max_bid + len(sched_dev) + 1)
sched_dev['building_id'] = idx
sched_dev = sched_dev.set_index('building_id')
from urbansim.developer.developer import Developer
merge = Developer(pd.DataFrame({})).merge
b = buildings.to_frame(buildings.local_columns)
all_buildings = merge(b, sched_dev[b.columns])
orca.add_table("buildings", all_buildings)
def developer_reprocess(buildings, year, years_per_iter, jobs,
parcels, summary, parcel_is_allowed_func):
# this takes new units that come out of the developer, both subsidized
# and non-subsidized and reprocesses them as required - please read
# comments to see what this means in detail
# 20% of base year buildings which are "residential" have job spaces - I
# mean, there is a ratio of job spaces to res units in residential
# buildings of 1 to 5 - this ratio should be kept for future year
# buildings
s = buildings.general_type == "Residential"
res_units = buildings.residential_units[s].sum()
job_spaces = buildings.job_spaces[s].sum()
to_add = res_units * .05 - job_spaces
if to_add > 0:
print "Adding %d job_spaces" % to_add
res_units = buildings.residential_units[s]
# bias selection of places to put job spaces based on res units
print res_units.describe()
print res_units[res_units < 0]
add_indexes = np.random.choice(res_units.index.values, size=to_add,
replace=True,
p=(res_units/res_units.sum()))
# collect same indexes
add_indexes = pd.Series(add_indexes).value_counts()
# this is sqft per job for residential bldgs
add_sizes = add_indexes * 400
print "Job spaces in res before adjustment: ", \
buildings.job_spaces[s].sum()
buildings.local.loc[add_sizes.index,
"non_residential_sqft"] += add_sizes.values
print "Job spaces in res after adjustment: ",\
buildings.job_spaces[s].sum()
# the second step here is to add retail to buildings that are greater than
# X stories tall - presumably this is a ground floor retail policy
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = old_buildings.query(
'%d == year_built and stories >= 4' % year)
print "Attempting to add ground floor retail to %d devs" % \
len(new_buildings)
retail = parcel_is_allowed_func("retail")
new_buildings = new_buildings[retail.loc[new_buildings.parcel_id].values]
print "Disallowing dev on these parcels:"
print " %d devs left after retail disallowed" % len(new_buildings)
# this is the key point - make these new buildings' nonres sqft equal
# to one story of the new buildings
new_buildings.non_residential_sqft = new_buildings.building_sqft / \
new_buildings.stories * .8
new_buildings["residential_units"] = 0
new_buildings["residential_sqft"] = 0
new_buildings["building_sqft"] = new_buildings.non_residential_sqft
new_buildings["stories"] = 1
new_buildings["building_type"] = "RB"
# this is a fairly arbitrary rule, but we're only adding ground floor
# retail in areas that are underserved right now - this is defined as
# the location where the retail ratio (ratio of income to retail sqft)
# is greater than the median
ratio = parcels.retail_ratio.loc[new_buildings.parcel_id]
new_buildings = new_buildings[ratio.values > ratio.median()]
print "Adding %d sqft of ground floor retail in %d locations" % \
(new_buildings.non_residential_sqft.sum(), len(new_buildings))
all_buildings = dev.merge(old_buildings, new_buildings)
orca.add_table("buildings", all_buildings)
new_buildings["form"] = "retail"
# this is sqft per job for retail use - this is all rather
# ad-hoc so I'm hard-coding
new_buildings["job_spaces"] = \
(new_buildings.non_residential_sqft / 445.0).astype('int')
new_buildings["net_units"] = new_buildings.job_spaces
summary.add_parcel_output(new_buildings)
# got to get the frame again because we just added rows
buildings = orca.get_table('buildings')
buildings_df = buildings.to_frame(
['year_built', 'building_sqft', 'general_type'])
sqft_by_gtype = buildings_df.query('year_built >= %d' % year).\
groupby('general_type').building_sqft.sum()
print "New square feet by general type in millions:\n",\
sqft_by_gtype / 1000000.0
def office_developer(feasibility, jobs, buildings, parcels, year,
settings, summary, form_to_btype_func, scenario,
add_extra_columns_func, parcels_geography,
limits_settings):
dev_settings = settings['non_residential_developer']
# I'm going to try a new way of computing this because the math the other
# way is simply too hard. Basically we used to try and apportion sectors
# into the demand for office, retail, and industrial, but there's just so
# much dirtyness to the data, for instance 15% of jobs are in residential
# buildings, and 15% in other buildings, it's just hard to know how much
# to build, we I think the right thing to do is to compute the number of
# job spaces that are required overall, and then to apportion that new dev
# into the three non-res types with a single set of coefficients
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
print "Total units to build = %d" % all_units
if all_units <= 0:
return
for typ in ["Office"]:
print "\nRunning for type: ", typ
num_units = all_units * float(dev_settings['type_splits'][typ])
targets = []
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or
# job_spaces
if year > 2015 and typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_job_spaces[
(juris_name == juris) & (parcels.newest_building > 2015)]\
.sum()
target = (year - 2015 + 1) * limit - current_total
if target <= 0:
print "Already met target for juris = %s" % juris
print " target = %d, current_total = %d" %\
(target, current_total)
continue
targets.append((juris_name == juris, target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index, num_units, "none"))
for parcel_mask, target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
print "Parcels in play:\n", pd.Series(parcel_mask).value_counts()
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
typ.lower(),
jobs,
buildings,
"job_spaces",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_job_spaces[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
residential=False,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**dev_settings['kwargs'])
if new_buildings is not None:
new_buildings["subsidized"] = False
summary.add_parcel_output(new_buildings)
def retail_developer(jobs, buildings, parcels, nodes, feasibility,
settings, summary, add_extra_columns_func, net):
dev_settings = settings['non_residential_developer']
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
target = all_units * float(dev_settings['type_splits']["Retail"])
# target here is in sqft
target *= settings["building_sqft_per_job"]["HS"]
feasibility = feasibility.to_frame().loc[:, "retail"]
feasibility = feasibility.dropna(subset=["max_profit"])
feasibility["non_residential_sqft"] = \
feasibility.non_residential_sqft.astype("int")
feasibility["retail_ratio"] = parcels.retail_ratio
feasibility = feasibility.reset_index()
# create features
f1 = feasibility.retail_ratio / feasibility.retail_ratio.max()
f2 = feasibility.max_profit / feasibility.max_profit.max()
# combine features in probability function - it's like combining expense
# of building the building with the market in the neighborhood
p = f1 * 1.5 + f2
p = p.clip(lower=1.0/len(p)/10)
print "Attempting to build {:,} retail sqft".format(target)
# order by weighted random sample
feasibility = feasibility.sample(frac=1.0, weights=p)
bldgs = buildings.to_frame(buildings.local_columns + ["general_type"])
devs = []
for dev_id, d in feasibility.iterrows():
if target <= 0:
break
# any special logic to filter these devs?
# remove new dev sqft from target
target -= d.non_residential_sqft
# add redeveloped sqft to target
filt = "general_type == 'Retail' and parcel_id == %d" % \
d["parcel_id"]
target += bldgs.query(filt).non_residential_sqft.sum()
devs.append(d)
if len(devs) == 0:
return
# record keeping - add extra columns to match building dataframe
# add the buidings and demolish old buildings, and add to debug output
devs = pd.DataFrame(devs, columns=feasibility.columns)
print "Building {:,} retail sqft in {:,} projects".format(
devs.non_residential_sqft.sum(), len(devs))
if target > 0:
print " WARNING: retail target not met"
devs["form"] = "retail"
devs = add_extra_columns_func(devs)
add_buildings(buildings, devs)
summary.add_parcel_output(devs)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year):
kwargs = settings['residential_developer']
target_vacancy = pd.read_csv("data/regional_controls.csv",
index_col="year").loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
overshoot = min(overshoot,
buildings.local.loc[index,
"residential_units"])
buildings.local.loc[index, "residential_units"] -= overshoot
summary.add_parcel_output(new_buildings)
def developer_reprocess(buildings, year, years_per_iter, jobs,
parcels, summary, parcel_is_allowed_func):
# this takes new units that come out of the developer, both subsidized
# and non-subsidized and reprocesses them as required - please read
# comments to see what this means in detail
# 20% of base year buildings which are "residential" have job spaces - I
# mean, there is a ratio of job spaces to res units in residential
# buildings of 1 to 5 - this ratio should be kept for future year
# buildings
s = buildings.general_type == "Residential"
res_units = buildings.residential_units[s].sum()
job_spaces = buildings.job_spaces[s].sum()
to_add = res_units * .05 - job_spaces
if to_add > 0:
print "Adding %d job_spaces" % to_add
res_units = buildings.residential_units[s]
# bias selection of places to put job spaces based on res units
print res_units.describe()
print res_units[res_units < 0]
add_indexes = np.random.choice(res_units.index.values, size=to_add,
replace=True,
p=(res_units / res_units.sum()))
# collect same indexes
add_indexes = pd.Series(add_indexes).value_counts()
# this is sqft per job for residential bldgs
add_sizes = add_indexes * 400
print "Job spaces in res before adjustment: ", \
buildings.job_spaces[s].sum()
buildings.local.loc[add_sizes.index,
"non_residential_sqft"] += add_sizes.values
print "Job spaces in res after adjustment: ",\
buildings.job_spaces[s].sum()
# the second step here is to add retail to buildings that are greater than
# X stories tall - presumably this is a ground floor retail policy
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = old_buildings.query(
'%d == year_built and stories >= 4' % year)
print "Attempting to add ground floor retail to %d devs" % \
len(new_buildings)
retail = parcel_is_allowed_func("retail")
new_buildings = new_buildings[retail.loc[new_buildings.parcel_id].values]
print "Disallowing dev on these parcels:"
print " %d devs left after retail disallowed" % len(new_buildings)
# this is the key point - make these new buildings' nonres sqft equal
# to one story of the new buildings
new_buildings.non_residential_sqft = new_buildings.building_sqft / \
new_buildings.stories * .8
new_buildings["residential_units"] = 0
new_buildings["residential_sqft"] = 0
new_buildings["deed_restricted_units"] = 0
new_buildings["building_sqft"] = new_buildings.non_residential_sqft
new_buildings["stories"] = 1
new_buildings["building_type"] = "RB"
# this is a fairly arbitrary rule, but we're only adding ground floor
# retail in areas that are underserved right now - this is defined as
# the location where the retail ratio (ratio of income to retail sqft)
# is greater than the median
ratio = parcels.retail_ratio.loc[new_buildings.parcel_id]
new_buildings = new_buildings[ratio.values > ratio.median()]
print "Adding %d sqft of ground floor retail in %d locations" % \
(new_buildings.non_residential_sqft.sum(), len(new_buildings))
all_buildings = dev.merge(old_buildings, new_buildings)
orca.add_table("buildings", all_buildings)
new_buildings["form"] = "retail"
# this is sqft per job for retail use - this is all rather
# ad-hoc so I'm hard-coding
new_buildings["job_spaces"] = \
(new_buildings.non_residential_sqft / 445.0).astype('int')
new_buildings["net_units"] = new_buildings.job_spaces
summary.add_parcel_output(new_buildings)
# got to get the frame again because we just added rows
buildings = orca.get_table('buildings')
buildings_df = buildings.to_frame(
['year_built', 'building_sqft', 'general_type'])
sqft_by_gtype = buildings_df.query('year_built >= %d' % year).\
groupby('general_type').building_sqft.sum()
print "New square feet by general type in millions:\n",\
sqft_by_gtype / 1000000.0
def office_developer(feasibility, jobs, buildings, parcels, year,
settings, summary, form_to_btype_func, scenario,
add_extra_columns_func, parcels_geography,
limits_settings):
dev_settings = settings['non_residential_developer']
# I'm going to try a new way of computing this because the math the other
# way is simply too hard. Basically we used to try and apportion sectors
# into the demand for office, retail, and industrial, but there's just so
# much dirtyness to the data, for instance 15% of jobs are in residential
# buildings, and 15% in other buildings, it's just hard to know how much
# to build, we I think the right thing to do is to compute the number of
# job spaces that are required overall, and then to apportion that new dev
# into the three non-res types with a single set of coefficients
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
print "Total units to build = %d" % all_units
if all_units <= 0:
return
for typ in ["Office"]:
print "\nRunning for type: ", typ
num_units = all_units * float(dev_settings['type_splits'][typ])
targets = []
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or
# job_spaces
if year > 2015 and typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_job_spaces[
(juris_name == juris) & (parcels.newest_building > 2015)]\
.sum()
target = (year - 2015 + 1) * limit - current_total
if target <= 0:
print "Already met target for juris = %s" % juris
print " target = %d, current_total = %d" %\
(target, current_total)
continue
targets.append((juris_name == juris, target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index, num_units, "none"))
for parcel_mask, target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
print "Parcels in play:\n", pd.Series(parcel_mask).value_counts()
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
typ.lower(),
jobs,
buildings,
"job_spaces",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_job_spaces[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
residential=False,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**dev_settings['kwargs'])
if new_buildings is not None:
new_buildings["subsidized"] = False
summary.add_parcel_output(new_buildings)
def retail_developer(jobs, buildings, parcels, nodes, feasibility,
settings, summary, add_extra_columns_func, net):
dev_settings = settings['non_residential_developer']
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
target = all_units * float(dev_settings['type_splits']["Retail"])
# target here is in sqft
target *= settings["building_sqft_per_job"]["HS"]
feasibility = feasibility.to_frame().loc[:, "retail"]
feasibility = feasibility.dropna(subset=["max_profit"])
feasibility["non_residential_sqft"] = \
feasibility.non_residential_sqft.astype("int")
feasibility["retail_ratio"] = parcels.retail_ratio
feasibility = feasibility.reset_index()
# create features
f1 = feasibility.retail_ratio / feasibility.retail_ratio.max()
f2 = feasibility.max_profit / feasibility.max_profit.max()
# combine features in probability function - it's like combining expense
# of building the building with the market in the neighborhood
p = f1 * 1.5 + f2
p = p.clip(lower=1.0 / len(p) / 10)
print "Attempting to build {:,} retail sqft".format(target)
# order by weighted random sample
feasibility = feasibility.sample(frac=1.0, weights=p)
foreign_columns = ["general_type"]
bldgs = buildings.to_frame(buildings.local_columns + foreign_columns)
devs = []
for dev_id, d in feasibility.iterrows():
if target <= 0:
break
# any special logic to filter these devs?
# remove new dev sqft from target
target -= d.non_residential_sqft
# add redeveloped sqft to target
filt = "general_type == 'Retail' and parcel_id == %d" % \
d["parcel_id"]
target += bldgs.query(filt).non_residential_sqft.sum()
devs.append(d)
if len(devs) == 0:
return
# record keeping - add extra columns to match building dataframe
# add the buidings and demolish old buildings, and add to debug output
devs = pd.DataFrame(devs, columns=buildings.local_columns)
print "Building {:,} retail sqft in {:,} projects".format(
devs.non_residential_sqft.sum(), len(devs))
if target > 0:
print " WARNING: retail target not met"
devs["form"] = "retail"
devs = add_extra_columns_func(devs)
add_buildings(buildings, devs)
summary.add_parcel_output(devs)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year,
regional_controls):
kwargs = settings['residential_developer']
rc = regional_controls.to_frame()
target_vacancy = rc.loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
current_units = buildings.local.loc[index, "residential_units"]
# only can reduce by as many units as we have
overshoot = min(overshoot, current_units)
# used below - this is the pct we need to reduce the building
overshoot_pct = \
(current_units - overshoot) / float(current_units)
buildings.local.loc[index, "residential_units"] -= overshoot
# we also need to fix the other columns so they make sense
for col in ["residential_sqft", "building_sqft",
"deed_restricted_units"]:
val = buildings.local.loc[index, col]
# reduce by pct but round to int
buildings.local.loc[index, col] = int(val * overshoot_pct)
summary.add_parcel_output(new_buildings)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year,
regional_controls):
kwargs = settings['residential_developer']
rc = regional_controls.to_frame()
target_vacancy = rc.loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print("Running developer for %s with target of %d" %
(str(juris), target))
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
print('Stats of buildings before run_developer(): \n{}'.format(
buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum()))
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=int(target),
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
print('Stats of buildings before run_developer(): \n{}'.format(
buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum()))
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
current_units = buildings.local.loc[index, "residential_units"]
# only can reduce by as many units as we have
overshoot = min(overshoot, current_units)
# used below - this is the pct we need to reduce the building
overshoot_pct = \
(current_units - overshoot) / float(current_units)
buildings.local.loc[index, "residential_units"] -= overshoot
# we also need to fix the other columns so they make sense
for col in ["residential_sqft", "building_sqft",
"deed_restricted_units", "inclusionary_units",
"subsidized_units"]:
val = buildings.local.loc[index, col]
# reduce by pct but round to int
buildings.local.loc[index, col] = int(val * overshoot_pct)
# also fix the corresponding columns in new_buildings
for col in ["residential_sqft","building_sqft",
"residential_units", "deed_restricted_units",
"inclusionary_units", "subsidized_units"]:
val = new_buildings.loc[index, col]
new_buildings.loc[index, col] = int(val * overshoot_pct)
for col in ["policy_based_revenue_reduction",
"max_profit"]:
val = new_buildings.loc[index, col]
new_buildings.loc[index, col] = val * overshoot_pct
summary.add_parcel_output(new_buildings)
def initialize_new_units(buildings, residential_units):
"""
This data maintenance step initializes units for buildings that have been
newly created, conforming to the data requirements of the
'residential_units' table.
Data expectations
-----------------
- 'buildings' table has the following columns:
- index that serves as its identifier
- 'residential_units' (int, count of units in building)
- 'residential_units' table has the following columns:
- index named 'unit_id' that serves as its identifier
- 'building_id' corresponding to the index of the 'buildings' table
Results
-------
- extends the 'residential_units' table, following the same schema as the
'initialize_residential_units' model step
"""
# Verify initial data characteristics
'''
ot.assert_orca_spec(OrcaSpec(
'',
TableSpec(
'buildings',
ColumnSpec('building_id', primary_key=True),
ColumnSpec('residential_units', min=0)),
TableSpec(
'residential_units',
ColumnSpec('unit_id', primary_key=True),
ColumnSpec('building_id', foreign_key='buildings.building_id'))))
'''
old_units = residential_units.to_frame(residential_units.local_columns)
bldgs = buildings.to_frame(['residential_units', 'deed_restricted_units'])
# Filter for residential buildings not currently represented in
# the units table, and create new units
new_bldgs = bldgs[~bldgs.index.isin(old_units.building_id)]
new_bldgs = new_bldgs[new_bldgs.residential_units > 0]
new_units = _create_empty_units(new_bldgs)
# Filter for residential buildings where ADUs were added and
# create new units
old_units_by_bldg = old_units.groupby(['building_id']).agg({
'unit_num':
max,
'num_units':
'count'
}).reset_index()
old_units_by_bldg.rename(columns={
'num_units': 'num_units_old',
'unit_num': 'max_num_old'
},
inplace=True)
old_bldgs = bldgs[bldgs.index.isin(old_units.building_id)]
old_bldgs = old_bldgs[old_bldgs.residential_units > 0]
adu_bldgs = pd.merge(old_bldgs,
old_units_by_bldg,
left_index=True,
right_on='building_id')
adu_bldgs['adu_count'] = \
adu_bldgs.residential_units - adu_bldgs.num_units_old
adu_bldgs = adu_bldgs[adu_bldgs.adu_count > 0]
if len(adu_bldgs) > 0:
new_adus = pd.DataFrame({
'unit_residential_price':
0.0,
'unit_residential_rent':
0.0,
'num_units':
1,
'building_id':
np.repeat(adu_bldgs.building_id,
adu_bldgs.adu_count.values.astype(int)),
# counter of the AUDs in a building
'unit_num_adu':
np.concatenate([
np.arange(num_adus)
for num_adus in adu_bldgs.adu_count.values.astype(int)
]),
# ADUs are not deed restricted
'deed_restricted':
0.0,
'adu_count_start':
np.repeat(adu_bldgs.max_num_old + 1,
adu_bldgs.adu_count.values.astype(int))
}).sort_values(by=['building_id', 'unit_num_adu']).reset_index(
drop=True)
# update unit_num of ADUs to continue with the previous
# unit_num of non-ADUs in the same buildings
new_adus.unit_num = \
new_adus.unit_num_adu + new_adus.adu_count_start
new_adus.drop(columns=['adu_count_start', 'unit_num_adu'],
inplace=True)
new_adus.index.name = 'unit_id'
new_units = dev.merge(new_units, new_adus)
else:
print('No ADUs were built.')
# Merge new units with old units and update the table
all_units = dev.merge(old_units, new_units)
all_units.index.name = 'unit_id'
print("Creating %d residential units for %d new buildings" %
(len(new_units), len(new_bldgs)))
orca.add_table('residential_units', all_units)
# Verify final data characteristics
'''
