def price_vars(net):
nodes2 = networks.from_yaml(net, "price_vars.yaml")
nodes2 = nodes2.fillna(0)
print nodes2.describe()
nodes = sim.get_table('nodes')
nodes = nodes.to_frame().join(nodes2)
sim.add_table("nodes", nodes)
def price_vars(net):
nodes2 = networks.from_yaml(net, "price_vars.yaml")
nodes2 = nodes2.fillna(0)
print nodes2.describe()
nodes = sim.get_table('nodes')
nodes = nodes.to_frame().join(nodes2)
sim.add_table("nodes", nodes)
def simple_transition(tbl, rate, location_fname):
"""
Run a simple growth rate transition model on the table passed in
Parameters
----------
tbl : DataFrameWrapper
Table to be transitioned
rate : float
Growth rate
location_fname : str
The field name in the resulting dataframe to set to -1 (to unplace
new agents)
Returns
-------
Nothing
"""
transition = GrowthRateTransition(rate)
df = tbl.to_frame(tbl.local_columns)
print "%d agents before transition" % len(df.index)
df, added, copied, removed = transition.transition(df, None)
print "%d agents after transition" % len(df.index)
df.loc[added, location_fname] = -1
sim.add_table(tbl.name, df)
def simple_transition(tbl, rate, location_fname):
"""
Run a simple growth rate transition model on the table passed in
Parameters
----------
tbl : DataFrameWrapper
Table to be transitioned
rate : float
Growth rate
location_fname : str
The field name in the resulting dataframe to set to -1 (to unplace
new agents)
Returns
-------
Nothing
"""
transition = GrowthRateTransition(rate)
df = tbl.to_frame(tbl.local_columns)
print "%d agents before transition" % len(df.index)
df, added, copied, removed = transition.transition(df, None)
print "%d agents after transition" % len(df.index)
df.loc[added, location_fname] = -1
sim.add_table(tbl.name, df)
def households_transition(households, annual_household_control_totals, year):
ct = annual_household_control_totals.to_frame()
tran = transition.TabularTotalsTransition(ct, 'total_number_of_households')
model = transition.TransitionModel(tran)
hh = households.to_frame(households.local_columns + ['activity_id', 'luz_id'])
new, added_hh_idx, empty_dict = \
model.transition(hh, year,)
new.loc[added_hh_idx, "building_id"] = -1
sim.add_table("households", new)
def simple_transition(tbl, rate, location_fname):
transition = GrowthRateTransition(rate)
df = tbl.to_frame(tbl.local_columns)
print "%d agents before transition" % len(df.index)
df, added, copied, removed = transition.transition(df, None)
print "%d agents after transition" % len(df.index)
df.loc[added, location_fname] = -1
sim.add_table(tbl.name, df)
def simple_transition(tbl, rate, location_fname):
transition = GrowthRateTransition(rate)
df = tbl.to_frame(tbl.local_columns)
print "%d agents before transition" % len(df.index)
df, added, copied, removed = transition.transition(df, None)
print "%d agents after transition" % len(df.index)
df.loc[added, location_fname] = -1
sim.add_table(tbl.name, df)
def households_transition(households, annual_household_control_totals, year):
ct = annual_household_control_totals.to_frame()
tran = transition.TabularTotalsTransition(ct, 'total_number_of_households')
model = transition.TransitionModel(tran)
hh = households.to_frame(households.local_columns +
['activity_id', 'luz_id'])
new, added_hh_idx, empty_dict = \
model.transition(hh, year,)
new.loc[added_hh_idx, "building_id"] = -1
sim.add_table("households", new)
def build_networks(parcels):
st = pd.HDFStore(os.path.join(misc.data_dir(), "osm_sandag.h5"), "r")
nodes, edges = st.nodes, st.edges
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
net.precompute(3000)
sim.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
sim.add_table("parcels", p)
def build_networks(parcels):
st = pd.HDFStore(os.path.join(misc.data_dir(), "osm_sandag.h5"), "r")
nodes, edges = st.nodes, st.edges
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
net.precompute(3000)
sim.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
sim.add_table("parcels", p)
def run_feasibility(parcels,
parcel_price_callback,
parcel_use_allowed_callback,
residential_to_yearly=True):
"""
Execute development feasibility on all parcels
Parameters
----------
parcels : DataFrame Wrapper
The data frame wrapper for the parcel data
parcel_price_callback : function
A callback which takes each use of the pro forma and returns a series
with index as parcel_id and value as yearly_rent
parcel_use_allowed_callback : function
A callback which takes each form of the pro forma and returns a series
with index as parcel_id and value and boolean whether the form
is allowed on the parcel
residential_to_yearly : boolean (default true)
Whether to use the cap rate to convert the residential price from total
sales price per sqft to rent per sqft
Returns
-------
Adds a table called feasibility to the sim object (returns nothing)
"""
pf = sqftproforma.SqFtProForma()
df = parcels.to_frame()
# add prices for each use
for use in pf.config.uses:
df[use] = parcel_price_callback(use)
# convert from cost to yearly rent
if residential_to_yearly:
df["residential"] *= pf.config.cap_rate
print "Describe of the yearly rent by use"
print df[pf.config.uses].describe()
d = {}
for form in pf.config.forms:
print "Computing feasibility for form %s" % form
d[form] = pf.lookup(form, df[parcel_use_allowed_callback(form)])
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def scheduled_development_events(buildings):
year = get_year()
sched_dev = pd.read_csv("./data/scheduled_development_events.csv")
sched_dev = sched_dev[sched_dev.year_built==year]
sched_dev['residential_sqft'] = sched_dev.sqft_per_unit*sched_dev.residential_units
sched_dev['job_spaces'] = sched_dev.non_residential_sqft/400
if len(sched_dev) > 0:
max_bid = buildings.index.values.max()
idx = np.arange(max_bid + 1,max_bid+len(sched_dev)+1)
sched_dev['building_id'] = idx
sched_dev = sched_dev.set_index('building_id')
from urbansim.developer.developer import Developer
merge = Developer(pd.DataFrame({})).merge
b = buildings.to_frame(buildings.local_columns)
all_buildings = merge(b,sched_dev[b.columns])
sim.add_table("buildings", all_buildings)
def run_feasibility(parcels, parcel_price_callback,
parcel_use_allowed_callback, residential_to_yearly=True):
"""
Execute development feasibility on all parcels
Parameters
----------
parcels : DataFrame Wrapper
The data frame wrapper for the parcel data
parcel_price_callback : function
A callback which takes each use of the pro forma and returns a series
with index as parcel_id and value as yearly_rent
parcel_use_allowed_callback : function
A callback which takes each form of the pro forma and returns a series
with index as parcel_id and value and boolean whether the form
is allowed on the parcel
residential_to_yearly : boolean (default true)
Whether to use the cap rate to convert the residential price from total
sales price per sqft to rent per sqft
Returns
-------
Adds a table called feasibility to the sim object (returns nothing)
"""
pf = sqftproforma.SqFtProForma()
df = parcels.to_frame()
# add prices for each use
for use in pf.config.uses:
df[use] = parcel_price_callback(use)
# convert from cost to yearly rent
if residential_to_yearly:
df["residential"] *= pf.config.cap_rate
print "Describe of the yearly rent by use"
print df[pf.config.uses].describe()
d = {}
for form in pf.config.forms:
print "Computing feasibility for form %s" % form
d[form] = pf.lookup(form, df[parcel_use_allowed_callback(form)])
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def scheduled_development_events(buildings):
year = get_year()
sched_dev = pd.read_csv("./data/scheduled_development_events.csv")
sched_dev = sched_dev[sched_dev.year_built == year]
sched_dev[
'residential_sqft'] = sched_dev.sqft_per_unit * sched_dev.residential_units
sched_dev['job_spaces'] = sched_dev.non_residential_sqft / 400
if len(sched_dev) > 0:
max_bid = buildings.index.values.max()
idx = np.arange(max_bid + 1, max_bid + len(sched_dev) + 1)
sched_dev['building_id'] = idx
sched_dev = sched_dev.set_index('building_id')
from urbansim.developer.developer import Developer
merge = Developer(pd.DataFrame({})).merge
b = buildings.to_frame(buildings.local_columns)
all_buildings = merge(b, sched_dev[b.columns])
sim.add_table("buildings", all_buildings)
def full_transition(agents, agent_controls, year, settings, location_fname):
"""
Run a transition model based on control totals specified in the usual
UrbanSim way
Parameters
----------
agents : DataFrameWrapper
Table to be transitioned
agent_controls : DataFrameWrapper
Table of control totals
year : int
The year, which will index into the controls
settings : dict
Contains the configuration for the transition model - is specified
down to the yaml level with a "total_column" which specifies the
control total and an "add_columns" param which specified which
columns to add when calling to_frame (should be a list of the columns
needed to do the transition
location_fname : str
The field name in the resulting dataframe to set to -1 (to unplace
new agents)
Returns
-------
Nothing
"""
ct = agent_controls.to_frame()
hh = agents.to_frame(agents.local_columns +
settings['add_columns'])
print "Total agents before transition: {}".format(len(hh))
tran = transition.TabularTotalsTransition(ct, settings['total_column'])
model = transition.TransitionModel(tran)
new, added_hh_idx, new_linked = model.transition(hh, year)
new.loc[added_hh_idx, location_fname] = -1
print "Total agents after transition: {}".format(len(new))
sim.add_table(agents.name, new)
def full_transition(agents, agent_controls, year, settings, location_fname):
"""
Run a transition model based on control totals specified in the usual
UrbanSim way
Parameters
----------
agents : DataFrameWrapper
Table to be transitioned
agent_controls : DataFrameWrapper
Table of control totals
year : int
The year, which will index into the controls
settings : dict
Contains the configuration for the transition model - is specified
down to the yaml level with a "total_column" which specifies the
control total and an "add_columns" param which specified which
columns to add when calling to_frame (should be a list of the columns
needed to do the transition
location_fname : str
The field name in the resulting dataframe to set to -1 (to unplace
new agents)
Returns
-------
Nothing
"""
ct = agent_controls.to_frame()
hh = agents.to_frame(agents.local_columns + settings['add_columns'])
print "Total agents before transition: {}".format(len(hh))
tran = transition.TabularTotalsTransition(ct, settings['total_column'])
model = transition.TransitionModel(tran)
new, added_hh_idx, new_linked = model.transition(hh, year)
new.loc[added_hh_idx, location_fname] = -1
print "Total agents after transition: {}".format(len(new))
sim.add_table(agents.name, new)
def run_developer(forms, agents, buildings, supply_fname, parcel_size,
ave_unit_size, total_units, feasibility, year=None,
target_vacancy=.1, form_to_btype_callback=None,
add_more_columns_callback=None, max_parcel_size=2000000,
residential=True, bldg_sqft_per_job=400.0,
min_unit_size=400, remove_developed_buildings=True,
unplace_agents=['households', 'jobs']):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
min_unit_size : float
Passed directly to dev.pick - min unit size that is valid
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
remove_redeveloped_buildings : optional, boolean (default True)
Remove all buildings on the parcels which are being developed on
unplace_agents : optional : list of strings (default ['households', 'jobs'])
For all tables in the list, will look for field building_id and set
it to -1 for buildings which are removed - only executed if
remove_developed_buildings is true
Returns
-------
Writes the result back to the buildings table and returns the new
buildings with available debugging information on each new building
"""
dev = developer.Developer(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
min_unit_size=min_unit_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
if new_buildings is None:
return
if len(new_buildings) == 0:
return new_buildings
if year is not None:
new_buildings["year_built"] = year
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings.\
apply(form_to_btype_callback, axis=1)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
ret_buildings = new_buildings
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in unplace_agents:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
return ret_buildings
def run_feasibility(parcels, parcel_price_callback,
parcel_use_allowed_callback, residential_to_yearly=True,
parcel_filter=None, only_built=True, forms_to_test=None,
config=None, pass_through=[]):
"""
Execute development feasibility on all parcels
Parameters
----------
parcels : DataFrame Wrapper
The data frame wrapper for the parcel data
parcel_price_callback : function
A callback which takes each use of the pro forma and returns a series
with index as parcel_id and value as yearly_rent
parcel_use_allowed_callback : function
A callback which takes each form of the pro forma and returns a series
with index as parcel_id and value and boolean whether the form
is allowed on the parcel
residential_to_yearly : boolean (default true)
Whether to use the cap rate to convert the residential price from total
sales price per sqft to rent per sqft
parcel_filter : string
A filter to apply to the parcels data frame to remove parcels from
consideration - is typically used to remove parcels with buildings
older than a certain date for historical preservation, but is
generally useful
only_built : boolean
Only return those buildings that are profitable - only those buildings
that "will be built"
forms_to_test : list of strings (optional)
Pass the list of the names of forms to test for feasibility - if set to
None will use all the forms available in ProFormaConfig
config : SqFtProFormaConfig configuration object. Optional. Defaults to
None
pass_through : list of strings
Will be passed to the feasibility lookup function - is used to pass
variables from the parcel dataframe to the output dataframe, usually
for debugging
Returns
-------
Adds a table called feasibility to the sim object (returns nothing)
"""
pf = sqftproforma.SqFtProForma(config) if config \
else sqftproforma.SqFtProForma()
df = parcels.to_frame()
if parcel_filter:
df = df.query(parcel_filter)
# add prices for each use
for use in pf.config.uses:
# assume we can get the 80th percentile price for new development
df[use] = parcel_price_callback(use)
# convert from cost to yearly rent
if residential_to_yearly:
df["residential"] *= pf.config.cap_rate
print "Describe of the yearly rent by use"
print df[pf.config.uses].describe()
d = {}
forms = forms_to_test or pf.config.forms
for form in forms:
print "Computing feasibility for form %s" % form
allowed = parcel_use_allowed_callback(form).loc[df.index]
d[form] = pf.lookup(form, df[allowed], only_built=only_built,
pass_through=pass_through)
if residential_to_yearly and "residential" in pass_through:
d[form]["residential"] /= pf.config.cap_rate
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def neighborhood_vars(net):
nodes = networks.from_yaml(net, "neighborhood_vars.yaml")
nodes = nodes.fillna(0)
print nodes.describe()
sim.add_table("nodes", nodes)
# Use codes were classified manually because the assessor classifications
# are meant for property tax purposes. These classifications should be
# reviewed and revised.
res_codes = {'single': [str(i) for i in range(11, 17)] + ['19'],
'multi': ['10', '17', '18', '61', '88'] +
[str(i) for i in range(20, 30)],
'mixed': ['48', '89']}
exempt_codes = []
## Register input tables.
tf = TableFrame(staging.parcels_cnc_pt)
sim.add_table('parcels_in', tf, copy_col=False)
## Register intermediate table and columns.
# The purpose of this intermediate table is to compute certain fields,
# like non_residential_sqft and residential_units, before grouping together
# records with the same parc_py_id. Thus, single-family condominium units
# would each be assumed to have one residential unit, and this count would
# be summed when grouping later.
@sim.table()
def parcels_in2(parcels_in):
return pd.DataFrame(index=parcels_in.index)
def run_developer(forms, agents, buildings, supply_fname, parcel_size,
ave_unit_size, total_units, feasibility, year=None,
target_vacancy=.1, form_to_btype_callback=None,
add_more_columns_callback=None, max_parcel_size=200000,
residential=True, bldg_sqft_per_job=400.0):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
Returns
-------
Writes the result back to the buildings table (returns nothing)
"""
dev = developer.Developer(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
if new_buildings is None:
return
if year is not None:
new_buildings["year_built"] = year
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings["form"].\
apply(form_to_btype_callback)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
all_buildings = dev.merge(buildings.to_frame(buildings.local_columns),
new_buildings[buildings.local_columns])
sim.add_table("buildings", all_buildings)
def residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
year = get_year()
targets = residential_space_targets()
print "{:,} feasible buildings before running developer".format(len(feas))
# LUZ overrides, if applicable
p = sim.get_table('parcels').to_frame(columns=[
'luz_id', 'total_residential_units', 'total_sfd_du', 'total_sfa_du',
'total_mfr_du'
])
feas['luz_id'] = p.luz_id
overrides = pd.read_csv('./data/overrides/luz_overrides.csv')
overrides = overrides[overrides.year == year]
controlled_luzes = np.unique(overrides.luz_id)
if len(overrides) > 0:
#Record existing LUZ values
existing_du = p.groupby('luz_id').total_residential_units.sum()
existing_sfd_du = p.groupby('luz_id').total_sfd_du.sum()
existing_sfa_du = p.groupby('luz_id').total_sfa_du.sum()
existing_mfr_du = p.groupby('luz_id').total_mfr_du.sum()
existing_df = pd.DataFrame({
'existing_du': existing_du,
19: existing_sfd_du,
20: existing_sfa_du,
21: existing_mfr_du
}).fillna(0)
mini_feases = []
for luz in controlled_luzes:
overrides_subset = overrides[overrides.luz_id == luz]
for devtype in np.unique(overrides_subset.development_type_id):
target = overrides_subset.target[
overrides_subset.development_type_id == devtype].values[0]
print 'LUZ %s has a DU override target for development type %s of %s.' % (
luz, devtype, target)
existing_du = existing_df[devtype][existing_df.index ==
luz].values[0]
print ' There are %s existing units of this type in this LUZ' % existing_du
difference = target - existing_du
if difference > 0:
feas_subset = feas[feas.luz_id == luz]
if len(feas_subset) > 0:
if devtype == 19: residential_form = 'sf_detached'
if devtype == 20: residential_form = 'sf_attached'
if devtype == 21: residential_form = 'mf_residential'
feasible_units = feas_subset[
residential_form].net_units.sum()
if feasible_units > 0:
if difference > feasible_units:
reallocate = difference - feasible_units
print ' Moving %s units to the uncontrolled bucket because only part of the target difference was feasible' % reallocate
targets[residential_form] = targets[
residential_form] + reallocate
dev_luz = developer.Developer(feas_subset)
run_developer(dev_luz,
residential_form,
difference,
year,
build=True)
#Store the unbuilt feasible parcels and add back into feasibility later
mini_feases.append(dev_luz.feasibility)
else:
print ' No profitable projects'
#Uncontrolled LUZs
print 'Running Developer for uncontrolled LUZs'
feas = feas[~feas.luz_id.isin(controlled_luzes)]
dev = developer.Developer(feas)
for residential_form in [
'mf_residential',
'sf_attached',
'sf_detached',
]:
if residential_form in targets.keys():
target = targets[residential_form]
run_developer(dev, residential_form, target, year, build=True)
#Remaining feasible parcels back to feas after running controlled LUZs so that nonres can be built on these parcels if multiple forms allowed and not already built on
if len(overrides) > 0:
feas_list = mini_feases.append(dev.feasibility)
feas = pd.concat(mini_feases)
sim.add_table("feasibility", feas)
else:
sim.add_table("feasibility", dev.feasibility)
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns)
b_sim = b[(b.note == 'simulated') * (b.year_built == year)]
print 'Simulated DU: %s' % b_sim.residential_units.sum()
print 'Target DU: %s' % (targets['sf_detached'] +
targets['mf_residential'] + targets['sf_attached']
) #Note: includes negative when target is lower.
def residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
year = get_year()
targets = residential_space_targets()
print "{:,} feasible buildings before running developer".format(
len(feas))
# LUZ overrides, if applicable
p = sim.get_table('parcels').to_frame(columns = ['luz_id', 'total_residential_units', 'total_sfd_du', 'total_sfa_du', 'total_mfr_du'])
feas['luz_id'] = p.luz_id
overrides = pd.read_csv('./data/overrides/luz_overrides.csv')
overrides = overrides[overrides.year == year]
controlled_luzes = np.unique(overrides.luz_id)
if len(overrides) > 0:
#Record existing LUZ values
existing_du = p.groupby('luz_id').total_residential_units.sum()
existing_sfd_du = p.groupby('luz_id').total_sfd_du.sum()
existing_sfa_du = p.groupby('luz_id').total_sfa_du.sum()
existing_mfr_du = p.groupby('luz_id').total_mfr_du.sum()
existing_df = pd.DataFrame({'existing_du':existing_du,
19:existing_sfd_du,
20:existing_sfa_du,
21:existing_mfr_du}).fillna(0)
mini_feases = []
for luz in controlled_luzes:
overrides_subset = overrides[overrides.luz_id == luz]
for devtype in np.unique(overrides_subset.development_type_id):
target = overrides_subset.target[overrides_subset.development_type_id == devtype].values[0]
print 'LUZ %s has a DU override target for development type %s of %s.' % (luz, devtype, target)
existing_du = existing_df[devtype][existing_df.index == luz].values[0]
print ' There are %s existing units of this type in this LUZ' % existing_du
difference = target - existing_du
if difference > 0:
feas_subset = feas[feas.luz_id == luz]
if len(feas_subset) > 0:
if devtype == 19: residential_form = 'sf_detached'
if devtype == 20: residential_form = 'sf_attached'
if devtype == 21: residential_form = 'mf_residential'
feasible_units = feas_subset[residential_form].net_units.sum()
if feasible_units > 0:
if difference > feasible_units:
reallocate = difference - feasible_units
print ' Moving %s units to the uncontrolled bucket because only part of the target difference was feasible' % reallocate
targets[residential_form] = targets[residential_form] + reallocate
dev_luz = developer.Developer(feas_subset)
run_developer(dev_luz, residential_form, difference, year, build = True)
#Store the unbuilt feasible parcels and add back into feasibility later
mini_feases.append(dev_luz.feasibility)
else:
print ' No profitable projects'
#Uncontrolled LUZs
print 'Running Developer for uncontrolled LUZs'
feas = feas[~feas.luz_id.isin(controlled_luzes)]
dev = developer.Developer(feas)
for residential_form in ['mf_residential', 'sf_attached', 'sf_detached', ]:
if residential_form in targets.keys():
target = targets[residential_form]
run_developer(dev, residential_form, target, year, build = True)
#Remaining feasible parcels back to feas after running controlled LUZs so that nonres can be built on these parcels if multiple forms allowed and not already built on
if len(overrides) > 0:
feas_list = mini_feases.append(dev.feasibility)
feas = pd.concat(mini_feases)
sim.add_table("feasibility", feas)
else:
sim.add_table("feasibility", dev.feasibility)
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns)
b_sim = b[(b.note == 'simulated') * (b.year_built == year)]
print 'Simulated DU: %s' % b_sim.residential_units.sum()
print 'Target DU: %s' % (targets['sf_detached'] + targets['mf_residential'] + targets['sf_attached']) #Note: includes negative when target is lower.
def run_developer(forms, agents, buildings, buildings_all, supply_fname, parcel_size,
ave_unit_size, total_units, feasibility, year=None,
target_vacancy=.1, form_to_btype_callback=None,
add_more_columns_callback=None, max_parcel_size=34647265,
residential=True, bldg_sqft_per_job=400.0,
min_unit_size=400, remove_developed_buildings=True,
unplace_agents=['households', 'jobs']):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
buildings_all:
Buildings for the entire region, used to write back to buildings table
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
min_unit_size : float
Passed directly to dev.pick - min unit size that is valid
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
remove_redeveloped_buildings : optional, boolean (default True)
Remove all buildings on the parcels which are being developed on
unplace_agents : optional : list of strings (default ['households', 'jobs'])
For all tables in the list, will look for field building_id and set
it to -1 for buildings which are removed - only executed if
remove_developed_buildings is true
Returns
-------
Writes the result back to the buildings table and returns the new
buildings with available debugging information on each new building
"""
dev = developer.Developer(feasibility.to_frame())
#dev = WFRCDeveloper.WFRCDeveloper(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
min_unit_size=min_unit_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
year = sim.get_injectable('year')
if new_buildings is None:
return
if len(new_buildings) == 0:
return new_buildings
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings.\
apply(form_to_btype_callback, axis=1)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
new_buildings["note"] = "simulated"
ret_buildings = new_buildings
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
if year is not None:
new_buildings["year_built"] = year
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
old_buildings = buildings.to_frame(buildings.local_columns)
old_buildings_all = buildings_all.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
redev_buildings_all = old_buildings_all.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
drop_buildings_all = old_buildings_all[redev_buildings_all]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
old_buildings_all = old_buildings_all[np.logical_not(redev_buildings_all)]
l2 = len(old_buildings)
print "before dropped l:" + str(l)
print "after dropped l2: " + str(l2)
#print redev_buildings
#print drop_buildings
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in unplace_agents:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
#displaced_agents = agents.building_id.isin(drop_buildings.index)
displaced_agents = agents.building_id.isin(drop_buildings_all.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
all_buildings = dev.merge(old_buildings_all, new_buildings)
sim.add_table("buildings", all_buildings)
return ret_buildings
def run_developer(dev, residential_form, target, year, build = False):
old_buildings = sim.get_table('buildings').to_frame(sim.get_table('buildings').local_columns)
parcels = sim.get_table('parcels')
print 'Residential unit target for %s is %s.' % (residential_form, target)
if target > 0:
print residential_form
drop_after_build = True if build else False
new_buildings = dev.pick(residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_residential_units,
max_parcel_size=2000000,
min_unit_size=400,
drop_after_build=True,
residential=True,
bldg_sqft_per_job=400.0)
if build:
print 'Constructed %s %s buildings, totaling %s new residential_units' % (len(new_buildings), residential_form, new_buildings.residential_units.sum())
overshoot = new_buildings.residential_units.sum() - target
print 'Overshot target by %s units' % (overshoot)
print 'Biggest development has %s units' % new_buildings.residential_units.max()
if overshoot > 1:
to_remove = new_buildings[['parcel_id', 'residential_units']].copy()
to_remove = to_remove[to_remove.residential_units < 4].set_index('parcel_id')
to_remove = to_remove.sort('residential_units')
to_remove['du_cumsum'] = to_remove.residential_units.cumsum()
idx_to_remove = np.searchsorted(to_remove.du_cumsum, overshoot)
parcel_ids_to_remove = to_remove.index.values[:(idx_to_remove[0] + 1)]
print 'Removing %s units to match target' % to_remove.residential_units.values[:(idx_to_remove[0] + 1)].sum()
new_buildings = new_buildings[~new_buildings.parcel_id.isin(parcel_ids_to_remove)]
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if residential_form == 'sf_detached':
new_buildings['development_type_id'] = 19
elif residential_form == 'sf_attached':
new_buildings['development_type_id'] = 20
elif residential_form == 'mf_residential':
new_buildings['development_type_id'] = 21
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
else:
return new_buildings.residential_units.sum()
def run_feasibility(parcels, parcel_price_callback,
parcel_use_allowed_callback, residential_to_yearly=True,
parcel_filter=None, only_built=True, forms_to_test=None,
config=None, pass_through=[]):
"""
Execute development feasibility on all parcels
Parameters
----------
parcels : DataFrame Wrapper
The data frame wrapper for the parcel data
parcel_price_callback : function
A callback which takes each use of the pro forma and returns a series
with index as parcel_id and value as yearly_rent
parcel_use_allowed_callback : function
A callback which takes each form of the pro forma and returns a series
with index as parcel_id and value and boolean whether the form
is allowed on the parcel
residential_to_yearly : boolean (default true)
Whether to use the cap rate to convert the residential price from total
sales price per sqft to rent per sqft
parcel_filter : string
A filter to apply to the parcels data frame to remove parcels from
consideration - is typically used to remove parcels with buildings
older than a certain date for historical preservation, but is
generally useful
only_built : boolean
Only return those buildings that are profitable - only those buildings
that "will be built"
forms_to_test : list of strings (optional)
Pass the list of the names of forms to test for feasibility - if set to
None will use all the forms available in ProFormaConfig
config : SqFtProFormaConfig configuration object. Optional. Defaults to
None
pass_through : list of strings
Will be passed to the feasibility lookup function - is used to pass
variables from the parcel dataframe to the output dataframe, usually
for debugging
Returns
-------
Adds a table called feasibility to the sim object (returns nothing)
"""
pf = sqftproforma.SqFtProForma(config) if config \
else sqftproforma.SqFtProForma()
df = parcels.to_frame()
if parcel_filter:
df = df.query(parcel_filter)
#print df.loc[765403]
#df.to_csv("select_parcels.csv")
# add prices for each use
for use in pf.config.uses:
# assume we can get the 80th percentile price for new development
df[use] = parcel_price_callback(use)
# convert from cost to yearly rent
if residential_to_yearly:
df["residential"] *= pf.config.cap_rate
print "Describe of the yearly rent by use"
print df[pf.config.uses].describe()
d = {}
forms = forms_to_test or pf.config.forms
for form in forms:
print "Computing feasibility for form %s" % form
allowed = parcel_use_allowed_callback(form).loc[df.index]
#allowed.to_csv(str(form) + "allow.csv")
#df[allowed].to_csv(str(form) + "allow.csv")
d[form] = pf.lookup(form, df[allowed], only_built=only_built,
pass_through=pass_through)
#d[form].to_csv(str(form) + "dform.csv")
if residential_to_yearly and "residential" in pass_through:
d[form]["residential"] /= pf.config.cap_rate
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
far_predictions['residential'].to_csv("residential_far_prediction.csv")
far_predictions['retail'].to_csv("retail_far_prediction.csv")
far_predictions['office'].to_csv("office_far_prediction.csv")
#far_predictions.to_csv("far_prediction.csv")
far_predictions['residential'].max_profit = far_predictions['residential'].max_profit / np.power(far_predictions['residential'].max_profit_far*far_predictions['residential'].shape_area,1)
far_predictions['industrial'].max_profit = far_predictions['industrial'].max_profit / np.power(far_predictions['industrial'].max_profit_far*far_predictions['industrial'].shape_area,1)
far_predictions['retail'].max_profit = far_predictions['retail'].max_profit / np.power(far_predictions['retail'].max_profit_far*far_predictions['retail'].shape_area,1)
far_predictions['office'].max_profit = far_predictions['office'].max_profit / np.power(far_predictions['office'].max_profit_far*far_predictions['office'].shape_area,1)
#far_predictions['residential'].max_profit = np.divide(far_predictions['residential'].max_profit,far_predictions['residential'].max_dua)
#far_predictions['residential'].max_profit[far_predictions['residential'].max_profit==-np.inf] = np.nan
sim.add_table("feasibility", far_predictions)
def feasibility(parcels, settings,
parcel_sales_price_sqft_func,
parcel_is_allowed_func):
# Fee table preprocessing
fee_schedule = sim.get_table('fee_schedule').to_frame()
parcel_fee_schedule = sim.get_table('parcel_fee_schedule').to_frame()
parcels = sim.get_table('parcels').to_frame(columns = ['zoning_id','development_type_id'])
fee_schedule = fee_schedule.groupby(['fee_schedule_id', 'development_type_id']).development_fee_per_unit_space_initial.mean().reset_index()
parcel_use_allowed_callback = sim.get_injectable('parcel_is_allowed_func')
def run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(sim.get_table('nodes')[use], sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv('.\\data\\calibration\\msa_shifters.csv').set_index('msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv('.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv('.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev, form, target_units, get_year(), build = False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form]/400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(feasibility.non_residential_sqft / 400.0)
feasibility['net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def residential_proforma(form, devtype_id, parking_rate):
print form
use = 'residential'
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = True
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use : 1.0}}
pfc.uses = [use]
pfc.residential_uses = [True]
pfc.parking_rates = {use : parking_rate}
pfc.costs = {use : [170.0, 190.0, 210.0, 240.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id == devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule, fee_schedule_devtype, left_on = 'fee_schedule_id', right_on = 'fee_schedule_id')
parcel_fee_schedule_devtype['development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial*parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby('parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data = parcel_fees_processed, index = parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
return run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = parcel_filter)
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use : 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use : parking_rate}
if use == 'retail':
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use : [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id == devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule, fee_schedule_devtype, left_on = 'fee_schedule_id', right_on = 'fee_schedule_id')
parcel_fee_schedule_devtype['development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial*parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby('parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data = parcel_fees_processed, index = parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees*pf.config.cap_rate
return run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = parcel_filter)
d = {}
##SF DETACHED proforma (devtype 19)
form = 'sf_detached'
devtype_id = 19
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##SF ATTACHED proforma (devtype 20)
form = 'sf_attached'
devtype_id = 20
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##MF_RESIDENTIAL proforma (devtype 21)
form = 'mf_residential'
devtype_id = 21
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##OFFICE (devtype 4)
form = 'office'
devtype_id = 4
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate = 1.0)
##RETAIL (devtype 5)
form = 'retail'
devtype_id = 5
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate = 2.0)
##LIGHT INDUSTRIAL (devtype 2)
form = 'light_industrial'
devtype_id = 2
d[form] = nonresidential_proforma(form, devtype_id, 'industrial', parking_rate = .6)
##HEAVY INDUSTRIAL (devtype 3)
form = 'heavy_industrial'
devtype_id = 3
d[form] = nonresidential_proforma(form, devtype_id, 'industrial', parking_rate = .6)
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def feasibility(parcels, settings, parcel_sales_price_sqft_func,
parcel_is_allowed_func):
# Fee table preprocessing
fee_schedule = sim.get_table('fee_schedule').to_frame()
parcel_fee_schedule = sim.get_table('parcel_fee_schedule').to_frame()
parcels = sim.get_table('parcels').to_frame(
columns=['zoning_id', 'development_type_id'])
fee_schedule = fee_schedule.groupby([
'fee_schedule_id', 'development_type_id'
]).development_fee_per_unit_space_initial.mean().reset_index()
parcel_use_allowed_callback = sim.get_injectable('parcel_is_allowed_func')
def run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(
sim.get_table('nodes')[use],
sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv(
'.\\data\\calibration\\msa_shifters.csv').set_index(
'msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv(
'.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv(
'.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][
parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev,
form,
target_units,
get_year(),
build=False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form] / 400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(
feasibility.non_residential_sqft / 400.0)
feasibility[
'net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def residential_proforma(form, devtype_id, parking_rate):
print form
use = 'residential'
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = True
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use: 1.0}}
pfc.uses = [use]
pfc.residential_uses = [True]
pfc.parking_rates = {use: parking_rate}
pfc.costs = {use: [170.0, 190.0, 210.0, 240.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id ==
devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule,
fee_schedule_devtype,
left_on='fee_schedule_id',
right_on='fee_schedule_id')
parcel_fee_schedule_devtype[
'development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial * parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby(
'parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data=parcel_fees_processed,
index=parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
return run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=parcel_filter)
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use: 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use: parking_rate}
if use == 'retail':
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use: [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id ==
devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule,
fee_schedule_devtype,
left_on='fee_schedule_id',
right_on='fee_schedule_id')
parcel_fee_schedule_devtype[
'development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial * parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby(
'parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data=parcel_fees_processed,
index=parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees * pf.config.cap_rate
return run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=parcel_filter)
d = {}
##SF DETACHED proforma (devtype 19)
form = 'sf_detached'
devtype_id = 19
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##SF ATTACHED proforma (devtype 20)
form = 'sf_attached'
devtype_id = 20
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##MF_RESIDENTIAL proforma (devtype 21)
form = 'mf_residential'
devtype_id = 21
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##OFFICE (devtype 4)
form = 'office'
devtype_id = 4
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate=1.0)
##RETAIL (devtype 5)
form = 'retail'
devtype_id = 5
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate=2.0)
##LIGHT INDUSTRIAL (devtype 2)
form = 'light_industrial'
devtype_id = 2
d[form] = nonresidential_proforma(form,
devtype_id,
'industrial',
parking_rate=.6)
##HEAVY INDUSTRIAL (devtype 3)
form = 'heavy_industrial'
devtype_id = 3
d[form] = nonresidential_proforma(form,
devtype_id,
'industrial',
parking_rate=.6)
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def run_developer(forms,
agents,
buildings,
supply_fname,
parcel_size,
ave_unit_size,
total_units,
feasibility,
year=None,
target_vacancy=.1,
form_to_btype_callback=None,
add_more_columns_callback=None,
max_parcel_size=200000,
residential=True,
bldg_sqft_per_job=400.0):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
Returns
-------
Writes the result back to the buildings table (returns nothing)
"""
dev = developer.Developer(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
if new_buildings is None:
return
if year is not None:
new_buildings["year_built"] = year
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings["form"].\
apply(form_to_btype_callback)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
all_buildings = dev.merge(buildings.to_frame(buildings.local_columns),
new_buildings[buildings.local_columns])
sim.add_table("buildings", all_buildings)
def non_residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
dev = developer.Developer(feas)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
year = get_year()
targets = non_residential_space_targets()
for non_residential_form in ['heavy_industrial', 'light_industrial', 'retail', 'office']:
if non_residential_form in targets.keys():
old_buildings = sim.get_table('buildings').to_frame(sim.get_table('buildings').local_columns)
target = targets[non_residential_form]
target = target/400
print 'Job space target for %s is %s.' % (non_residential_form, target)
if target > 0:
new_buildings = dev.pick(non_residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_job_spaces,
max_parcel_size=2000000,
min_unit_size=0,
drop_after_build=True,
residential=False,
bldg_sqft_per_job=400.0)
print 'Constructed %s %s buildings, totaling %s new job spaces.' % (len(new_buildings), non_residential_form, new_buildings.non_residential_sqft.sum()/400)
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if non_residential_form == 'light_industrial':
new_buildings['development_type_id'] = 2
elif non_residential_form == 'heavy_industrial':
new_buildings['development_type_id'] = 3
elif non_residential_form == 'office':
new_buildings['development_type_id'] = 4
elif non_residential_form == 'retail':
new_buildings['development_type_id'] = 5
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
sim.add_table("feasibility", dev.feasibility)
def non_residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
dev = developer.Developer(feas)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
year = get_year()
targets = non_residential_space_targets()
for non_residential_form in [
'heavy_industrial', 'light_industrial', 'retail', 'office'
]:
if non_residential_form in targets.keys():
old_buildings = sim.get_table('buildings').to_frame(
sim.get_table('buildings').local_columns)
target = targets[non_residential_form]
target = target / 400
print 'Job space target for %s is %s.' % (non_residential_form,
target)
if target > 0:
new_buildings = dev.pick(non_residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_job_spaces,
max_parcel_size=2000000,
min_unit_size=0,
drop_after_build=True,
residential=False,
bldg_sqft_per_job=400.0)
print 'Constructed %s %s buildings, totaling %s new job spaces.' % (
len(new_buildings), non_residential_form,
new_buildings.non_residential_sqft.sum() / 400)
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if non_residential_form == 'light_industrial':
new_buildings['development_type_id'] = 2
elif non_residential_form == 'heavy_industrial':
new_buildings['development_type_id'] = 3
elif non_residential_form == 'office':
new_buildings['development_type_id'] = 4
elif non_residential_form == 'retail':
new_buildings['development_type_id'] = 5
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(
new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2 - l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(
drop_buildings.index)
print "Unplaced {} before: {}".format(
tbl, len(agents.query("building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(
tbl, len(agents.query("building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
sim.add_table("feasibility", dev.feasibility)
def run_developer(dev, residential_form, target, year, build=False):
old_buildings = sim.get_table('buildings').to_frame(
sim.get_table('buildings').local_columns)
parcels = sim.get_table('parcels')
print 'Residential unit target for %s is %s.' % (residential_form, target)
if target > 0:
print residential_form
drop_after_build = True if build else False
new_buildings = dev.pick(residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_residential_units,
max_parcel_size=2000000,
min_unit_size=400,
drop_after_build=True,
residential=True,
bldg_sqft_per_job=400.0)
if build:
print 'Constructed %s %s buildings, totaling %s new residential_units' % (
len(new_buildings), residential_form,
new_buildings.residential_units.sum())
overshoot = new_buildings.residential_units.sum() - target
print 'Overshot target by %s units' % (overshoot)
print 'Biggest development has %s units' % new_buildings.residential_units.max(
)
if overshoot > 1:
to_remove = new_buildings[['parcel_id',
'residential_units']].copy()
to_remove = to_remove[
to_remove.residential_units < 4].set_index('parcel_id')
to_remove = to_remove.sort('residential_units')
to_remove['du_cumsum'] = to_remove.residential_units.cumsum()
idx_to_remove = np.searchsorted(to_remove.du_cumsum, overshoot)
parcel_ids_to_remove = to_remove.index.values[:(
idx_to_remove[0] + 1)]
print 'Removing %s units to match target' % to_remove.residential_units.values[:(
idx_to_remove[0] + 1)].sum()
new_buildings = new_buildings[~new_buildings.parcel_id.
isin(parcel_ids_to_remove)]
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if residential_form == 'sf_detached':
new_buildings['development_type_id'] = 19
elif residential_form == 'sf_attached':
new_buildings['development_type_id'] = 20
elif residential_form == 'mf_residential':
new_buildings['development_type_id'] = 21
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(
new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2 - l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(
drop_buildings.index)
print "Unplaced {} before: {}".format(
tbl, len(agents.query("building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(
tbl, len(agents.query("building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
else:
return new_buildings.residential_units.sum()
# Use codes were classified manually because the assessor classifications
# are meant for property tax purposes. These classifications should be
# reviewed and revised.
res_codes = {
'single': ['01', '51', '52', '53'],
'multi':
[string.zfill(i, 2) for i in range(2, 6) + range(7, 10) + range(89, 99)],
'mixed': []
}
exempt_codes = []
## Register input tables.
tf = TableFrame(staging.parcels_smt, index_col='apn')
sim.add_table('parcels_in', tf, copy_col=False)
@sim.table(cache=True)
def roll():
mdb = loader.get_path(
'built/parcel/2010/smt/Property Characteristics/ASSESSOR_ROLL.mdb')
csv = os.path.splitext(mdb)[0] + '.csv'
if not os.path.exists(csv):
with open(csv, 'w') as f:
# Export of MS Access database requires mdbtools.
subprocess.check_call(['mdb-export', mdb, '2009_ROLL'], stdout=f)
df = pd.read_csv(csv, dtype={'APN': str}, low_memory=False)
def neighborhood_vars(net):
nodes = networks.from_yaml(net, "neighborhood_vars.yaml")
nodes = nodes.fillna(0)
print nodes.describe()
sim.add_table("nodes", nodes)
