def conditional_upzone(scenario, scenario_inputs, attr_name, upzone_name):
"""
Parameters
----------
scenario : str
The name of the active scenario (set to "baseline" if no scenario
zoning)
scenario_inputs : dict
Dictionary of scenario options - keys are scenario names and values
are also dictionaries of key-value paris for scenario inputs. Right
now "zoning_table_name" should be set to the table that contains the
scenario based zoning for that scenario
attr_name : str
The name of the attribute in the baseline zoning table
upzone_name : str
The name of the attribute in the scenario zoning table
Returns
-------
The new zoning per parcel which is increased if the scenario based
zoning is higher than the baseline zoning
"""
zoning_baseline = sim.get_table(
scenario_inputs["baseline"]["zoning_table_name"])
attr = zoning_baseline[attr_name]
if scenario != "baseline":
zoning_scenario = sim.get_table(
scenario_inputs[scenario]["zoning_table_name"])
upzone = zoning_scenario[upzone_name].dropna()
attr = pd.concat([attr, upzone], axis=1).max(skipna=True, axis=1)
return attr
def conditional_upzone(scenario, scenario_inputs, attr_name, upzone_name):
"""
Parameters
----------
scenario : str
The name of the active scenario (set to "baseline" if no scenario
zoning)
scenario_inputs : dict
Dictionary of scenario options - keys are scenario names and values
are also dictionaries of key-value paris for scenario inputs. Right
now "zoning_table_name" should be set to the table that contains the
scenario based zoning for that scenario
attr_name : str
The name of the attribute in the baseline zoning table
upzone_name : str
The name of the attribute in the scenario zoning table
Returns
-------
The new zoning per parcel which is increased if the scenario based
zoning is higher than the baseline zoning
"""
zoning_baseline = sim.get_table(
scenario_inputs["baseline"]["zoning_table_name"])
attr = zoning_baseline[attr_name]
if scenario != "baseline":
zoning_scenario = sim.get_table(
scenario_inputs[scenario]["zoning_table_name"])
upzone = zoning_scenario[upzone_name].dropna()
attr = pd.concat([attr, upzone], axis=1).max(skipna=True, axis=1)
return attr
def model_integration_indicators():
year = get_year()
#Households by MGRA to PASEF
print 'Exporting indicators: households by MGRA to PASEF'
hh = sim.get_table('households')
hh = hh.to_frame(hh.local_columns + ['mgra_id', 'activity_id'])
pasef_indicators = hh.groupby(['mgra_id',
'activity_id']).size().reset_index()
pasef_indicators.columns = [
'mgra_id', 'activity_id', 'number_of_households'
]
pasef_indicators.to_csv('./data/pasef/mgra_hh_%s.csv' % year, index=False)
#Space by LUZ to PECAS
print 'Exporting indicators: space by LUZ to PECAS'
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns + ['luz_id'])
pecas_res_indicators = b[b.residential_units > 0].groupby(
['luz_id',
'development_type_id']).residential_units.sum().reset_index()
pecas_res_indicators.columns = [
'luz_id', 'development_type_id', 'residential_units'
]
pecas_res_indicators.to_csv(
'./data/pecas_urbansim_exchange/luz_du_%s.csv' % year, index=False)
pecas_nonres_indicators = b[b.non_residential_sqft > 0].groupby(
['luz_id',
'development_type_id']).non_residential_sqft.sum().reset_index()
pecas_nonres_indicators.columns = [
'luz_id', 'development_type_id', 'non_residential_sqft'
]
pecas_nonres_indicators.to_csv(
'./data/pecas_urbansim_exchange/luz_nrsf_%s.csv' % year, index=False)
def parcel_is_allowed(form):
form_to_btype = sim.get_injectable("form_to_btype")
# we have zoning by building type but want
# to know if specific forms are allowed
allowed = [sim.get_table('zoning_baseline')
['type%d' % typ] == 't' for typ in form_to_btype[form]]
return pd.concat(allowed, axis=1).max(axis=1).\
reindex(sim.get_table('parcels').index).fillna(False)
def parcel_is_allowed(form):
form_to_btype = sim.get_injectable("form_to_btype")
# we have zoning by building type but want
# to know if specific forms are allowed
allowed = [sim.get_table('zoning_baseline')
['type%d' % typ] == 't' for typ in form_to_btype[form]]
return pd.concat(allowed, axis=1).max(axis=1).\
reindex(sim.get_table('parcels').index).fillna(False)
def conditional_upzone(scenario, attr_name, upzone_name):
scenario_inputs = sim.get_injectable("scenario_inputs")
zoning_baseline = sim.get_table(
scenario_inputs["baseline"]["zoning_table_name"])
attr = zoning_baseline[attr_name]
if scenario != "baseline":
zoning_scenario = sim.get_table(
scenario_inputs[scenario]["zoning_table_name"])
upzone = zoning_scenario[upzone_name].dropna()
attr = pd.concat([attr, upzone], axis=1).max(skipna=True, axis=1)
return attr
def conditional_upzone(scenario, attr_name, upzone_name):
scenario_inputs = sim.get_injectable("scenario_inputs")
zoning_baseline = sim.get_table(
scenario_inputs["baseline"]["zoning_table_name"])
attr = zoning_baseline[attr_name]
if scenario != "baseline":
zoning_scenario = sim.get_table(
scenario_inputs[scenario]["zoning_table_name"])
upzone = zoning_scenario[upzone_name].dropna()
attr = pd.concat([attr, upzone], axis=1).max(skipna=True, axis=1)
return attr
def price_vars(net):
nodes2 = networks.from_yaml(net, "price_vars.yaml")
nodes2 = nodes2.fillna(0)
print nodes2.describe()
nodes = sim.get_table('nodes')
nodes = nodes.to_frame().join(nodes2)
sim.add_table("nodes", nodes)
def residential_space_targets():
defm_resunit_controls = pd.read_csv('data/defm_res_unit_controls.csv')
buildings = sim.get_table('buildings').to_frame(
columns=['development_type_id', 'residential_units'])
number_sf_units = buildings[buildings.development_type_id.isin(
[19])].residential_units.sum()
number_sfa_units = buildings[buildings.development_type_id ==
20].residential_units.sum()
number_mf_units = buildings[buildings.development_type_id ==
21].residential_units.sum()
year = get_year()
sf_target = defm_resunit_controls[
defm_resunit_controls.year ==
year].to_dict()['single_family'].itervalues().next()
sfa_target = defm_resunit_controls[
defm_resunit_controls.year ==
year].to_dict()['sf_attached'].itervalues().next()
mf_target = defm_resunit_controls[
defm_resunit_controls.year ==
year].to_dict()['multi_family'].itervalues().next()
mf_difference = mf_target - number_mf_units
sf_difference = sf_target - number_sf_units
sfa_difference = sfa_target - number_sfa_units
targets = {
'mf_residential': mf_difference,
'sf_detached': sf_difference,
'sf_attached': sfa_difference
}
return targets
def price_vars(net):
nodes2 = networks.from_yaml(net, "price_vars.yaml")
nodes2 = nodes2.fillna(0)
print nodes2.describe()
nodes = sim.get_table('nodes')
nodes = nodes.to_frame().join(nodes2)
sim.add_table("nodes", nodes)
def non_residential_space_targets():
vacancy_multiplier = 1.7
defm_nonres_controls = pd.read_csv('data/non_res_space_control.csv')
buildings = sim.get_table('buildings').to_frame(columns = ['development_type_id', 'non_residential_sqft'])
light_industrial_sqft = buildings[buildings.development_type_id == 2].non_residential_sqft.sum()
heavy_industrial_sqft = buildings[buildings.development_type_id == 3].non_residential_sqft.sum()
office_sqft = buildings[buildings.development_type_id == 4].non_residential_sqft.sum()
retail_sqft = buildings[buildings.development_type_id == 5].non_residential_sqft.sum()
year = get_year()
defm_nonres_controls = defm_nonres_controls[defm_nonres_controls.yr == year]
light_industrial_target = defm_nonres_controls[defm_nonres_controls.development_type_id == 2].total_min_sqft.values[0] * vacancy_multiplier
heavy_industrial_target = defm_nonres_controls[defm_nonres_controls.development_type_id == 3].total_min_sqft.values[0] * vacancy_multiplier
office_target = defm_nonres_controls[defm_nonres_controls.development_type_id == 4].total_min_sqft.values[0] * vacancy_multiplier
retail_target = defm_nonres_controls[defm_nonres_controls.development_type_id == 5].total_min_sqft.values[0] * vacancy_multiplier
light_industrial_difference = light_industrial_target - light_industrial_sqft
heavy_industrial_difference = heavy_industrial_target - heavy_industrial_sqft
office_difference = office_target - office_sqft
retail_difference = retail_target - retail_sqft
targets = {'light_industrial':light_industrial_difference, 'heavy_industrial':heavy_industrial_difference, 'office':office_difference, 'retail':retail_difference}
return targets
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use : 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use : parking_rate}
if use == 'retail':
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use : [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id == devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule, fee_schedule_devtype, left_on = 'fee_schedule_id', right_on = 'fee_schedule_id')
parcel_fee_schedule_devtype['development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial*parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby('parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data = parcel_fees_processed, index = parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees*pf.config.cap_rate
return run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = parcel_filter)
def plot(table_names=None):
"""
Plot relationships between columns and tables using Graphviz.
Parameters
----------
table_names : iterable of str, optional
Names of UrbanSim registered tables to plot.
Defaults to all registered tables.
Returns
-------
graph : pygraphviz.AGraph
PyGraphviz graph object.
"""
if not table_names:
# Default to all registered tables.
table_names = simulation.list_tables()
graph = AGraph(directed=True)
graph.graph_attr['fontname'] = 'Sans'
graph.graph_attr['fontsize'] = 28
graph.node_attr['shape'] = 'box'
graph.node_attr['fontname'] = 'Sans'
graph.node_attr['fontcolor'] = 'blue'
graph.edge_attr['weight'] = 2
# Add each registered table as a subgraph with columns as nodes.
for table_name in table_names:
subgraph = graph.add_subgraph(name='cluster_' + table_name,
label=table_name,
fontcolor='red')
table = simulation.get_table(table_name)
for column_name in table.columns:
full_name = table_name + '.' + column_name
subgraph.add_node(full_name, label=column_name)
# Iterate over computed columns to build nodes.
for key, wrapped_col in simulation._COLUMNS.items():
table_name = key[0]
column_name = key[1]
# Combine inputs from argument names and argument default values.
args = list(wrapped_col._argspec.args)
if wrapped_col._argspec.defaults:
default_args = list(wrapped_col._argspec.defaults)
else:
default_args = []
inputs = args[:len(args) - len(default_args)] + default_args
# Create edge from each input column to the computed column.
for input_name in inputs:
full_name = table_name + '.' + column_name
graph.add_edge(input_name, full_name)
graph.layout(prog='dot')
return graph
def plot(table_names=None):
"""
Plot relationships between columns and tables using Graphviz.
Parameters
----------
table_names : iterable of str, optional
Names of UrbanSim registered tables to plot.
Defaults to all registered tables.
Returns
-------
graph : pygraphviz.AGraph
PyGraphviz graph object.
"""
if not table_names:
# Default to all registered tables.
table_names = simulation.list_tables()
graph = AGraph(directed=True)
graph.graph_attr['fontname'] = 'Sans'
graph.graph_attr['fontsize'] = 28
graph.node_attr['shape'] = 'box'
graph.node_attr['fontname'] = 'Sans'
graph.node_attr['fontcolor'] = 'blue'
graph.edge_attr['weight'] = 2
# Add each registered table as a subgraph with columns as nodes.
for table_name in table_names:
subgraph = graph.add_subgraph(name='cluster_' + table_name,
label=table_name, fontcolor='red')
table = simulation.get_table(table_name)
for column_name in table.columns:
full_name = table_name + '.' + column_name
subgraph.add_node(full_name, label=column_name)
# Iterate over computed columns to build nodes.
for key, wrapped_col in simulation._COLUMNS.items():
table_name = key[0]
column_name = key[1]
# Combine inputs from argument names and argument default values.
args = list(wrapped_col._argspec.args)
if wrapped_col._argspec.defaults:
default_args = list(wrapped_col._argspec.defaults)
else:
default_args = []
inputs = args[:len(args) - len(default_args)] + default_args
# Create edge from each input column to the computed column.
for input_name in inputs:
full_name = table_name + '.' + column_name
graph.add_edge(input_name, full_name)
graph.layout(prog='dot')
return graph
def from_yaml(cfgname):
print "Computing accessibility variables"
cfg = yaml.load(open(misc.config(cfgname)))
nodes = pd.DataFrame(index=NETWORKS.external_nodeids)
node_col = cfg.get('node_col', None)
for variable in cfg['variable_definitions']:
name = variable["name"]
print "Computing %s" % name
decay = {
"exponential": "DECAY_EXP",
"linear": "DECAY_LINEAR",
"flat": "DECAY_FLAT"
}.get(variable.get("decay", "linear"))
agg = {
"sum": "AGG_SUM",
"average": "AGG_AVE",
"stddev": "AGG_STDDEV"
}.get(variable.get("aggregation", "sum"))
vname = variable.get("varname", None)
radius = variable["radius"]
dfname = variable["dataframe"]
flds = [vname] if vname else []
if 'add_fields' in variable:
flds += variable['add_fields']
if node_col:
flds.append(node_col)
logger.info(" Fields available to accvar =", ', '.join(flds))
df = sim.get_table(dfname).to_frame(flds)
if "filters" in variable:
df = util.apply_filter_query(df, variable["filters"])
logger.info(" Filters = %s" % variable["filters"])
logger.info(" dataframe = %s, varname=%s" % (dfname, vname))
logger.info(" radius = %s, aggregation = %s, decay = %s" % (
radius, agg, decay))
nodes[name] = NETWORKS.accvar(
df, radius, node_ids=node_col, agg=agg, decay=decay,
vname=vname).astype('float').values
if "apply" in variable:
nodes[name] = nodes[name].apply(eval(variable["apply"]))
return nodes
def job_spaces():
store = sim.get_injectable('store')
b = sim.get_table('buildings').to_frame(['luz_id', 'development_type_id','non_residential_sqft'])
bsqft_job = store['building_sqft_per_job']
merged = pd.merge(b.reset_index(), bsqft_job, left_on = ['luz_id', 'development_type_id'], right_on = ['luz_id', 'development_type_id'])
merged = merged.set_index('building_id')
merged.sqft_per_emp[merged.sqft_per_emp < 40] = 40
merged['job_spaces'] = np.ceil(merged.non_residential_sqft / merged.sqft_per_emp)
job_spaces = pd.Series(merged.job_spaces, index = b.index)
b['job_spaces'] = job_spaces
b.job_spaces[b.job_spaces.isnull()] = np.ceil(b.non_residential_sqft/200.0)
return b.job_spaces
def model_integration_indicators():
year = get_year()
#Households by MGRA to PASEF
print 'Exporting indicators: households by MGRA to PASEF'
hh = sim.get_table('households')
hh = hh.to_frame(hh.local_columns + ['mgra_id', 'activity_id'])
pasef_indicators = hh.groupby(['mgra_id', 'activity_id']).size().reset_index()
pasef_indicators.columns = ['mgra_id', 'activity_id', 'number_of_households']
pasef_indicators.to_csv('./data/pasef/mgra_hh_%s.csv'%year, index = False)
#Space by LUZ to PECAS
print 'Exporting indicators: space by LUZ to PECAS'
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns + ['luz_id'])
pecas_res_indicators = b[b.residential_units > 0].groupby(['luz_id', 'development_type_id']).residential_units.sum().reset_index()
pecas_res_indicators.columns = ['luz_id', 'development_type_id', 'residential_units']
pecas_res_indicators.to_csv('./data/pecas_urbansim_exchange/luz_du_%s.csv'%year, index = False)
pecas_nonres_indicators = b[b.non_residential_sqft > 0].groupby(['luz_id', 'development_type_id']).non_residential_sqft.sum().reset_index()
pecas_nonres_indicators.columns = ['luz_id', 'development_type_id', 'non_residential_sqft']
pecas_nonres_indicators.to_csv('./data/pecas_urbansim_exchange/luz_nrsf_%s.csv'%year, index = False)
def zoning_allowed_uses(store, parcels):
parcels_allowed = store['zoning_allowed_uses']
parcels = sim.get_table('parcels').to_frame(columns = ['zoning_id',])
allowed_df = pd.DataFrame(index = parcels.index)
for devtype in np.unique(parcels_allowed.development_type_id):
devtype_allowed = parcels_allowed[parcels_allowed.development_type_id == devtype].set_index('zoning_id')
allowed = misc.reindex(devtype_allowed.development_type_id, parcels.zoning_id)
df = pd.DataFrame(index=allowed.index)
df['allowed'] = False
df[~allowed.isnull()] = True
allowed_df[devtype] = df.allowed
return allowed_df
def write_parcel_output(self,
add_xy=None):
"""
Write the parcel-level output to a csv file
Parameters
----------
add_xy : dictionary (optional)
Used to add x, y values to the output - an example dictionary is
pasted below - the parameters should be fairly self explanatory.
Note that from_epsg and to_epsg can be omitted in which case the
coordinate system is not changed. NOTE: pyproj is required
if changing coordinate systems::
{
"xy_table": "parcels",
"foreign_key": "parcel_id",
"x_col": "x",
"y_col": "y",
"from_epsg": 3740,
"to_epsg": 4326
}
Returns
-------
Nothing
"""
if self.parcel_output is None:
return
po = self.parcel_output
if add_xy is not None:
x_name, y_name = add_xy["x_col"], add_xy["y_col"]
xy_joinname = add_xy["foreign_key"]
xy_df = sim.get_table(add_xy["xy_table"])
po[x_name] = misc.reindex(xy_df[x_name], po[xy_joinname])
po[y_name] = misc.reindex(xy_df[y_name], po[xy_joinname])
if "from_epsg" in add_xy and "to_epsg" in add_xy:
import pyproj
p1 = pyproj.Proj('+init=epsg:%d' % add_xy["from_epsg"])
p2 = pyproj.Proj('+init=epsg:%d' % add_xy["to_epsg"])
x2, y2 = pyproj.transform(p1, p2,
po[x_name].values,
po[y_name].values)
po[x_name], po[y_name] = x2, y2
po.to_csv(self.parcel_indicator_file, index_label="development_id")
def write_parcel_output(self,
add_xy=None):
"""
Write the parcel-level output to a csv file
Parameters
----------
add_xy : dictionary (optional)
Used to add x, y values to the output - an example dictionary is
pasted below - the parameters should be fairly self explanatory.
Note that from_epsg and to_epsg can be omitted in which case the
coordinate system is not changed. NOTE: pyproj is required
if changing coordinate systems::
{
"xy_table": "parcels",
"foreign_key": "parcel_id",
"x_col": "x",
"y_col": "y",
"from_epsg": 3740,
"to_epsg": 4326
}
Returns
-------
Nothing
"""
if self.parcel_output is None:
return
po = self.parcel_output
if add_xy is not None:
x_name, y_name = add_xy["x_col"], add_xy["y_col"]
xy_joinname = add_xy["foreign_key"]
xy_df = sim.get_table(add_xy["xy_table"])
po[x_name] = misc.reindex(xy_df[x_name], po[xy_joinname])
po[y_name] = misc.reindex(xy_df[y_name], po[xy_joinname])
if "from_epsg" in add_xy and "to_epsg" in add_xy:
import pyproj
p1 = pyproj.Proj('+init=epsg:%d' % add_xy["from_epsg"])
p2 = pyproj.Proj('+init=epsg:%d' % add_xy["to_epsg"])
x2, y2 = pyproj.transform(p1, p2,
po[x_name].values,
po[y_name].values)
po[x_name], po[y_name] = x2, y2
po.to_csv(self.parcel_indicator_file, index_label="development_id")
def parcel_is_allowed(form):
parcels = sim.get_table('parcels')
zoning_allowed_uses = sim.get_table('zoning_allowed_uses').to_frame()
if form == 'sf_detached':
allowed = zoning_allowed_uses[19]
elif form == 'sf_attached':
allowed = zoning_allowed_uses[20]
elif form == 'mf_residential':
allowed = zoning_allowed_uses[21]
elif form == 'light_industrial':
allowed = zoning_allowed_uses[2]
elif form == 'heavy_industrial':
allowed = zoning_allowed_uses[3]
elif form == 'office':
allowed = zoning_allowed_uses[4]
elif form == 'retail':
allowed = zoning_allowed_uses[5]
else:
df = pd.DataFrame(index=parcels.index)
df['allowed'] = True
allowed = df.allowed
return allowed
def parcel_is_allowed(form):
parcels = sim.get_table('parcels')
zoning_allowed_uses = sim.get_table('zoning_allowed_uses').to_frame()
if form == 'sf_detached':
allowed = zoning_allowed_uses[19]
elif form == 'sf_attached':
allowed = zoning_allowed_uses[20]
elif form == 'mf_residential':
allowed = zoning_allowed_uses[21]
elif form == 'light_industrial':
allowed = zoning_allowed_uses[2]
elif form == 'heavy_industrial':
allowed = zoning_allowed_uses[3]
elif form == 'office':
allowed = zoning_allowed_uses[4]
elif form == 'retail':
allowed = zoning_allowed_uses[5]
else:
df = pd.DataFrame(index=parcels.index)
df['allowed'] = True
allowed = df.allowed
return allowed
def from_yaml(net, cfgname):
print "Computing accessibility variables"
cfg = yaml.load(open(misc.config(cfgname)))
nodes = pd.DataFrame(index=net.node_ids)
assert "node_col" in cfg, "Need to specify from where to take the node id"
node_col = cfg.get('node_col')
for variable in cfg['variable_definitions']:
name = variable["name"]
print "Computing %s" % name
decay = variable.get("decay", "linear")
agg = variable.get("aggregation", "sum")
vname = variable.get("varname", None)
radius = variable["radius"]
dfname = variable["dataframe"]
flds = [vname] if vname else []
flds.append(node_col)
if "filters" in variable:
flds += util.columns_in_filters(variable["filters"])
logger.info(" Fields available to aggregate = " + ', '.join(flds))
df = sim.get_table(dfname).to_frame(flds)
if "filters" in variable:
df = util.apply_filter_query(df, variable["filters"])
logger.info(" Filters = %s" % variable["filters"])
logger.info(" dataframe = %s, varname=%s" % (dfname, vname))
logger.info(" radius = %s, aggregation = %s, decay = %s" % (
radius, agg, decay))
# set the variable
net.set(df[node_col], variable=df[vname] if vname else None)
# aggregate it
nodes[name] = net.aggregate(radius, type=agg, decay=decay)
if "apply" in variable:
nodes[name] = nodes[name].apply(eval(variable["apply"]))
return nodes
def zoning_allowed_uses(store, parcels):
parcels_allowed = store['zoning_allowed_uses']
parcels = sim.get_table('parcels').to_frame(columns=[
'zoning_id',
])
allowed_df = pd.DataFrame(index=parcels.index)
for devtype in np.unique(parcels_allowed.development_type_id):
devtype_allowed = parcels_allowed[parcels_allowed.development_type_id
== devtype].set_index('zoning_id')
allowed = misc.reindex(devtype_allowed.development_type_id,
parcels.zoning_id)
df = pd.DataFrame(index=allowed.index)
df['allowed'] = False
df[~allowed.isnull()] = True
allowed_df[devtype] = df.allowed
return allowed_df
def job_spaces():
store = sim.get_injectable('store')
b = sim.get_table('buildings').to_frame(
['luz_id', 'development_type_id', 'non_residential_sqft'])
bsqft_job = store['building_sqft_per_job']
merged = pd.merge(b.reset_index(),
bsqft_job,
left_on=['luz_id', 'development_type_id'],
right_on=['luz_id', 'development_type_id'])
merged = merged.set_index('building_id')
merged.sqft_per_emp[merged.sqft_per_emp < 40] = 40
merged['job_spaces'] = np.ceil(merged.non_residential_sqft /
merged.sqft_per_emp)
job_spaces = pd.Series(merged.job_spaces, index=b.index)
b['job_spaces'] = job_spaces
b.job_spaces[b.job_spaces.isnull()] = np.ceil(b.non_residential_sqft /
200.0)
return b.job_spaces
def non_residential_space_targets():
vacancy_multiplier = 1.7
defm_nonres_controls = pd.read_csv('data/non_res_space_control.csv')
buildings = sim.get_table('buildings').to_frame(
columns=['development_type_id', 'non_residential_sqft'])
light_industrial_sqft = buildings[buildings.development_type_id ==
2].non_residential_sqft.sum()
heavy_industrial_sqft = buildings[buildings.development_type_id ==
3].non_residential_sqft.sum()
office_sqft = buildings[buildings.development_type_id ==
4].non_residential_sqft.sum()
retail_sqft = buildings[buildings.development_type_id ==
5].non_residential_sqft.sum()
year = get_year()
defm_nonres_controls = defm_nonres_controls[defm_nonres_controls.yr ==
year]
light_industrial_target = defm_nonres_controls[
defm_nonres_controls.development_type_id ==
2].total_min_sqft.values[0] * vacancy_multiplier
heavy_industrial_target = defm_nonres_controls[
defm_nonres_controls.development_type_id ==
3].total_min_sqft.values[0] * vacancy_multiplier
office_target = defm_nonres_controls[
defm_nonres_controls.development_type_id ==
4].total_min_sqft.values[0] * vacancy_multiplier
retail_target = defm_nonres_controls[
defm_nonres_controls.development_type_id ==
5].total_min_sqft.values[0] * vacancy_multiplier
light_industrial_difference = light_industrial_target - light_industrial_sqft
heavy_industrial_difference = heavy_industrial_target - heavy_industrial_sqft
office_difference = office_target - office_sqft
retail_difference = retail_target - retail_sqft
targets = {
'light_industrial': light_industrial_difference,
'heavy_industrial': heavy_industrial_difference,
'office': office_difference,
'retail': retail_difference
}
return targets
def residential_space_targets():
defm_resunit_controls = pd.read_csv('data/defm_res_unit_controls.csv')
buildings = sim.get_table('buildings').to_frame(columns = ['development_type_id', 'residential_units'])
number_sf_units = buildings[buildings.development_type_id.isin([19])].residential_units.sum()
number_sfa_units = buildings[buildings.development_type_id == 20].residential_units.sum()
number_mf_units = buildings[buildings.development_type_id == 21].residential_units.sum()
year = get_year()
sf_target = defm_resunit_controls[defm_resunit_controls.year == year].to_dict()['single_family'].itervalues().next()
sfa_target = defm_resunit_controls[defm_resunit_controls.year == year].to_dict()['sf_attached'].itervalues().next()
mf_target = defm_resunit_controls[defm_resunit_controls.year == year].to_dict()['multi_family'].itervalues().next()
mf_difference = mf_target - number_mf_units
sf_difference = sf_target - number_sf_units
sfa_difference = sfa_target - number_sfa_units
targets = {'mf_residential':mf_difference, 'sf_detached':sf_difference, 'sf_attached':sfa_difference}
return targets
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use: 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use: parking_rate}
if use == 'retail':
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use: [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id ==
devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule,
fee_schedule_devtype,
left_on='fee_schedule_id',
right_on='fee_schedule_id')
parcel_fee_schedule_devtype[
'development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial * parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby(
'parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data=parcel_fees_processed,
index=parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees * pf.config.cap_rate
return run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=parcel_filter)
def lcm_simulate(cfg, choosers, buildings, join_tbls, out_fname,
supply_fname, vacant_fname,
enable_supply_correction=None):
"""
Simulate the location choices for the specified choosers
Parameters
----------
cfg : string
The name of the yaml config file from which to read the location
choice model
choosers : DataFrameWrapper
A dataframe of agents doing the choosing
buildings : DataFrameWrapper
A dataframe of buildings which the choosers are locating in and which
have a supply
join_tbls : list of strings
A list of land use dataframes to give neighborhood info around the
buildings - will be joined to the buildings using existing broadcasts.
out_fname : string
The column name to write the simulated location to
supply_fname : string
The string in the buildings table that indicates the amount of
available units there are for choosers, vacant or not
vacant_fname : string
The string in the buildings table that indicates the amount of vacant
units there will be for choosers
enable_supply_correction : Python dict
Should contain keys "price_col" and "submarket_col" which are set to
the column names in buildings which contain the column for prices and
an identifier which segments buildings into submarkets
"""
cfg = misc.config(cfg)
choosers_df = utils.to_frame(choosers, [], cfg, additional_columns=[out_fname])
additional_columns = [supply_fname, vacant_fname]
if enable_supply_correction is not None and \
"submarket_col" in enable_supply_correction:
additional_columns += [enable_supply_correction["submarket_col"]]
if enable_supply_correction is not None and \
"price_col" in enable_supply_correction:
additional_columns += [enable_supply_correction["price_col"]]
locations_df = utils.to_frame(buildings, join_tbls, cfg,
additional_columns=additional_columns)
available_units = buildings[supply_fname]
vacant_units = buildings[vacant_fname]
print "There are %d total available units" % available_units.sum()
print " and %d total choosers" % len(choosers)
print " but there are %d overfull buildings" % \
len(vacant_units[vacant_units < 0])
vacant_units = vacant_units[vacant_units > 0]
# sometimes there are vacant units for buildings that are not in the
# locations_df, which happens for reasons explained in the warning below
indexes = np.repeat(vacant_units.index.values,
vacant_units.values.astype('int'))
isin = pd.Series(indexes).isin(locations_df.index)
missing = len(isin[isin == False])
indexes = indexes[isin.values]
units = locations_df.loc[indexes].reset_index()
utils.check_nas(units)
print " for a total of %d temporarily empty units" % vacant_units.sum()
print " in %d buildings total in the region" % len(vacant_units)
if missing > 0:
print "WARNING: %d indexes aren't found in the locations df -" % \
missing
print " this is usually because of a few records that don't join "
print " correctly between the locations df and the aggregations tables"
movers = choosers_df[choosers_df[out_fname] == -1]
print "There are %d total movers for this LCM" % len(movers)
if enable_supply_correction is not None:
assert isinstance(enable_supply_correction, dict)
assert "price_col" in enable_supply_correction
price_col = enable_supply_correction["price_col"]
assert "submarket_col" in enable_supply_correction
submarket_col = enable_supply_correction["submarket_col"]
lcm = utils.yaml_to_class(cfg).from_yaml(str_or_buffer=cfg)
if enable_supply_correction.get("warm_start", False) is True:
raise NotImplementedError()
multiplier_func = enable_supply_correction.get("multiplier_func", None)
if multiplier_func is not None:
multiplier_func = sim.get_injectable(multiplier_func)
kwargs = enable_supply_correction.get('kwargs', {})
new_prices, submarkets_ratios = supply_and_demand(
lcm,
movers,
units,
submarket_col,
price_col,
base_multiplier=None,
multiplier_func=multiplier_func,
**kwargs)
# we will only get back new prices for those alternatives
# that pass the filter - might need to specify the table in
# order to get the complete index of possible submarkets
submarket_table = enable_supply_correction.get("submarket_table", None)
if submarket_table is not None:
submarkets_ratios = submarkets_ratios.reindex(
sim.get_table(submarket_table).index).fillna(1)
# write final shifters to the submarket_table for use in debugging
sim.get_table(submarket_table)["price_shifters"] = submarkets_ratios
print "Running supply and demand"
print "Simulated Prices"
print buildings[price_col].describe()
print "Submarket Price Shifters"
print submarkets_ratios.describe()
# we want new prices on the buildings, not on the units, so apply
# shifters directly to buildings and ignore unit prices
sim.add_column(buildings.name,
price_col+"_hedonic", buildings[price_col])
new_prices = buildings[price_col] * \
submarkets_ratios.loc[buildings[submarket_col]].values
buildings.update_col_from_series(price_col, new_prices)
print "Adjusted Prices"
print buildings[price_col].describe()
#if len(movers) > vacant_units.sum():
# print "WARNING: Not enough locations for movers"
# print " reducing locations to size of movers for performance gain"
# movers = movers.head(vacant_units.sum())
new_units, _ = utils.yaml_to_class(cfg).predict_from_cfg(movers, units, cfg, location_ratio=100.0)
# new_units returns nans when there aren't enough units,
# get rid of them and they'll stay as -1s
new_units = new_units.dropna()
# go from units back to buildings
new_buildings = pd.Series(units.loc[new_units.values][out_fname].values,
index=new_units.index)
choosers.update_col_from_series(out_fname, new_buildings)
utils._print_number_unplaced(choosers, out_fname)
if enable_supply_correction is not None:
new_prices = buildings[price_col]
if "clip_final_price_low" in enable_supply_correction:
new_prices = new_prices.clip(lower=enable_supply_correction[
"clip_final_price_low"])
if "clip_final_price_high" in enable_supply_correction:
new_prices = new_prices.clip(upper=enable_supply_correction[
"clip_final_price_high"])
buildings.update_col_from_series(price_col, new_prices)
vacant_units = buildings[vacant_fname]
print " and there are now %d empty units" % vacant_units.sum()
print " and %d overfull buildings" % len(vacant_units[vacant_units < 0])
import pandas as pd, numpy as np
import models
import urbansim.sim.simulation as sim
np.random.seed(1)
# Simulation run
##Single-year
# sim.run(["build_networks", "neighborhood_vars", "rsh_simulate", "nrh_simulate", "nrh_simulate2",
# "price_vars", "feasibility", "residential_developer", "non_residential_developer"])
##Multi-year
sim.run(["build_networks"]) #initialize network accessibility engine
sim.run(["neighborhood_vars", #"scheduled_development_events", #scheduled events and accessibility variables
"rsh_simulate", "nrh_simulate", "nrh_simulate2", #price models
"jobs_transition", "elcm_simulate", "households_transition", "hlcm_luz_simulate", #demand/location models
"price_vars", "feasibility", "residential_developer", "non_residential_developer", #supply/proforma models
], years=[2013, 2014, 2015,])
# Summarize results at MSA level
b = sim.get_table('buildings').to_frame(columns = ['msa_id', 'mgra_id', 'residential_units', 'non_residential_sqft', 'note'])
new_du_by_msa = b[b.note == 'simulated'].groupby('msa_id').residential_units.sum()
new_nrsf_by_msa = b[b.note == 'simulated'].groupby('msa_id').non_residential_sqft.sum()
proportion_du_by_msa = new_du_by_msa / new_du_by_msa.sum()
proportion_nrsf_by_msa = new_nrsf_by_msa / new_nrsf_by_msa.sum()
# Write out indicators to calibration directory
proportion_du_by_msa.to_csv('.\\data\\calibration\\msa_du_simulated.csv', header = True)
proportion_nrsf_by_msa.to_csv('.\\data\\calibration\\msa_nrsf_simulated.csv', header = True)
from psrc_urbansim.accessibility.utils import load_network, assign_nodes_to_dataset
import pandas as pd
import os
from urbansim.utils import misc
# this imports all the parcels variables we need
import psrc_urbansim.variables
# this computes the variables
import urbansim.sim.simulation as sim
distances = [500, 1000, 3000]
attributes = ["number_of_households", "number_of_jobs"]
output_file = "parcel_accessibilities.h5"
net = load_network(precompute=distances)
parcels = sim.get_table("parcels")
assign_nodes_to_dataset(parcels, net)
outstore = pd.HDFStore(output_file)
for attr in attributes:
net.set(parcels["node_ids"], variable=parcels[attr], name=attr)
for dist in distances:
res_name = "%s_%s" % (attr, dist)
sim.add_column("parcels", res_name, net.aggregate(dist, type="sum", decay="linear", name=attr))
outstore["parcels"] = parcels.to_frame()
outstore.close()
def aggregations(settings):
if "aggregation_tables" not in settings or \
settings["aggregation_tables"] is None:
return []
return [sim.get_table(tbl) for tbl in settings["aggregation_tables"]]
def run_developer(forms, agents, buildings, supply_fname, parcel_size,
ave_unit_size, total_units, feasibility, year=None,
target_vacancy=.1, form_to_btype_callback=None,
add_more_columns_callback=None, max_parcel_size=2000000,
residential=True, bldg_sqft_per_job=400.0,
min_unit_size=400, remove_developed_buildings=True,
unplace_agents=['households', 'jobs']):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
min_unit_size : float
Passed directly to dev.pick - min unit size that is valid
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
remove_redeveloped_buildings : optional, boolean (default True)
Remove all buildings on the parcels which are being developed on
unplace_agents : optional : list of strings (default ['households', 'jobs'])
For all tables in the list, will look for field building_id and set
it to -1 for buildings which are removed - only executed if
remove_developed_buildings is true
Returns
-------
Writes the result back to the buildings table and returns the new
buildings with available debugging information on each new building
"""
dev = developer.Developer(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
min_unit_size=min_unit_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
if new_buildings is None:
return
if len(new_buildings) == 0:
return new_buildings
if year is not None:
new_buildings["year_built"] = year
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings.\
apply(form_to_btype_callback, axis=1)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
ret_buildings = new_buildings
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
old_buildings = buildings.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in unplace_agents:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
return ret_buildings
from flask import Flask, jsonify
from flask.ext.cors import CORS
import pandas as pd
import urbansim.sim.simulation as sim
app = Flask(__name__)
app.debug = True
cors = CORS(app)
MAX_PARCELS_RETURNED = 5000
print "Loading"
store = pd.HDFStore('data/bayarea_v3.h5')
parcels = sim.get_table('parcels').to_frame()
buildings = sim.get_table('buildings').to_frame()
#households = sim.get_table('households').to_frame()
#jobs = sim.get_table('jobs').to_frame()
zoning = sim.get_table('zoning_baseline').to_frame()
print "Ready"
@app.route('/extract/<query>')
def get_data(query):
print "Got query:", query
#global parcels, buildings, households, jobs, zoning_baseline
global parcels, buildings, zoning_baseline
def run_developer(forms, agents, buildings, buildings_all, supply_fname, parcel_size,
ave_unit_size, total_units, feasibility, year=None,
target_vacancy=.1, form_to_btype_callback=None,
add_more_columns_callback=None, max_parcel_size=34647265,
residential=True, bldg_sqft_per_job=400.0,
min_unit_size=400, remove_developed_buildings=True,
unplace_agents=['households', 'jobs']):
"""
Run the developer model to pick and build buildings
Parameters
----------
forms : string or list of strings
Passed directly dev.pick
agents : DataFrame Wrapper
Used to compute the current demand for units/floorspace in the area
buildings : DataFrame Wrapper
Used to compute the current supply of units/floorspace in the area
buildings_all:
Buildings for the entire region, used to write back to buildings table
supply_fname : string
Identifies the column in buildings which indicates the supply of
units/floorspace
parcel_size : Series
Passed directly to dev.pick
ave_unit_size : Series
Passed directly to dev.pick - average residential unit size
total_units : Series
Passed directly to dev.pick - total current residential_units /
job_spaces
feasibility : DataFrame Wrapper
The output from feasibility above (the table called 'feasibility')
year : int
The year of the simulation - will be assigned to 'year_built' on the
new buildings
target_vacancy : float
The target vacancy rate - used to determine how much to build
form_to_btype_callback : function
Will be used to convert the 'forms' in the pro forma to
'building_type_id' in the larger model
add_more_columns_callback : function
Takes a dataframe and returns a dataframe - is used to make custom
modifications to the new buildings that get added
max_parcel_size : float
Passed directly to dev.pick - max parcel size to consider
min_unit_size : float
Passed directly to dev.pick - min unit size that is valid
residential : boolean
Passed directly to dev.pick - switches between adding/computing
residential_units and job_spaces
bldg_sqft_per_job : float
Passed directly to dev.pick - specified the multiplier between
floor spaces and job spaces for this form (does not vary by parcel
as ave_unit_size does)
remove_redeveloped_buildings : optional, boolean (default True)
Remove all buildings on the parcels which are being developed on
unplace_agents : optional : list of strings (default ['households', 'jobs'])
For all tables in the list, will look for field building_id and set
it to -1 for buildings which are removed - only executed if
remove_developed_buildings is true
Returns
-------
Writes the result back to the buildings table and returns the new
buildings with available debugging information on each new building
"""
dev = developer.Developer(feasibility.to_frame())
#dev = WFRCDeveloper.WFRCDeveloper(feasibility.to_frame())
target_units = dev.\
compute_units_to_build(len(agents),
buildings[supply_fname].sum(),
target_vacancy)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
new_buildings = dev.pick(forms,
target_units,
parcel_size,
ave_unit_size,
total_units,
max_parcel_size=max_parcel_size,
min_unit_size=min_unit_size,
drop_after_build=True,
residential=residential,
bldg_sqft_per_job=bldg_sqft_per_job)
sim.add_table("feasibility", dev.feasibility)
year = sim.get_injectable('year')
if new_buildings is None:
return
if len(new_buildings) == 0:
return new_buildings
if not isinstance(forms, list):
# form gets set only if forms is a list
new_buildings["form"] = forms
if form_to_btype_callback is not None:
new_buildings["building_type_id"] = new_buildings.\
apply(form_to_btype_callback, axis=1)
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
new_buildings["note"] = "simulated"
ret_buildings = new_buildings
if add_more_columns_callback is not None:
new_buildings = add_more_columns_callback(new_buildings)
if year is not None:
new_buildings["year_built"] = year
print "Adding {:,} buildings with {:,} {}".\
format(len(new_buildings),
int(new_buildings[supply_fname].sum()),
supply_fname)
print "{:,} feasible buildings after running developer".format(
len(dev.feasibility))
old_buildings = buildings.to_frame(buildings.local_columns)
old_buildings_all = buildings_all.to_frame(buildings.local_columns)
new_buildings = new_buildings[buildings.local_columns]
if remove_developed_buildings:
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
redev_buildings_all = old_buildings_all.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
drop_buildings_all = old_buildings_all[redev_buildings_all]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
old_buildings_all = old_buildings_all[np.logical_not(redev_buildings_all)]
l2 = len(old_buildings)
print "before dropped l:" + str(l)
print "after dropped l2: " + str(l2)
#print redev_buildings
#print drop_buildings
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in unplace_agents:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
#displaced_agents = agents.building_id.isin(drop_buildings.index)
displaced_agents = agents.building_id.isin(drop_buildings_all.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
all_buildings = dev.merge(old_buildings_all, new_buildings)
sim.add_table("buildings", all_buildings)
return ret_buildings
def run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(
sim.get_table('nodes')[use],
sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv(
'.\\data\\calibration\\msa_shifters.csv').set_index(
'msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv(
'.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv(
'.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][
parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev,
form,
target_units,
get_year(),
build=False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form] / 400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(
feasibility.non_residential_sqft / 400.0)
feasibility[
'net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(sim.get_table('nodes')[use], sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv('.\\data\\calibration\\msa_shifters.csv').set_index('msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv('.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv('.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev, form, target_units, get_year(), build = False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form]/400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(feasibility.non_residential_sqft / 400.0)
feasibility['net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def feasibility(parcels, settings,
parcel_sales_price_sqft_func,
parcel_is_allowed_func):
# Fee table preprocessing
fee_schedule = sim.get_table('fee_schedule').to_frame()
parcel_fee_schedule = sim.get_table('parcel_fee_schedule').to_frame()
parcels = sim.get_table('parcels').to_frame(columns = ['zoning_id','development_type_id'])
fee_schedule = fee_schedule.groupby(['fee_schedule_id', 'development_type_id']).development_fee_per_unit_space_initial.mean().reset_index()
parcel_use_allowed_callback = sim.get_injectable('parcel_is_allowed_func')
def run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(sim.get_table('nodes')[use], sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv('.\\data\\calibration\\msa_shifters.csv').set_index('msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv('.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv('.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev, form, target_units, get_year(), build = False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form]/400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(feasibility.non_residential_sqft / 400.0)
feasibility['net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form, parcels[allowed], only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def residential_proforma(form, devtype_id, parking_rate):
print form
use = 'residential'
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = True
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use : 1.0}}
pfc.uses = [use]
pfc.residential_uses = [True]
pfc.parking_rates = {use : parking_rate}
pfc.costs = {use : [170.0, 190.0, 210.0, 240.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id == devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule, fee_schedule_devtype, left_on = 'fee_schedule_id', right_on = 'fee_schedule_id')
parcel_fee_schedule_devtype['development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial*parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby('parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data = parcel_fees_processed, index = parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
return run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = parcel_filter)
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use : 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use : parking_rate}
if use == 'retail':
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use : [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use : [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id == devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule, fee_schedule_devtype, left_on = 'fee_schedule_id', right_on = 'fee_schedule_id')
parcel_fee_schedule_devtype['development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial*parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby('parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data = parcel_fees_processed, index = parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees*pf.config.cap_rate
return run_proforma_lookup(parcels, fees, pf, use, form, residential_to_yearly, parcel_filter = parcel_filter)
d = {}
##SF DETACHED proforma (devtype 19)
form = 'sf_detached'
devtype_id = 19
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##SF ATTACHED proforma (devtype 20)
form = 'sf_attached'
devtype_id = 20
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##MF_RESIDENTIAL proforma (devtype 21)
form = 'mf_residential'
devtype_id = 21
d[form] = residential_proforma(form, devtype_id, parking_rate = 1.0)
##OFFICE (devtype 4)
form = 'office'
devtype_id = 4
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate = 1.0)
##RETAIL (devtype 5)
form = 'retail'
devtype_id = 5
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate = 2.0)
##LIGHT INDUSTRIAL (devtype 2)
form = 'light_industrial'
devtype_id = 2
d[form] = nonresidential_proforma(form, devtype_id, 'industrial', parking_rate = .6)
##HEAVY INDUSTRIAL (devtype 3)
form = 'heavy_industrial'
devtype_id = 3
d[form] = nonresidential_proforma(form, devtype_id, 'industrial', parking_rate = .6)
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
(options, args) = parser.parse_args()
geo = options.geo
port = options.port
for tbl in ["zones", "fazes", "tractcity"]:
if tbl in tables[geo]:
variables2[tbl] = common_vars2
import psrc_urbansim.models
import urbansim.sim.simulation as sim
from psrc_urbansim.utils import change_store
change_store(data_file)
#import psrc_urbansim.accessibility.variables
# create a dictionary of pandas frames
d = {tbl: sim.get_table(tbl).to_frame() for tbl in tables[geo]}
# remove unwanted columns
for tbl, cols in columns_to_remove.iteritems():
if tbl in d.keys():
d[tbl].drop(cols, axis=1, inplace=True)
change_store(data_file2)
sim.clear_cache()
#import urbansim.sim.simulation as sim
#import psrc_urbansim.accessibility.variables
for tbl in variables2.keys():
d2 = sim.get_table(tbl).to_frame(columns=variables2[tbl])
d2.columns = map(lambda x: x + "_2", d2.columns)
d[tbl] = pd.merge(d[tbl], d2, left_index=True, right_index=True)
# add the id column since the join does not work if the id is an index
def parcel_average_price(use):
return misc.reindex(sim.get_table('zones_prices')[use],
sim.get_table('parcels').zone_id)
def run_developer(dev, residential_form, target, year, build = False):
old_buildings = sim.get_table('buildings').to_frame(sim.get_table('buildings').local_columns)
parcels = sim.get_table('parcels')
print 'Residential unit target for %s is %s.' % (residential_form, target)
if target > 0:
print residential_form
drop_after_build = True if build else False
new_buildings = dev.pick(residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_residential_units,
max_parcel_size=2000000,
min_unit_size=400,
drop_after_build=True,
residential=True,
bldg_sqft_per_job=400.0)
if build:
print 'Constructed %s %s buildings, totaling %s new residential_units' % (len(new_buildings), residential_form, new_buildings.residential_units.sum())
overshoot = new_buildings.residential_units.sum() - target
print 'Overshot target by %s units' % (overshoot)
print 'Biggest development has %s units' % new_buildings.residential_units.max()
if overshoot > 1:
to_remove = new_buildings[['parcel_id', 'residential_units']].copy()
to_remove = to_remove[to_remove.residential_units < 4].set_index('parcel_id')
to_remove = to_remove.sort('residential_units')
to_remove['du_cumsum'] = to_remove.residential_units.cumsum()
idx_to_remove = np.searchsorted(to_remove.du_cumsum, overshoot)
parcel_ids_to_remove = to_remove.index.values[:(idx_to_remove[0] + 1)]
print 'Removing %s units to match target' % to_remove.residential_units.values[:(idx_to_remove[0] + 1)].sum()
new_buildings = new_buildings[~new_buildings.parcel_id.isin(parcel_ids_to_remove)]
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if residential_form == 'sf_detached':
new_buildings['development_type_id'] = 19
elif residential_form == 'sf_attached':
new_buildings['development_type_id'] = 20
elif residential_form == 'mf_residential':
new_buildings['development_type_id'] = 21
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
else:
return new_buildings.residential_units.sum()
def parcel_average_price(use):
return misc.reindex(
sim.get_table('zones_prices')[use],
sim.get_table('parcels').zone_id)
def residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
year = get_year()
targets = residential_space_targets()
print "{:,} feasible buildings before running developer".format(
len(feas))
# LUZ overrides, if applicable
p = sim.get_table('parcels').to_frame(columns = ['luz_id', 'total_residential_units', 'total_sfd_du', 'total_sfa_du', 'total_mfr_du'])
feas['luz_id'] = p.luz_id
overrides = pd.read_csv('./data/overrides/luz_overrides.csv')
overrides = overrides[overrides.year == year]
controlled_luzes = np.unique(overrides.luz_id)
if len(overrides) > 0:
#Record existing LUZ values
existing_du = p.groupby('luz_id').total_residential_units.sum()
existing_sfd_du = p.groupby('luz_id').total_sfd_du.sum()
existing_sfa_du = p.groupby('luz_id').total_sfa_du.sum()
existing_mfr_du = p.groupby('luz_id').total_mfr_du.sum()
existing_df = pd.DataFrame({'existing_du':existing_du,
19:existing_sfd_du,
20:existing_sfa_du,
21:existing_mfr_du}).fillna(0)
mini_feases = []
for luz in controlled_luzes:
overrides_subset = overrides[overrides.luz_id == luz]
for devtype in np.unique(overrides_subset.development_type_id):
target = overrides_subset.target[overrides_subset.development_type_id == devtype].values[0]
print 'LUZ %s has a DU override target for development type %s of %s.' % (luz, devtype, target)
existing_du = existing_df[devtype][existing_df.index == luz].values[0]
print ' There are %s existing units of this type in this LUZ' % existing_du
difference = target - existing_du
if difference > 0:
feas_subset = feas[feas.luz_id == luz]
if len(feas_subset) > 0:
if devtype == 19: residential_form = 'sf_detached'
if devtype == 20: residential_form = 'sf_attached'
if devtype == 21: residential_form = 'mf_residential'
feasible_units = feas_subset[residential_form].net_units.sum()
if feasible_units > 0:
if difference > feasible_units:
reallocate = difference - feasible_units
print ' Moving %s units to the uncontrolled bucket because only part of the target difference was feasible' % reallocate
targets[residential_form] = targets[residential_form] + reallocate
dev_luz = developer.Developer(feas_subset)
run_developer(dev_luz, residential_form, difference, year, build = True)
#Store the unbuilt feasible parcels and add back into feasibility later
mini_feases.append(dev_luz.feasibility)
else:
print ' No profitable projects'
#Uncontrolled LUZs
print 'Running Developer for uncontrolled LUZs'
feas = feas[~feas.luz_id.isin(controlled_luzes)]
dev = developer.Developer(feas)
for residential_form in ['mf_residential', 'sf_attached', 'sf_detached', ]:
if residential_form in targets.keys():
target = targets[residential_form]
run_developer(dev, residential_form, target, year, build = True)
#Remaining feasible parcels back to feas after running controlled LUZs so that nonres can be built on these parcels if multiple forms allowed and not already built on
if len(overrides) > 0:
feas_list = mini_feases.append(dev.feasibility)
feas = pd.concat(mini_feases)
sim.add_table("feasibility", feas)
else:
sim.add_table("feasibility", dev.feasibility)
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns)
b_sim = b[(b.note == 'simulated') * (b.year_built == year)]
print 'Simulated DU: %s' % b_sim.residential_units.sum()
print 'Target DU: %s' % (targets['sf_detached'] + targets['mf_residential'] + targets['sf_attached']) #Note: includes negative when target is lower.
def lcm_simulate(cfg, choosers, buildings, join_tbls, out_fname,
supply_fname, vacant_fname,
enable_supply_correction=None):
"""
Simulate the location choices for the specified choosers
Parameters
----------
cfg : string
The name of the yaml config file from which to read the location
choice model
choosers : DataFrameWrapper
A dataframe of agents doing the choosing
buildings : DataFrameWrapper
A dataframe of buildings which the choosers are locating in and which
have a supply
join_tbls : list of strings
A list of land use dataframes to give neighborhood info around the
buildings - will be joined to the buildings using existing broadcasts.
out_fname : string
The column name to write the simulated location to
supply_fname : string
The string in the buildings table that indicates the amount of
available units there are for choosers, vacant or not
vacant_fname : string
The string in the buildings table that indicates the amount of vacant
units there will be for choosers
enable_supply_correction : Python dict
Should contain keys "price_col" and "submarket_col" which are set to
the column names in buildings which contain the column for prices and
an identifier which segments buildings into submarkets
"""
cfg = misc.config(cfg)
choosers_df = to_frame(choosers, [], cfg, additional_columns=[out_fname])
additional_columns = [supply_fname, vacant_fname]
if enable_supply_correction is not None and \
"submarket_col" in enable_supply_correction:
additional_columns += [enable_supply_correction["submarket_col"]]
if enable_supply_correction is not None and \
"price_col" in enable_supply_correction:
additional_columns += [enable_supply_correction["price_col"]]
locations_df = to_frame(buildings, join_tbls, cfg,
additional_columns=additional_columns)
available_units = buildings[supply_fname]
vacant_units = buildings[vacant_fname]
print "There are %d total available units" % available_units.sum()
print " and %d total choosers" % len(choosers)
print " but there are %d overfull buildings" % \
len(vacant_units[vacant_units < 0])
vacant_units = vacant_units[vacant_units > 0]
# sometimes there are vacant units for buildings that are not in the
# locations_df, which happens for reasons explained in the warning below
indexes = np.repeat(vacant_units.index.values,
vacant_units.values.astype('int'))
isin = pd.Series(indexes).isin(locations_df.index)
missing = len(isin[isin == False])
indexes = indexes[isin.values]
units = locations_df.loc[indexes].reset_index()
check_nas(units)
print " for a total of %d temporarily empty units" % vacant_units.sum()
print " in %d buildings total in the region" % len(vacant_units)
if missing > 0:
print "WARNING: %d indexes aren't found in the locations df -" % \
missing
print " this is usually because of a few records that don't join "
print " correctly between the locations df and the aggregations tables"
movers = choosers_df[choosers_df[out_fname] == -1]
print "There are %d total movers for this LCM" % len(movers)
if enable_supply_correction is not None:
assert isinstance(enable_supply_correction, dict)
assert "price_col" in enable_supply_correction
price_col = enable_supply_correction["price_col"]
assert "submarket_col" in enable_supply_correction
submarket_col = enable_supply_correction["submarket_col"]
lcm = yaml_to_class(cfg).from_yaml(str_or_buffer=cfg)
if enable_supply_correction.get("warm_start", False) is True:
raise NotImplementedError()
multiplier_func = enable_supply_correction.get("multiplier_func", None)
if multiplier_func is not None:
multiplier_func = sim.get_injectable(multiplier_func)
kwargs = enable_supply_correction.get('kwargs', {})
new_prices, submarkets_ratios = supply_and_demand(
lcm,
movers,
units,
submarket_col,
price_col,
base_multiplier=None,
multiplier_func=multiplier_func,
**kwargs)
# we will only get back new prices for those alternatives
# that pass the filter - might need to specify the table in
# order to get the complete index of possible submarkets
submarket_table = enable_supply_correction.get("submarket_table", None)
if submarket_table is not None:
submarkets_ratios = submarkets_ratios.reindex(
sim.get_table(submarket_table).index).fillna(1)
# write final shifters to the submarket_table for use in debugging
sim.get_table(submarket_table)["price_shifters"] = submarkets_ratios
print "Running supply and demand"
print "Simulated Prices"
print buildings[price_col].describe()
print "Submarket Price Shifters"
print submarkets_ratios.describe()
# we want new prices on the buildings, not on the units, so apply
# shifters directly to buildings and ignore unit prices
sim.add_column(buildings.name,
price_col+"_hedonic", buildings[price_col])
new_prices = buildings[price_col] * \
submarkets_ratios.loc[buildings[submarket_col]].values
buildings.update_col_from_series(price_col, new_prices)
print "Adjusted Prices"
print buildings[price_col].describe()
if len(movers) > vacant_units.sum():
print "WARNING: Not enough locations for movers"
print " reducing locations to size of movers for performance gain"
movers = movers.head(vacant_units.sum())
new_units, _ = yaml_to_class(cfg).predict_from_cfg(movers, units, cfg)
# new_units returns nans when there aren't enough units,
# get rid of them and they'll stay as -1s
new_units = new_units.dropna()
# go from units back to buildings
new_buildings = pd.Series(units.loc[new_units.values][out_fname].values,
index=new_units.index)
choosers.update_col_from_series(out_fname, new_buildings)
_print_number_unplaced(choosers, out_fname)
if enable_supply_correction is not None:
new_prices = buildings[price_col]
if "clip_final_price_low" in enable_supply_correction:
new_prices = new_prices.clip(lower=enable_supply_correction[
"clip_final_price_low"])
if "clip_final_price_high" in enable_supply_correction:
new_prices = new_prices.clip(upper=enable_supply_correction[
"clip_final_price_high"])
buildings.update_col_from_series(price_col, new_prices)
vacant_units = buildings[vacant_fname]
print " and there are now %d empty units" % vacant_units.sum()
print " and %d overfull buildings" % len(vacant_units[vacant_units < 0])
def non_residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
dev = developer.Developer(feas)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
year = get_year()
targets = non_residential_space_targets()
for non_residential_form in ['heavy_industrial', 'light_industrial', 'retail', 'office']:
if non_residential_form in targets.keys():
old_buildings = sim.get_table('buildings').to_frame(sim.get_table('buildings').local_columns)
target = targets[non_residential_form]
target = target/400
print 'Job space target for %s is %s.' % (non_residential_form, target)
if target > 0:
new_buildings = dev.pick(non_residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_job_spaces,
max_parcel_size=2000000,
min_unit_size=0,
drop_after_build=True,
residential=False,
bldg_sqft_per_job=400.0)
print 'Constructed %s %s buildings, totaling %s new job spaces.' % (len(new_buildings), non_residential_form, new_buildings.non_residential_sqft.sum()/400)
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if non_residential_form == 'light_industrial':
new_buildings['development_type_id'] = 2
elif non_residential_form == 'heavy_industrial':
new_buildings['development_type_id'] = 3
elif non_residential_form == 'office':
new_buildings['development_type_id'] = 4
elif non_residential_form == 'retail':
new_buildings['development_type_id'] = 5
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2-l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(drop_buildings.index)
print "Unplaced {} before: {}".format(tbl, len(agents.query(
"building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(tbl, len(agents.query(
"building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
sim.add_table("feasibility", dev.feasibility)
def aggregations(settings):
if "aggregation_tables" not in settings or \
settings["aggregation_tables"] is None:
return []
return [sim.get_table(tbl) for tbl in settings["aggregation_tables"]]
def feasibility(parcels, settings, parcel_sales_price_sqft_func,
parcel_is_allowed_func):
# Fee table preprocessing
fee_schedule = sim.get_table('fee_schedule').to_frame()
parcel_fee_schedule = sim.get_table('parcel_fee_schedule').to_frame()
parcels = sim.get_table('parcels').to_frame(
columns=['zoning_id', 'development_type_id'])
fee_schedule = fee_schedule.groupby([
'fee_schedule_id', 'development_type_id'
]).development_fee_per_unit_space_initial.mean().reset_index()
parcel_use_allowed_callback = sim.get_injectable('parcel_is_allowed_func')
def run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=None):
if parcel_filter:
parcels = parcels.query(parcel_filter)
# add prices for each use (rents). Apply fees
parcels[use] = misc.reindex(
sim.get_table('nodes')[use],
sim.get_table('parcels').node_id) - fees
#Calibration shifters
calibration_shifters = pd.read_csv(
'.\\data\\calibration\\msa_shifters.csv').set_index(
'msa_id').to_dict()
if use == 'residential':
shifter_name = 'res_price_shifter'
else:
shifter_name = 'nonres_price_shifter'
parcels[shifter_name] = 1.0
shifters = calibration_shifters[shifter_name]
for msa_id in shifters.keys():
shift = shifters[msa_id]
parcels[shifter_name][parcels.msa_id == msa_id] = shift
parcels[use] = parcels[use] * parcels[shifter_name]
#LUZ shifter
if use == 'residential':
target_luz = pd.read_csv(
'.\\data\\calibration\\target_luz.csv').values.flatten()
luz_shifter = pd.read_csv(
'.\\data\\calibration\\luz_du_shifter.csv').values[0][0]
parcels[use][parcels.luz_id.isin(target_luz)] = parcels[use][
parcels.luz_id.isin(target_luz)] * luz_shifter
# convert from cost to yearly rent
if residential_to_yearly:
parcels[use] *= pf.config.cap_rate
# Price minimum if hedonic predicts outlier
parcels[use][parcels[use] <= .5] = .5
parcels[use][parcels[use].isnull()] = .5
print "Describe of the yearly rent by use"
print parcels[use].describe()
allowed = parcel_use_allowed_callback(form).loc[parcels.index]
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
if use == 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
target_units = residential_space_targets()[form]
#Calculate number of profitable units
d = {}
d[form] = feasibility
feas = pd.concat(d.values(), keys=d.keys(), axis=1)
dev = developer.Developer(feas)
profitable_units = run_developer(dev,
form,
target_units,
get_year(),
build=False)
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
price_scaling_factor += .1
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
elif use != 'residential':
def iter_feasibility(feasibility, price_scaling_factor):
if price_scaling_factor > 3.0:
return feasibility
# Get targets
targets = non_residential_space_targets()
target_units = targets[form] / 400
#Calculate number of profitable units
feasibility['current_units'] = parcels.total_job_spaces
feasibility["parcel_size"] = parcels.parcel_size
feasibility = feasibility[feasibility.parcel_size < 200000]
feasibility['job_spaces'] = np.round(
feasibility.non_residential_sqft / 400.0)
feasibility[
'net_units'] = feasibility.job_spaces - feasibility.current_units
feasibility.net_units = feasibility.net_units.fillna(0)
profitable_units = int(feasibility.net_units.sum())
print 'Feasibility given current prices/zonining indicates %s profitable units and target of %s' % (
profitable_units, target_units)
if profitable_units < target_units:
price_scaling_factor += .1
print 'Scaling prices up by factor of %s' % price_scaling_factor
parcels[use] = parcels[use] * price_scaling_factor
feasibility = pf.lookup(form,
parcels[allowed],
only_built=True,
pass_through=[])
return iter_feasibility(feasibility, price_scaling_factor)
else:
return feasibility
feasibility = iter_feasibility(feasibility, 1.0)
print len(feasibility)
return feasibility
def residential_proforma(form, devtype_id, parking_rate):
print form
use = 'residential'
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = True
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use: 1.0}}
pfc.uses = [use]
pfc.residential_uses = [True]
pfc.parking_rates = {use: parking_rate}
pfc.costs = {use: [170.0, 190.0, 210.0, 240.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id ==
devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule,
fee_schedule_devtype,
left_on='fee_schedule_id',
right_on='fee_schedule_id')
parcel_fee_schedule_devtype[
'development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial * parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby(
'parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data=parcel_fees_processed,
index=parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
return run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=parcel_filter)
def nonresidential_proforma(form, devtype_id, use, parking_rate):
print form
parcels = sim.get_table('parcels').to_frame()
residential_to_yearly = False
parcel_filter = settings['feasibility']['parcel_filter']
#parcel_filter = None
pfc = sqftproforma.SqFtProFormaConfig()
pfc.forms = {form: {use: 1.0}}
pfc.uses = [use]
pfc.residential_uses = [False]
pfc.parking_rates = {use: parking_rate}
if use == 'retail':
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
elif use == 'industrial':
pfc.costs = {use: [140.0, 175.0, 200.0, 230.0]}
else: #office
pfc.costs = {use: [160.0, 175.0, 200.0, 230.0]}
#Fees
fee_schedule_devtype = fee_schedule[fee_schedule.development_type_id ==
devtype_id]
parcel_fee_schedule_devtype = pd.merge(parcel_fee_schedule,
fee_schedule_devtype,
left_on='fee_schedule_id',
right_on='fee_schedule_id')
parcel_fee_schedule_devtype[
'development_fee_per_unit'] = parcel_fee_schedule_devtype.development_fee_per_unit_space_initial * parcel_fee_schedule_devtype.portion
parcel_fees_processed = parcel_fee_schedule_devtype.groupby(
'parcel_id').development_fee_per_unit.sum()
fees = pd.Series(data=parcel_fees_processed,
index=parcels.index).fillna(0)
pf = sqftproforma.SqFtProForma(pfc)
fees = fees * pf.config.cap_rate
return run_proforma_lookup(parcels,
fees,
pf,
use,
form,
residential_to_yearly,
parcel_filter=parcel_filter)
d = {}
##SF DETACHED proforma (devtype 19)
form = 'sf_detached'
devtype_id = 19
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##SF ATTACHED proforma (devtype 20)
form = 'sf_attached'
devtype_id = 20
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##MF_RESIDENTIAL proforma (devtype 21)
form = 'mf_residential'
devtype_id = 21
d[form] = residential_proforma(form, devtype_id, parking_rate=1.0)
##OFFICE (devtype 4)
form = 'office'
devtype_id = 4
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate=1.0)
##RETAIL (devtype 5)
form = 'retail'
devtype_id = 5
d[form] = nonresidential_proforma(form, devtype_id, form, parking_rate=2.0)
##LIGHT INDUSTRIAL (devtype 2)
form = 'light_industrial'
devtype_id = 2
d[form] = nonresidential_proforma(form,
devtype_id,
'industrial',
parking_rate=.6)
##HEAVY INDUSTRIAL (devtype 3)
form = 'heavy_industrial'
devtype_id = 3
d[form] = nonresidential_proforma(form,
devtype_id,
'industrial',
parking_rate=.6)
far_predictions = pd.concat(d.values(), keys=d.keys(), axis=1)
sim.add_table("feasibility", far_predictions)
def residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
year = get_year()
targets = residential_space_targets()
print "{:,} feasible buildings before running developer".format(len(feas))
# LUZ overrides, if applicable
p = sim.get_table('parcels').to_frame(columns=[
'luz_id', 'total_residential_units', 'total_sfd_du', 'total_sfa_du',
'total_mfr_du'
])
feas['luz_id'] = p.luz_id
overrides = pd.read_csv('./data/overrides/luz_overrides.csv')
overrides = overrides[overrides.year == year]
controlled_luzes = np.unique(overrides.luz_id)
if len(overrides) > 0:
#Record existing LUZ values
existing_du = p.groupby('luz_id').total_residential_units.sum()
existing_sfd_du = p.groupby('luz_id').total_sfd_du.sum()
existing_sfa_du = p.groupby('luz_id').total_sfa_du.sum()
existing_mfr_du = p.groupby('luz_id').total_mfr_du.sum()
existing_df = pd.DataFrame({
'existing_du': existing_du,
19: existing_sfd_du,
20: existing_sfa_du,
21: existing_mfr_du
}).fillna(0)
mini_feases = []
for luz in controlled_luzes:
overrides_subset = overrides[overrides.luz_id == luz]
for devtype in np.unique(overrides_subset.development_type_id):
target = overrides_subset.target[
overrides_subset.development_type_id == devtype].values[0]
print 'LUZ %s has a DU override target for development type %s of %s.' % (
luz, devtype, target)
existing_du = existing_df[devtype][existing_df.index ==
luz].values[0]
print ' There are %s existing units of this type in this LUZ' % existing_du
difference = target - existing_du
if difference > 0:
feas_subset = feas[feas.luz_id == luz]
if len(feas_subset) > 0:
if devtype == 19: residential_form = 'sf_detached'
if devtype == 20: residential_form = 'sf_attached'
if devtype == 21: residential_form = 'mf_residential'
feasible_units = feas_subset[
residential_form].net_units.sum()
if feasible_units > 0:
if difference > feasible_units:
reallocate = difference - feasible_units
print ' Moving %s units to the uncontrolled bucket because only part of the target difference was feasible' % reallocate
targets[residential_form] = targets[
residential_form] + reallocate
dev_luz = developer.Developer(feas_subset)
run_developer(dev_luz,
residential_form,
difference,
year,
build=True)
#Store the unbuilt feasible parcels and add back into feasibility later
mini_feases.append(dev_luz.feasibility)
else:
print ' No profitable projects'
#Uncontrolled LUZs
print 'Running Developer for uncontrolled LUZs'
feas = feas[~feas.luz_id.isin(controlled_luzes)]
dev = developer.Developer(feas)
for residential_form in [
'mf_residential',
'sf_attached',
'sf_detached',
]:
if residential_form in targets.keys():
target = targets[residential_form]
run_developer(dev, residential_form, target, year, build=True)
#Remaining feasible parcels back to feas after running controlled LUZs so that nonres can be built on these parcels if multiple forms allowed and not already built on
if len(overrides) > 0:
feas_list = mini_feases.append(dev.feasibility)
feas = pd.concat(mini_feases)
sim.add_table("feasibility", feas)
else:
sim.add_table("feasibility", dev.feasibility)
b = sim.get_table('buildings')
b = b.to_frame(b.local_columns)
b_sim = b[(b.note == 'simulated') * (b.year_built == year)]
print 'Simulated DU: %s' % b_sim.residential_units.sum()
print 'Target DU: %s' % (targets['sf_detached'] +
targets['mf_residential'] + targets['sf_attached']
) #Note: includes negative when target is lower.
def run_developer(dev, residential_form, target, year, build=False):
old_buildings = sim.get_table('buildings').to_frame(
sim.get_table('buildings').local_columns)
parcels = sim.get_table('parcels')
print 'Residential unit target for %s is %s.' % (residential_form, target)
if target > 0:
print residential_form
drop_after_build = True if build else False
new_buildings = dev.pick(residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_residential_units,
max_parcel_size=2000000,
min_unit_size=400,
drop_after_build=True,
residential=True,
bldg_sqft_per_job=400.0)
if build:
print 'Constructed %s %s buildings, totaling %s new residential_units' % (
len(new_buildings), residential_form,
new_buildings.residential_units.sum())
overshoot = new_buildings.residential_units.sum() - target
print 'Overshot target by %s units' % (overshoot)
print 'Biggest development has %s units' % new_buildings.residential_units.max(
)
if overshoot > 1:
to_remove = new_buildings[['parcel_id',
'residential_units']].copy()
to_remove = to_remove[
to_remove.residential_units < 4].set_index('parcel_id')
to_remove = to_remove.sort('residential_units')
to_remove['du_cumsum'] = to_remove.residential_units.cumsum()
idx_to_remove = np.searchsorted(to_remove.du_cumsum, overshoot)
parcel_ids_to_remove = to_remove.index.values[:(
idx_to_remove[0] + 1)]
print 'Removing %s units to match target' % to_remove.residential_units.values[:(
idx_to_remove[0] + 1)].sum()
new_buildings = new_buildings[~new_buildings.parcel_id.
isin(parcel_ids_to_remove)]
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if residential_form == 'sf_detached':
new_buildings['development_type_id'] = 19
elif residential_form == 'sf_attached':
new_buildings['development_type_id'] = 20
elif residential_form == 'mf_residential':
new_buildings['development_type_id'] = 21
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(
new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2 - l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(
drop_buildings.index)
print "Unplaced {} before: {}".format(
tbl, len(agents.query("building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(
tbl, len(agents.query("building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
else:
return new_buildings.residential_units.sum()
"jobs_transition",
"elcm_simulate",
"households_transition",
"hlcm_luz_simulate", #demand/location models
"price_vars",
"feasibility",
"residential_developer",
"non_residential_developer", #supply/proforma models
],
years=[
2013,
2014,
2015,
])
# Summarize results at MSA level
b = sim.get_table('buildings').to_frame(columns=[
'msa_id', 'mgra_id', 'residential_units', 'non_residential_sqft', 'note'
])
new_du_by_msa = b[b.note == 'simulated'].groupby(
'msa_id').residential_units.sum()
new_nrsf_by_msa = b[b.note == 'simulated'].groupby(
'msa_id').non_residential_sqft.sum()
proportion_du_by_msa = new_du_by_msa / new_du_by_msa.sum()
proportion_nrsf_by_msa = new_nrsf_by_msa / new_nrsf_by_msa.sum()
# Write out indicators to calibration directory
proportion_du_by_msa.to_csv('.\\data\\calibration\\msa_du_simulated.csv',
header=True)
proportion_nrsf_by_msa.to_csv('.\\data\\calibration\\msa_nrsf_simulated.csv',
header=True)
def non_residential_developer(parcels):
feas = sim.get_table('feasibility').to_frame()
dev = developer.Developer(feas)
print "{:,} feasible buildings before running developer".format(
len(dev.feasibility))
year = get_year()
targets = non_residential_space_targets()
for non_residential_form in [
'heavy_industrial', 'light_industrial', 'retail', 'office'
]:
if non_residential_form in targets.keys():
old_buildings = sim.get_table('buildings').to_frame(
sim.get_table('buildings').local_columns)
target = targets[non_residential_form]
target = target / 400
print 'Job space target for %s is %s.' % (non_residential_form,
target)
if target > 0:
new_buildings = dev.pick(non_residential_form,
target,
parcels.parcel_size,
parcels.ave_sqft_per_unit,
parcels.total_job_spaces,
max_parcel_size=2000000,
min_unit_size=0,
drop_after_build=True,
residential=False,
bldg_sqft_per_job=400.0)
print 'Constructed %s %s buildings, totaling %s new job spaces.' % (
len(new_buildings), non_residential_form,
new_buildings.non_residential_sqft.sum() / 400)
new_buildings["year_built"] = year
new_buildings["stories"] = new_buildings.stories.apply(np.ceil)
if non_residential_form == 'light_industrial':
new_buildings['development_type_id'] = 2
elif non_residential_form == 'heavy_industrial':
new_buildings['development_type_id'] = 3
elif non_residential_form == 'office':
new_buildings['development_type_id'] = 4
elif non_residential_form == 'retail':
new_buildings['development_type_id'] = 5
new_buildings['improvement_value'] = 0
new_buildings['note'] = 'simulated'
new_buildings['res_price_per_sqft'] = 0.0
new_buildings['nonres_rent_per_sqft'] = 0.0
new_buildings = new_buildings[old_buildings.columns]
# Remove redeveloped buildings
redev_buildings = old_buildings.parcel_id.isin(
new_buildings.parcel_id)
l = len(old_buildings)
drop_buildings = old_buildings[redev_buildings]
old_buildings = old_buildings[np.logical_not(redev_buildings)]
l2 = len(old_buildings)
if l2 - l > 0:
print "Dropped {} buildings because they were redeveloped".\
format(l2-l)
for tbl in ['households', 'jobs']:
agents = sim.get_table(tbl)
agents = agents.to_frame(agents.local_columns)
displaced_agents = agents.building_id.isin(
drop_buildings.index)
print "Unplaced {} before: {}".format(
tbl, len(agents.query("building_id == -1")))
agents.building_id[displaced_agents] = -1
print "Unplaced {} after: {}".format(
tbl, len(agents.query("building_id == -1")))
sim.add_table(tbl, agents)
# Update buildings table
all_buildings = dev.merge(old_buildings, new_buildings)
sim.add_table("buildings", all_buildings)
sim.add_table("feasibility", dev.feasibility)
import models
import pandas as pd
import urbansim.sim.simulation as sim
parcels = sim.get_table("parcels")
buildings = sim.get_table("buildings")
households = sim.get_table("households")
jobs = sim.get_table("jobs")
s = buildings.parcel_id.isin(parcels.index)
print "Building's parcel id in parcel index\n", s.value_counts()
s = households.building_id.isin(buildings.index)
print "Households with no building assigned: \n", \
(households.building_id == -1).value_counts()
print "Household's building id in building index\n", s.value_counts()
s = jobs.building_id.isin(buildings.index)
print "Jobs with no building assigned: \n", \
(jobs.building_id == -1).value_counts()
print "Job's building id in building index\n", s.value_counts()
print "Len jobs"
print len(jobs)
print "Num job spaces"
# Keys correspond to the resulting parcel columns (minus suffix).
# Values correspond the names in the add-on dataset.
network_attributes = {"tstops": "busstops"}
intersections = {"nodes1": "1-way", "nodes3": "3-way", "nodes4": "4-way"}
pois = {"lbus": "busstops", "ebus": "busstops",
"fry": "ferry", "crt": "railway", "lrt": "lightrail"} # will compute nearest distance to these
output_file = "parcels_buffered.h5"
# get input parcel data (modify the path here)
instore = pd.HDFStore('/Users/hana/workspace/data/soundcast/urbansim_outputs/2010/parcels.h5', "r")
#instore = pd.HDFStore(os.path.join(misc.data_dir(), "parcels.h5"), "r")
parcels = instore["parcels"]
# merge in latitude and longitude columns (this parcels table is taken from data/base_year.h5)
parcels_with_lat_long = sim.get_table("parcels").to_frame(['lat', 'long'])
parcels = pd.merge(parcels, parcels_with_lat_long, left_index=True, right_index=True)
# load network & the various addons and assign parcels to the network
net = load_network(precompute=distances)
load_network_addons(network=net)
assign_nodes_to_dataset(parcels, net)
def process_net_attribute(network, attr, fun):
print "Processing %s" % attr
newdf = None
for dist_index, dist in distances.iteritems():
res_name = "%s_%s" % (re.sub("_?p$", "", attr), dist_index) # remove '_p' if present
aggr = network.aggregate(dist, type=fun, decay="linear", name=attr)
if newdf is None:
newdf = pd.DataFrame({res_name: aggr, "node_ids": aggr.index.values})
