def retail_developer(jobs, buildings, parcels, nodes, feasibility,
settings, summary, add_extra_columns_func, net):
dev_settings = settings['non_residential_developer']
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
target = all_units * float(dev_settings['type_splits']["Retail"])
# target here is in sqft
target *= settings["building_sqft_per_job"]["HS"]
feasibility = feasibility.to_frame().loc[:, "retail"]
feasibility = feasibility.dropna(subset=["max_profit"])
feasibility["non_residential_sqft"] = \
feasibility.non_residential_sqft.astype("int")
feasibility["retail_ratio"] = parcels.retail_ratio
feasibility = feasibility.reset_index()
# create features
f1 = feasibility.retail_ratio / feasibility.retail_ratio.max()
f2 = feasibility.max_profit / feasibility.max_profit.max()
# combine features in probability function - it's like combining expense
# of building the building with the market in the neighborhood
p = f1 * 1.5 + f2
p = p.clip(lower=1.0/len(p)/10)
print "Attempting to build {:,} retail sqft".format(target)
# order by weighted random sample
feasibility = feasibility.sample(frac=1.0, weights=p)
bldgs = buildings.to_frame(buildings.local_columns + ["general_type"])
devs = []
for dev_id, d in feasibility.iterrows():
if target <= 0:
break
# any special logic to filter these devs?
# remove new dev sqft from target
target -= d.non_residential_sqft
# add redeveloped sqft to target
filt = "general_type == 'Retail' and parcel_id == %d" % \
d["parcel_id"]
target += bldgs.query(filt).non_residential_sqft.sum()
devs.append(d)
if len(devs) == 0:
return
# record keeping - add extra columns to match building dataframe
# add the buidings and demolish old buildings, and add to debug output
devs = pd.DataFrame(devs, columns=feasibility.columns)
print "Building {:,} retail sqft in {:,} projects".format(
devs.non_residential_sqft.sum(), len(devs))
if target > 0:
print " WARNING: retail target not met"
devs["form"] = "retail"
devs = add_extra_columns_func(devs)
add_buildings(buildings, devs)
summary.add_parcel_output(devs)
def office_developer(feasibility, jobs, buildings, parcels, year,
settings, summary, form_to_btype_func, scenario,
add_extra_columns_func, parcels_geography,
limits_settings):
dev_settings = settings['non_residential_developer']
# I'm going to try a new way of computing this because the math the other
# way is simply too hard. Basically we used to try and apportion sectors
# into the demand for office, retail, and industrial, but there's just so
# much dirtyness to the data, for instance 15% of jobs are in residential
# buildings, and 15% in other buildings, it's just hard to know how much
# to build, we I think the right thing to do is to compute the number of
# job spaces that are required overall, and then to apportion that new dev
# into the three non-res types with a single set of coefficients
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
print "Total units to build = %d" % all_units
if all_units <= 0:
return
for typ in ["Office"]:
print "\nRunning for type: ", typ
num_units = all_units * float(dev_settings['type_splits'][typ])
targets = []
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or
# job_spaces
if year > 2015 and typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_job_spaces[
(juris_name == juris) & (parcels.newest_building > 2015)]\
.sum()
target = (year - 2015 + 1) * limit - current_total
if target <= 0:
print "Already met target for juris = %s" % juris
print " target = %d, current_total = %d" %\
(target, current_total)
continue
targets.append((juris_name == juris, target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index, num_units, "none"))
for parcel_mask, target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
print "Parcels in play:\n", pd.Series(parcel_mask).value_counts()
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
typ.lower(),
jobs,
buildings,
"job_spaces",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_job_spaces[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
residential=False,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**dev_settings['kwargs'])
if new_buildings is not None:
new_buildings["subsidized"] = False
summary.add_parcel_output(new_buildings)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year):
kwargs = settings['residential_developer']
target_vacancy = pd.read_csv("data/regional_controls.csv",
index_col="year").loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
overshoot = min(overshoot,
buildings.local.loc[index,
"residential_units"])
buildings.local.loc[index, "residential_units"] -= overshoot
summary.add_parcel_output(new_buildings)
def office_developer(feasibility, jobs, buildings, parcels, year,
settings, summary, form_to_btype_func, scenario,
add_extra_columns_func, parcels_geography,
limits_settings):
dev_settings = settings['non_residential_developer']
# I'm going to try a new way of computing this because the math the other
# way is simply too hard. Basically we used to try and apportion sectors
# into the demand for office, retail, and industrial, but there's just so
# much dirtyness to the data, for instance 15% of jobs are in residential
# buildings, and 15% in other buildings, it's just hard to know how much
# to build, we I think the right thing to do is to compute the number of
# job spaces that are required overall, and then to apportion that new dev
# into the three non-res types with a single set of coefficients
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
print "Total units to build = %d" % all_units
if all_units <= 0:
return
for typ in ["Office"]:
print "\nRunning for type: ", typ
num_units = all_units * float(dev_settings['type_splits'][typ])
targets = []
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or
# job_spaces
if year > 2015 and typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_job_spaces[
(juris_name == juris) & (parcels.newest_building > 2015)]\
.sum()
target = (year - 2015 + 1) * limit - current_total
if target <= 0:
print "Already met target for juris = %s" % juris
print " target = %d, current_total = %d" %\
(target, current_total)
continue
targets.append((juris_name == juris, target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index, num_units, "none"))
for parcel_mask, target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
print "Parcels in play:\n", pd.Series(parcel_mask).value_counts()
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
typ.lower(),
jobs,
buildings,
"job_spaces",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_job_spaces[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
residential=False,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**dev_settings['kwargs'])
if new_buildings is not None:
new_buildings["subsidized"] = False
summary.add_parcel_output(new_buildings)
def retail_developer(jobs, buildings, parcels, nodes, feasibility,
settings, summary, add_extra_columns_func, net):
dev_settings = settings['non_residential_developer']
all_units = dev.compute_units_to_build(
len(jobs),
buildings.job_spaces.sum(),
dev_settings['kwargs']['target_vacancy'])
target = all_units * float(dev_settings['type_splits']["Retail"])
# target here is in sqft
target *= settings["building_sqft_per_job"]["HS"]
feasibility = feasibility.to_frame().loc[:, "retail"]
feasibility = feasibility.dropna(subset=["max_profit"])
feasibility["non_residential_sqft"] = \
feasibility.non_residential_sqft.astype("int")
feasibility["retail_ratio"] = parcels.retail_ratio
feasibility = feasibility.reset_index()
# create features
f1 = feasibility.retail_ratio / feasibility.retail_ratio.max()
f2 = feasibility.max_profit / feasibility.max_profit.max()
# combine features in probability function - it's like combining expense
# of building the building with the market in the neighborhood
p = f1 * 1.5 + f2
p = p.clip(lower=1.0 / len(p) / 10)
print "Attempting to build {:,} retail sqft".format(target)
# order by weighted random sample
feasibility = feasibility.sample(frac=1.0, weights=p)
foreign_columns = ["general_type"]
bldgs = buildings.to_frame(buildings.local_columns + foreign_columns)
devs = []
for dev_id, d in feasibility.iterrows():
if target <= 0:
break
# any special logic to filter these devs?
# remove new dev sqft from target
target -= d.non_residential_sqft
# add redeveloped sqft to target
filt = "general_type == 'Retail' and parcel_id == %d" % \
d["parcel_id"]
target += bldgs.query(filt).non_residential_sqft.sum()
devs.append(d)
if len(devs) == 0:
return
# record keeping - add extra columns to match building dataframe
# add the buidings and demolish old buildings, and add to debug output
devs = pd.DataFrame(devs, columns=buildings.local_columns)
print "Building {:,} retail sqft in {:,} projects".format(
devs.non_residential_sqft.sum(), len(devs))
if target > 0:
print " WARNING: retail target not met"
devs["form"] = "retail"
devs = add_extra_columns_func(devs)
add_buildings(buildings, devs)
summary.add_parcel_output(devs)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year,
regional_controls):
kwargs = settings['residential_developer']
rc = regional_controls.to_frame()
target_vacancy = rc.loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print "Running developer for %s with target of %d" % \
(str(juris), target)
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=target,
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
current_units = buildings.local.loc[index, "residential_units"]
# only can reduce by as many units as we have
overshoot = min(overshoot, current_units)
# used below - this is the pct we need to reduce the building
overshoot_pct = \
(current_units - overshoot) / float(current_units)
buildings.local.loc[index, "residential_units"] -= overshoot
# we also need to fix the other columns so they make sense
for col in ["residential_sqft", "building_sqft",
"deed_restricted_units"]:
val = buildings.local.loc[index, col]
# reduce by pct but round to int
buildings.local.loc[index, col] = int(val * overshoot_pct)
summary.add_parcel_output(new_buildings)
def residential_developer(feasibility, households, buildings, parcels, year,
settings, summary, form_to_btype_func,
add_extra_columns_func, parcels_geography,
limits_settings, final_year,
regional_controls):
kwargs = settings['residential_developer']
rc = regional_controls.to_frame()
target_vacancy = rc.loc[year].st_res_vac
num_units = dev.compute_units_to_build(
len(households),
buildings["residential_units"].sum(),
target_vacancy)
targets = []
typ = "Residential"
# now apply limits - limits are assumed to be yearly, apply to an
# entire jurisdiction and be in terms of residential_units or job_spaces
if typ in limits_settings:
juris_name = parcels_geography.juris_name.\
reindex(parcels.index).fillna('Other')
juris_list = limits_settings[typ].keys()
for juris, limit in limits_settings[typ].items():
# the actual target is the limit times the number of years run
# so far in the simulation (plus this year), minus the amount
# built in previous years - in other words, you get rollover
# and development is lumpy
current_total = parcels.total_residential_units[
(juris_name == juris) & (parcels.newest_building >= 2010)]\
.sum()
target = (year - 2010 + 1) * limit - current_total
# make sure we don't overshoot the total development of the limit
# for the horizon year - for instance, in Half Moon Bay we have
# a very low limit and a single development in a far out year can
# easily build over the limit for the total simulation
max_target = (final_year - 2010 + 1) * limit - current_total
if target <= 0:
continue
targets.append((juris_name == juris, target, max_target, juris))
num_units -= target
# other cities not in the targets get the remaining target
targets.append((~juris_name.isin(juris_list), num_units, None, "none"))
else:
# otherwise use all parcels with total number of units
targets.append((parcels.index == parcels.index,
num_units, None, "none"))
for parcel_mask, target, final_target, juris in targets:
print("Running developer for %s with target of %d" %
(str(juris), target))
# this was a fairly heinous bug - have to get the building wrapper
# again because the buildings df gets modified by the run_developer
# method below
buildings = orca.get_table('buildings')
print('Stats of buildings before run_developer(): \n{}'.format(
buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum()))
new_buildings = utils.run_developer(
"residential",
households,
buildings,
"residential_units",
parcels.parcel_size[parcel_mask],
parcels.ave_sqft_per_unit[parcel_mask],
parcels.total_residential_units[parcel_mask],
feasibility,
year=year,
form_to_btype_callback=form_to_btype_func,
add_more_columns_callback=add_extra_columns_func,
num_units_to_build=int(target),
profit_to_prob_func=subsidies.profit_to_prob_func,
**kwargs)
print('Stats of buildings before run_developer(): \n{}'.format(
buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum()))
buildings = orca.get_table('buildings')
if new_buildings is not None:
new_buildings["subsidized"] = False
if final_target is not None and new_buildings is not None:
# make sure we don't overbuild the target for the whole simulation
overshoot = new_buildings.net_units.sum() - final_target
if overshoot > 0:
index = new_buildings.tail(1).index[0]
index = int(index)
# make sure we don't get into a negative unit situation
current_units = buildings.local.loc[index, "residential_units"]
# only can reduce by as many units as we have
overshoot = min(overshoot, current_units)
# used below - this is the pct we need to reduce the building
overshoot_pct = \
(current_units - overshoot) / float(current_units)
buildings.local.loc[index, "residential_units"] -= overshoot
# we also need to fix the other columns so they make sense
for col in ["residential_sqft", "building_sqft",
"deed_restricted_units", "inclusionary_units",
"subsidized_units"]:
val = buildings.local.loc[index, col]
# reduce by pct but round to int
buildings.local.loc[index, col] = int(val * overshoot_pct)
# also fix the corresponding columns in new_buildings
for col in ["residential_sqft","building_sqft",
"residential_units", "deed_restricted_units",
"inclusionary_units", "subsidized_units"]:
val = new_buildings.loc[index, col]
new_buildings.loc[index, col] = int(val * overshoot_pct)
for col in ["policy_based_revenue_reduction",
"max_profit"]:
val = new_buildings.loc[index, col]
new_buildings.loc[index, col] = val * overshoot_pct
summary.add_parcel_output(new_buildings)
