def retail_developer(jobs, buildings, parcels, nodes, feasibility, settings, summary, add_extra_columns_func, net): dev_settings = settings['non_residential_developer'] all_units = dev.compute_units_to_build( len(jobs), buildings.job_spaces.sum(), dev_settings['kwargs']['target_vacancy']) target = all_units * float(dev_settings['type_splits']["Retail"]) # target here is in sqft target *= settings["building_sqft_per_job"]["HS"] feasibility = feasibility.to_frame().loc[:, "retail"] feasibility = feasibility.dropna(subset=["max_profit"]) feasibility["non_residential_sqft"] = \ feasibility.non_residential_sqft.astype("int") feasibility["retail_ratio"] = parcels.retail_ratio feasibility = feasibility.reset_index() # create features f1 = feasibility.retail_ratio / feasibility.retail_ratio.max() f2 = feasibility.max_profit / feasibility.max_profit.max() # combine features in probability function - it's like combining expense # of building the building with the market in the neighborhood p = f1 * 1.5 + f2 p = p.clip(lower=1.0/len(p)/10) print "Attempting to build {:,} retail sqft".format(target) # order by weighted random sample feasibility = feasibility.sample(frac=1.0, weights=p) bldgs = buildings.to_frame(buildings.local_columns + ["general_type"]) devs = [] for dev_id, d in feasibility.iterrows(): if target <= 0: break # any special logic to filter these devs? # remove new dev sqft from target target -= d.non_residential_sqft # add redeveloped sqft to target filt = "general_type == 'Retail' and parcel_id == %d" % \ d["parcel_id"] target += bldgs.query(filt).non_residential_sqft.sum() devs.append(d) if len(devs) == 0: return # record keeping - add extra columns to match building dataframe # add the buidings and demolish old buildings, and add to debug output devs = pd.DataFrame(devs, columns=feasibility.columns) print "Building {:,} retail sqft in {:,} projects".format( devs.non_residential_sqft.sum(), len(devs)) if target > 0: print " WARNING: retail target not met" devs["form"] = "retail" devs = add_extra_columns_func(devs) add_buildings(buildings, devs) summary.add_parcel_output(devs)
def office_developer(feasibility, jobs, buildings, parcels, year, settings, summary, form_to_btype_func, scenario, add_extra_columns_func, parcels_geography, limits_settings): dev_settings = settings['non_residential_developer'] # I'm going to try a new way of computing this because the math the other # way is simply too hard. Basically we used to try and apportion sectors # into the demand for office, retail, and industrial, but there's just so # much dirtyness to the data, for instance 15% of jobs are in residential # buildings, and 15% in other buildings, it's just hard to know how much # to build, we I think the right thing to do is to compute the number of # job spaces that are required overall, and then to apportion that new dev # into the three non-res types with a single set of coefficients all_units = dev.compute_units_to_build( len(jobs), buildings.job_spaces.sum(), dev_settings['kwargs']['target_vacancy']) print "Total units to build = %d" % all_units if all_units <= 0: return for typ in ["Office"]: print "\nRunning for type: ", typ num_units = all_units * float(dev_settings['type_splits'][typ]) targets = [] # now apply limits - limits are assumed to be yearly, apply to an # entire jurisdiction and be in terms of residential_units or # job_spaces if year > 2015 and typ in limits_settings: juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') juris_list = limits_settings[typ].keys() for juris, limit in limits_settings[typ].items(): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount # built in previous years - in other words, you get rollover # and development is lumpy current_total = parcels.total_job_spaces[ (juris_name == juris) & (parcels.newest_building > 2015)]\ .sum() target = (year - 2015 + 1) * limit - current_total if target <= 0: print "Already met target for juris = %s" % juris print " target = %d, current_total = %d" %\ (target, current_total) continue targets.append((juris_name == juris, target, juris)) num_units -= target # other cities not in the targets get the remaining target targets.append((~juris_name.isin(juris_list), num_units, "none")) else: # otherwise use all parcels with total number of units targets.append((parcels.index == parcels.index, num_units, "none")) for parcel_mask, target, juris in targets: print "Running developer for %s with target of %d" % \ (str(juris), target) print "Parcels in play:\n", pd.Series(parcel_mask).value_counts() # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer # method below buildings = orca.get_table('buildings') new_buildings = utils.run_developer( typ.lower(), jobs, buildings, "job_spaces", parcels.parcel_size[parcel_mask], parcels.ave_sqft_per_unit[parcel_mask], parcels.total_job_spaces[parcel_mask], feasibility, year=year, form_to_btype_callback=form_to_btype_func, add_more_columns_callback=add_extra_columns_func, residential=False, num_units_to_build=target, profit_to_prob_func=subsidies.profit_to_prob_func, **dev_settings['kwargs']) if new_buildings is not None: new_buildings["subsidized"] = False summary.add_parcel_output(new_buildings)
def residential_developer(feasibility, households, buildings, parcels, year, settings, summary, form_to_btype_func, add_extra_columns_func, parcels_geography, limits_settings, final_year): kwargs = settings['residential_developer'] target_vacancy = pd.read_csv("data/regional_controls.csv", index_col="year").loc[year].st_res_vac num_units = dev.compute_units_to_build( len(households), buildings["residential_units"].sum(), target_vacancy) targets = [] typ = "Residential" # now apply limits - limits are assumed to be yearly, apply to an # entire jurisdiction and be in terms of residential_units or job_spaces if typ in limits_settings: juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') juris_list = limits_settings[typ].keys() for juris, limit in limits_settings[typ].items(): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount # built in previous years - in other words, you get rollover # and development is lumpy current_total = parcels.total_residential_units[ (juris_name == juris) & (parcels.newest_building >= 2010)]\ .sum() target = (year - 2010 + 1) * limit - current_total # make sure we don't overshoot the total development of the limit # for the horizon year - for instance, in Half Moon Bay we have # a very low limit and a single development in a far out year can # easily build over the limit for the total simulation max_target = (final_year - 2010 + 1) * limit - current_total if target <= 0: continue targets.append((juris_name == juris, target, max_target, juris)) num_units -= target # other cities not in the targets get the remaining target targets.append((~juris_name.isin(juris_list), num_units, None, "none")) else: # otherwise use all parcels with total number of units targets.append((parcels.index == parcels.index, num_units, None, "none")) for parcel_mask, target, final_target, juris in targets: print "Running developer for %s with target of %d" % \ (str(juris), target) # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer # method below buildings = orca.get_table('buildings') new_buildings = utils.run_developer( "residential", households, buildings, "residential_units", parcels.parcel_size[parcel_mask], parcels.ave_sqft_per_unit[parcel_mask], parcels.total_residential_units[parcel_mask], feasibility, year=year, form_to_btype_callback=form_to_btype_func, add_more_columns_callback=add_extra_columns_func, num_units_to_build=target, profit_to_prob_func=subsidies.profit_to_prob_func, **kwargs) buildings = orca.get_table('buildings') if new_buildings is not None: new_buildings["subsidized"] = False if final_target is not None and new_buildings is not None: # make sure we don't overbuild the target for the whole simulation overshoot = new_buildings.net_units.sum() - final_target if overshoot > 0: index = new_buildings.tail(1).index[0] index = int(index) # make sure we don't get into a negative unit situation overshoot = min(overshoot, buildings.local.loc[index, "residential_units"]) buildings.local.loc[index, "residential_units"] -= overshoot summary.add_parcel_output(new_buildings)
def office_developer(feasibility, jobs, buildings, parcels, year, settings, summary, form_to_btype_func, scenario, add_extra_columns_func, parcels_geography, limits_settings): dev_settings = settings['non_residential_developer'] # I'm going to try a new way of computing this because the math the other # way is simply too hard. Basically we used to try and apportion sectors # into the demand for office, retail, and industrial, but there's just so # much dirtyness to the data, for instance 15% of jobs are in residential # buildings, and 15% in other buildings, it's just hard to know how much # to build, we I think the right thing to do is to compute the number of # job spaces that are required overall, and then to apportion that new dev # into the three non-res types with a single set of coefficients all_units = dev.compute_units_to_build( len(jobs), buildings.job_spaces.sum(), dev_settings['kwargs']['target_vacancy']) print "Total units to build = %d" % all_units if all_units <= 0: return for typ in ["Office"]: print "\nRunning for type: ", typ num_units = all_units * float(dev_settings['type_splits'][typ]) targets = [] # now apply limits - limits are assumed to be yearly, apply to an # entire jurisdiction and be in terms of residential_units or # job_spaces if year > 2015 and typ in limits_settings: juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') juris_list = limits_settings[typ].keys() for juris, limit in limits_settings[typ].items(): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount # built in previous years - in other words, you get rollover # and development is lumpy current_total = parcels.total_job_spaces[ (juris_name == juris) & (parcels.newest_building > 2015)]\ .sum() target = (year - 2015 + 1) * limit - current_total if target <= 0: print "Already met target for juris = %s" % juris print " target = %d, current_total = %d" %\ (target, current_total) continue targets.append((juris_name == juris, target, juris)) num_units -= target # other cities not in the targets get the remaining target targets.append((~juris_name.isin(juris_list), num_units, "none")) else: # otherwise use all parcels with total number of units targets.append((parcels.index == parcels.index, num_units, "none")) for parcel_mask, target, juris in targets: print "Running developer for %s with target of %d" % \ (str(juris), target) print "Parcels in play:\n", pd.Series(parcel_mask).value_counts() # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer # method below buildings = orca.get_table('buildings') new_buildings = utils.run_developer( typ.lower(), jobs, buildings, "job_spaces", parcels.parcel_size[parcel_mask], parcels.ave_sqft_per_unit[parcel_mask], parcels.total_job_spaces[parcel_mask], feasibility, year=year, form_to_btype_callback=form_to_btype_func, add_more_columns_callback=add_extra_columns_func, residential=False, num_units_to_build=target, profit_to_prob_func=subsidies.profit_to_prob_func, **dev_settings['kwargs']) if new_buildings is not None: new_buildings["subsidized"] = False summary.add_parcel_output(new_buildings)
def retail_developer(jobs, buildings, parcels, nodes, feasibility, settings, summary, add_extra_columns_func, net): dev_settings = settings['non_residential_developer'] all_units = dev.compute_units_to_build( len(jobs), buildings.job_spaces.sum(), dev_settings['kwargs']['target_vacancy']) target = all_units * float(dev_settings['type_splits']["Retail"]) # target here is in sqft target *= settings["building_sqft_per_job"]["HS"] feasibility = feasibility.to_frame().loc[:, "retail"] feasibility = feasibility.dropna(subset=["max_profit"]) feasibility["non_residential_sqft"] = \ feasibility.non_residential_sqft.astype("int") feasibility["retail_ratio"] = parcels.retail_ratio feasibility = feasibility.reset_index() # create features f1 = feasibility.retail_ratio / feasibility.retail_ratio.max() f2 = feasibility.max_profit / feasibility.max_profit.max() # combine features in probability function - it's like combining expense # of building the building with the market in the neighborhood p = f1 * 1.5 + f2 p = p.clip(lower=1.0 / len(p) / 10) print "Attempting to build {:,} retail sqft".format(target) # order by weighted random sample feasibility = feasibility.sample(frac=1.0, weights=p) foreign_columns = ["general_type"] bldgs = buildings.to_frame(buildings.local_columns + foreign_columns) devs = [] for dev_id, d in feasibility.iterrows(): if target <= 0: break # any special logic to filter these devs? # remove new dev sqft from target target -= d.non_residential_sqft # add redeveloped sqft to target filt = "general_type == 'Retail' and parcel_id == %d" % \ d["parcel_id"] target += bldgs.query(filt).non_residential_sqft.sum() devs.append(d) if len(devs) == 0: return # record keeping - add extra columns to match building dataframe # add the buidings and demolish old buildings, and add to debug output devs = pd.DataFrame(devs, columns=buildings.local_columns) print "Building {:,} retail sqft in {:,} projects".format( devs.non_residential_sqft.sum(), len(devs)) if target > 0: print " WARNING: retail target not met" devs["form"] = "retail" devs = add_extra_columns_func(devs) add_buildings(buildings, devs) summary.add_parcel_output(devs)
def residential_developer(feasibility, households, buildings, parcels, year, settings, summary, form_to_btype_func, add_extra_columns_func, parcels_geography, limits_settings, final_year, regional_controls): kwargs = settings['residential_developer'] rc = regional_controls.to_frame() target_vacancy = rc.loc[year].st_res_vac num_units = dev.compute_units_to_build( len(households), buildings["residential_units"].sum(), target_vacancy) targets = [] typ = "Residential" # now apply limits - limits are assumed to be yearly, apply to an # entire jurisdiction and be in terms of residential_units or job_spaces if typ in limits_settings: juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') juris_list = limits_settings[typ].keys() for juris, limit in limits_settings[typ].items(): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount # built in previous years - in other words, you get rollover # and development is lumpy current_total = parcels.total_residential_units[ (juris_name == juris) & (parcels.newest_building >= 2010)]\ .sum() target = (year - 2010 + 1) * limit - current_total # make sure we don't overshoot the total development of the limit # for the horizon year - for instance, in Half Moon Bay we have # a very low limit and a single development in a far out year can # easily build over the limit for the total simulation max_target = (final_year - 2010 + 1) * limit - current_total if target <= 0: continue targets.append((juris_name == juris, target, max_target, juris)) num_units -= target # other cities not in the targets get the remaining target targets.append((~juris_name.isin(juris_list), num_units, None, "none")) else: # otherwise use all parcels with total number of units targets.append((parcels.index == parcels.index, num_units, None, "none")) for parcel_mask, target, final_target, juris in targets: print "Running developer for %s with target of %d" % \ (str(juris), target) # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer # method below buildings = orca.get_table('buildings') new_buildings = utils.run_developer( "residential", households, buildings, "residential_units", parcels.parcel_size[parcel_mask], parcels.ave_sqft_per_unit[parcel_mask], parcels.total_residential_units[parcel_mask], feasibility, year=year, form_to_btype_callback=form_to_btype_func, add_more_columns_callback=add_extra_columns_func, num_units_to_build=target, profit_to_prob_func=subsidies.profit_to_prob_func, **kwargs) buildings = orca.get_table('buildings') if new_buildings is not None: new_buildings["subsidized"] = False if final_target is not None and new_buildings is not None: # make sure we don't overbuild the target for the whole simulation overshoot = new_buildings.net_units.sum() - final_target if overshoot > 0: index = new_buildings.tail(1).index[0] index = int(index) # make sure we don't get into a negative unit situation current_units = buildings.local.loc[index, "residential_units"] # only can reduce by as many units as we have overshoot = min(overshoot, current_units) # used below - this is the pct we need to reduce the building overshoot_pct = \ (current_units - overshoot) / float(current_units) buildings.local.loc[index, "residential_units"] -= overshoot # we also need to fix the other columns so they make sense for col in ["residential_sqft", "building_sqft", "deed_restricted_units"]: val = buildings.local.loc[index, col] # reduce by pct but round to int buildings.local.loc[index, col] = int(val * overshoot_pct) summary.add_parcel_output(new_buildings)
def residential_developer(feasibility, households, buildings, parcels, year, settings, summary, form_to_btype_func, add_extra_columns_func, parcels_geography, limits_settings, final_year, regional_controls): kwargs = settings['residential_developer'] rc = regional_controls.to_frame() target_vacancy = rc.loc[year].st_res_vac num_units = dev.compute_units_to_build( len(households), buildings["residential_units"].sum(), target_vacancy) targets = [] typ = "Residential" # now apply limits - limits are assumed to be yearly, apply to an # entire jurisdiction and be in terms of residential_units or job_spaces if typ in limits_settings: juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') juris_list = limits_settings[typ].keys() for juris, limit in limits_settings[typ].items(): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount # built in previous years - in other words, you get rollover # and development is lumpy current_total = parcels.total_residential_units[ (juris_name == juris) & (parcels.newest_building >= 2010)]\ .sum() target = (year - 2010 + 1) * limit - current_total # make sure we don't overshoot the total development of the limit # for the horizon year - for instance, in Half Moon Bay we have # a very low limit and a single development in a far out year can # easily build over the limit for the total simulation max_target = (final_year - 2010 + 1) * limit - current_total if target <= 0: continue targets.append((juris_name == juris, target, max_target, juris)) num_units -= target # other cities not in the targets get the remaining target targets.append((~juris_name.isin(juris_list), num_units, None, "none")) else: # otherwise use all parcels with total number of units targets.append((parcels.index == parcels.index, num_units, None, "none")) for parcel_mask, target, final_target, juris in targets: print("Running developer for %s with target of %d" % (str(juris), target)) # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer # method below buildings = orca.get_table('buildings') print('Stats of buildings before run_developer(): \n{}'.format( buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum())) new_buildings = utils.run_developer( "residential", households, buildings, "residential_units", parcels.parcel_size[parcel_mask], parcels.ave_sqft_per_unit[parcel_mask], parcels.total_residential_units[parcel_mask], feasibility, year=year, form_to_btype_callback=form_to_btype_func, add_more_columns_callback=add_extra_columns_func, num_units_to_build=int(target), profit_to_prob_func=subsidies.profit_to_prob_func, **kwargs) print('Stats of buildings before run_developer(): \n{}'.format( buildings.to_frame()[['deed_restricted_units','preserved_units','inclusionary_units']].sum())) buildings = orca.get_table('buildings') if new_buildings is not None: new_buildings["subsidized"] = False if final_target is not None and new_buildings is not None: # make sure we don't overbuild the target for the whole simulation overshoot = new_buildings.net_units.sum() - final_target if overshoot > 0: index = new_buildings.tail(1).index[0] index = int(index) # make sure we don't get into a negative unit situation current_units = buildings.local.loc[index, "residential_units"] # only can reduce by as many units as we have overshoot = min(overshoot, current_units) # used below - this is the pct we need to reduce the building overshoot_pct = \ (current_units - overshoot) / float(current_units) buildings.local.loc[index, "residential_units"] -= overshoot # we also need to fix the other columns so they make sense for col in ["residential_sqft", "building_sqft", "deed_restricted_units", "inclusionary_units", "subsidized_units"]: val = buildings.local.loc[index, col] # reduce by pct but round to int buildings.local.loc[index, col] = int(val * overshoot_pct) # also fix the corresponding columns in new_buildings for col in ["residential_sqft","building_sqft", "residential_units", "deed_restricted_units", "inclusionary_units", "subsidized_units"]: val = new_buildings.loc[index, col] new_buildings.loc[index, col] = int(val * overshoot_pct) for col in ["policy_based_revenue_reduction", "max_profit"]: val = new_buildings.loc[index, col] new_buildings.loc[index, col] = val * overshoot_pct summary.add_parcel_output(new_buildings)