def run(mgra_dataframe):
output_dir = config.parameters['output_directory']
simulation_years = config.parameters['simulation_years']
simulation_begin = config.parameters['simulation_begin']
debug = config.parameters['debug']
if debug:
print('input frame description')
analyze_mgra(mgra_dataframe, output_dir)
# the number of tqdm progress bar steps per simulation year
checkpoints = 8
progress = tqdm(desc='progress',
total=simulation_years * checkpoints,
position=0)
for i in range(simulation_years):
forecast_year = simulation_begin + i + 1
progress.set_description('starting year {}'.format(i + 1))
if debug:
print('simulating year {} ({})'.format(i + 1, forecast_year))
mgra_dataframe, progress = step(mgra_dataframe, progress)
progress.update()
if debug:
print('updated frame:')
analyze_mgra(mgra_dataframe, output_dir)
progress.set_description('saving year {}'.format(i + 1))
save_to_file(mgra_dataframe, output_dir,
'year{}_{}.csv'.format(i + 1, forecast_year))
progress.update()
progress.close()
return
def run(mgra_dataframe, planned_sites):
output_dir = parameters['output_directory']
simulation_begin = parameters['simulation_begin']
forecast_year = simulation_begin + 1
# add scheduled development if available
if planned_sites is not None:
print('adding scheduled development:')
add_scheduled_development(mgra_dataframe,
planned_sites,
year=simulation_begin)
# finish meeting demand as needed
print('developing to meet remaining demand:')
mgra_dataframe = develop(mgra_dataframe)
if mgra_dataframe is None:
print('program terminated early')
return
# save output file
save_to_file(mgra_dataframe, output_dir,
'forecasted_year_{}.csv'.format(forecast_year))
# create aa export if crosswalk is available
print('creating AA commodity export file ...')
export_luz_data(mgra_dataframe)
return
def make_profitability_file():
frame_dict = {mgra_labels.MGRA: [i + 1 for i in range(MGRA_COUNT)]}
for product_type in product_types():
frame_dict[
ProductTypeLabels(product_type).profitability_adjust
] = [0]*MGRA_COUNT
frame = pandas.DataFrame(frame_dict)
save_to_file(frame, 'data', 'profitability.csv', force=True)
def run(mgra_dataframe, planned_sites):
output_dir = parameters['output_directory']
forecast_year = parameters['simulation_year']
# add scheduled development if available
if planned_sites is not None:
print('adding scheduled development:')
add_scheduled_development(mgra_dataframe,
planned_sites,
year=forecast_year)
# year=simulation_begin)
sched_dev_csv = os.path.join(output_dir, "scheduled_development_added.csv")
if os.path.exists(sched_dev_csv):
# save scheduled development report for forecast_year
# otherwise it gets deleted when develop() is called
sched_df = pandas.read_csv(sched_dev_csv)
input_df = pandas.read_csv(
os.path.join(output_dir, "..",
"supply_input_" + str(forecast_year) + ".csv"))
merge_df = pandas.merge(input_df, sched_df, on="MGRA")
merge_df["dev_sf"] = merge_df["dev_sf_y"] - merge_df["dev_sf_x"]
merge_df["dev_mf"] = merge_df["dev_mf_y"] - merge_df["dev_mf_x"]
merge_df["dev_mh"] = merge_df["dev_mh_y"] - merge_df["dev_mh_x"]
merge_df[
"dev_indus"] = merge_df["dev_indus_y"] - merge_df["dev_indus_x"]
merge_df["dev_comm"] = merge_df["dev_comm_y"] - merge_df["dev_comm_x"]
merge_df[
"dev_office"] = merge_df["dev_office_y"] - merge_df["dev_office_x"]
merge_df["dev_oth"] = merge_df["dev_oth_y"] - merge_df["dev_oth_x"]
merge_df = merge_df[[
"MGRA", "dev_sf", "dev_mf", "dev_mh", "dev_indus", "dev_comm",
"dev_office", "dev_oth"
]]
merge_df.to_csv(os.path.join(
output_dir, "..",
"scheduled_development_" + str(forecast_year) + ".csv"),
index=False)
#os.replace(
# os.path.join(output_dir,"scheduled_development_added.csv"),
# os.path.join(output_dir,"..","scheduled_development_"+str(forecast_year)+".csv"),)
# finish meeting demand as needed
print('developing to meet remaining demand:')
mgra_dataframe = develop(mgra_dataframe)
if mgra_dataframe is None:
print('program terminated early')
return
# save output file
save_to_file(mgra_dataframe, output_dir,
'forecasted_year_{}.csv'.format(forecast_year))
# create aa export if crosswalk is available
print('updating AA floorspace file ...')
update_floorspace(mgra_dataframe, forecast_year)
return
def create_version_4point1():
original_frame = pandas.read_csv("data/SRF_Input_Base_V4.csv")
# add interpolated variables
interpolated_frame = load_interpolated().drop([mgra_labels.LUZ], axis=1)
combined_frame = pandas.merge(original_frame,
interpolated_frame,
on=mgra_labels.MGRA)
# add job spaces
frame_with_job_spaces = add_job_spaces_columns(combined_frame)
# fix capacity
frame_with_fixed_capacity = fix_capacity(frame_with_job_spaces)
# remove unused columns
final_frame = frame_with_fixed_capacity[all_columns()]
save_to_file(final_frame, 'data', 'SRF_Input_Base_V4.1.csv')
return
def run(mgra_dataframe):
output_dir = parameters['output_directory']
simulation_begin = parameters['simulation_begin']
forecast_year = simulation_begin + 1
print('simulating year {}'.format(forecast_year))
# develop enough land to meet demand for this year.
mgra_dataframe = develop(mgra_dataframe)
if mgra_dataframe is None:
print('program terminated early')
return
# save output file
save_to_file(mgra_dataframe, output_dir,
'forecasted_year_{}.csv'.format(forecast_year))
return
def export_aa(frame):
cache = SquareFootageCache(frame)
output_rows = []
with open('data/CRE2FSC.csv') as f:
dict_reader = csv.DictReader(f)
# for each row in crosswalk; add row to output after finding quantity
# and expanding commodity code
# FIXME: it looks like the dict reader might be the reason for the
# slowdown, as the cache now makes for only 23000 pandas operations
with tqdm(total=CROSSWALK_ENTRY_COUNT) as progress:
for row in dict_reader:
mgra = int(row["MGRA"])
commodity = codes_labels[int(row["FSC_num"])]
quantity = calculate_quantity(cache, mgra, row)
if quantity != 0:
output_rows.append(create_row(mgra, commodity, quantity))
progress.update()
output_frame = pandas.DataFrame(output_rows)
save_to_file(output_frame, 'data/output', 'aa_export.csv')
def configure():
args = get_args()
try:
if args.test:
# only meet_demand.py should use a test argument
parameters = load_yaml('test_parameters.yaml')
else:
parameters = load_yaml('parameters.yaml')
except (FileNotFoundError):
parameters = None
if parameters is not None:
parameters['input_filename'] = None
parameters['include_integration_tests'] = args.include
if args.filename is not None:
parameters['input_filename'] = args.filename
parameters['use_database'] = False
if args.year is not None:
parameters['simulation_year'] = args.year
# TODO: add the control totals here
use_control_totals(parameters)
# prep output directory
output_dir = parameters['output_directory']
empty_folder(output_dir)
save_to_file(parameters,
output_dir,
'parameters_used.yaml',
as_yaml=True,
output_status=False)
# configure logging level
if parameters['debug']:
if parameters['to_file']:
sys.stderr = open(os.path.join(output_dir, 'debug_output'),
mode='x')
logging.basicConfig(level=logging.DEBUG)
else:
print('could not load parameters, exiting')
return parameters
def run(mgras, intersections, year=None):
output_dir = parameters['output_directory']
if year is not None:
intersections = remove_other_years(intersections, year)
if len(intersections) == 0:
print('no scheduled development found for year {}'.format(year))
return
# add each site. we actually have a frame of intersections
# find max siteID
# for i = 1 through max_siteID
# return instances of siteID in intersections
sites = []
max_siteID = intersections.siteid.max()
progress_bar = tqdm(range(max_siteID))
progress_bar.set_description("splitting sites on mgras")
for i in progress_bar:
sites.append(find_sites(intersections, i + 1))
# we now have sites as a list of dataframes.
# for each frame
# if there are no entries, skip
# if there is one, put all development on the mgra
# if there are multiple determine split.
progress_bar = tqdm(sites)
progress_bar.set_description('allocating development for each site')
for frame in progress_bar:
if len(frame) == 0:
pass
elif len(frame) == 1:
add_to_mgra(mgras, frame)
else:
distribute_units(mgras, frame)
save_to_file(mgras, output_dir, 'scheduled_development_added.csv')
return
def export_luz_data(frame):
luz_dict = luz_dict_from_crosswalk()
output_dict = {}
focus_columns = [
mgra_labels.LUZ,
ProductTypeLabels('industrial').square_footage,
ProductTypeLabels('commercial').square_footage,
ProductTypeLabels('office').square_footage
]
focus_frame = frame[focus_columns]
# iterating through a pandas frame returns a python namedtuple
for row in focus_frame.itertuples(index=False): # for each mgra
# use the namedtuple._asdict to access with the labels we are used to.
row_dict = row._asdict()
square_footages = [row_dict[focus_columns[1]],
row_dict[focus_columns[2]],
row_dict[focus_columns[3]]
]
if sum(square_footages) != 0:
luz = row[0] # find matching LUZ
for commodity_info in luz_dict[luz]: # check each commodity type
commodity_name = commodity_info['commodity']
ratios = commodity_info['ratios']
quantity = calculate_quantity(square_footages, ratios)
if quantity != 0: # if there is some commodity space, add it.
# differentiate commodities for each luz with a tuple as a
# dict key
luz_commodity_tuple = (luz, commodity_name)
if luz_commodity_tuple not in output_dict:
output_dict[luz_commodity_tuple] = create_row(
luz, commodity_name, quantity)
else:
output_dict[
luz_commodity_tuple
]['Quantity'] += quantity
output_frame = pandas.DataFrame(output_dict.values())
output_frame.sort_values(by=['Commodity', 'TAZ'], inplace=True)
save_to_file(output_frame, 'data/output', 'aa_export.csv')
def create_version_4point2():
starting_frame = pandas.read_csv('data/SRF_Input_Base_V4.1.csv')
ending_frame = fix_non_residential_capacity(starting_frame)
save_to_file(ending_frame, 'data', 'SRF_Input_Base_V4.2.csv')
def test_save_to_file(self):
item = 'hello there'
save_to_file(item, 'temp', 'hello.txt')
self.assertEquals(item, open('temp/hello.txt', 'r').read())
from utils.interface import save_to_file
# from utils.access_labels import mgra_labels
# from modeling.filters import generic_filter
import pandas
# Run with python -m utils.extract_test_data to select
# mgra_count random MGRA's with some vacant land to test with
mgra_count = 5000
dataframe = pandas.read_csv('data/SRF_Input_Base_V4.1.csv')
# dataframe = generic_filter(
# dataframe, [mgra_labels.VACANT_ACRES], filter_nans=False)
sampled = dataframe.sample(n=mgra_count).reset_index(drop=True)
save_to_file(sampled, './test_data', 'random_MGRA.csv')
progress.set_description('saving year {}'.format(i + 1))
save_to_file(mgra_dataframe, output_dir,
'year{}_{}.csv'.format(i + 1, forecast_year))
progress.update()
progress.close()
return
if __name__ == "__main__":
# load parameters
config.parameters = load_parameters('test_parameters.yaml')
# re-enable normal parameters once full dataset is available
# args = get_args()
# if args.test:
# config.parameters = load_parameters('test_parameters.yaml')
# else:
# config.parameters = load_parameters('parameters.yaml')
if config.parameters is not None:
output_dir = config.parameters['output_directory']
empty_folder(output_dir)
save_to_file(config.parameters, output_dir, 'parameters.txt')
mgra_dataframe = pandas.read_csv(config.parameters['input_filename'])
run(mgra_dataframe)
else:
print('could not load parameters, exiting')
def updateSupply(combined_location,combined_rent,old_supply_output,new_supply_input=None):
print("update supply input data")
r_df = pandas.read_csv(combined_rent, dtype={'Value': np.float64}, na_values = np.NaN, index_col=['Realestate','Zone'])
r_df = r_df.replace([np.inf, -np.inf], np.nan)
#r_df = r_df.replace(np.inf, max(r_df[r_df.Value!=np.inf].Value))
#r_df = r_df.replace(-np.inf, min(r_df[r_df.Value!=-np.inf].Value))
#r_df = r_df.clip(lower=0.1)
#r_df = r_df.clip(upper=50000)
s_df = pandas.read_csv(old_supply_output,dtype={'valueSFmea': np.float64, 'valueMFmea': np.float64,
'valueINDme': np.float64, 'valueCOMme': np.float64,
'valueOFCme': np.float64}, index_col='MGRA')
cl_df_wide = pandas.read_csv(combined_location, dtype=np.float64)
cl_df_long = pandas.melt(cl_df_wide, id_vars=['Realestate','Zone'], var_name='AgentType', value_name='Value').set_index(['Realestate','Zone'])
#agentXtype = cl_df_long.agg(func=np.sum, axis='index').pivot(index='Zone', columns='Realestate', values='Value')
agentXtype = cl_df_long.groupby(["Realestate","Zone"]).agg({"Value": np.sum}).unstack(level="Realestate").fillna(0)
agentXtype.columns = ['hh_sf','hh_mf','hh_mh','emp_ind','emp_com','emp_ofc','emp_oth']
agentXtype = agentXtype.replace([np.inf, -np.inf], np.nan)
#agentXtype.set_index('mgra')
s_df = s_df.join(agentXtype, rsuffix="_new", sort=True)
s_df['hh_sf'] = s_df['hh_sf_new']
s_df['hh_mf'] = s_df['hh_mf_new']
s_df['hh_mh'] = s_df['hh_mh_new']
s_df['hh'] = s_df['hh_sf'] + s_df['hh_mf'] + s_df['hh_mh']
s_df['emp_indus_'] = s_df['emp_ind']
s_df['emp_office'] = s_df['emp_ofc']
s_df['emp_comm_l'] = s_df['emp_com']
s_df['emp_other_'] = s_df['emp_oth']
s_df.drop(['hh_sf_new','hh_mf_new','hh_mh_new','emp_ind','emp_ofc','emp_com','emp_oth'], axis=1)
## TODO: Update employment by real estate type as well
# =============================================================================
# s_df['valueSFmea'] = r_df.xs(1)['Value'].fillna(s_df['valueSFmea'])
# s_df['valueMFmea'] = r_df.xs(2)['Value'].fillna(s_df['valueMFmea'])
# s_df['valueINDme'] = r_df.xs(4)['Value'].fillna(s_df['valueINDme'])
# s_df['valueCOMme'] = r_df.xs(5)['Value'].fillna(s_df['valueCOMme'])
# s_df['valueOFCme'] = r_df.xs(6)['Value'].fillna(s_df['valueOFCme'])
# =============================================================================
min_sdf = s_df[['valueSFmea','valueMFmea','valueINDme','valueCOMme','valueOFCme']].min()
max_sdf = s_df[['valueSFmea','valueMFmea','valueINDme','valueCOMme','valueOFCme']].max()
s_df['valueSFmea'] = r_df.xs(1)['Value'].fillna(s_df['valueSFmea']).clip(lower=0.8*min_sdf['valueSFmea']).clip(upper=1.2*max_sdf['valueSFmea'])
s_df['valueMFmea'] = r_df.xs(2)['Value'].fillna(s_df['valueMFmea']).clip(lower=0.8*min_sdf['valueMFmea']).clip(upper=1.2*max_sdf['valueSFmea'])
s_df['valueINDme'] = r_df.xs(4)['Value'].fillna(s_df['valueINDme']).clip(lower=0.8*min_sdf['valueINDme']).clip(upper=1.2*max_sdf['valueSFmea'])
s_df['valueCOMme'] = r_df.xs(5)['Value'].fillna(s_df['valueCOMme']).clip(lower=0.8*min_sdf['valueCOMme']).clip(upper=1.2*max_sdf['valueSFmea'])
s_df['valueOFCme'] = r_df.xs(6)['Value'].fillna(s_df['valueOFCme']).clip(lower=0.8*min_sdf['valueOFCme']).clip(upper=1.2*max_sdf['valueSFmea'])
# =============================================================================
# s_df['valueSFmea'] = np.maximum(0.8*s_df['valueSFmea'],np.minimum(s_df['valueSFmea']*1.2,r_df.xs(1)['Value'].fillna(s_df['valueSFmea'])))
# s_df['valueMFmea'] = np.maximum(0.8*s_df['valueMFmea'],np.minimum(s_df['valueMFmea']*1.2,r_df.xs(2)['Value'].fillna(s_df['valueMFmea'])))
# s_df['valueINDme'] = np.maximum(0.8*s_df['valueINDme'],np.minimum(s_df['valueINDme']*1.2,r_df.xs(4)['Value'].fillna(s_df['valueINDme'])))
# s_df['valueCOMme'] = np.maximum(0.8*s_df['valueCOMme'],np.minimum(s_df['valueCOMme']*1.2,r_df.xs(5)['Value'].fillna(s_df['valueCOMme'])))
# s_df['valueOFCme'] = np.maximum(0.8*s_df['valueOFCme'],np.minimum(s_df['valueOFCme']*1.2,r_df.xs(6)['Value'].fillna(s_df['valueOFCme'])))
#
# =============================================================================
s_df = s_df.reset_index()
if (new_supply_input is not None):
outdir, fname = os.path.split(new_supply_input)
save_to_file(s_df, outdir, fname, force=True)
return s_df