def household_relocation_rates(scenario, policy):
if scenario in policy['futures_scenarios']:
if scenario in policy['reloc_fr2_enable']:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates_fr2.csv"))
orca.add_injectable("hh_reloc", 'activated')
print("File used is: household_relocation_rates_fr2.csv")
else:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates_fr_base.csv"))
orca.add_injectable("hh_reloc", 'not activated')
print("File used is: household_relocation_rates_fr_base.csv")
elif scenario in policy['reloc_db_enable']:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates_db_var.csv"))
orca.add_injectable("hh_reloc", 'activated')
print("File used is: household_relocation_rates_db_var.csv")
elif scenario in policy['reloc_fb_enable']:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates_fb.csv"))
orca.add_injectable("hh_reloc", 'activated')
print("File used is: household_relocation_rates_fb.csv")
else:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates_db_base.csv"))
orca.add_injectable("hh_reloc", 'not activated')
print("File used is: household_relocation_rates_db_base.csv")
return df
def add_canonical_dirs():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
orca.add_injectable("configs_dir", configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
orca.add_injectable("output_dir", output_dir)
def setup_orca(dfa, dfb, dfa_col, dfb_col, dfa_factor, dfb_factor):
orca.add_injectable('a_factor', dfa_factor)
@orca.injectable()
def b_factor():
return dfb_factor
orca.add_table('dfa', dfa)
@orca.table('dfb')
def dfb_table():
return dfb
orca.add_column('dfa', 'acol', dfa_col)
orca.add_column('dfb', 'bcol', dfb_col)
@orca.column('dfa')
def extra_acol(a_factor):
return dfa_col * a_factor
@orca.column('dfb')
def extra_bcol(b_factor):
return dfb_col * b_factor
orca.broadcast('dfb', 'dfa', cast_on='a_id', onto_index=True)
@orca.step()
def test_step(dfa, dfb):
pass
def large_area_id(jobs, buildings):
job_la = "jobs_large_area_lookup"
if (not orca.is_injectable(job_la)) or (len(orca.get_injectable(job_la)) == 0):
orca.add_injectable(job_la,
misc.reindex(buildings.large_area_id, jobs.building_id),
autocall=False, cache=True)
return orca.get_injectable(job_la).loc[jobs.index]
def slr_remove_dev(buildings, destroy_parcels, year, parcels,
households, jobs):
slr_demolish = buildings.local[buildings.parcel_id.isin
(destroy_parcels.index)]
orca.add_table("slr_demolish", slr_demolish)
print "Demolishing %d buildings" % len(slr_demolish)
households = households.to_frame()
hh_unplaced = households[households["building_id"] == -1]
jobs = jobs.to_frame()
jobs_unplaced = jobs[jobs["building_id"] == -1]
l1 = len(buildings)
buildings = utils._remove_developed_buildings(
buildings.to_frame(buildings.local_columns),
slr_demolish,
unplace_agents=["households", "jobs"])
households = orca.get_table("households")
households = households.to_frame()
hh_unplaced_slr = households[households["building_id"] == -1]
hh_unplaced_slr = hh_unplaced_slr[~hh_unplaced_slr.index.isin
(hh_unplaced.index)]
orca.add_injectable("hh_unplaced_slr", hh_unplaced_slr)
jobs = orca.get_table("jobs")
jobs = jobs.to_frame()
jobs_unplaced_slr = jobs[jobs["building_id"] == -1]
jobs_unplaced_slr = jobs_unplaced_slr[~jobs_unplaced_slr.index.isin
(jobs_unplaced.index)]
orca.add_injectable("jobs_unplaced_slr", jobs_unplaced_slr)
orca.add_table("buildings", buildings)
buildings = orca.get_table("buildings")
print "Demolished %d buildings" % (l1 - len(buildings))
def slr_remove_dev(buildings, destroy_parcels, year, parcels, households,
jobs):
slr_demolish = buildings.local[buildings.parcel_id.isin(
destroy_parcels.index)]
orca.add_table("slr_demolish", slr_demolish)
print "Demolishing %d buildings" % len(slr_demolish)
households = households.to_frame()
hh_unplaced = households[households["building_id"] == -1]
jobs = jobs.to_frame()
jobs_unplaced = jobs[jobs["building_id"] == -1]
l1 = len(buildings)
buildings = utils._remove_developed_buildings(
buildings.to_frame(buildings.local_columns),
slr_demolish,
unplace_agents=["households", "jobs"])
households = orca.get_table("households")
households = households.to_frame()
hh_unplaced_slr = households[households["building_id"] == -1]
hh_unplaced_slr = hh_unplaced_slr[~hh_unplaced_slr.index.isin(hh_unplaced.
index)]
orca.add_injectable("hh_unplaced_slr", hh_unplaced_slr)
jobs = orca.get_table("jobs")
jobs = jobs.to_frame()
jobs_unplaced_slr = jobs[jobs["building_id"] == -1]
jobs_unplaced_slr = jobs_unplaced_slr[~jobs_unplaced_slr.index.
isin(jobs_unplaced.index)]
orca.add_injectable("jobs_unplaced_slr", jobs_unplaced_slr)
orca.add_table("buildings", buildings)
buildings = orca.get_table("buildings")
print "Demolished %d buildings" % (l1 - len(buildings))
def test_assign_variables_failing(capsys, data):
close_handlers()
output_dir = os.path.join(os.path.dirname(__file__), 'output')
orca.add_injectable("output_dir", output_dir)
tracing.config_logger(basic=True)
spec_name = \
os.path.join(os.path.dirname(__file__), 'data', 'assignment_spec_failing.csv')
spec = assign.read_assignment_spec(spec_name)
locals_d = {
'CONSTANT': 7,
'_shadow': 99,
'log': np.log,
}
with pytest.raises(NameError) as excinfo:
results, trace_results = assign.assign_variables(spec,
data,
locals_d,
trace_rows=None)
out, err = capsys.readouterr()
# don't consume output
print out
# undefined variable should raise error
assert "'undefined_variable' is not defined" in str(excinfo.value)
def buildings(store):
df = store['buildings']
# Todo: combine two sqft prices into one and set non use sqft price to 0
df.loc[df.improvement_value < 0, 'improvement_value'] = 0
df['sqft_price_nonres'] = df.improvement_value * 1.0 / 0.7 / df.non_residential_sqft
df.loc[df.sqft_price_nonres > 1000, 'sqft_price_nonres'] = 0
df.loc[df.sqft_price_nonres < 0, 'sqft_price_nonres'] = 0
df['sqft_price_res'] = df.improvement_value * 1.25 / 0.7 / (
df.sqft_per_unit.astype(int) * df.residential_units)
df.loc[df.sqft_price_res > 1000, 'sqft_price_res'] = 0
df.loc[df.sqft_price_res < 0, 'sqft_price_res'] = 0
df.fillna(0, inplace=True)
orca.add_injectable("max_building_id", 10000000)
df['hu_filter'] = 0
cites = [551, 1155, 1100, 3130, 6020, 6040]
sample = df[df.residential_units > 0]
sample = sample[~(sample.index.isin(store['households'].building_id))]
for c in sample.b_city_id.unique():
frac = 0.9 if c in cites else 0.5
df.loc[sample[sample.b_city_id ==
c].sample(frac=frac, replace=False).index.values,
'hu_filter'] = 1
return df
def test_no_config(capsys):
configs_dir = os.path.join(os.path.dirname(__file__))
orca.add_injectable("configs_dir", configs_dir)
# remove existing hanlers or basicConfig is a NOP
logging.getLogger().handlers = []
tracing.config_logger(basic=True)
logger = logging.getLogger()
file_handlers = [h for h in logger.handlers if type(h) is logging.FileHandler]
assert len(file_handlers) == 0
logger = logging.getLogger('activitysim')
logger.debug('log_debug')
logger.info('log_info')
logger.warn('log_warn')
out, err = capsys.readouterr()
# don't consume output
print out
assert 'log_warn' in out
assert 'log_info' in out
assert 'log_debug' not in out
close_handlers()
def test_bad_custom_config_file(capsys):
configs_dir = os.path.join(os.path.dirname(__file__))
orca.add_injectable("configs_dir", configs_dir)
custom_config_file = os.path.join(os.path.dirname(__file__), 'configs', 'xlogging.yaml')
tracing.config_logger(custom_config_file=custom_config_file)
logger = logging.getLogger('activitysim')
file_handlers = [h for h in logger.handlers if type(h) is logging.FileHandler]
assert len(file_handlers) == 1
asim_logger_baseFilename = file_handlers[0].baseFilename
logger = logging.getLogger(__name__)
logger.info('log_info')
logger.warn('log_warn1')
out, err = capsys.readouterr()
# don't consume output
print out
assert "could not find conf file" in out
assert 'log_warn1' in out
assert 'log_info' not in out
close_handlers()
logger.warn('log_warn2')
with open(asim_logger_baseFilename, 'r') as content_file:
content = content_file.read()
assert 'log_warn1' in content
assert 'log_warn2' not in content
def build_networks(settings , store, parcels, intersections):
edges, nodes = store['edges'], store['nodes']
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
max_distance = settings['build_networks']['max_distance']
net.precompute(max_distance)
#SETUP POI COMPONENTS
on_ramp_nodes = nodes[nodes.on_ramp]
net.init_pois(num_categories=4, max_dist=max_distance, max_pois=1)
net.set_pois('onramps', on_ramp_nodes.x, on_ramp_nodes.y)
parks = store.parks
net.set_pois('parks', parks.x, parks.y)
schools = store.schools
net.set_pois('schools', schools.x, schools.y)
transit = store.transit
net.set_pois('transit', transit.x, transit.y)
orca.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
i = intersections.to_frame(intersections.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
i['node_id'] = net.get_node_ids(i['x'], i['y'])
#p.to_csv('data/parcels.csv')
orca.add_table("parcels", p)
orca.add_table("intersections", i)
def open_pipeline_store(overwrite=False):
"""
Open the pipeline checkpoint store and add an orca injectable to access it
Parameters
----------
overwrite : bool
delete file before opening (unless resuming)
"""
if orca.is_injectable('pipeline_store'):
raise RuntimeError("Pipeline store is already open!")
pipeline_file_path = orca.get_injectable('pipeline_path')
if overwrite:
try:
if os.path.isfile(pipeline_file_path):
logger.debug("removing pipeline store: %s" %
pipeline_file_path)
os.unlink(pipeline_file_path)
except Exception as e:
print(e)
logger.warn("Error removing %s: %s" % (e, ))
store = pd.HDFStore(pipeline_file_path, mode='a')
orca.add_injectable('pipeline_store', store)
logger.debug("opened pipeline_store")
def register_households(df, trace_hh_id):
"""
Register with orca households for tracing
Parameters
----------
df: pandas.DataFrame
traced dataframe
trace_hh_id: int
household id we are tracing
Returns
-------
Nothing
"""
print("tracing household id %s in %s households" %
(trace_hh_id, len(df.index)))
if trace_hh_id not in df.index:
print("trace_hh_id %s not in dataframe" % trace_hh_id)
# inject persons_index name of person dataframe index
if df.index.name is None:
df.index.names = ['household_id']
print("households table index had no"
" name. renamed index '%s'" % df.index.name)
orca.add_injectable("hh_index_name", df.index.name)
print("register_households injected hh_index_name '%s'" % df.index.name)
def register_households(df, trace_hh_id):
"""
Register with orca households for tracing
Parameters
----------
df: pandas.DataFrame
traced dataframe
trace_hh_id: int
household ID to trace
Returns
-------
Nothing
"""
if trace_hh_id is None:
logger.error("register_households called with null trace_hh_id")
return
logger.info("tracing household id %s in %s households" % (trace_hh_id, len(df.index)))
if trace_hh_id not in df.index:
logger.warn("trace_hh_id %s not in dataframe" % trace_hh_id)
# inject persons_index name of person dataframe index
if df.index.name is None:
df.index.names = ['household_id']
logger.warn("households table index had no name. renamed index '%s'" % df.index.name)
orca.add_injectable("hh_index_name", df.index.name)
logger.debug("register_households injected hh_index_name '%s'" % df.index.name)
def test_simple_simulate(data, spec):
orca.add_injectable("check_for_variability", False)
choices = simulate.simple_simulate(data, spec, nest_spec=None)
expected = pd.Series([1, 1, 1], index=data.index)
pdt.assert_series_equal(choices, expected)
def setup_orca(dfa, dfb, dfa_col, dfb_col, dfa_factor, dfb_factor):
orca.add_injectable('a_factor', dfa_factor)
@orca.injectable()
def b_factor():
return dfb_factor
orca.add_table('dfa', dfa)
@orca.table('dfb')
def dfb_table():
return dfb
orca.add_column('dfa', 'acol', dfa_col)
orca.add_column('dfb', 'bcol', dfb_col)
@orca.column('dfa')
def extra_acol(a_factor):
return dfa_col * a_factor
@orca.column('dfb')
def extra_bcol(b_factor):
return dfb_col * b_factor
orca.broadcast('dfb', 'dfa', cast_on='a_id', onto_index=True)
@orca.step()
def test_step(dfa, dfb):
pass
def reinject_decorated_tables():
"""
reinject the decorated tables (and columns)
"""
logger.info("reinject_decorated_tables")
# need to clear any non-decorated tables that were added during the previous run
_ORCA._TABLES.clear()
_ORCA._COLUMNS.clear()
_ORCA._TABLE_CACHE.clear()
_ORCA._COLUMN_CACHE.clear()
for name, func in _DECORATED_TABLES.items():
logger.debug("reinject decorated table %s" % name)
orca.add_table(name, func)
for column_key, args in _DECORATED_COLUMNS.items():
table_name, column_name = column_key
logger.debug("reinject decorated column %s.%s" % (table_name, column_name))
orca.add_column(table_name, column_name, args['func'], cache=args['cache'])
for name, args in _DECORATED_INJECTABLES.items():
logger.debug("reinject decorated injectable %s" % name)
orca.add_injectable(name, args['func'], cache=args['cache'])
def non_mandatory_accessibility():
fname = get_logsum_file('non_mandatory')
orca.add_injectable("nonmand_acc_file_2010", fname)
df = pd.read_csv(os.path.join(misc.data_dir(), fname))
df.loc[df.subzone == 0, 'subzone'] = 'c' # no walk
df.loc[df.subzone == 1, 'subzone'] = 'a' # short walk
df.loc[df.subzone == 2, 'subzone'] = 'b' # long walk
df['taz_sub'] = df.taz.astype('str') + df.subzone
return df.set_index('taz_sub')
def inputs(input_pcl, input_bld, input_jobs, input_gcl, input_settings):
#orca.clear_all()
orca.add_table('parcels', input_pcl)
orca.add_table('buildings', input_bld)
orca.add_table('jobs', input_jobs)
orca.add_table('gridcells', input_gcl)
orca.add_injectable('settings', input_settings)
import psrc_urbansim.vars.variables_jobs
import psrc_urbansim.vars.variables_parcels
def scheduled_development_events(buildings, development_projects,
demolish_events, summary, year, parcels,
mapping, years_per_iter, parcels_geography,
building_sqft_per_job, vmt_fee_categories,
static_parcels):
# first demolish
demolish = demolish_events.to_frame().\
query("%d <= year_built < %d" % (year, year + years_per_iter))
print("Demolishing/building %d buildings" % len(demolish))
l1 = len(buildings)
buildings = utils._remove_developed_buildings(
buildings.to_frame(buildings.local_columns),
demolish,
unplace_agents=["households", "jobs"])
orca.add_injectable(
'static_parcels',
np.append(static_parcels, demolish.loc[demolish.action == 'build',
'parcel_id']))
orca.add_table("buildings", buildings)
buildings = orca.get_table("buildings")
print("Demolished %d buildings" % (l1 - len(buildings)))
print(" (this number is smaller when parcel has no existing buildings)")
# then build
dps = development_projects.to_frame().\
query("%d <= year_built < %d" % (year, year + years_per_iter))
if len(dps) == 0:
return
new_buildings = utils.scheduled_development_events(
buildings,
dps,
remove_developed_buildings=False,
unplace_agents=['households', 'jobs'])
new_buildings["form"] = new_buildings.building_type.map(
mapping['building_type_map']).str.lower()
new_buildings["job_spaces"] = new_buildings.non_residential_sqft / \
new_buildings.building_type.fillna("OF").map(building_sqft_per_job)
new_buildings["job_spaces"] = new_buildings.job_spaces.\
fillna(0).astype('int')
new_buildings["geom_id"] = parcel_id_to_geom_id(new_buildings.parcel_id)
new_buildings["SDEM"] = True
new_buildings["subsidized"] = False
new_buildings["zone_id"] = misc.reindex(parcels.zone_id,
new_buildings.parcel_id)
new_buildings["vmt_res_cat"] = misc.reindex(vmt_fee_categories.res_cat,
new_buildings.zone_id)
del new_buildings["zone_id"]
new_buildings["pda"] = parcels_geography.pda_id.loc[
new_buildings.parcel_id].values
new_buildings["juris_trich"] = parcels_geography.juris_trich.loc[
new_buildings.parcel_id].values
summary.add_parcel_output(new_buildings)
def household_relocation_rates(scenario, policy):
if scenario in policy['reloc_fr2_enable']:
df = pd.read_csv(
os.path.join("data", "household_relocation_rates_fr2.csv"))
orca.add_injectable("hh_reloc", 'activated')
else:
df = pd.read_csv(os.path.join("data",
"household_relocation_rates.csv"))
orca.add_injectable("hh_reloc", 'not activated')
return df
def large_area_id(households, buildings):
hh_la = "households_large_area_lookup"
if (not orca.is_injectable(hh_la)) or (len(orca.get_injectable(hh_la))
== 0):
orca.add_injectable(hh_la,
misc.reindex(buildings.large_area_id,
households.building_id),
autocall=False,
cache=True)
return orca.get_injectable(hh_la).loc[households.index]
def get_val_from_uc_db(query):
try:
result = sql.read_frame(query, conn)
return result.values[0][0]
except:
conn=psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
result = sql.read_frame(query, conn)
return result.values[0][0]
def test_integerizer():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
orca.add_injectable("configs_dir", configs_dir)
# data_dir = os.path.join(os.path.dirname(__file__), 'data')
# orca.add_injectable("data_dir", data_dir)
#
# output_dir = os.path.join(os.path.dirname(__file__), 'output')
# orca.add_injectable("output_dir", output_dir)
# rows are elements for which factors are calculated, columns are constraints to be satisfied
incidence_table = pd.DataFrame({
'num_hh': [1, 1, 1, 1, 1, 1, 1, 1],
'hh_1': [1, 1, 1, 0, 0, 0, 0, 0],
'hh_2': [0, 0, 0, 1, 1, 1, 1, 1],
'p1': [1, 1, 2, 1, 0, 1, 2, 1],
'p2': [1, 0, 1, 0, 2, 1, 1, 1],
'p3': [1, 1, 0, 2, 1, 0, 2, 0],
'float_weights': [
1.362893, 25.658290, 7.978812, 27.789651, 18.451021, 8.641589,
1.476104, 8.641589
]
})
control_cols = ['num_hh', 'hh_1', 'hh_2', 'p1', 'p2', 'p3']
control_spec = pd.DataFrame({
'seed_table': [
'households', 'households', 'households', 'persons', 'persons',
'persons'
],
'target':
control_cols,
'importance': [10000000, 1000, 1000, 1000, 1000, 1000]
})
# column totals which the final weighted incidence table sums must satisfy
control_totals = pd.Series([100, 35, 65, 91, 65, 104],
index=control_spec.target.values)
integerized_weights, status = integerizer.do_integerizing(
trace_label='label',
control_spec=control_spec,
control_totals=control_totals,
incidence_table=incidence_table[control_cols],
float_weights=incidence_table['float_weights'],
total_hh_control_col='num_hh')
print "do_integerizing status", status
print "sum", integerized_weights.sum()
print "do_integerizing integerized_weights\n", integerized_weights
assert integerized_weights.sum() == 100
def build_networks(parcels, net_store):
st = net_store
nodes, edges = st.nodes, st.edges
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
net.precompute(3000)
orca.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
orca.add_table("parcels", p)
def exec_sql_uc(query):
try:
cur.execute(query)
conn.commit()
except:
conn = psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
cur.execute(query)
conn.commit()
def accessibilities_segmentation():
fname = get_logsum_file('segmentation')
orca.add_injectable("acc_seg_file_2010", fname)
df = pd.read_csv(os.path.join(misc.data_dir(), fname))
df['AV'] = df['hasAV'].apply(lambda x: 'AV' if x == 1 else 'noAV')
df['label'] = (df['incQ_label'] + '_' + df['autoSuff_label'] + '_' +
df['AV'])
df = df.groupby('label').sum()
df['prop'] = df['num_persons'] / df['num_persons'].sum()
df = df[['prop']].transpose().reset_index(drop=True)
return df
def inject_settings(configs_dir, households_sample_size, preload_3d_skims=None, chunk_size=None):
with open(os.path.join(configs_dir, "configs", "settings.yaml")) as f:
settings = yaml.load(f)
settings['households_sample_size'] = households_sample_size
if preload_3d_skims is not None:
settings['preload_3d_skims'] = preload_3d_skims
if chunk_size is not None:
settings['chunk_size'] = chunk_size
orca.add_injectable("settings", settings)
def build_networks(parcels):
st = pd.HDFStore(os.path.join(misc.data_dir(), "osm_sandag.h5"), "r")
nodes, edges = st.nodes, st.edges
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
net.precompute(3000)
orca.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
orca.add_table("parcels", p)
def close():
"""
Close any known open files
"""
close_open_files()
orca.get_injectable('pipeline_store').close()
orca.add_injectable('pipeline_store', None)
logger.info("close_pipeline")
def exec_sql_uc(query):
try:
cur.execute(query)
conn.commit()
except:
conn = psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
cur.execute(query)
conn.commit()
def get_val_from_uc_db(query):
try:
result = sql.read_frame(query, conn)
return result.values[0][0]
except:
conn = psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
result = sql.read_frame(query, conn)
return result.values[0][0]
def build_networks(parcels):
st = pd.HDFStore(os.path.join(misc.data_dir(), "osm_sandag.h5"), "r")
nodes, edges = st.nodes, st.edges
net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"],
edges[["weight"]])
net.precompute(3000)
orca.add_injectable("net", net)
p = parcels.to_frame(parcels.local_columns)
p['node_id'] = net.get_node_ids(p['x'], p['y'])
orca.add_table("parcels", p)
def simulate(self,
choice_function=None,
save_probabilities=False,
**kwargs):
"""
Computing choices, with arbitrary function for handling simulation
strategy.
Parameters
----------
choice_function : function
Function defining how to simulate choices based on fitted model.
Function must accept the following 3 arguments: model object,
choosers DataFrame, and alternatives DataFrame. Additional optional
keyword args can be utilized by function if needed (kwargs).
save_probabilities : bool
If true, will save the calculated probabilities underlying the
simulation as an orca injectable with name
'probabilities_modelname_itervar'.
Returns
-------
choices : pandas.Series
Mapping of chooser ID to alternative ID. Some choosers
will map to a nan value when there are not enough alternatives
for all the choosers.
"""
choosers, alternatives = self.calculate_model_variables()
choosers, alternatives = self.apply_predict_filters(
choosers, alternatives)
# By convention, choosers are denoted by a -1 value
# in the choice column
choosers = choosers[choosers[self.choice_column] == -1]
print("{} agents are making a choice.".format(len(choosers)))
if choice_function:
choices = choice_function(self, choosers, alternatives, **kwargs)
else:
choices = self.predict(choosers, alternatives, debug=True)
if save_probabilities:
if not self.sim_pdf:
probabilities = self.calculate_probabilities(
choosers, alternatives)
else:
probabilities = self.sim_pdf.reset_index().set_index(
'alternative_id')[0]
orca.add_injectable(
'probabilities_{}_{}'.format(self.name,
orca.get_injectable('iter_var')),
probabilities)
return choices
def get_logsum_file(type='mandatory'):
logsums = orca.get_injectable('settings')['logsums'][type]
sc = orca.get_injectable('scenario')
yr = orca.get_injectable('year')
try:
prev_type = orca.get_injectable('previous_{}_logsum_type'.format(type))
if prev_type == 'generic':
return orca.get_injectable('previous_{}_logsum_file'.format(type))
elif prev_type == 'year':
if 'logsum_{}'.format(yr) in logsums:
ls = logsums['logsum_{}'.format(yr)]
orca.add_injectable('previous_{}_logsum_file'.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
elif prev_type == 'scenario':
if 'logsum_s{}'.format(sc) in logsums:
ls = logsums['logsum_s{}'.format(sc)]
orca.add_injectable('previous_{}_logsum_file'
.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
else:
if 'logsum_{}_s{}'.format(yr, sc) in logsums:
ls = logsums['logsum_{}_s{}'.format(yr, sc)]
orca.add_injectable('previous_{}_logsum_file'
.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
except:
if 'logsum' in logsums:
ls = logsums['logsum']
ls_type = 'generic'
if 'logsum_{}'.format(yr) in logsums:
ls = logsums['logsum_{}'.format(yr)]
ls_type = 'year'
if 'logsum_s{}'.format(sc) in logsums:
ls = logsums['logsum_s{}'.format(sc)]
ls_type = 'scenario'
if 'logsum_{}_s{}'.format(yr, sc) in logsums:
ls = logsums['logsum_{}_s{}'.format(yr, sc)]
ls_type = 'year_scenario'
orca.add_injectable('previous_{}_logsum_type'.format(type),
ls_type)
orca.add_injectable('previous_{}_logsum_file'.format(type),
ls)
return ls
def get_logsum_file(type='mandatory'):
logsums = orca.get_injectable('settings')['logsums'][type]
sc = orca.get_injectable('scenario')
yr = orca.get_injectable('year')
try:
prev_type = orca.get_injectable('previous_{}_logsum_type'.format(type))
if prev_type == 'generic':
return orca.get_injectable('previous_{}_logsum_file'.format(type))
elif prev_type == 'year':
if 'logsum_{}'.format(yr) in logsums:
ls = logsums['logsum_{}'.format(yr)]
orca.add_injectable('previous_{}_logsum_file'.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
elif prev_type == 'scenario':
if 'logsum_s{}'.format(sc) in logsums:
ls = logsums['logsum_s{}'.format(sc)]
orca.add_injectable('previous_{}_logsum_file'
.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
else:
if 'logsum_{}_s{}'.format(yr, sc) in logsums:
ls = logsums['logsum_{}_s{}'.format(yr, sc)]
orca.add_injectable('previous_{}_logsum_file'
.format(type), ls)
return ls
else:
return orca.get_injectable('previous_{}_logsum_file'
.format(type))
except:
if 'logsum' in logsums:
ls = logsums['logsum']
ls_type = 'generic'
if 'logsum_{}'.format(yr) in logsums:
ls = logsums['logsum_{}'.format(yr)]
ls_type = 'year'
if 'logsum_s{}'.format(sc) in logsums:
ls = logsums['logsum_s{}'.format(sc)]
ls_type = 'scenario'
if 'logsum_{}_s{}'.format(yr, sc) in logsums:
ls = logsums['logsum_{}_s{}'.format(yr, sc)]
ls_type = 'year_scenario'
orca.add_injectable('previous_{}_logsum_type'.format(type),
ls_type)
orca.add_injectable('previous_{}_logsum_file'.format(type),
ls)
return ls
def get_val_from_uc_db(query):
try:
result = sql.read_frame(query, conn)
return result.values[0][0]
except:
conn=psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
result = sql.read_frame(query, conn)
result2 = sql.read_frame("select column_name from Information_schema.columns where table_name like 'building' ", conn)
print result2
return result.values[0][0]
def buildings(store):
df = store['buildings']
# Todo: combine two sqft prices into one and set non use sqft price to 0
df.loc[df.improvement_value < 0, 'improvement_value'] = 0
df['sqft_price_nonres'] = df.improvement_value * 1.0 / 0.7 / df.non_residential_sqft
df.loc[df.sqft_price_nonres > 1000, 'sqft_price_nonres'] = 0
df.loc[df.sqft_price_nonres < 0, 'sqft_price_nonres'] = 0
df['sqft_price_res'] = df.improvement_value * 1.25 / 0.7 / (df.sqft_per_unit.astype(int) * df.residential_units)
df.loc[df.sqft_price_res > 1000, 'sqft_price_res'] = 0
df.loc[df.sqft_price_res < 0, 'sqft_price_res'] = 0
df.fillna(0, inplace=True)
orca.add_injectable("max_building_id", 10000000)
return df
def get_dev_projects_table(scenario, parcels):
# requires the user has MTC's urban_data_internal
# repository alongside bayarea_urbansim
urban_data_repo = ("../urban_data_internal/development_projects/")
file = "2021_0309_1939_development_projects.csv"
print('Version of development_projects: {}'.format(file))
current_dev_proj = (file)
orca.add_injectable("dev_proj_file", current_dev_proj)
df = pd.read_csv(os.path.join(urban_data_repo, current_dev_proj))
df = reprocess_dev_projects(df)
orca.add_injectable("devproj_len", len(df))
# this filters project by scenario
scen = 'scen' + str(scenario)
if scen in df:
# df[scenario] is 1s and 0s indicating whether to include it
df = df[df[scen].astype('bool')]
orca.add_injectable("devproj_len_scen", len(df))
df = df.dropna(subset=['geom_id'])
cnts = df.geom_id.isin(parcels.geom_id).value_counts()
if False in cnts.index:
print("%d MISSING GEOMIDS!" % cnts.loc[False])
df = df[df.geom_id.isin(parcels.geom_id)]
geom_id = df.geom_id # save for later
df = df.set_index("geom_id")
df = geom_id_to_parcel_id(df, parcels).reset_index() # use parcel id
df["geom_id"] = geom_id.values # add it back again cause it goes away
orca.add_injectable("devproj_len_geomid", len(df))
return df
def run(forecast_year=2035, random_seed=False):
"""
Set up and run simulation.
Parameters
----------
forecast_year : int, optional
Year to simulate to. If year argument is passed from the terminal, then
that year is applied here, otherwise a default value is applied.
random_seed : int, optional
Random seed.
Returns
-------
_ : None
No return value for now.
"""
# Record start time
start_time = time.time()
orca.add_injectable('forecast_year', forecast_year)
# Set value of optional random seed
if random_seed:
np.random.seed(random_seed)
# Model names
transition_models = ['household_transition', 'job_transition']
price_models = [
'repm_sf_detached', 'repm_duplex_townhome', 'repm_multifamily',
'repm_industrial', 'repm_retail', 'repm_office'
]
developer_models = [
'feasibility', 'residential_developer', 'non_residential_developer'
]
location_models = [
'hlcm1', 'hlcm2', 'elcm1', 'elcm2', 'elcm3', 'elcm4', 'elcm5', 'elcm6',
'elcm7', 'elcm8', 'elcm9', 'elcm10', 'elcm11', 'elcm12', 'elcm13',
'elcm14'
]
end_of_year_models = ['generate_indicators']
# Simulate
orca.run(['build_networks', 'generate_indicators'])
orca.run(transition_models + price_models + developer_models +
location_models + end_of_year_models,
iter_vars=list(range(2011, forecast_year + 1)))
# Record end time
end_time = time.time()
time_elapsed = end_time - start_time
print('Simulation duration: %s minutes' % (time_elapsed / 60))
def superdistricts(scenario):
sd_scenario_file = os.path.join(
misc.data_dir(), ("superdistricts_s{}.csv").format(scenario))
# scenarios could contain policies (eg telework) and/or other modifications
if os.path.isfile(sd_scenario_file):
superdistricts = pd.read_csv(sd_scenario_file, index_col="number")
orca.add_injectable("sqft_per_job_settings", "for this scenario")
# the default includes a telework assumption and SD adjustments
else:
superdistricts = pd.read_csv(os.path.join(misc.data_dir(),
"superdistricts.csv"),
index_col="number")
orca.add_injectable("sqft_per_job_settings", "default")
return superdistricts
def test_full_run(store):
orca.add_injectable("configs_dir",
os.path.join(os.path.dirname(__file__), '..', '..',
'..', 'example'))
orca.add_injectable("store", store)
orca.add_injectable("nonmotskm_matrix", np.ones((1454, 1454)))
orca.add_injectable("set_random_seed", set_random_seed)
# grab some of the tables
orca.get_table("land_use").to_frame().info()
orca.get_table("households").to_frame().info()
orca.get_table("persons").to_frame().info()
assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
# run the models in the expected order
orca.run(["workplace_location_simulate"])
orca.run(["auto_ownership_simulate"])
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
orca.get_table("mandatory_tours").tour_type.value_counts()
orca.run(['non_mandatory_tour_frequency'])
orca.get_table("non_mandatory_tours").tour_type.value_counts()
orca.run(["destination_choice"])
orca.run(["mandatory_scheduling"])
orca.run(["non_mandatory_scheduling"])
orca.run(["mode_choice_simulate"])
orca.clear_cache()
def decorator(func):
name = func.__name__
logger.debug("inject injectable %s" % name)
# insist on explicit override to ensure multiple definitions occur in correct order
assert override or not _DECORATED_INJECTABLES.get(name, False), \
"injectable '%s' already defined. not overridden" % name
_DECORATED_INJECTABLES[name] = {'func': func, 'cache': cache}
orca.add_injectable(name, func, cache=cache)
return func
def calibrate(iter_var):
orca.add_injectable('mult_val', iter_var)
orca.run([
'rsh_simulate',
'nrh_simulate',
'emp_transition',
'emp_relocation',
'elcm_simulate',
'hh_transition',
'hh_relocation',
'hlcm_simulate',
'feasibility',
'residential_developer',
'non_residential_developer',
'indicator_export',
'reset_to_base',
], iter_vars=[2015])
def parcels_zoning_by_scenario(parcels, parcels_zoning_calculations,
zoning_baseline):
df = pd.DataFrame(index=parcels.index)
df["baseline_dua"] = zoning_baseline.max_dua
df["baseline_far"] = zoning_baseline.max_far
df["baseline_height"] = zoning_baseline.max_height
df["zoning_name"] = zoning_baseline["name"]
df["zoning_source"] = zoning_baseline["tablename"]
for scenario in [str(i) for i in range(4)]:
orca.clear_cache()
orca.add_injectable("scenario", scenario)
z = orca.get_table("parcels_zoning_calculations")
df["max_dua_%s" % scenario] = z.effective_max_dua
df["max_far_%s" % scenario] = z.effective_max_far
df["du_underbuild_%s" % scenario] = z.zoned_du_underbuild
df["non_res_cat_%s" % scenario] = z.non_res_categories
return df
def full_run(store, omx_file, preload_3d_skims, chunk_size=0):
configs_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'example')
orca.add_injectable("configs_dir", configs_dir)
inject_settings(configs_dir,
households_sample_size=HOUSEHOLDS_SAMPLE_SIZE,
preload_3d_skims=preload_3d_skims,
chunk_size=chunk_size)
orca.add_injectable("omx_file", omx_file)
orca.add_injectable("store", store)
orca.add_injectable("set_random_seed", set_random_seed)
orca.clear_cache()
# grab some of the tables
orca.get_table("land_use").to_frame().info()
orca.get_table("households").to_frame().info()
orca.get_table("persons").to_frame().info()
assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
assert orca.get_injectable("chunk_size") == chunk_size
# run the models in the expected order
orca.run(["school_location_simulate"])
orca.run(["workplace_location_simulate"])
orca.run(["auto_ownership_simulate"])
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
orca.get_table("mandatory_tours").tour_type.value_counts()
orca.run(['non_mandatory_tour_frequency'])
orca.get_table("non_mandatory_tours").tour_type.value_counts()
orca.run(["destination_choice"])
orca.run(["mandatory_scheduling"])
orca.run(["non_mandatory_scheduling"])
orca.run(["patch_mandatory_tour_destination"])
orca.run(["tour_mode_choice_simulate"])
orca.run(["trip_mode_choice_simulate"])
tours_merged = orca.get_table("tours_merged").to_frame()
tour_count = len(tours_merged.index)
orca.clear_cache()
return tour_count
def test_mini_run(store, omx_file, random_seed):
configs_dir = os.path.join(os.path.dirname(__file__))
orca.add_injectable("configs_dir", configs_dir)
inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
orca.add_injectable("omx_file", omx_file)
orca.add_injectable("store", store)
orca.add_injectable("set_random_seed", set_random_seed)
orca.clear_cache()
assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
# run the models in the expected order
orca.run(["workplace_location_simulate"])
orca.run(["auto_ownership_simulate"])
# this is a regression test so that we know if these numbers change
auto_choice = orca.get_table('households').get_column('auto_ownership')
hh_ids = [2124015, 961042, 1583271]
choices = [1, 2, 2]
print "auto_choice\n", auto_choice.head(3)
pdt.assert_series_equal(
auto_choice[hh_ids],
pd.Series(choices, index=pd.Index(hh_ids, name="HHID")))
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
mtf_choice = orca.get_table('persons').get_column('mandatory_tour_frequency')
per_ids = [172616, 172781, 172782]
choices = ['work1', 'school1', 'work_and_school']
print "mtf_choice\n", mtf_choice.head(20)
pdt.assert_series_equal(
mtf_choice[per_ids],
pd.Series(choices, index=pd.Index(per_ids, name='PERID')))
orca.clear_cache()
def test_mini_run(store, random_seed):
orca.add_injectable("configs_dir",
os.path.join(os.path.dirname(__file__)))
orca.add_injectable("store", store)
orca.add_injectable("nonmotskm_matrix", np.ones((1454, 1454)))
orca.add_injectable("set_random_seed", set_random_seed)
assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
# run the models in the expected order
orca.run(["workplace_location_simulate"])
orca.run(["auto_ownership_simulate"])
# this is a regression test so that we know if these numbers change
auto_choice = orca.get_table('households').get_column('auto_ownership')
print auto_choice[[2306822, 652072, 651907]]
pdt.assert_series_equal(
auto_choice[[2306822, 652072, 651907]],
pd.Series(
[2, 1, 1], index=pd.Index([2306822, 652072, 651907], name='HHID')))
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
mtf_choice = orca.get_table('persons').get_column(
'mandatory_tour_frequency')
pdt.assert_series_equal(
mtf_choice[[146642, 642922, 642921]],
pd.Series(
['school1', 'work1', 'school2'],
index=pd.Index([146642, 642922, 642921], name='PERID')))
orca.clear_cache()
import orca
from activitysim import defaults
import pandas as pd
import numpy as np
import os
orca.add_injectable("output_dir", 'output')
defaults.tracing.config_logger()
orca.run(["school_location_simulate"])
orca.run(["workplace_location_simulate"])
print orca.get_table("persons").distance_to_work.describe()
orca.run(["auto_ownership_simulate"])
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
orca.get_table("mandatory_tours").tour_type.value_counts()
orca.run(["mandatory_scheduling"])
orca.run(['non_mandatory_tour_frequency'])
orca.get_table("non_mandatory_tours").tour_type.value_counts()
orca.run(["destination_choice"])
orca.run(["non_mandatory_scheduling"])
# FIXME - jwd - choose more felicitous name or do this elsewhere?
orca.run(["patch_mandatory_tour_destination"])
orca.run(['tour_mode_choice_simulate'])
orca.run(['trip_mode_choice_simulate'])
def change_store(store_name):
orca.add_injectable("store",
pd.HDFStore(os.path.join(misc.data_dir(),
store_name), mode="r"))
EVERY_NTH_YEAR = 5
CURRENT_COMMIT = os.popen('git rev-parse HEAD').read()
COMPARE_TO_NO_PROJECT = True
NO_PROJECT = 611
IN_YEAR, OUT_YEAR = 2010, 2040
LAST_KNOWN_GOOD_RUNS = {
"0": 1057,
"1": 1058,
"2": 1059,
"3": 1060,
"4": 1059,
"5": 1059
}
orca.add_injectable("years_per_iter", EVERY_NTH_YEAR)
if len(args) and args[0] == "-i":
SLACK = MAPS = LOGS = False
INTERACT = True
if len(args) and args[0] == "-s":
orca.add_injectable("scenario", args[1])
SCENARIO = orca.get_injectable("scenario")
if INTERACT:
import code
code.interact(local=locals())
sys.exit()
def eq_code_buildings(buildings, year, earthquake):
if year == 2035 and earthquake:
# tags buildings that exist in 2035 with a fragility coefficient
# keeping in-model adds run time, but is important given developer
# model stochastisitcy, that will change the building stock in 2035
# this also allows us to change the building codes when retrofitting
# policies are applied, thus changing fragility coefficients
buildings = buildings.to_frame()
code = []
fragilities = []
for i in buildings.index:
if (buildings['building_type'][i] == 'HS' and
buildings['year_built'][i] <= 2015):
a = 'SF'
if buildings['stories'][i] == 1:
b = '01'
if buildings['year_built'][i] <= 1940:
c = 'G1'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G2'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1995):
c = 'G3'
elif (buildings['year_built'][i] >= 1996 and
buildings['year_built'][i] <= 2015):
c = 'G4'
elif buildings['stories'][i] >= 2:
b = '2P'
if buildings['year_built'][i] <= 1920:
c = 'G1'
elif (buildings['year_built'][i] >= 1921 and
buildings['year_built'][i] <= 1940):
c = 'G2'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G3'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1995):
c = 'G4'
elif (buildings['year_built'][i] >= 1996 and
buildings['year_built'][i] <= 2015):
c = 'G5'
elif ((buildings['building_type'][i] == 'HM' or
buildings['building_type'][i] == 'MR') and
buildings['year_built'][i] <= 2015):
if (buildings['residential_units'][i] == 2 or
buildings['residential_units'][i] == 3 or
buildings['residential_units'][i] == 4):
a = 'DU' # 2, 3, & 4 units
# are considered duplex/triplex/quadplex
if buildings['stories'][i] == 1:
b = '01'
if buildings['year_built'][i] <= 1940:
c = 'G1'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G2'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1995):
c = 'G3'
elif (buildings['year_built'][i] >= 1996 and
buildings['year_built'][i] <= 2015):
c = 'G4'
if buildings['stories'][i] >= 2:
b = '2P'
if buildings['year_built'][i] <= 1920:
c = 'G1'
elif (buildings['year_built'][i] >= 1921 and
buildings['year_built'][i] <= 1940):
c = 'G2'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G3'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1977):
c = 'G4'
elif (buildings['year_built'][i] >= 1978 and
buildings['year_built'][i] <= 1991):
c = 'G5'
elif (buildings['year_built'][i] >= 1992 and
buildings['year_built'][i] <= 2015):
c = 'G6'
else: # this assumes one-unit HM/MR buildings
# are also 5+ units (multifamily split by parcels)
a = 'MF'
if buildings['stories'][i] == 1:
b = '01'
if buildings['year_built'][i] <= 1920:
c = 'G1'
elif (buildings['year_built'][i] >= 1921 and
buildings['year_built'][i] <= 1940):
c = 'G2'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G3'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1995):
c = 'G4'
elif (buildings['year_built'][i] >= 1996 and
buildings['year_built'][i] <= 2015):
c = 'G5'
elif (buildings['stories'][i] >= 2 and
buildings['stories'][i] <= 5):
b = '25'
if buildings['year_built'][i] <= 1920:
c = 'G1'
elif (buildings['year_built'][i] >= 1921 and
buildings['year_built'][i] <= 1940):
c = 'G2'
elif (buildings['year_built'][i] >= 1941 and
buildings['year_built'][i] <= 1960):
c = 'G3'
elif (buildings['year_built'][i] >= 1961 and
buildings['year_built'][i] <= 1977):
c = 'G4'
elif (buildings['year_built'][i] >= 1978 and
buildings['year_built'][i] <= 1991):
c = 'G5'
elif (buildings['year_built'][i] >= 1992 and
buildings['year_built'][i] <= 2015):
c = 'G6'
elif buildings['stories'][i] >= 6:
b = '5P'
if buildings['year_built'][i] <= 1950:
c = 'G1'
elif (buildings['year_built'][i] >= 1951 and
buildings['year_built'][i] <= 1971):
c = 'G2'
elif (buildings['year_built'][i] >= 1972 and
buildings['year_built'][i] <= 1995):
c = 'G3'
elif (buildings['year_built'][i] >= 1996 and
buildings['year_built'][i] <= 2006):
c = 'G4'
elif (buildings['year_built'][i] >= 2007 and
buildings['year_built'][i] <= 2015):
c = 'G5'
elif buildings['year_built'][i] <= 2015:
a = 'OT'
b = 'NN'
if buildings['year_built'][i] <= 1933:
c = 'G1'
elif (buildings['year_built'][i] >= 1934 and
buildings['year_built'][i] <= 1950):
c = 'G2'
elif (buildings['year_built'][i] >= 1951 and
buildings['year_built'][i] <= 1972):
c = 'G3'
elif (buildings['year_built'][i] >= 1973 and
buildings['year_built'][i] <= 1996):
c = 'G4'
elif (buildings['year_built'][i] >= 1997 and
buildings['year_built'][i] <= 2006):
c = 'G5'
elif (buildings['year_built'][i] >= 2007 and
buildings['year_built'][i] <= 2015):
c = 'G6'
# new buildings built by the developer model
elif buildings['year_built'][i] > 2015:
a = 'NN'
b = 'NN'
c = 'NN'
# alternative if retrofitted: d = 'R'
d = 'N'
code_i = a+b+c+d
code.append(code_i)
# assign a fragility coefficient based on building code
if (code_i == 'SF01G4N' or code_i == 'SF2PG5N' or
code_i == 'DU2PG6N' or code_i == 'MF5PG5N' or
code_i == 'DU01G4N' or code_i == 'MF25G6N' or
code_i == 'MF01G5N' or code_i == 'OTNNG6N'):
fragility = 1
elif (code_i == 'SF01G3N' or code_i == 'DU01G3N' or
code_i == 'DU2PG5N' or code_i == 'MF25G5N' or
code_i == 'MF01G4N' or code_i == 'OTNNG5N' or
code_i == 'MF5PG4N'):
fragility = 1.2
elif (code_i == 'SF2PG4N' or code_i == 'MF5PG3N' or
code_i == 'OTNNG4N'):
fragility = 1.3
elif (code_i == 'MF5PG1N' or code_i == 'OTNNG2N'):
fragility = 1.4
elif (code_i == 'MF01G3N' or code_i == 'MF5PG2N' or
code_i == 'SF01G2N' or code_i == 'DU01G2N' or
code_i == 'OTNNG3N'):
fragility = 1.5
elif (code_i == 'DU2PG3N' or code_i == 'DU2PG4N'):
fragility = 1.75
elif (code_i == 'SF2PG3N' or code_i == 'DU01G1N' or
code_i == 'DU2PG2N' or code_i == 'MF01G2N' or
code_i == 'OTNNG1N'):
fragility = 2
elif (code_i == 'SF2PG2N'):
fragility = 2.25
elif (code_i == 'DU2PG1N' or code_i == 'SF01G1N' or
code_i == 'SF2PG1N' or code_i == 'MF01G1N' or
code_i == 'MF25G1N'):
fragility = 2.5
elif (code_i == 'MF25G2N' or code_i == 'MF25G3N' or
code_i == 'MF25G4N'):
fragility = 3
elif (code_i == 'NNNNNNN'):
fragility = 0
fragilities.append(fragility)
orca.add_injectable("code", code)
orca.add_injectable("fragilities", fragilities)
# add codes and fragilities as orca columns
code = pd.Series(code, buildings.index)
orca.add_column('buildings', 'earthquake_code', code)
fragility = pd.Series(fragilities, buildings.index)
orca.add_column('buildings', 'fragility_coef', fragility)
# generate random number, multiply by fragilities
buildings = orca.get_table('buildings')
rand_eq = np.random.random(len(buildings))
destroy_eq = pd.Series(rand_eq*fragility)
orca.add_column('buildings', 'eq_destroy', destroy_eq)
# generate random number for fire
rand_fire = pd.Series(np.random.random(len(buildings)))
orca.add_column('buildings', 'fire_destroy', rand_fire)
def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings,
households, jobs, residential_units, year, earthquake):
if year == 2035 and earthquake:
# assign each parcel to a census tract
# using the lookup table created with "parcel_tract_assignment.ipynb"
census_tract = pd.Series(parcels_tract['census_tract'],
parcels_tract.index)
print "Number of parcels with census tracts is: %d" % len(census_tract)
orca.add_column('parcels', 'tract', census_tract)
# group parcels by their census tract
parcels_tract['parcel_id'] = parcels_tract.index
parcels_tract = parcels_tract.to_frame(columns=['parcel_id',
'census_tract'])
parcels_tract = parcels_tract[['census_tract', 'parcel_id']]
tract_parcels_grp = []
tracts = []
parcels_tract = sorted(parcels_tract.values, key=itemgetter(0))
for tract, parcels in itertools.groupby(parcels_tract,
key=itemgetter(0)):
tract_parcels_grp.append(list(parcels))
tracts.append(tract)
print "Number of census tract groups is: %d" % len(tract_parcels_grp)
# for the parcels in each tract, destroy X% of parcels in that tract
tracts_earthquake = tracts_earthquake.to_frame()
tracts_earthquake = tracts_earthquake.sort_values(by=['tract_ba'])
tracts_earthquake = tracts_earthquake.reset_index(drop=True)
buildings = buildings.to_frame()
eq_buildings = []
existing_buildings = []
new_buildings = []
fire_buildings = []
for i in range(len(tracts)):
grp = [x[1] for x in tract_parcels_grp[i]]
buildings_i = buildings[buildings['parcel_id'].isin(grp)]
# existing buildings
# select the buildings with highest fragility co-efficient
# (and random no.) based on census tract pct to be destroyed
existing_pct = tracts_earthquake['prop_eq'][i]
build_frag = buildings_i['eq_destroy'].sort_values(ascending=False)
top_build_frag = build_frag[: int(round(
len(build_frag) * existing_pct))]
# add to a list of buildings to destroy
buildings_top = top_build_frag.index
existing_buildings.extend(buildings_top)
eq_buildings.extend(buildings_top)
# new buildings
# translate MMI to a probability
# in-model is also nice if probabilities associated with
# new buildings change
buildings_new = buildings_i[buildings_i['year_built'] > 2015]
if len(buildings_new) > 0:
mmi = int(round(tracts_earthquake['shaking'][i]))
if mmi < 6:
new_pct = 0
elif mmi == 7:
new_pct = .002
elif mmi == 8:
new_pct = .01
elif mmi == 9:
new_pct = .05
# randomly select buildings to be destroyed based on
# percentages
new_no = int(round(len(buildings_new)*new_pct))
buildings_new_rand = np.random.choice(buildings_new.index,
new_no, replace=False)
# add to a list of buildings to destroy
if len(buildings_new_rand) > 0:
new_buildings.extend(buildings_new_rand)
eq_buildings.extend(buildings_new_rand)
# fire buildings
# select buildings to be destroyed by fire by looking only at
# remaining buildings
fire_pct = tracts_earthquake['prop_fire'][i]
buildings_i_remain = buildings_i[~buildings_i.index.isin
(buildings_top)]
if len(buildings_new) > 0:
buildings_i_remain = buildings_i_remain[~buildings_i_remain.
index.isin
(buildings_new_rand)]
# select buildings to be destroyed based on random number
# and census tract pct
fire_buildings_rand = buildings_i_remain['fire_destroy']. \
sort_values(ascending=False)
top_fire_buildings = fire_buildings_rand[: int(round(
len(fire_buildings_rand) * fire_pct))]
# add to a list of buildings to destroy
buildings_fire = top_fire_buildings.index
fire_buildings.extend(buildings_fire)
eq_buildings.extend(buildings_fire)
print "Total number of buildings being destroyed is: %d" \
% len(eq_buildings)
orca.add_injectable("eq_buildings", eq_buildings)
orca.add_injectable("existing_buildings", existing_buildings)
orca.add_injectable("new_buildings", new_buildings)
orca.add_injectable("fire_buildings", fire_buildings)
# remove buildings, unplace agents
buildings = orca.get_table('buildings')
eq_demolish = buildings.local[buildings.index.isin
(eq_buildings)]
orca.add_table("eq_demolish", eq_demolish)
print "Demolishing %d buildings" % len(eq_demolish)
households = households.to_frame()
hh_unplaced = households[households["building_id"] == -1]
jobs = jobs.to_frame()
jobs_unplaced = jobs[jobs["building_id"] == -1]
l1 = len(buildings)
# currently destroying more buildings than it is being
# passed- why?
buildings = utils._remove_developed_buildings(
buildings.to_frame(buildings.local_columns),
eq_demolish,
unplace_agents=["households", "jobs"])
households = orca.get_table("households")
households = households.to_frame()
hh_unplaced_eq = households[households["building_id"] == -1]
hh_unplaced_eq = hh_unplaced_eq[~hh_unplaced_eq.index.isin
(hh_unplaced.index)]
orca.add_injectable("hh_unplaced_eq", hh_unplaced_eq)
jobs = orca.get_table("jobs")
jobs = jobs.to_frame()
jobs_unplaced_eq = jobs[jobs["building_id"] == -1]
jobs_unplaced_eq = jobs_unplaced_eq[~jobs_unplaced_eq.index.isin
(jobs_unplaced.index)]
orca.add_injectable("jobs_unplaced_eq", jobs_unplaced_eq)
orca.add_table("buildings", buildings)
buildings = orca.get_table("buildings")
print "Demolished %d buildings" % (l1 - len(buildings))
def topsheet(households, jobs, buildings, parcels, zones, year,
run_number, taz_geography, parcels_zoning_calculations,
summary, settings, parcels_geography):
hh_by_subregion = misc.reindex(taz_geography.subregion,
households.zone_id).value_counts()
households_df = orca.merge_tables(
'households',
[parcels_geography, buildings, households],
columns=['pda_id', 'tpp_id', 'income'])
hh_by_inpda = households_df.pda_id.notnull().value_counts()
hhincome_by_intpp = households_df.income.groupby(
households_df.tpp_id.notnull()).mean()
# round to nearest 100s
hhincome_by_intpp = (hhincome_by_intpp/100).round()*100
jobs_by_subregion = misc.reindex(taz_geography.subregion,
jobs.zone_id).value_counts()
jobs_df = orca.merge_tables(
'jobs',
[parcels, buildings, jobs],
columns=['pda'])
jobs_by_inpda = jobs_df.pda.notnull().value_counts()
capacity = parcels_zoning_calculations.\
zoned_du_underbuild_nodev.groupby(parcels.subregion).sum()
if year == 2010:
# save some info for computing growth measures
orca.add_injectable("base_year_measures", {
"hh_by_subregion": hh_by_subregion,
"jobs_by_subregion": jobs_by_subregion,
"hh_by_inpda": hh_by_inpda,
"jobs_by_inpda": jobs_by_inpda,
"hhincome_by_intpp": hhincome_by_intpp,
"capacity": capacity
})
# if year != 2040:
# return
base_year_measures = orca.get_injectable("base_year_measures")
f = open(os.path.join("runs", "run%d_topsheet_%d.log" %
(run_number, year)), "w")
def write(s):
# print s
f.write(s + "\n\n")
def norm_and_round(s):
# normalize and round a series
return str((s/s.sum()).round(2))
nhh = len(households)
write("Number of households = %d" % nhh)
nj = len(jobs)
write("Number of jobs = %d" % nj)
n = len(households.building_id[households.building_id == -1])
write("Number of unplaced households = %d" % n)
n = len(jobs.building_id[jobs.building_id == -1])
write("Number of unplaced jobs = %d" % n)
du = buildings.residential_units.sum()
write("Number of residential units = %d" % du)
write("Residential vacancy rate = %.2f" % (1-0 - float(nhh)/du))
du = buildings.deed_restricted_units.sum()
write("Number of deed restricted units = %d" % du)
write("Base year mean income by whether household is in tpp:\n%s" %
base_year_measures["hhincome_by_intpp"])
write("Horizon year mean income by whether household is in tpp:\n%s" %
hhincome_by_intpp)
jsp = buildings.job_spaces.sum()
write("Number of job spaces = %d" % jsp)
write("Non-residential vacancy rate = %.2f" % (1-0 - float(nj)/jsp))
tmp = base_year_measures["hh_by_subregion"]
write("Households base year share by subregion:\n%s" %
norm_and_round(tmp))
write("Households share by subregion:\n%s" %
norm_and_round(hh_by_subregion))
diff = hh_by_subregion - base_year_measures["hh_by_subregion"]
write("Households pct of regional growth by subregion:\n%s" %
norm_and_round(diff))
tmp = base_year_measures["jobs_by_subregion"]
write("Jobs base year share by subregion:\n%s" %
norm_and_round(tmp))
write("Jobs share by subregion:\n%s" %
norm_and_round(jobs_by_subregion))
diff = jobs_by_subregion - base_year_measures["jobs_by_subregion"]
write("Jobs pct of regional growth by subregion:\n%s" %
norm_and_round(diff))
tmp = base_year_measures["hh_by_inpda"]
write("Households base year share in pdas:\n%s" %
norm_and_round(tmp))
write("Households share in pdas:\n%s" %
norm_and_round(hh_by_inpda))
diff = hh_by_inpda - base_year_measures["hh_by_inpda"]
write("Households pct of regional growth in pdas:\n%s" %
norm_and_round(diff))
tmp = base_year_measures["jobs_by_inpda"]
write("Jobs base year share in pdas:\n%s" %
norm_and_round(tmp))
write("Jobs share in pdas:\n%s" %
norm_and_round(jobs_by_inpda))
diff = jobs_by_inpda - base_year_measures["jobs_by_inpda"]
write("Jobs pct of regional growth in pdas:\n%s" %
norm_and_round(diff))
write("Base year dwelling unit raw capacity:\n%s" %
base_year_measures["capacity"])
write("Dwelling unit raw capacity:\n%s" % capacity)
if summary.parcel_output is not None:
df = summary.parcel_output
# we mark greenfield as a parcel with less than 500 current sqft
greenfield = df.total_sqft < 500
write("Current share of projects which are greenfield development:\n%s"
% norm_and_round(greenfield.value_counts()))
write("Current share of units which are greenfield development:\n%s" %
norm_and_round(df.residential_units.groupby(greenfield).sum()))
cmap = settings["county_id_tm_map"]
jobs_by_county = jobs.zone_id.map(taz_geography.county)\
.map(cmap).value_counts()
households_by_county = households.zone_id.map(taz_geography.county)\
.map(cmap).value_counts()
jobs_by_housing = jobs_by_county / households_by_county.replace(0, 1)
write("Jobs/housing balance:\n" + str(jobs_by_housing))
f.close()
def buildings_to_uc(new_buildings, year):
"""
Parameters
----------
new_buildings : pandas.DataFrame
DataFrame of buildings to export.
year : int
Simulation year
Returns
-------
None
"""
# Checking that building_id is index or exists as column
if (new_buildings.index.name != 'building_id') & ('building_id' not in new_buildings.columns):
print 'Index of buildings must be "building_id" or "building_id" column must exist. Skipping export-to-Urban-Canvas.'
return None
if 'building_id' not in new_buildings.columns:
new_buildings = new_buildings.reset_index()
# Urban Canvas database connection
conn_string = orca.get_injectable('conn_string')
if len(conn_string) == 0:
print 'A "conn_string" injectable must be registered and populated. Skipping export-to-Urban-Canvas.'
return None
if 'uc_conn' not in orca.list_injectables():
conn = psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
else:
conn = orca.get_injectable('uc_conn')
cur = orca.get_injectable('uc_cur')
def exec_sql_uc(query):
try:
cur.execute(query)
conn.commit()
except:
conn = psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
cur.execute(query)
conn.commit()
def get_val_from_uc_db(query):
try:
result = sql.read_frame(query, conn)
return result.values[0][0]
except:
conn=psycopg2.connect(conn_string)
cur = conn.cursor()
orca.add_injectable('uc_conn', conn)
orca.add_injectable('uc_cur', cur)
result = sql.read_frame(query, conn)
result2 = sql.read_frame("select column_name from Information_schema.columns where table_name like 'building' ", conn)
print result2
return result.values[0][0]
max_bid = get_val_from_uc_db("select max(building_id) FROM building where building_id<100000000;")
new_buildings.building_id = np.arange(max_bid+1, max_bid+1+len(new_buildings))
if 'projects_num' not in orca.list_injectables():
exec_sql_uc("INSERT INTO scenario(id, name, type) select nextval('scenario_id_seq'), 'Run #' || cast(currval('scenario_id_seq') as character varying), 1;")
nextval = get_val_from_uc_db("SELECT MAX(ID) FROM SCENARIO WHERE ID < 100000;")
exec_sql_uc("INSERT INTO scenario(id, name, type) select nextval('scenario_id_seq'), 'Run #' || cast(currval('scenario_id_seq') as character varying), 1;")
nextval = get_val_from_uc_db("SELECT MAX(ID) FROM SCENARIO WHERE ID < 1000000;")
orca.add_injectable('projects_num', nextval)
exec_sql_uc("INSERT INTO scenario_project(scenario, project) VALUES(%s, 1);" % nextval)
exec_sql_uc("INSERT INTO scenario_project(scenario, project) VALUES(%s, %s);" % (nextval, nextval))
else:
nextval = orca.get_injectable('projects_num')
nextval = '{'+str(nextval)+ '}'
new_buildings['projects'] = nextval
valid_from = '{'+ str(year) + '-1-1}'
new_buildings['valid_from'] = valid_from
print 'Exporting %s buildings to Urban Canvas database for project %s and year %s.' % (len(new_buildings),nextval,year)
output = cStringIO.StringIO()
new_buildings.to_csv('buildings_for_eddie.csv')
new_buildings.to_csv(output, sep='\t', header=False, index=False)
output.seek(0)
cur.copy_from(output, 'building', columns=tuple(new_buildings.columns.values.astype('U').tolist()))
test = pd.read_sql("select projects from building where (year_built>2010)", conn)
print test
conn.commit()
def change_scenario(scenario):
assert scenario in orca.get_injectable("scenario_inputs"), \
"Invalid scenario name"
print "Changing scenario to '%s'" % scenario
orca.add_injectable("scenario", scenario)
NO_PROJECT = 611
EARTHQUAKE = False
IN_YEAR, OUT_YEAR = 2010, 2050
COMPARE_AGAINST_LAST_KNOWN_GOOD = False
LAST_KNOWN_GOOD_RUNS = {
"0": 1057,
"1": 1058,
"2": 1059,
"3": 1060,
"4": 1059,
"5": 1059
}
orca.add_injectable("years_per_iter", EVERY_NTH_YEAR)
orca.add_injectable("earthquake", EARTHQUAKE)
parser = argparse.ArgumentParser(description='Run UrbanSim models.')
parser.add_argument(
'-c', action='store_true', dest='console',
help='run from the console (logs to stdout), no slack or maps')
parser.add_argument('-i', action='store_true', dest='interactive',
help='enter interactive mode after imports')
parser.add_argument('-s', action='store', dest='scenario',
help='specify which scenario to run')
import os
import orca
import pandas as pd
import numpy as np
import urbansim_defaults.utils as utils
from urbansim.utils import misc
from urbansim_defaults import datasources
import warnings
warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning)
orca.add_injectable('base_year', 2014)
@orca.injectable()
def year(base_year):
if 'iter_var' in orca.list_injectables():
year = orca.get_injectable('iter_var')
if year is not None:
return year
# outside of a run, return the base/default
return base_year
@orca.injectable()
def store_table_names_dict():
# Dictionary with pairs of orca.table name and the associated name in storage.
# Only entries where the names differ.
return {'employment_controls': "annual_employment_control_totals",
'employment_sector_group_definitions': "employment_adhoc_sector_group_definitions",
'employment_sector_groups': "employment_adhoc_sector_groups",
'household_controls': "annual_household_control_totals",
'household_relocation_rates': 'annual_household_relocation_rates',
'job_relocation_rates': 'annual_job_relocation_rates',
__author__ = 'JMartinez'
import orca
import pandas as pd
import numpy as np
from sqlalchemy import engine
#add your data path here
orca.add_injectable("store",pd.HDFStore('C:/urbansim_new/urbansim/urbansim_drcog/config/drcog.h5', mode='r'))
#register tables
@orca.table('buildings', cache=True)
def buildings(store):
df = store['buildings']
return df
@orca.table('parcels', cache=True)
def parcels(store):
df = store['parcels']
return df
@orca.table('households', cache=True)
def households(store):
df = store['households']
#df.drop('zone_id', axis=1, inplace=True)
return df
@orca.table('zones', cache=True)
def zones(store):
df = store['zones']
amenities = pd.read_csv('c:/urbansim_new/urbansim/urbansim_drcog/config/amenities.csv',index_col=0)
df = pd.merge(df, amenities, left_index=True, right_index=True)
def run_feasibility(parcels, parcel_price_callback,
parcel_use_allowed_callback, pipeline=False,
cfg=None, **kwargs):
"""
Execute development feasibility on all development sites
Parameters
----------
parcels : DataFrame Wrapper
The data frame wrapper for the parcel data
parcel_price_callback : function
A callback which takes each use of the pro forma and returns a series
with index as parcel_id and value as yearly_rent
parcel_use_allowed_callback : function
A callback which takes each form of the pro forma and returns a series
with index as parcel_id and value and boolean whether the form
is allowed on the parcel
pipeline : bool, optional
If True, removes parcels from consideration if already in dev_sites
table
cfg : str, optional
The name of the yaml file to read pro forma configurations from
"""
cfg = misc.config(cfg)
# Create default SqFtProForma
pf = (sqftproforma.SqFtProForma.from_yaml(str_or_buffer=cfg)
if cfg else sqftproforma.SqFtProForma.from_defaults())
# Update default values using templates and store
pf = update_sqftproforma(pf, cfg, **kwargs)
orca.add_injectable("pf_config", pf)
sites = (pl.remove_pipelined_sites(parcels) if pipeline
else parcels.to_frame(parcels.local_columns))
#df = apply_parcel_callbacks(sites, parcel_price_callback,
# pf, **kwargs)
# compute price for each use
df = sites
for use in pf.uses:
df[use] = parcel_price_callback(use, pf)
#feasibility = lookup_by_form(df, parcel_use_allowed_callback, pf, **kwargs)
print "Describe of the yearly rent by use"
print df[pf.uses].describe()
# Computing actual feasibility
d = {}
forms = pf.forms_to_test or pf.forms
for form in forms:
print "Computing feasibility for form %s" % form
#if parcel_id_col is not None:
# parcels = df[parcel_id_col].unique()
# allowed = (parcel_use_allowed_callback(form).loc[parcels])
# newdf = df.loc[misc.reindex(allowed, df.parcel_id)]
#else:
allowed = parcel_use_allowed_callback(form).loc[df.index]
newdf = df[allowed]
# Core function - computes profitability
d[form] = pf.lookup(form, newdf, only_built = pf.only_built,
pass_through = pf.pass_through)
# Collect results
if pf.proposals_to_keep > 1:
# feasibility is in long format
form_feas = []
for form_name in d.keys():
df_feas_form = d[form_name]
df_feas_form['form'] = form_name
form_feas.append(df_feas_form)
feasibility = pd.concat(form_feas, sort=False)
if pf.percent_of_max_profit > 0:
feasibility['max_profit_parcel'] = feasibility.groupby([feasibility.index, 'form'])['max_profit'].transform(max)
feasibility['ratio'] = feasibility.max_profit/feasibility.max_profit_parcel
feasibility = feasibility[feasibility.ratio >= pf.percent_of_max_profit / 100.]
feasibility.drop(['max_profit_parcel', 'ratio'], axis=1, inplace = True)
feasibility.index.name = 'parcel_id'
# add attribute that enumerates proposals (can be used as a unique index)
feasibility["feasibility_id"] = np.arange(1, len(feasibility)+1, dtype = "int32")
# create a dataset with disaggregated sqft by building type
feas_bt = pd.merge(feasibility.loc[:, ["form", "feasibility_id", "residential_sqft", "non_residential_sqft"]], pf.forms_df, left_on = "form", right_index = True)
feas_bt.set_index(['form'], append = True, inplace = True)
feas_bt[pf.uses[pf.residential_uses.values == 1]] = feas_bt[pf.uses[pf.residential_uses.values == 1]].multiply(feas_bt.residential_sqft, axis = "index")
feas_bt[pf.uses[pf.residential_uses.values == 0]] = feas_bt[pf.uses[pf.residential_uses.values == 0]].multiply(feas_bt.non_residential_sqft, axis = "index")
orca.add_table('feasibility_bt', feas_bt)
else:
# feasibility is in wide format
feasibility = pd.concat(d.values(), keys = d.keys(), axis=1)
orca.add_table('feasibility', feasibility)
return feasibility
