def household_relocation_rates(scenario, policy): if scenario in policy['futures_scenarios']: if scenario in policy['reloc_fr2_enable']: df = pd.read_csv(os.path.join("data", "household_relocation_rates_fr2.csv")) orca.add_injectable("hh_reloc", 'activated') print("File used is: household_relocation_rates_fr2.csv") else: df = pd.read_csv(os.path.join("data", "household_relocation_rates_fr_base.csv")) orca.add_injectable("hh_reloc", 'not activated') print("File used is: household_relocation_rates_fr_base.csv") elif scenario in policy['reloc_db_enable']: df = pd.read_csv(os.path.join("data", "household_relocation_rates_db_var.csv")) orca.add_injectable("hh_reloc", 'activated') print("File used is: household_relocation_rates_db_var.csv") elif scenario in policy['reloc_fb_enable']: df = pd.read_csv(os.path.join("data", "household_relocation_rates_fb.csv")) orca.add_injectable("hh_reloc", 'activated') print("File used is: household_relocation_rates_fb.csv") else: df = pd.read_csv(os.path.join("data", "household_relocation_rates_db_base.csv")) orca.add_injectable("hh_reloc", 'not activated') print("File used is: household_relocation_rates_db_base.csv") return df
def add_canonical_dirs(): configs_dir = os.path.join(os.path.dirname(__file__), 'configs') orca.add_injectable("configs_dir", configs_dir) output_dir = os.path.join(os.path.dirname(__file__), 'output') orca.add_injectable("output_dir", output_dir)
def setup_orca(dfa, dfb, dfa_col, dfb_col, dfa_factor, dfb_factor): orca.add_injectable('a_factor', dfa_factor) @orca.injectable() def b_factor(): return dfb_factor orca.add_table('dfa', dfa) @orca.table('dfb') def dfb_table(): return dfb orca.add_column('dfa', 'acol', dfa_col) orca.add_column('dfb', 'bcol', dfb_col) @orca.column('dfa') def extra_acol(a_factor): return dfa_col * a_factor @orca.column('dfb') def extra_bcol(b_factor): return dfb_col * b_factor orca.broadcast('dfb', 'dfa', cast_on='a_id', onto_index=True) @orca.step() def test_step(dfa, dfb): pass
def large_area_id(jobs, buildings): job_la = "jobs_large_area_lookup" if (not orca.is_injectable(job_la)) or (len(orca.get_injectable(job_la)) == 0): orca.add_injectable(job_la, misc.reindex(buildings.large_area_id, jobs.building_id), autocall=False, cache=True) return orca.get_injectable(job_la).loc[jobs.index]
def slr_remove_dev(buildings, destroy_parcels, year, parcels, households, jobs): slr_demolish = buildings.local[buildings.parcel_id.isin (destroy_parcels.index)] orca.add_table("slr_demolish", slr_demolish) print "Demolishing %d buildings" % len(slr_demolish) households = households.to_frame() hh_unplaced = households[households["building_id"] == -1] jobs = jobs.to_frame() jobs_unplaced = jobs[jobs["building_id"] == -1] l1 = len(buildings) buildings = utils._remove_developed_buildings( buildings.to_frame(buildings.local_columns), slr_demolish, unplace_agents=["households", "jobs"]) households = orca.get_table("households") households = households.to_frame() hh_unplaced_slr = households[households["building_id"] == -1] hh_unplaced_slr = hh_unplaced_slr[~hh_unplaced_slr.index.isin (hh_unplaced.index)] orca.add_injectable("hh_unplaced_slr", hh_unplaced_slr) jobs = orca.get_table("jobs") jobs = jobs.to_frame() jobs_unplaced_slr = jobs[jobs["building_id"] == -1] jobs_unplaced_slr = jobs_unplaced_slr[~jobs_unplaced_slr.index.isin (jobs_unplaced.index)] orca.add_injectable("jobs_unplaced_slr", jobs_unplaced_slr) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") print "Demolished %d buildings" % (l1 - len(buildings))
def slr_remove_dev(buildings, destroy_parcels, year, parcels, households, jobs): slr_demolish = buildings.local[buildings.parcel_id.isin( destroy_parcels.index)] orca.add_table("slr_demolish", slr_demolish) print "Demolishing %d buildings" % len(slr_demolish) households = households.to_frame() hh_unplaced = households[households["building_id"] == -1] jobs = jobs.to_frame() jobs_unplaced = jobs[jobs["building_id"] == -1] l1 = len(buildings) buildings = utils._remove_developed_buildings( buildings.to_frame(buildings.local_columns), slr_demolish, unplace_agents=["households", "jobs"]) households = orca.get_table("households") households = households.to_frame() hh_unplaced_slr = households[households["building_id"] == -1] hh_unplaced_slr = hh_unplaced_slr[~hh_unplaced_slr.index.isin(hh_unplaced. index)] orca.add_injectable("hh_unplaced_slr", hh_unplaced_slr) jobs = orca.get_table("jobs") jobs = jobs.to_frame() jobs_unplaced_slr = jobs[jobs["building_id"] == -1] jobs_unplaced_slr = jobs_unplaced_slr[~jobs_unplaced_slr.index. isin(jobs_unplaced.index)] orca.add_injectable("jobs_unplaced_slr", jobs_unplaced_slr) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") print "Demolished %d buildings" % (l1 - len(buildings))
def test_assign_variables_failing(capsys, data): close_handlers() output_dir = os.path.join(os.path.dirname(__file__), 'output') orca.add_injectable("output_dir", output_dir) tracing.config_logger(basic=True) spec_name = \ os.path.join(os.path.dirname(__file__), 'data', 'assignment_spec_failing.csv') spec = assign.read_assignment_spec(spec_name) locals_d = { 'CONSTANT': 7, '_shadow': 99, 'log': np.log, } with pytest.raises(NameError) as excinfo: results, trace_results = assign.assign_variables(spec, data, locals_d, trace_rows=None) out, err = capsys.readouterr() # don't consume output print out # undefined variable should raise error assert "'undefined_variable' is not defined" in str(excinfo.value)
def buildings(store): df = store['buildings'] # Todo: combine two sqft prices into one and set non use sqft price to 0 df.loc[df.improvement_value < 0, 'improvement_value'] = 0 df['sqft_price_nonres'] = df.improvement_value * 1.0 / 0.7 / df.non_residential_sqft df.loc[df.sqft_price_nonres > 1000, 'sqft_price_nonres'] = 0 df.loc[df.sqft_price_nonres < 0, 'sqft_price_nonres'] = 0 df['sqft_price_res'] = df.improvement_value * 1.25 / 0.7 / ( df.sqft_per_unit.astype(int) * df.residential_units) df.loc[df.sqft_price_res > 1000, 'sqft_price_res'] = 0 df.loc[df.sqft_price_res < 0, 'sqft_price_res'] = 0 df.fillna(0, inplace=True) orca.add_injectable("max_building_id", 10000000) df['hu_filter'] = 0 cites = [551, 1155, 1100, 3130, 6020, 6040] sample = df[df.residential_units > 0] sample = sample[~(sample.index.isin(store['households'].building_id))] for c in sample.b_city_id.unique(): frac = 0.9 if c in cites else 0.5 df.loc[sample[sample.b_city_id == c].sample(frac=frac, replace=False).index.values, 'hu_filter'] = 1 return df
def test_no_config(capsys): configs_dir = os.path.join(os.path.dirname(__file__)) orca.add_injectable("configs_dir", configs_dir) # remove existing hanlers or basicConfig is a NOP logging.getLogger().handlers = [] tracing.config_logger(basic=True) logger = logging.getLogger() file_handlers = [h for h in logger.handlers if type(h) is logging.FileHandler] assert len(file_handlers) == 0 logger = logging.getLogger('activitysim') logger.debug('log_debug') logger.info('log_info') logger.warn('log_warn') out, err = capsys.readouterr() # don't consume output print out assert 'log_warn' in out assert 'log_info' in out assert 'log_debug' not in out close_handlers()
def test_bad_custom_config_file(capsys): configs_dir = os.path.join(os.path.dirname(__file__)) orca.add_injectable("configs_dir", configs_dir) custom_config_file = os.path.join(os.path.dirname(__file__), 'configs', 'xlogging.yaml') tracing.config_logger(custom_config_file=custom_config_file) logger = logging.getLogger('activitysim') file_handlers = [h for h in logger.handlers if type(h) is logging.FileHandler] assert len(file_handlers) == 1 asim_logger_baseFilename = file_handlers[0].baseFilename logger = logging.getLogger(__name__) logger.info('log_info') logger.warn('log_warn1') out, err = capsys.readouterr() # don't consume output print out assert "could not find conf file" in out assert 'log_warn1' in out assert 'log_info' not in out close_handlers() logger.warn('log_warn2') with open(asim_logger_baseFilename, 'r') as content_file: content = content_file.read() assert 'log_warn1' in content assert 'log_warn2' not in content
def build_networks(settings , store, parcels, intersections): edges, nodes = store['edges'], store['nodes'] net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"], edges[["weight"]]) max_distance = settings['build_networks']['max_distance'] net.precompute(max_distance) #SETUP POI COMPONENTS on_ramp_nodes = nodes[nodes.on_ramp] net.init_pois(num_categories=4, max_dist=max_distance, max_pois=1) net.set_pois('onramps', on_ramp_nodes.x, on_ramp_nodes.y) parks = store.parks net.set_pois('parks', parks.x, parks.y) schools = store.schools net.set_pois('schools', schools.x, schools.y) transit = store.transit net.set_pois('transit', transit.x, transit.y) orca.add_injectable("net", net) p = parcels.to_frame(parcels.local_columns) i = intersections.to_frame(intersections.local_columns) p['node_id'] = net.get_node_ids(p['x'], p['y']) i['node_id'] = net.get_node_ids(i['x'], i['y']) #p.to_csv('data/parcels.csv') orca.add_table("parcels", p) orca.add_table("intersections", i)
def open_pipeline_store(overwrite=False): """ Open the pipeline checkpoint store and add an orca injectable to access it Parameters ---------- overwrite : bool delete file before opening (unless resuming) """ if orca.is_injectable('pipeline_store'): raise RuntimeError("Pipeline store is already open!") pipeline_file_path = orca.get_injectable('pipeline_path') if overwrite: try: if os.path.isfile(pipeline_file_path): logger.debug("removing pipeline store: %s" % pipeline_file_path) os.unlink(pipeline_file_path) except Exception as e: print(e) logger.warn("Error removing %s: %s" % (e, )) store = pd.HDFStore(pipeline_file_path, mode='a') orca.add_injectable('pipeline_store', store) logger.debug("opened pipeline_store")
def register_households(df, trace_hh_id): """ Register with orca households for tracing Parameters ---------- df: pandas.DataFrame traced dataframe trace_hh_id: int household id we are tracing Returns ------- Nothing """ print("tracing household id %s in %s households" % (trace_hh_id, len(df.index))) if trace_hh_id not in df.index: print("trace_hh_id %s not in dataframe" % trace_hh_id) # inject persons_index name of person dataframe index if df.index.name is None: df.index.names = ['household_id'] print("households table index had no" " name. renamed index '%s'" % df.index.name) orca.add_injectable("hh_index_name", df.index.name) print("register_households injected hh_index_name '%s'" % df.index.name)
def register_households(df, trace_hh_id): """ Register with orca households for tracing Parameters ---------- df: pandas.DataFrame traced dataframe trace_hh_id: int household ID to trace Returns ------- Nothing """ if trace_hh_id is None: logger.error("register_households called with null trace_hh_id") return logger.info("tracing household id %s in %s households" % (trace_hh_id, len(df.index))) if trace_hh_id not in df.index: logger.warn("trace_hh_id %s not in dataframe" % trace_hh_id) # inject persons_index name of person dataframe index if df.index.name is None: df.index.names = ['household_id'] logger.warn("households table index had no name. renamed index '%s'" % df.index.name) orca.add_injectable("hh_index_name", df.index.name) logger.debug("register_households injected hh_index_name '%s'" % df.index.name)
def test_simple_simulate(data, spec): orca.add_injectable("check_for_variability", False) choices = simulate.simple_simulate(data, spec, nest_spec=None) expected = pd.Series([1, 1, 1], index=data.index) pdt.assert_series_equal(choices, expected)
def reinject_decorated_tables(): """ reinject the decorated tables (and columns) """ logger.info("reinject_decorated_tables") # need to clear any non-decorated tables that were added during the previous run _ORCA._TABLES.clear() _ORCA._COLUMNS.clear() _ORCA._TABLE_CACHE.clear() _ORCA._COLUMN_CACHE.clear() for name, func in _DECORATED_TABLES.items(): logger.debug("reinject decorated table %s" % name) orca.add_table(name, func) for column_key, args in _DECORATED_COLUMNS.items(): table_name, column_name = column_key logger.debug("reinject decorated column %s.%s" % (table_name, column_name)) orca.add_column(table_name, column_name, args['func'], cache=args['cache']) for name, args in _DECORATED_INJECTABLES.items(): logger.debug("reinject decorated injectable %s" % name) orca.add_injectable(name, args['func'], cache=args['cache'])
def non_mandatory_accessibility(): fname = get_logsum_file('non_mandatory') orca.add_injectable("nonmand_acc_file_2010", fname) df = pd.read_csv(os.path.join(misc.data_dir(), fname)) df.loc[df.subzone == 0, 'subzone'] = 'c' # no walk df.loc[df.subzone == 1, 'subzone'] = 'a' # short walk df.loc[df.subzone == 2, 'subzone'] = 'b' # long walk df['taz_sub'] = df.taz.astype('str') + df.subzone return df.set_index('taz_sub')
def inputs(input_pcl, input_bld, input_jobs, input_gcl, input_settings): #orca.clear_all() orca.add_table('parcels', input_pcl) orca.add_table('buildings', input_bld) orca.add_table('jobs', input_jobs) orca.add_table('gridcells', input_gcl) orca.add_injectable('settings', input_settings) import psrc_urbansim.vars.variables_jobs import psrc_urbansim.vars.variables_parcels
def scheduled_development_events(buildings, development_projects, demolish_events, summary, year, parcels, mapping, years_per_iter, parcels_geography, building_sqft_per_job, vmt_fee_categories, static_parcels): # first demolish demolish = demolish_events.to_frame().\ query("%d <= year_built < %d" % (year, year + years_per_iter)) print("Demolishing/building %d buildings" % len(demolish)) l1 = len(buildings) buildings = utils._remove_developed_buildings( buildings.to_frame(buildings.local_columns), demolish, unplace_agents=["households", "jobs"]) orca.add_injectable( 'static_parcels', np.append(static_parcels, demolish.loc[demolish.action == 'build', 'parcel_id'])) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") print("Demolished %d buildings" % (l1 - len(buildings))) print(" (this number is smaller when parcel has no existing buildings)") # then build dps = development_projects.to_frame().\ query("%d <= year_built < %d" % (year, year + years_per_iter)) if len(dps) == 0: return new_buildings = utils.scheduled_development_events( buildings, dps, remove_developed_buildings=False, unplace_agents=['households', 'jobs']) new_buildings["form"] = new_buildings.building_type.map( mapping['building_type_map']).str.lower() new_buildings["job_spaces"] = new_buildings.non_residential_sqft / \ new_buildings.building_type.fillna("OF").map(building_sqft_per_job) new_buildings["job_spaces"] = new_buildings.job_spaces.\ fillna(0).astype('int') new_buildings["geom_id"] = parcel_id_to_geom_id(new_buildings.parcel_id) new_buildings["SDEM"] = True new_buildings["subsidized"] = False new_buildings["zone_id"] = misc.reindex(parcels.zone_id, new_buildings.parcel_id) new_buildings["vmt_res_cat"] = misc.reindex(vmt_fee_categories.res_cat, new_buildings.zone_id) del new_buildings["zone_id"] new_buildings["pda"] = parcels_geography.pda_id.loc[ new_buildings.parcel_id].values new_buildings["juris_trich"] = parcels_geography.juris_trich.loc[ new_buildings.parcel_id].values summary.add_parcel_output(new_buildings)
def household_relocation_rates(scenario, policy): if scenario in policy['reloc_fr2_enable']: df = pd.read_csv( os.path.join("data", "household_relocation_rates_fr2.csv")) orca.add_injectable("hh_reloc", 'activated') else: df = pd.read_csv(os.path.join("data", "household_relocation_rates.csv")) orca.add_injectable("hh_reloc", 'not activated') return df
def large_area_id(households, buildings): hh_la = "households_large_area_lookup" if (not orca.is_injectable(hh_la)) or (len(orca.get_injectable(hh_la)) == 0): orca.add_injectable(hh_la, misc.reindex(buildings.large_area_id, households.building_id), autocall=False, cache=True) return orca.get_injectable(hh_la).loc[households.index]
def get_val_from_uc_db(query): try: result = sql.read_frame(query, conn) return result.values[0][0] except: conn=psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) result = sql.read_frame(query, conn) return result.values[0][0]
def test_integerizer(): configs_dir = os.path.join(os.path.dirname(__file__), 'configs') orca.add_injectable("configs_dir", configs_dir) # data_dir = os.path.join(os.path.dirname(__file__), 'data') # orca.add_injectable("data_dir", data_dir) # # output_dir = os.path.join(os.path.dirname(__file__), 'output') # orca.add_injectable("output_dir", output_dir) # rows are elements for which factors are calculated, columns are constraints to be satisfied incidence_table = pd.DataFrame({ 'num_hh': [1, 1, 1, 1, 1, 1, 1, 1], 'hh_1': [1, 1, 1, 0, 0, 0, 0, 0], 'hh_2': [0, 0, 0, 1, 1, 1, 1, 1], 'p1': [1, 1, 2, 1, 0, 1, 2, 1], 'p2': [1, 0, 1, 0, 2, 1, 1, 1], 'p3': [1, 1, 0, 2, 1, 0, 2, 0], 'float_weights': [ 1.362893, 25.658290, 7.978812, 27.789651, 18.451021, 8.641589, 1.476104, 8.641589 ] }) control_cols = ['num_hh', 'hh_1', 'hh_2', 'p1', 'p2', 'p3'] control_spec = pd.DataFrame({ 'seed_table': [ 'households', 'households', 'households', 'persons', 'persons', 'persons' ], 'target': control_cols, 'importance': [10000000, 1000, 1000, 1000, 1000, 1000] }) # column totals which the final weighted incidence table sums must satisfy control_totals = pd.Series([100, 35, 65, 91, 65, 104], index=control_spec.target.values) integerized_weights, status = integerizer.do_integerizing( trace_label='label', control_spec=control_spec, control_totals=control_totals, incidence_table=incidence_table[control_cols], float_weights=incidence_table['float_weights'], total_hh_control_col='num_hh') print "do_integerizing status", status print "sum", integerized_weights.sum() print "do_integerizing integerized_weights\n", integerized_weights assert integerized_weights.sum() == 100
def build_networks(parcels, net_store): st = net_store nodes, edges = st.nodes, st.edges net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"], edges[["weight"]]) net.precompute(3000) orca.add_injectable("net", net) p = parcels.to_frame(parcels.local_columns) p['node_id'] = net.get_node_ids(p['x'], p['y']) orca.add_table("parcels", p)
def exec_sql_uc(query): try: cur.execute(query) conn.commit() except: conn = psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) cur.execute(query) conn.commit()
def accessibilities_segmentation(): fname = get_logsum_file('segmentation') orca.add_injectable("acc_seg_file_2010", fname) df = pd.read_csv(os.path.join(misc.data_dir(), fname)) df['AV'] = df['hasAV'].apply(lambda x: 'AV' if x == 1 else 'noAV') df['label'] = (df['incQ_label'] + '_' + df['autoSuff_label'] + '_' + df['AV']) df = df.groupby('label').sum() df['prop'] = df['num_persons'] / df['num_persons'].sum() df = df[['prop']].transpose().reset_index(drop=True) return df
def inject_settings(configs_dir, households_sample_size, preload_3d_skims=None, chunk_size=None): with open(os.path.join(configs_dir, "configs", "settings.yaml")) as f: settings = yaml.load(f) settings['households_sample_size'] = households_sample_size if preload_3d_skims is not None: settings['preload_3d_skims'] = preload_3d_skims if chunk_size is not None: settings['chunk_size'] = chunk_size orca.add_injectable("settings", settings)
def build_networks(parcels): st = pd.HDFStore(os.path.join(misc.data_dir(), "osm_sandag.h5"), "r") nodes, edges = st.nodes, st.edges net = pdna.Network(nodes["x"], nodes["y"], edges["from"], edges["to"], edges[["weight"]]) net.precompute(3000) orca.add_injectable("net", net) p = parcels.to_frame(parcels.local_columns) p['node_id'] = net.get_node_ids(p['x'], p['y']) orca.add_table("parcels", p)
def close(): """ Close any known open files """ close_open_files() orca.get_injectable('pipeline_store').close() orca.add_injectable('pipeline_store', None) logger.info("close_pipeline")
def get_val_from_uc_db(query): try: result = sql.read_frame(query, conn) return result.values[0][0] except: conn = psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) result = sql.read_frame(query, conn) return result.values[0][0]
def simulate(self, choice_function=None, save_probabilities=False, **kwargs): """ Computing choices, with arbitrary function for handling simulation strategy. Parameters ---------- choice_function : function Function defining how to simulate choices based on fitted model. Function must accept the following 3 arguments: model object, choosers DataFrame, and alternatives DataFrame. Additional optional keyword args can be utilized by function if needed (kwargs). save_probabilities : bool If true, will save the calculated probabilities underlying the simulation as an orca injectable with name 'probabilities_modelname_itervar'. Returns ------- choices : pandas.Series Mapping of chooser ID to alternative ID. Some choosers will map to a nan value when there are not enough alternatives for all the choosers. """ choosers, alternatives = self.calculate_model_variables() choosers, alternatives = self.apply_predict_filters( choosers, alternatives) # By convention, choosers are denoted by a -1 value # in the choice column choosers = choosers[choosers[self.choice_column] == -1] print("{} agents are making a choice.".format(len(choosers))) if choice_function: choices = choice_function(self, choosers, alternatives, **kwargs) else: choices = self.predict(choosers, alternatives, debug=True) if save_probabilities: if not self.sim_pdf: probabilities = self.calculate_probabilities( choosers, alternatives) else: probabilities = self.sim_pdf.reset_index().set_index( 'alternative_id')[0] orca.add_injectable( 'probabilities_{}_{}'.format(self.name, orca.get_injectable('iter_var')), probabilities) return choices
def get_logsum_file(type='mandatory'): logsums = orca.get_injectable('settings')['logsums'][type] sc = orca.get_injectable('scenario') yr = orca.get_injectable('year') try: prev_type = orca.get_injectable('previous_{}_logsum_type'.format(type)) if prev_type == 'generic': return orca.get_injectable('previous_{}_logsum_file'.format(type)) elif prev_type == 'year': if 'logsum_{}'.format(yr) in logsums: ls = logsums['logsum_{}'.format(yr)] orca.add_injectable('previous_{}_logsum_file'.format(type), ls) return ls else: return orca.get_injectable('previous_{}_logsum_file' .format(type)) elif prev_type == 'scenario': if 'logsum_s{}'.format(sc) in logsums: ls = logsums['logsum_s{}'.format(sc)] orca.add_injectable('previous_{}_logsum_file' .format(type), ls) return ls else: return orca.get_injectable('previous_{}_logsum_file' .format(type)) else: if 'logsum_{}_s{}'.format(yr, sc) in logsums: ls = logsums['logsum_{}_s{}'.format(yr, sc)] orca.add_injectable('previous_{}_logsum_file' .format(type), ls) return ls else: return orca.get_injectable('previous_{}_logsum_file' .format(type)) except: if 'logsum' in logsums: ls = logsums['logsum'] ls_type = 'generic' if 'logsum_{}'.format(yr) in logsums: ls = logsums['logsum_{}'.format(yr)] ls_type = 'year' if 'logsum_s{}'.format(sc) in logsums: ls = logsums['logsum_s{}'.format(sc)] ls_type = 'scenario' if 'logsum_{}_s{}'.format(yr, sc) in logsums: ls = logsums['logsum_{}_s{}'.format(yr, sc)] ls_type = 'year_scenario' orca.add_injectable('previous_{}_logsum_type'.format(type), ls_type) orca.add_injectable('previous_{}_logsum_file'.format(type), ls) return ls
def get_val_from_uc_db(query): try: result = sql.read_frame(query, conn) return result.values[0][0] except: conn=psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) result = sql.read_frame(query, conn) result2 = sql.read_frame("select column_name from Information_schema.columns where table_name like 'building' ", conn) print result2 return result.values[0][0]
def buildings(store): df = store['buildings'] # Todo: combine two sqft prices into one and set non use sqft price to 0 df.loc[df.improvement_value < 0, 'improvement_value'] = 0 df['sqft_price_nonres'] = df.improvement_value * 1.0 / 0.7 / df.non_residential_sqft df.loc[df.sqft_price_nonres > 1000, 'sqft_price_nonres'] = 0 df.loc[df.sqft_price_nonres < 0, 'sqft_price_nonres'] = 0 df['sqft_price_res'] = df.improvement_value * 1.25 / 0.7 / (df.sqft_per_unit.astype(int) * df.residential_units) df.loc[df.sqft_price_res > 1000, 'sqft_price_res'] = 0 df.loc[df.sqft_price_res < 0, 'sqft_price_res'] = 0 df.fillna(0, inplace=True) orca.add_injectable("max_building_id", 10000000) return df
def get_dev_projects_table(scenario, parcels): # requires the user has MTC's urban_data_internal # repository alongside bayarea_urbansim urban_data_repo = ("../urban_data_internal/development_projects/") file = "2021_0309_1939_development_projects.csv" print('Version of development_projects: {}'.format(file)) current_dev_proj = (file) orca.add_injectable("dev_proj_file", current_dev_proj) df = pd.read_csv(os.path.join(urban_data_repo, current_dev_proj)) df = reprocess_dev_projects(df) orca.add_injectable("devproj_len", len(df)) # this filters project by scenario scen = 'scen' + str(scenario) if scen in df: # df[scenario] is 1s and 0s indicating whether to include it df = df[df[scen].astype('bool')] orca.add_injectable("devproj_len_scen", len(df)) df = df.dropna(subset=['geom_id']) cnts = df.geom_id.isin(parcels.geom_id).value_counts() if False in cnts.index: print("%d MISSING GEOMIDS!" % cnts.loc[False]) df = df[df.geom_id.isin(parcels.geom_id)] geom_id = df.geom_id # save for later df = df.set_index("geom_id") df = geom_id_to_parcel_id(df, parcels).reset_index() # use parcel id df["geom_id"] = geom_id.values # add it back again cause it goes away orca.add_injectable("devproj_len_geomid", len(df)) return df
def run(forecast_year=2035, random_seed=False): """ Set up and run simulation. Parameters ---------- forecast_year : int, optional Year to simulate to. If year argument is passed from the terminal, then that year is applied here, otherwise a default value is applied. random_seed : int, optional Random seed. Returns ------- _ : None No return value for now. """ # Record start time start_time = time.time() orca.add_injectable('forecast_year', forecast_year) # Set value of optional random seed if random_seed: np.random.seed(random_seed) # Model names transition_models = ['household_transition', 'job_transition'] price_models = [ 'repm_sf_detached', 'repm_duplex_townhome', 'repm_multifamily', 'repm_industrial', 'repm_retail', 'repm_office' ] developer_models = [ 'feasibility', 'residential_developer', 'non_residential_developer' ] location_models = [ 'hlcm1', 'hlcm2', 'elcm1', 'elcm2', 'elcm3', 'elcm4', 'elcm5', 'elcm6', 'elcm7', 'elcm8', 'elcm9', 'elcm10', 'elcm11', 'elcm12', 'elcm13', 'elcm14' ] end_of_year_models = ['generate_indicators'] # Simulate orca.run(['build_networks', 'generate_indicators']) orca.run(transition_models + price_models + developer_models + location_models + end_of_year_models, iter_vars=list(range(2011, forecast_year + 1))) # Record end time end_time = time.time() time_elapsed = end_time - start_time print('Simulation duration: %s minutes' % (time_elapsed / 60))
def superdistricts(scenario): sd_scenario_file = os.path.join( misc.data_dir(), ("superdistricts_s{}.csv").format(scenario)) # scenarios could contain policies (eg telework) and/or other modifications if os.path.isfile(sd_scenario_file): superdistricts = pd.read_csv(sd_scenario_file, index_col="number") orca.add_injectable("sqft_per_job_settings", "for this scenario") # the default includes a telework assumption and SD adjustments else: superdistricts = pd.read_csv(os.path.join(misc.data_dir(), "superdistricts.csv"), index_col="number") orca.add_injectable("sqft_per_job_settings", "default") return superdistricts
def test_full_run(store): orca.add_injectable("configs_dir", os.path.join(os.path.dirname(__file__), '..', '..', '..', 'example')) orca.add_injectable("store", store) orca.add_injectable("nonmotskm_matrix", np.ones((1454, 1454))) orca.add_injectable("set_random_seed", set_random_seed) # grab some of the tables orca.get_table("land_use").to_frame().info() orca.get_table("households").to_frame().info() orca.get_table("persons").to_frame().info() assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE # run the models in the expected order orca.run(["workplace_location_simulate"]) orca.run(["auto_ownership_simulate"]) orca.run(["cdap_simulate"]) orca.run(['mandatory_tour_frequency']) orca.get_table("mandatory_tours").tour_type.value_counts() orca.run(['non_mandatory_tour_frequency']) orca.get_table("non_mandatory_tours").tour_type.value_counts() orca.run(["destination_choice"]) orca.run(["mandatory_scheduling"]) orca.run(["non_mandatory_scheduling"]) orca.run(["mode_choice_simulate"]) orca.clear_cache()
def decorator(func): name = func.__name__ logger.debug("inject injectable %s" % name) # insist on explicit override to ensure multiple definitions occur in correct order assert override or not _DECORATED_INJECTABLES.get(name, False), \ "injectable '%s' already defined. not overridden" % name _DECORATED_INJECTABLES[name] = {'func': func, 'cache': cache} orca.add_injectable(name, func, cache=cache) return func
def calibrate(iter_var): orca.add_injectable('mult_val', iter_var) orca.run([ 'rsh_simulate', 'nrh_simulate', 'emp_transition', 'emp_relocation', 'elcm_simulate', 'hh_transition', 'hh_relocation', 'hlcm_simulate', 'feasibility', 'residential_developer', 'non_residential_developer', 'indicator_export', 'reset_to_base', ], iter_vars=[2015])
def parcels_zoning_by_scenario(parcels, parcels_zoning_calculations, zoning_baseline): df = pd.DataFrame(index=parcels.index) df["baseline_dua"] = zoning_baseline.max_dua df["baseline_far"] = zoning_baseline.max_far df["baseline_height"] = zoning_baseline.max_height df["zoning_name"] = zoning_baseline["name"] df["zoning_source"] = zoning_baseline["tablename"] for scenario in [str(i) for i in range(4)]: orca.clear_cache() orca.add_injectable("scenario", scenario) z = orca.get_table("parcels_zoning_calculations") df["max_dua_%s" % scenario] = z.effective_max_dua df["max_far_%s" % scenario] = z.effective_max_far df["du_underbuild_%s" % scenario] = z.zoned_du_underbuild df["non_res_cat_%s" % scenario] = z.non_res_categories return df
def full_run(store, omx_file, preload_3d_skims, chunk_size=0): configs_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'example') orca.add_injectable("configs_dir", configs_dir) inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE, preload_3d_skims=preload_3d_skims, chunk_size=chunk_size) orca.add_injectable("omx_file", omx_file) orca.add_injectable("store", store) orca.add_injectable("set_random_seed", set_random_seed) orca.clear_cache() # grab some of the tables orca.get_table("land_use").to_frame().info() orca.get_table("households").to_frame().info() orca.get_table("persons").to_frame().info() assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE assert orca.get_injectable("chunk_size") == chunk_size # run the models in the expected order orca.run(["school_location_simulate"]) orca.run(["workplace_location_simulate"]) orca.run(["auto_ownership_simulate"]) orca.run(["cdap_simulate"]) orca.run(['mandatory_tour_frequency']) orca.get_table("mandatory_tours").tour_type.value_counts() orca.run(['non_mandatory_tour_frequency']) orca.get_table("non_mandatory_tours").tour_type.value_counts() orca.run(["destination_choice"]) orca.run(["mandatory_scheduling"]) orca.run(["non_mandatory_scheduling"]) orca.run(["patch_mandatory_tour_destination"]) orca.run(["tour_mode_choice_simulate"]) orca.run(["trip_mode_choice_simulate"]) tours_merged = orca.get_table("tours_merged").to_frame() tour_count = len(tours_merged.index) orca.clear_cache() return tour_count
def test_mini_run(store, omx_file, random_seed): configs_dir = os.path.join(os.path.dirname(__file__)) orca.add_injectable("configs_dir", configs_dir) inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE) orca.add_injectable("omx_file", omx_file) orca.add_injectable("store", store) orca.add_injectable("set_random_seed", set_random_seed) orca.clear_cache() assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE # run the models in the expected order orca.run(["workplace_location_simulate"]) orca.run(["auto_ownership_simulate"]) # this is a regression test so that we know if these numbers change auto_choice = orca.get_table('households').get_column('auto_ownership') hh_ids = [2124015, 961042, 1583271] choices = [1, 2, 2] print "auto_choice\n", auto_choice.head(3) pdt.assert_series_equal( auto_choice[hh_ids], pd.Series(choices, index=pd.Index(hh_ids, name="HHID"))) orca.run(["cdap_simulate"]) orca.run(['mandatory_tour_frequency']) mtf_choice = orca.get_table('persons').get_column('mandatory_tour_frequency') per_ids = [172616, 172781, 172782] choices = ['work1', 'school1', 'work_and_school'] print "mtf_choice\n", mtf_choice.head(20) pdt.assert_series_equal( mtf_choice[per_ids], pd.Series(choices, index=pd.Index(per_ids, name='PERID'))) orca.clear_cache()
def test_mini_run(store, random_seed): orca.add_injectable("configs_dir", os.path.join(os.path.dirname(__file__))) orca.add_injectable("store", store) orca.add_injectable("nonmotskm_matrix", np.ones((1454, 1454))) orca.add_injectable("set_random_seed", set_random_seed) assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE # run the models in the expected order orca.run(["workplace_location_simulate"]) orca.run(["auto_ownership_simulate"]) # this is a regression test so that we know if these numbers change auto_choice = orca.get_table('households').get_column('auto_ownership') print auto_choice[[2306822, 652072, 651907]] pdt.assert_series_equal( auto_choice[[2306822, 652072, 651907]], pd.Series( [2, 1, 1], index=pd.Index([2306822, 652072, 651907], name='HHID'))) orca.run(["cdap_simulate"]) orca.run(['mandatory_tour_frequency']) mtf_choice = orca.get_table('persons').get_column( 'mandatory_tour_frequency') pdt.assert_series_equal( mtf_choice[[146642, 642922, 642921]], pd.Series( ['school1', 'work1', 'school2'], index=pd.Index([146642, 642922, 642921], name='PERID'))) orca.clear_cache()
import orca from activitysim import defaults import pandas as pd import numpy as np import os orca.add_injectable("output_dir", 'output') defaults.tracing.config_logger() orca.run(["school_location_simulate"]) orca.run(["workplace_location_simulate"]) print orca.get_table("persons").distance_to_work.describe() orca.run(["auto_ownership_simulate"]) orca.run(["cdap_simulate"]) orca.run(['mandatory_tour_frequency']) orca.get_table("mandatory_tours").tour_type.value_counts() orca.run(["mandatory_scheduling"]) orca.run(['non_mandatory_tour_frequency']) orca.get_table("non_mandatory_tours").tour_type.value_counts() orca.run(["destination_choice"]) orca.run(["non_mandatory_scheduling"]) # FIXME - jwd - choose more felicitous name or do this elsewhere? orca.run(["patch_mandatory_tour_destination"]) orca.run(['tour_mode_choice_simulate']) orca.run(['trip_mode_choice_simulate'])
def change_store(store_name): orca.add_injectable("store", pd.HDFStore(os.path.join(misc.data_dir(), store_name), mode="r"))
EVERY_NTH_YEAR = 5 CURRENT_COMMIT = os.popen('git rev-parse HEAD').read() COMPARE_TO_NO_PROJECT = True NO_PROJECT = 611 IN_YEAR, OUT_YEAR = 2010, 2040 LAST_KNOWN_GOOD_RUNS = { "0": 1057, "1": 1058, "2": 1059, "3": 1060, "4": 1059, "5": 1059 } orca.add_injectable("years_per_iter", EVERY_NTH_YEAR) if len(args) and args[0] == "-i": SLACK = MAPS = LOGS = False INTERACT = True if len(args) and args[0] == "-s": orca.add_injectable("scenario", args[1]) SCENARIO = orca.get_injectable("scenario") if INTERACT: import code code.interact(local=locals()) sys.exit()
def eq_code_buildings(buildings, year, earthquake): if year == 2035 and earthquake: # tags buildings that exist in 2035 with a fragility coefficient # keeping in-model adds run time, but is important given developer # model stochastisitcy, that will change the building stock in 2035 # this also allows us to change the building codes when retrofitting # policies are applied, thus changing fragility coefficients buildings = buildings.to_frame() code = [] fragilities = [] for i in buildings.index: if (buildings['building_type'][i] == 'HS' and buildings['year_built'][i] <= 2015): a = 'SF' if buildings['stories'][i] == 1: b = '01' if buildings['year_built'][i] <= 1940: c = 'G1' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G2' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1995): c = 'G3' elif (buildings['year_built'][i] >= 1996 and buildings['year_built'][i] <= 2015): c = 'G4' elif buildings['stories'][i] >= 2: b = '2P' if buildings['year_built'][i] <= 1920: c = 'G1' elif (buildings['year_built'][i] >= 1921 and buildings['year_built'][i] <= 1940): c = 'G2' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G3' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1995): c = 'G4' elif (buildings['year_built'][i] >= 1996 and buildings['year_built'][i] <= 2015): c = 'G5' elif ((buildings['building_type'][i] == 'HM' or buildings['building_type'][i] == 'MR') and buildings['year_built'][i] <= 2015): if (buildings['residential_units'][i] == 2 or buildings['residential_units'][i] == 3 or buildings['residential_units'][i] == 4): a = 'DU' # 2, 3, & 4 units # are considered duplex/triplex/quadplex if buildings['stories'][i] == 1: b = '01' if buildings['year_built'][i] <= 1940: c = 'G1' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G2' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1995): c = 'G3' elif (buildings['year_built'][i] >= 1996 and buildings['year_built'][i] <= 2015): c = 'G4' if buildings['stories'][i] >= 2: b = '2P' if buildings['year_built'][i] <= 1920: c = 'G1' elif (buildings['year_built'][i] >= 1921 and buildings['year_built'][i] <= 1940): c = 'G2' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G3' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1977): c = 'G4' elif (buildings['year_built'][i] >= 1978 and buildings['year_built'][i] <= 1991): c = 'G5' elif (buildings['year_built'][i] >= 1992 and buildings['year_built'][i] <= 2015): c = 'G6' else: # this assumes one-unit HM/MR buildings # are also 5+ units (multifamily split by parcels) a = 'MF' if buildings['stories'][i] == 1: b = '01' if buildings['year_built'][i] <= 1920: c = 'G1' elif (buildings['year_built'][i] >= 1921 and buildings['year_built'][i] <= 1940): c = 'G2' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G3' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1995): c = 'G4' elif (buildings['year_built'][i] >= 1996 and buildings['year_built'][i] <= 2015): c = 'G5' elif (buildings['stories'][i] >= 2 and buildings['stories'][i] <= 5): b = '25' if buildings['year_built'][i] <= 1920: c = 'G1' elif (buildings['year_built'][i] >= 1921 and buildings['year_built'][i] <= 1940): c = 'G2' elif (buildings['year_built'][i] >= 1941 and buildings['year_built'][i] <= 1960): c = 'G3' elif (buildings['year_built'][i] >= 1961 and buildings['year_built'][i] <= 1977): c = 'G4' elif (buildings['year_built'][i] >= 1978 and buildings['year_built'][i] <= 1991): c = 'G5' elif (buildings['year_built'][i] >= 1992 and buildings['year_built'][i] <= 2015): c = 'G6' elif buildings['stories'][i] >= 6: b = '5P' if buildings['year_built'][i] <= 1950: c = 'G1' elif (buildings['year_built'][i] >= 1951 and buildings['year_built'][i] <= 1971): c = 'G2' elif (buildings['year_built'][i] >= 1972 and buildings['year_built'][i] <= 1995): c = 'G3' elif (buildings['year_built'][i] >= 1996 and buildings['year_built'][i] <= 2006): c = 'G4' elif (buildings['year_built'][i] >= 2007 and buildings['year_built'][i] <= 2015): c = 'G5' elif buildings['year_built'][i] <= 2015: a = 'OT' b = 'NN' if buildings['year_built'][i] <= 1933: c = 'G1' elif (buildings['year_built'][i] >= 1934 and buildings['year_built'][i] <= 1950): c = 'G2' elif (buildings['year_built'][i] >= 1951 and buildings['year_built'][i] <= 1972): c = 'G3' elif (buildings['year_built'][i] >= 1973 and buildings['year_built'][i] <= 1996): c = 'G4' elif (buildings['year_built'][i] >= 1997 and buildings['year_built'][i] <= 2006): c = 'G5' elif (buildings['year_built'][i] >= 2007 and buildings['year_built'][i] <= 2015): c = 'G6' # new buildings built by the developer model elif buildings['year_built'][i] > 2015: a = 'NN' b = 'NN' c = 'NN' # alternative if retrofitted: d = 'R' d = 'N' code_i = a+b+c+d code.append(code_i) # assign a fragility coefficient based on building code if (code_i == 'SF01G4N' or code_i == 'SF2PG5N' or code_i == 'DU2PG6N' or code_i == 'MF5PG5N' or code_i == 'DU01G4N' or code_i == 'MF25G6N' or code_i == 'MF01G5N' or code_i == 'OTNNG6N'): fragility = 1 elif (code_i == 'SF01G3N' or code_i == 'DU01G3N' or code_i == 'DU2PG5N' or code_i == 'MF25G5N' or code_i == 'MF01G4N' or code_i == 'OTNNG5N' or code_i == 'MF5PG4N'): fragility = 1.2 elif (code_i == 'SF2PG4N' or code_i == 'MF5PG3N' or code_i == 'OTNNG4N'): fragility = 1.3 elif (code_i == 'MF5PG1N' or code_i == 'OTNNG2N'): fragility = 1.4 elif (code_i == 'MF01G3N' or code_i == 'MF5PG2N' or code_i == 'SF01G2N' or code_i == 'DU01G2N' or code_i == 'OTNNG3N'): fragility = 1.5 elif (code_i == 'DU2PG3N' or code_i == 'DU2PG4N'): fragility = 1.75 elif (code_i == 'SF2PG3N' or code_i == 'DU01G1N' or code_i == 'DU2PG2N' or code_i == 'MF01G2N' or code_i == 'OTNNG1N'): fragility = 2 elif (code_i == 'SF2PG2N'): fragility = 2.25 elif (code_i == 'DU2PG1N' or code_i == 'SF01G1N' or code_i == 'SF2PG1N' or code_i == 'MF01G1N' or code_i == 'MF25G1N'): fragility = 2.5 elif (code_i == 'MF25G2N' or code_i == 'MF25G3N' or code_i == 'MF25G4N'): fragility = 3 elif (code_i == 'NNNNNNN'): fragility = 0 fragilities.append(fragility) orca.add_injectable("code", code) orca.add_injectable("fragilities", fragilities) # add codes and fragilities as orca columns code = pd.Series(code, buildings.index) orca.add_column('buildings', 'earthquake_code', code) fragility = pd.Series(fragilities, buildings.index) orca.add_column('buildings', 'fragility_coef', fragility) # generate random number, multiply by fragilities buildings = orca.get_table('buildings') rand_eq = np.random.random(len(buildings)) destroy_eq = pd.Series(rand_eq*fragility) orca.add_column('buildings', 'eq_destroy', destroy_eq) # generate random number for fire rand_fire = pd.Series(np.random.random(len(buildings))) orca.add_column('buildings', 'fire_destroy', rand_fire)
def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, households, jobs, residential_units, year, earthquake): if year == 2035 and earthquake: # assign each parcel to a census tract # using the lookup table created with "parcel_tract_assignment.ipynb" census_tract = pd.Series(parcels_tract['census_tract'], parcels_tract.index) print "Number of parcels with census tracts is: %d" % len(census_tract) orca.add_column('parcels', 'tract', census_tract) # group parcels by their census tract parcels_tract['parcel_id'] = parcels_tract.index parcels_tract = parcels_tract.to_frame(columns=['parcel_id', 'census_tract']) parcels_tract = parcels_tract[['census_tract', 'parcel_id']] tract_parcels_grp = [] tracts = [] parcels_tract = sorted(parcels_tract.values, key=itemgetter(0)) for tract, parcels in itertools.groupby(parcels_tract, key=itemgetter(0)): tract_parcels_grp.append(list(parcels)) tracts.append(tract) print "Number of census tract groups is: %d" % len(tract_parcels_grp) # for the parcels in each tract, destroy X% of parcels in that tract tracts_earthquake = tracts_earthquake.to_frame() tracts_earthquake = tracts_earthquake.sort_values(by=['tract_ba']) tracts_earthquake = tracts_earthquake.reset_index(drop=True) buildings = buildings.to_frame() eq_buildings = [] existing_buildings = [] new_buildings = [] fire_buildings = [] for i in range(len(tracts)): grp = [x[1] for x in tract_parcels_grp[i]] buildings_i = buildings[buildings['parcel_id'].isin(grp)] # existing buildings # select the buildings with highest fragility co-efficient # (and random no.) based on census tract pct to be destroyed existing_pct = tracts_earthquake['prop_eq'][i] build_frag = buildings_i['eq_destroy'].sort_values(ascending=False) top_build_frag = build_frag[: int(round( len(build_frag) * existing_pct))] # add to a list of buildings to destroy buildings_top = top_build_frag.index existing_buildings.extend(buildings_top) eq_buildings.extend(buildings_top) # new buildings # translate MMI to a probability # in-model is also nice if probabilities associated with # new buildings change buildings_new = buildings_i[buildings_i['year_built'] > 2015] if len(buildings_new) > 0: mmi = int(round(tracts_earthquake['shaking'][i])) if mmi < 6: new_pct = 0 elif mmi == 7: new_pct = .002 elif mmi == 8: new_pct = .01 elif mmi == 9: new_pct = .05 # randomly select buildings to be destroyed based on # percentages new_no = int(round(len(buildings_new)*new_pct)) buildings_new_rand = np.random.choice(buildings_new.index, new_no, replace=False) # add to a list of buildings to destroy if len(buildings_new_rand) > 0: new_buildings.extend(buildings_new_rand) eq_buildings.extend(buildings_new_rand) # fire buildings # select buildings to be destroyed by fire by looking only at # remaining buildings fire_pct = tracts_earthquake['prop_fire'][i] buildings_i_remain = buildings_i[~buildings_i.index.isin (buildings_top)] if len(buildings_new) > 0: buildings_i_remain = buildings_i_remain[~buildings_i_remain. index.isin (buildings_new_rand)] # select buildings to be destroyed based on random number # and census tract pct fire_buildings_rand = buildings_i_remain['fire_destroy']. \ sort_values(ascending=False) top_fire_buildings = fire_buildings_rand[: int(round( len(fire_buildings_rand) * fire_pct))] # add to a list of buildings to destroy buildings_fire = top_fire_buildings.index fire_buildings.extend(buildings_fire) eq_buildings.extend(buildings_fire) print "Total number of buildings being destroyed is: %d" \ % len(eq_buildings) orca.add_injectable("eq_buildings", eq_buildings) orca.add_injectable("existing_buildings", existing_buildings) orca.add_injectable("new_buildings", new_buildings) orca.add_injectable("fire_buildings", fire_buildings) # remove buildings, unplace agents buildings = orca.get_table('buildings') eq_demolish = buildings.local[buildings.index.isin (eq_buildings)] orca.add_table("eq_demolish", eq_demolish) print "Demolishing %d buildings" % len(eq_demolish) households = households.to_frame() hh_unplaced = households[households["building_id"] == -1] jobs = jobs.to_frame() jobs_unplaced = jobs[jobs["building_id"] == -1] l1 = len(buildings) # currently destroying more buildings than it is being # passed- why? buildings = utils._remove_developed_buildings( buildings.to_frame(buildings.local_columns), eq_demolish, unplace_agents=["households", "jobs"]) households = orca.get_table("households") households = households.to_frame() hh_unplaced_eq = households[households["building_id"] == -1] hh_unplaced_eq = hh_unplaced_eq[~hh_unplaced_eq.index.isin (hh_unplaced.index)] orca.add_injectable("hh_unplaced_eq", hh_unplaced_eq) jobs = orca.get_table("jobs") jobs = jobs.to_frame() jobs_unplaced_eq = jobs[jobs["building_id"] == -1] jobs_unplaced_eq = jobs_unplaced_eq[~jobs_unplaced_eq.index.isin (jobs_unplaced.index)] orca.add_injectable("jobs_unplaced_eq", jobs_unplaced_eq) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") print "Demolished %d buildings" % (l1 - len(buildings))
def topsheet(households, jobs, buildings, parcels, zones, year, run_number, taz_geography, parcels_zoning_calculations, summary, settings, parcels_geography): hh_by_subregion = misc.reindex(taz_geography.subregion, households.zone_id).value_counts() households_df = orca.merge_tables( 'households', [parcels_geography, buildings, households], columns=['pda_id', 'tpp_id', 'income']) hh_by_inpda = households_df.pda_id.notnull().value_counts() hhincome_by_intpp = households_df.income.groupby( households_df.tpp_id.notnull()).mean() # round to nearest 100s hhincome_by_intpp = (hhincome_by_intpp/100).round()*100 jobs_by_subregion = misc.reindex(taz_geography.subregion, jobs.zone_id).value_counts() jobs_df = orca.merge_tables( 'jobs', [parcels, buildings, jobs], columns=['pda']) jobs_by_inpda = jobs_df.pda.notnull().value_counts() capacity = parcels_zoning_calculations.\ zoned_du_underbuild_nodev.groupby(parcels.subregion).sum() if year == 2010: # save some info for computing growth measures orca.add_injectable("base_year_measures", { "hh_by_subregion": hh_by_subregion, "jobs_by_subregion": jobs_by_subregion, "hh_by_inpda": hh_by_inpda, "jobs_by_inpda": jobs_by_inpda, "hhincome_by_intpp": hhincome_by_intpp, "capacity": capacity }) # if year != 2040: # return base_year_measures = orca.get_injectable("base_year_measures") f = open(os.path.join("runs", "run%d_topsheet_%d.log" % (run_number, year)), "w") def write(s): # print s f.write(s + "\n\n") def norm_and_round(s): # normalize and round a series return str((s/s.sum()).round(2)) nhh = len(households) write("Number of households = %d" % nhh) nj = len(jobs) write("Number of jobs = %d" % nj) n = len(households.building_id[households.building_id == -1]) write("Number of unplaced households = %d" % n) n = len(jobs.building_id[jobs.building_id == -1]) write("Number of unplaced jobs = %d" % n) du = buildings.residential_units.sum() write("Number of residential units = %d" % du) write("Residential vacancy rate = %.2f" % (1-0 - float(nhh)/du)) du = buildings.deed_restricted_units.sum() write("Number of deed restricted units = %d" % du) write("Base year mean income by whether household is in tpp:\n%s" % base_year_measures["hhincome_by_intpp"]) write("Horizon year mean income by whether household is in tpp:\n%s" % hhincome_by_intpp) jsp = buildings.job_spaces.sum() write("Number of job spaces = %d" % jsp) write("Non-residential vacancy rate = %.2f" % (1-0 - float(nj)/jsp)) tmp = base_year_measures["hh_by_subregion"] write("Households base year share by subregion:\n%s" % norm_and_round(tmp)) write("Households share by subregion:\n%s" % norm_and_round(hh_by_subregion)) diff = hh_by_subregion - base_year_measures["hh_by_subregion"] write("Households pct of regional growth by subregion:\n%s" % norm_and_round(diff)) tmp = base_year_measures["jobs_by_subregion"] write("Jobs base year share by subregion:\n%s" % norm_and_round(tmp)) write("Jobs share by subregion:\n%s" % norm_and_round(jobs_by_subregion)) diff = jobs_by_subregion - base_year_measures["jobs_by_subregion"] write("Jobs pct of regional growth by subregion:\n%s" % norm_and_round(diff)) tmp = base_year_measures["hh_by_inpda"] write("Households base year share in pdas:\n%s" % norm_and_round(tmp)) write("Households share in pdas:\n%s" % norm_and_round(hh_by_inpda)) diff = hh_by_inpda - base_year_measures["hh_by_inpda"] write("Households pct of regional growth in pdas:\n%s" % norm_and_round(diff)) tmp = base_year_measures["jobs_by_inpda"] write("Jobs base year share in pdas:\n%s" % norm_and_round(tmp)) write("Jobs share in pdas:\n%s" % norm_and_round(jobs_by_inpda)) diff = jobs_by_inpda - base_year_measures["jobs_by_inpda"] write("Jobs pct of regional growth in pdas:\n%s" % norm_and_round(diff)) write("Base year dwelling unit raw capacity:\n%s" % base_year_measures["capacity"]) write("Dwelling unit raw capacity:\n%s" % capacity) if summary.parcel_output is not None: df = summary.parcel_output # we mark greenfield as a parcel with less than 500 current sqft greenfield = df.total_sqft < 500 write("Current share of projects which are greenfield development:\n%s" % norm_and_round(greenfield.value_counts())) write("Current share of units which are greenfield development:\n%s" % norm_and_round(df.residential_units.groupby(greenfield).sum())) cmap = settings["county_id_tm_map"] jobs_by_county = jobs.zone_id.map(taz_geography.county)\ .map(cmap).value_counts() households_by_county = households.zone_id.map(taz_geography.county)\ .map(cmap).value_counts() jobs_by_housing = jobs_by_county / households_by_county.replace(0, 1) write("Jobs/housing balance:\n" + str(jobs_by_housing)) f.close()
def buildings_to_uc(new_buildings, year): """ Parameters ---------- new_buildings : pandas.DataFrame DataFrame of buildings to export. year : int Simulation year Returns ------- None """ # Checking that building_id is index or exists as column if (new_buildings.index.name != 'building_id') & ('building_id' not in new_buildings.columns): print 'Index of buildings must be "building_id" or "building_id" column must exist. Skipping export-to-Urban-Canvas.' return None if 'building_id' not in new_buildings.columns: new_buildings = new_buildings.reset_index() # Urban Canvas database connection conn_string = orca.get_injectable('conn_string') if len(conn_string) == 0: print 'A "conn_string" injectable must be registered and populated. Skipping export-to-Urban-Canvas.' return None if 'uc_conn' not in orca.list_injectables(): conn = psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) else: conn = orca.get_injectable('uc_conn') cur = orca.get_injectable('uc_cur') def exec_sql_uc(query): try: cur.execute(query) conn.commit() except: conn = psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) cur.execute(query) conn.commit() def get_val_from_uc_db(query): try: result = sql.read_frame(query, conn) return result.values[0][0] except: conn=psycopg2.connect(conn_string) cur = conn.cursor() orca.add_injectable('uc_conn', conn) orca.add_injectable('uc_cur', cur) result = sql.read_frame(query, conn) result2 = sql.read_frame("select column_name from Information_schema.columns where table_name like 'building' ", conn) print result2 return result.values[0][0] max_bid = get_val_from_uc_db("select max(building_id) FROM building where building_id<100000000;") new_buildings.building_id = np.arange(max_bid+1, max_bid+1+len(new_buildings)) if 'projects_num' not in orca.list_injectables(): exec_sql_uc("INSERT INTO scenario(id, name, type) select nextval('scenario_id_seq'), 'Run #' || cast(currval('scenario_id_seq') as character varying), 1;") nextval = get_val_from_uc_db("SELECT MAX(ID) FROM SCENARIO WHERE ID < 100000;") exec_sql_uc("INSERT INTO scenario(id, name, type) select nextval('scenario_id_seq'), 'Run #' || cast(currval('scenario_id_seq') as character varying), 1;") nextval = get_val_from_uc_db("SELECT MAX(ID) FROM SCENARIO WHERE ID < 1000000;") orca.add_injectable('projects_num', nextval) exec_sql_uc("INSERT INTO scenario_project(scenario, project) VALUES(%s, 1);" % nextval) exec_sql_uc("INSERT INTO scenario_project(scenario, project) VALUES(%s, %s);" % (nextval, nextval)) else: nextval = orca.get_injectable('projects_num') nextval = '{'+str(nextval)+ '}' new_buildings['projects'] = nextval valid_from = '{'+ str(year) + '-1-1}' new_buildings['valid_from'] = valid_from print 'Exporting %s buildings to Urban Canvas database for project %s and year %s.' % (len(new_buildings),nextval,year) output = cStringIO.StringIO() new_buildings.to_csv('buildings_for_eddie.csv') new_buildings.to_csv(output, sep='\t', header=False, index=False) output.seek(0) cur.copy_from(output, 'building', columns=tuple(new_buildings.columns.values.astype('U').tolist())) test = pd.read_sql("select projects from building where (year_built>2010)", conn) print test conn.commit()
def change_scenario(scenario): assert scenario in orca.get_injectable("scenario_inputs"), \ "Invalid scenario name" print "Changing scenario to '%s'" % scenario orca.add_injectable("scenario", scenario)
NO_PROJECT = 611 EARTHQUAKE = False IN_YEAR, OUT_YEAR = 2010, 2050 COMPARE_AGAINST_LAST_KNOWN_GOOD = False LAST_KNOWN_GOOD_RUNS = { "0": 1057, "1": 1058, "2": 1059, "3": 1060, "4": 1059, "5": 1059 } orca.add_injectable("years_per_iter", EVERY_NTH_YEAR) orca.add_injectable("earthquake", EARTHQUAKE) parser = argparse.ArgumentParser(description='Run UrbanSim models.') parser.add_argument( '-c', action='store_true', dest='console', help='run from the console (logs to stdout), no slack or maps') parser.add_argument('-i', action='store_true', dest='interactive', help='enter interactive mode after imports') parser.add_argument('-s', action='store', dest='scenario', help='specify which scenario to run')
import os import orca import pandas as pd import numpy as np import urbansim_defaults.utils as utils from urbansim.utils import misc from urbansim_defaults import datasources import warnings warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning) orca.add_injectable('base_year', 2014) @orca.injectable() def year(base_year): if 'iter_var' in orca.list_injectables(): year = orca.get_injectable('iter_var') if year is not None: return year # outside of a run, return the base/default return base_year @orca.injectable() def store_table_names_dict(): # Dictionary with pairs of orca.table name and the associated name in storage. # Only entries where the names differ. return {'employment_controls': "annual_employment_control_totals", 'employment_sector_group_definitions': "employment_adhoc_sector_group_definitions", 'employment_sector_groups': "employment_adhoc_sector_groups", 'household_controls': "annual_household_control_totals", 'household_relocation_rates': 'annual_household_relocation_rates', 'job_relocation_rates': 'annual_job_relocation_rates',
__author__ = 'JMartinez' import orca import pandas as pd import numpy as np from sqlalchemy import engine #add your data path here orca.add_injectable("store",pd.HDFStore('C:/urbansim_new/urbansim/urbansim_drcog/config/drcog.h5', mode='r')) #register tables @orca.table('buildings', cache=True) def buildings(store): df = store['buildings'] return df @orca.table('parcels', cache=True) def parcels(store): df = store['parcels'] return df @orca.table('households', cache=True) def households(store): df = store['households'] #df.drop('zone_id', axis=1, inplace=True) return df @orca.table('zones', cache=True) def zones(store): df = store['zones'] amenities = pd.read_csv('c:/urbansim_new/urbansim/urbansim_drcog/config/amenities.csv',index_col=0) df = pd.merge(df, amenities, left_index=True, right_index=True)
def run_feasibility(parcels, parcel_price_callback, parcel_use_allowed_callback, pipeline=False, cfg=None, **kwargs): """ Execute development feasibility on all development sites Parameters ---------- parcels : DataFrame Wrapper The data frame wrapper for the parcel data parcel_price_callback : function A callback which takes each use of the pro forma and returns a series with index as parcel_id and value as yearly_rent parcel_use_allowed_callback : function A callback which takes each form of the pro forma and returns a series with index as parcel_id and value and boolean whether the form is allowed on the parcel pipeline : bool, optional If True, removes parcels from consideration if already in dev_sites table cfg : str, optional The name of the yaml file to read pro forma configurations from """ cfg = misc.config(cfg) # Create default SqFtProForma pf = (sqftproforma.SqFtProForma.from_yaml(str_or_buffer=cfg) if cfg else sqftproforma.SqFtProForma.from_defaults()) # Update default values using templates and store pf = update_sqftproforma(pf, cfg, **kwargs) orca.add_injectable("pf_config", pf) sites = (pl.remove_pipelined_sites(parcels) if pipeline else parcels.to_frame(parcels.local_columns)) #df = apply_parcel_callbacks(sites, parcel_price_callback, # pf, **kwargs) # compute price for each use df = sites for use in pf.uses: df[use] = parcel_price_callback(use, pf) #feasibility = lookup_by_form(df, parcel_use_allowed_callback, pf, **kwargs) print "Describe of the yearly rent by use" print df[pf.uses].describe() # Computing actual feasibility d = {} forms = pf.forms_to_test or pf.forms for form in forms: print "Computing feasibility for form %s" % form #if parcel_id_col is not None: # parcels = df[parcel_id_col].unique() # allowed = (parcel_use_allowed_callback(form).loc[parcels]) # newdf = df.loc[misc.reindex(allowed, df.parcel_id)] #else: allowed = parcel_use_allowed_callback(form).loc[df.index] newdf = df[allowed] # Core function - computes profitability d[form] = pf.lookup(form, newdf, only_built = pf.only_built, pass_through = pf.pass_through) # Collect results if pf.proposals_to_keep > 1: # feasibility is in long format form_feas = [] for form_name in d.keys(): df_feas_form = d[form_name] df_feas_form['form'] = form_name form_feas.append(df_feas_form) feasibility = pd.concat(form_feas, sort=False) if pf.percent_of_max_profit > 0: feasibility['max_profit_parcel'] = feasibility.groupby([feasibility.index, 'form'])['max_profit'].transform(max) feasibility['ratio'] = feasibility.max_profit/feasibility.max_profit_parcel feasibility = feasibility[feasibility.ratio >= pf.percent_of_max_profit / 100.] feasibility.drop(['max_profit_parcel', 'ratio'], axis=1, inplace = True) feasibility.index.name = 'parcel_id' # add attribute that enumerates proposals (can be used as a unique index) feasibility["feasibility_id"] = np.arange(1, len(feasibility)+1, dtype = "int32") # create a dataset with disaggregated sqft by building type feas_bt = pd.merge(feasibility.loc[:, ["form", "feasibility_id", "residential_sqft", "non_residential_sqft"]], pf.forms_df, left_on = "form", right_index = True) feas_bt.set_index(['form'], append = True, inplace = True) feas_bt[pf.uses[pf.residential_uses.values == 1]] = feas_bt[pf.uses[pf.residential_uses.values == 1]].multiply(feas_bt.residential_sqft, axis = "index") feas_bt[pf.uses[pf.residential_uses.values == 0]] = feas_bt[pf.uses[pf.residential_uses.values == 0]].multiply(feas_bt.non_residential_sqft, axis = "index") orca.add_table('feasibility_bt', feas_bt) else: # feasibility is in wide format feasibility = pd.concat(d.values(), keys = d.keys(), axis=1) orca.add_table('feasibility', feasibility) return feasibility