def auto_ownership_simulate(households_merged, auto_ownership_spec,
auto_ownership_settings, trace_hh_id):
"""
Auto ownership is a standard model which predicts how many cars a household
with given characteristics owns
"""
logger.info("Running auto_ownership_simulate with %d households" %
len(households_merged))
nest_spec = config.get_logit_model_settings(auto_ownership_settings)
constants = config.get_model_constants(auto_ownership_settings)
choices = asim.simple_simulate(choosers=households_merged.to_frame(),
spec=auto_ownership_spec,
nest_spec=nest_spec,
locals_d=constants,
trace_label=trace_hh_id
and 'auto_ownership',
trace_choice_name='auto_ownership')
tracing.print_summary('auto_ownership', choices, value_counts=True)
orca.add_column('households', 'auto_ownership', choices)
pipeline.add_dependent_columns('households', 'households_autoown')
if trace_hh_id:
trace_columns = ['auto_ownership'
] + orca.get_table('households_autoown').columns
tracing.trace_df(orca.get_table('households').to_frame(),
label='auto_ownership',
columns=trace_columns,
warn_if_empty=True)
def auto_ownership_simulate(households,
households_merged,
chunk_size,
trace_hh_id):
"""
Auto ownership is a standard model which predicts how many cars a household
with given characteristics owns
"""
trace_label = 'auto_ownership_simulate'
model_settings_file_name = 'auto_ownership.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
estimator = estimation.manager.begin_estimation('auto_ownership')
model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])
coefficients_df = simulate.read_model_coefficients(model_settings)
model_spec = simulate.eval_coefficients(model_spec, coefficients_df, estimator)
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choosers = households_merged.to_frame()
logger.info("Running %s with %d households", trace_label, len(choosers))
if estimator:
estimator.write_model_settings(model_settings, model_settings_file_name)
estimator.write_spec(model_settings)
estimator.write_coefficients(coefficients_df)
estimator.write_choosers(choosers)
choices = simulate.simple_simulate(
choosers=choosers,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='auto_ownership',
estimator=estimator)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(choices, 'households', 'auto_ownership')
estimator.write_override_choices(choices)
estimator.end_estimation()
households = households.to_frame()
# no need to reindex as we used all households
households['auto_ownership'] = choices
pipeline.replace_table("households", households)
tracing.print_summary('auto_ownership', households.auto_ownership, value_counts=True)
if trace_hh_id:
tracing.trace_df(households,
label='auto_ownership',
warn_if_empty=True)
def free_parking(
persons_merged, persons, households,
skim_dict, skim_stack,
chunk_size, trace_hh_id, locutor):
"""
"""
trace_label = 'free_parking'
model_settings = config.read_model_settings('free_parking.yaml')
choosers = persons_merged.to_frame()
choosers = choosers[choosers.workplace_taz > -1]
logger.info("Running %s with %d persons", trace_label, len(choosers))
constants = config.get_model_constants(model_settings)
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_d = {}
if constants is not None:
locals_d.update(constants)
expressions.assign_columns(
df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_d,
trace_label=trace_label)
model_spec = simulate.read_model_spec(file_name='free_parking.csv')
nest_spec = config.get_logit_model_settings(model_settings)
choices = simulate.simple_simulate(
choosers=choosers,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='free_parking_at_work')
persons = persons.to_frame()
# no need to reindex as we used all households
free_parking_alt = model_settings['FREE_PARKING_ALT']
choices = (choices == free_parking_alt)
persons['free_parking_at_work'] = choices.reindex(persons.index).fillna(0).astype(bool)
pipeline.replace_table("persons", persons)
tracing.print_summary('free_parking', persons.free_parking_at_work, value_counts=True)
if trace_hh_id:
tracing.trace_df(persons,
label=trace_label,
warn_if_empty=True)
def free_parking(persons_merged, persons, households, skim_dict, skim_stack,
chunk_size, trace_hh_id, locutor):
"""
"""
trace_label = 'free_parking'
model_settings = config.read_model_settings('free_parking.yaml')
choosers = persons_merged.to_frame()
choosers = choosers[choosers.workplace_taz > -1]
logger.info("Running %s with %d persons", trace_label, len(choosers))
constants = config.get_model_constants(model_settings)
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_d = {}
if constants is not None:
locals_d.update(constants)
expressions.assign_columns(df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_d,
trace_label=trace_label)
model_spec = simulate.read_model_spec(file_name='free_parking.csv')
nest_spec = config.get_logit_model_settings(model_settings)
choices = simulate.simple_simulate(
choosers=choosers,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='free_parking_at_work')
persons = persons.to_frame()
# no need to reindex as we used all households
free_parking_alt = model_settings['FREE_PARKING_ALT']
choices = (choices == free_parking_alt)
persons['free_parking_at_work'] = choices.reindex(
persons.index).fillna(0).astype(bool)
pipeline.replace_table("persons", persons)
tracing.print_summary('free_parking',
persons.free_parking_at_work,
value_counts=True)
if trace_hh_id:
tracing.trace_df(persons, label=trace_label, warn_if_empty=True)
def atwork_subtour_destination(
tours,
persons_merged,
skim_dict,
skim_stack,
land_use, size_terms,
chunk_size, trace_hh_id):
persons_merged = persons_merged.to_frame()
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results('atwork_subtour_destination')
return
# interaction_sample_simulate insists choosers appear in same order as alts
subtours = subtours.sort_index()
destination_size_terms = tour_destination_size_terms(land_use, size_terms, 'atwork')
destination_sample = atwork_subtour_destination_sample(
subtours,
persons_merged,
skim_dict,
destination_size_terms,
chunk_size, trace_hh_id)
destination_sample = atwork_subtour_destination_logsums(
persons_merged,
destination_sample,
skim_dict, skim_stack,
chunk_size, trace_hh_id)
choices = atwork_subtour_destination_simulate(
subtours,
persons_merged,
destination_sample,
skim_dict,
destination_size_terms,
chunk_size, trace_hh_id)
subtours['destination'] = choices
assign_in_place(tours, subtours[['destination']])
pipeline.replace_table("tours", tours)
tracing.print_summary('subtour destination', subtours.destination, describe=True)
if trace_hh_id:
tracing.trace_df(tours,
label='atwork_subtour_destination',
columns=['destination'])
def mandatory_scheduling(mandatory_tours_merged, tdd_alts, tdd_school_spec,
tdd_work_spec, mandatory_scheduling_settings,
chunk_size, trace_hh_id):
"""
This model predicts the departure time and duration of each activity for
mandatory tours
"""
tours = mandatory_tours_merged.to_frame()
alts = tdd_alts.to_frame()
constants = config.get_model_constants(mandatory_scheduling_settings)
school_spec = tdd_school_spec.to_frame()
school_tours = tours[tours.tour_type == "school"]
logger.info("Running mandatory_scheduling school_tours with %d tours" %
len(school_tours))
school_choices = vectorize_tour_scheduling(
school_tours,
alts,
school_spec,
constants=constants,
chunk_size=chunk_size,
trace_label='mandatory_scheduling.school')
work_spec = tdd_work_spec.to_frame()
work_tours = tours[tours.tour_type == "work"]
logger.info("Running %d work tour scheduling choices" % len(work_tours))
work_choices = vectorize_tour_scheduling(
work_tours,
alts,
work_spec,
constants=constants,
chunk_size=chunk_size,
trace_label='mandatory_scheduling.work')
choices = pd.concat([school_choices, work_choices])
tracing.print_summary('mandatory_scheduling tour_departure_and_duration',
choices,
describe=True)
orca.add_column("mandatory_tours", "tour_departure_and_duration", choices)
if trace_hh_id:
tracing.trace_df(orca.get_table('mandatory_tours').to_frame(),
label="mandatory_tours",
slicer='person_id',
index_label='tour',
columns=None,
warn_if_empty=True)
def mandatory_tour_frequency(persons_merged,
mandatory_tour_frequency_spec,
mandatory_tour_frequency_settings,
chunk_size,
trace_hh_id):
"""
This model predicts the frequency of making mandatory trips (see the
alternatives above) - these trips include work and school in some combination.
"""
trace_label = 'mandatory_tour_frequency'
choosers = persons_merged.to_frame()
# filter based on results of CDAP
choosers = choosers[choosers.cdap_activity == 'M']
logger.info("Running mandatory_tour_frequency with %d persons" % len(choosers))
nest_spec = config.get_logit_model_settings(mandatory_tour_frequency_settings)
constants = config.get_model_constants(mandatory_tour_frequency_settings)
choices = simulate.simple_simulate(
choosers,
spec=mandatory_tour_frequency_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='mandatory_tour_frequency')
# convert indexes to alternative names
choices = pd.Series(
mandatory_tour_frequency_spec.columns[choices.values],
index=choices.index).reindex(persons_merged.local.index)
tracing.print_summary('mandatory_tour_frequency', choices, value_counts=True)
inject.add_column("persons", "mandatory_tour_frequency", choices)
create_mandatory_tours(trace_hh_id)
# add mandatory_tour-dependent columns (e.g. tour counts) to persons
pipeline.add_dependent_columns("persons", "persons_mtf")
if trace_hh_id:
trace_columns = ['mandatory_tour_frequency']
tracing.trace_df(inject.get_table('persons').to_frame(),
label="mandatory_tour_frequency.persons",
# columns=trace_columns,
warn_if_empty=True)
def cdap_simulate(persons_merged, cdap_settings, cdap_indiv_spec,
cdap_interaction_coefficients,
cdap_fixed_relative_proportions, chunk_size, trace_hh_id):
"""
CDAP stands for Coordinated Daily Activity Pattern, which is a choice of
high-level activity pattern for each person, in a coordinated way with other
members of a person's household.
Because Python requires vectorization of computation, there are some specialized
routines in the cdap directory of activitysim for this purpose. This module
simply applies those utilities using the simulation framework.
"""
persons_df = persons_merged.to_frame()
constants = config.get_model_constants(cdap_settings)
logger.info("Running cdap_simulate with %d persons" %
len(persons_df.index))
choices = run_cdap(
persons=persons_df,
cdap_indiv_spec=cdap_indiv_spec,
cdap_interaction_coefficients=cdap_interaction_coefficients,
cdap_fixed_relative_proportions=cdap_fixed_relative_proportions,
locals_d=constants,
chunk_size=chunk_size,
trace_hh_id=trace_hh_id,
trace_label='cdap')
tracing.print_summary('cdap_activity',
choices.cdap_activity,
value_counts=True)
print pd.crosstab(persons_df.ptype, choices.cdap_activity, margins=True)
choices = choices.reindex(persons_merged.index)
orca.add_column("persons", "cdap_activity", choices.cdap_activity)
orca.add_column("persons", "cdap_rank", choices.cdap_rank)
pipeline.add_dependent_columns("persons", "persons_cdap")
pipeline.add_dependent_columns("households", "households_cdap")
if trace_hh_id:
tracing.trace_df(orca.get_table('persons_merged').to_frame(),
label="cdap",
columns=['ptype', 'cdap_rank', 'cdap_activity'],
warn_if_empty=True)
def mandatory_tour_frequency(persons_merged,
mandatory_tour_frequency_spec,
mandatory_tour_frequency_settings,
trace_hh_id):
"""
This model predicts the frequency of making mandatory trips (see the
alternatives above) - these trips include work and school in some combination.
"""
choosers = persons_merged.to_frame()
# filter based on results of CDAP
choosers = choosers[choosers.cdap_activity == 'M']
logger.info("Running mandatory_tour_frequency with %d persons" % len(choosers))
nest_spec = config.get_logit_model_settings(mandatory_tour_frequency_settings)
constants = config.get_model_constants(mandatory_tour_frequency_settings)
choices = asim.simple_simulate(
choosers,
spec=mandatory_tour_frequency_spec,
nest_spec=nest_spec,
locals_d=constants,
trace_label=trace_hh_id and 'mandatory_tour_frequency',
trace_choice_name='mandatory_tour_frequency')
# convert indexes to alternative names
choices = pd.Series(
mandatory_tour_frequency_spec.columns[choices.values],
index=choices.index).reindex(persons_merged.local.index)
tracing.print_summary('mandatory_tour_frequency', choices, value_counts=True)
orca.add_column("persons", "mandatory_tour_frequency", choices)
pipeline.add_dependent_columns("persons", "persons_mtf")
create_mandatory_tours_table()
# FIXME - test prng repeatability
r = pipeline.get_rn_generator().random_for_df(choices)
orca.add_column("persons", "mtf_rand", [item for sublist in r for item in sublist])
if trace_hh_id:
trace_columns = ['mandatory_tour_frequency']
tracing.trace_df(orca.get_table('persons_merged').to_frame(),
label="mandatory_tour_frequency",
columns=trace_columns,
warn_if_empty=True)
def atwork_subtour_destination(tours, persons_merged, skim_dict, skim_stack,
land_use, size_terms, chunk_size, trace_hh_id):
persons_merged = persons_merged.to_frame()
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results('atwork_subtour_destination')
return
# interaction_sample_simulate insists choosers appear in same order as alts
subtours = subtours.sort_index()
destination_size_terms = tour_destination_size_terms(
land_use, size_terms, 'atwork')
destination_sample = atwork_subtour_destination_sample(
subtours, persons_merged, skim_dict, destination_size_terms,
chunk_size, trace_hh_id)
destination_sample = atwork_subtour_destination_logsums(
persons_merged, destination_sample, skim_dict, skim_stack, chunk_size,
trace_hh_id)
choices = atwork_subtour_destination_simulate(subtours, persons_merged,
destination_sample,
skim_dict,
destination_size_terms,
chunk_size, trace_hh_id)
subtours['destination'] = choices
assign_in_place(tours, subtours[['destination']])
pipeline.replace_table("tours", tours)
tracing.print_summary('subtour destination',
subtours.destination,
describe=True)
if trace_hh_id:
tracing.trace_df(tours,
label='atwork_subtour_destination',
columns=['destination'])
def auto_ownership_simulate(households,
households_merged,
chunk_size,
trace_hh_id):
"""
Auto ownership is a standard model which predicts how many cars a household
with given characteristics owns
"""
trace_label = 'auto_ownership_simulate'
model_settings = config.read_model_settings('auto_ownership.yaml')
logger.info("Running %s with %d households", trace_label, len(households_merged))
model_spec = simulate.read_model_spec(file_name='auto_ownership.csv')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choices = simulate.simple_simulate(
choosers=households_merged.to_frame(),
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='auto_ownership')
households = households.to_frame()
# no need to reindex as we used all households
households['auto_ownership'] = choices
pipeline.replace_table("households", households)
tracing.print_summary('auto_ownership', households.auto_ownership, value_counts=True)
if trace_hh_id:
tracing.trace_df(households,
label='auto_ownership',
warn_if_empty=True)
def auto_ownership_simulate(households, households_merged, chunk_size,
trace_hh_id):
"""
Auto ownership is a standard model which predicts how many cars a household
with given characteristics owns
"""
trace_label = 'auto_ownership_simulate'
model_settings = config.read_model_settings('auto_ownership.yaml')
logger.info("Running %s with %d households", trace_label,
len(households_merged))
model_spec = simulate.read_model_spec(file_name='auto_ownership.csv')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choices = simulate.simple_simulate(choosers=households_merged.to_frame(),
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='auto_ownership')
households = households.to_frame()
# no need to reindex as we used all households
households['auto_ownership'] = choices
pipeline.replace_table("households", households)
tracing.print_summary('auto_ownership',
households.auto_ownership,
value_counts=True)
if trace_hh_id:
tracing.trace_df(households,
label='auto_ownership',
warn_if_empty=True)
def non_mandatory_tour_frequency(persons, persons_merged,
chunk_size,
trace_hh_id):
"""
This model predicts the frequency of making non-mandatory trips
(alternatives for this model come from a separate csv file which is
configured by the user) - these trips include escort, shopping, othmaint,
othdiscr, eatout, and social trips in various combination.
"""
trace_label = 'non_mandatory_tour_frequency'
model_settings = config.read_model_settings('non_mandatory_tour_frequency.yaml')
model_spec = simulate.read_model_spec(file_name='non_mandatory_tour_frequency.csv')
alternatives = simulate.read_model_alts(
config.config_file_path('non_mandatory_tour_frequency_alternatives.csv'),
set_index=None)
choosers = persons_merged.to_frame()
# FIXME kind of tacky both that we know to add this here and del it below
# 'tot_tours' is used in model_spec expressions
alternatives['tot_tours'] = alternatives.sum(axis=1)
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {
'person_max_window': person_max_window
}
expressions.assign_columns(
df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
# filter based on results of CDAP
choosers = choosers[choosers.cdap_activity.isin(['M', 'N'])]
logger.info("Running non_mandatory_tour_frequency with %d persons", len(choosers))
constants = config.get_model_constants(model_settings)
choices_list = []
# segment by person type and pick the right spec for each person type
for ptype, segment in choosers.groupby('ptype'):
name = PTYPE_NAME[ptype]
# pick the spec column for the segment
spec = model_spec[[name]]
# drop any zero-valued rows
spec = spec[spec[name] != 0]
logger.info("Running segment '%s' of size %d", name, len(segment))
choices = interaction_simulate(
segment,
alternatives,
spec=spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label='non_mandatory_tour_frequency.%s' % name,
trace_choice_name='non_mandatory_tour_frequency')
choices_list.append(choices)
# FIXME - force garbage collection?
# force_garbage_collect()
choices = pd.concat(choices_list)
del alternatives['tot_tours'] # del tot_tours column we added above
# - add non_mandatory_tour_frequency column to persons
persons = persons.to_frame()
# need to reindex as we only handled persons with cdap_activity in ['M', 'N']
# (we expect there to be an alt with no tours - which we can use to backfill non-travelers)
no_tours_alt = (alternatives.sum(axis=1) == 0).index[0]
persons['non_mandatory_tour_frequency'] = \
choices.reindex(persons.index).fillna(no_tours_alt).astype(np.int8)
"""
We have now generated non-mandatory tours, but they are attributes of the person table
Now we create a "tours" table which has one row per tour that has been generated
(and the person id it is associated with)
"""
# - get counts of each of the alternatives (so we can extend)
# (choices is just the index values for the chosen alts)
"""
escort shopping othmaint othdiscr eatout social
parent_id
2588676 2 0 0 1 1 0
2588677 0 1 0 1 0 0
"""
tour_counts = alternatives.loc[choices]
tour_counts.index = choices.index # assign person ids to the index
prev_tour_count = tour_counts.sum().sum()
# - extend_tour_counts
tour_counts = extend_tour_counts(choosers, tour_counts, alternatives,
trace_hh_id,
tracing.extend_trace_label(trace_label, 'extend_tour_counts'))
extended_tour_count = tour_counts.sum().sum()
logging.info("extend_tour_counts increased nmtf tour count by %s from %s to %s" %
(extended_tour_count - prev_tour_count, prev_tour_count, extended_tour_count))
# - create the non_mandatory tours
non_mandatory_tours = process_non_mandatory_tours(persons, tour_counts)
assert len(non_mandatory_tours) == extended_tour_count
pipeline.extend_table("tours", non_mandatory_tours)
tracing.register_traceable_table('tours', non_mandatory_tours)
pipeline.get_rn_generator().add_channel('tours', non_mandatory_tours)
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=trace_label)
pipeline.replace_table("persons", persons)
tracing.print_summary('non_mandatory_tour_frequency',
persons.non_mandatory_tour_frequency, value_counts=True)
if trace_hh_id:
tracing.trace_df(non_mandatory_tours,
label="non_mandatory_tour_frequency.non_mandatory_tours",
warn_if_empty=True)
tracing.trace_df(choosers,
label="non_mandatory_tour_frequency.choosers",
warn_if_empty=True)
tracing.trace_df(persons,
label="non_mandatory_tour_frequency.annotated_persons",
warn_if_empty=True)
def joint_tour_destination(tours, persons_merged, households_merged,
network_los, chunk_size, trace_hh_id):
"""
Given the tour generation from the above, each tour needs to have a
destination, so in this case tours are the choosers (with the associated
person that's making the tour)
"""
trace_label = 'joint_tour_destination'
model_settings_file_name = 'joint_tour_destination.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('DEST_CHOICE_LOGSUM_COLUMN_NAME')
want_logsums = logsum_column_name is not None
sample_table_name = model_settings.get('DEST_CHOICE_SAMPLE_TABLE_NAME')
want_sample_table = config.setting(
'want_dest_choice_sample_tables') and sample_table_name is not None
# choosers are tours - in a sense tours are choosing their destination
tours = tours.to_frame()
joint_tours = tours[tours.tour_category == 'joint']
persons_merged = persons_merged.to_frame()
# - if no joint tours
if joint_tours.shape[0] == 0:
tracing.no_results('joint_tour_destination')
return
estimator = estimation.manager.begin_estimation('joint_tour_destination')
if estimator:
estimator.write_coefficients(model_settings=model_settings)
# estimator.write_spec(model_settings, tag='SAMPLE_SPEC')
estimator.write_spec(model_settings, tag='SPEC')
estimator.set_alt_id(model_settings["ALT_DEST_COL_NAME"])
estimator.write_table(inject.get_injectable('size_terms'),
'size_terms',
append=False)
estimator.write_table(inject.get_table('land_use').to_frame(),
'landuse',
append=False)
estimator.write_model_settings(model_settings,
model_settings_file_name)
choices_df, save_sample_df = tour_destination.run_tour_destination(
tours, persons_merged, want_logsums, want_sample_table, model_settings,
network_los, estimator, chunk_size, trace_hh_id, trace_label)
if estimator:
estimator.write_choices(choices_df.choice)
choices_df.choice = estimator.get_survey_values(
choices_df.choice, 'tours', 'destination')
estimator.write_override_choices(choices_df.choice)
estimator.end_estimation()
# add column as we want joint_tours table for tracing.
joint_tours['destination'] = choices_df.choice
assign_in_place(tours, joint_tours[['destination']])
pipeline.replace_table("tours", tours)
if want_logsums:
joint_tours[logsum_column_name] = choices_df['logsum']
assign_in_place(tours, joint_tours[[logsum_column_name]])
tracing.print_summary('destination',
joint_tours.destination,
describe=True)
if want_sample_table:
assert len(save_sample_df.index.get_level_values(0).unique()) == len(
choices_df)
# save_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'], append=True, inplace=True)
pipeline.extend_table(sample_table_name, save_sample_df)
if trace_hh_id:
tracing.trace_df(joint_tours,
label="joint_tour_destination.joint_tours")
def mandatory_tour_frequency(persons_merged,
chunk_size,
trace_hh_id):
"""
This model predicts the frequency of making mandatory trips (see the
alternatives above) - these trips include work and school in some combination.
"""
trace_label = 'mandatory_tour_frequency'
model_settings = config.read_model_settings('mandatory_tour_frequency.yaml')
model_spec = simulate.read_model_spec(file_name='mandatory_tour_frequency.csv')
alternatives = simulate.read_model_alts(
config.config_file_path('mandatory_tour_frequency_alternatives.csv'), set_index='alt')
choosers = persons_merged.to_frame()
# filter based on results of CDAP
choosers = choosers[choosers.cdap_activity == 'M']
logger.info("Running mandatory_tour_frequency with %d persons", len(choosers))
# - if no mandatory tours
if choosers.shape[0] == 0:
add_null_results(trace_label, model_settings)
return
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {}
expressions.assign_columns(
df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choices = simulate.simple_simulate(
choosers=choosers,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='mandatory_tour_frequency')
# convert indexes to alternative names
choices = pd.Series(
model_spec.columns[choices.values],
index=choices.index).reindex(persons_merged.local.index)
# - create mandatory tours
"""
This reprocesses the choice of index of the mandatory tour frequency
alternatives into an actual dataframe of tours. Ending format is
the same as got non_mandatory_tours except trip types are "work" and "school"
"""
choosers['mandatory_tour_frequency'] = choices
mandatory_tours = process_mandatory_tours(
persons=choosers,
mandatory_tour_frequency_alts=alternatives
)
tours = pipeline.extend_table("tours", mandatory_tours)
tracing.register_traceable_table('tours', mandatory_tours)
pipeline.get_rn_generator().add_channel('tours', mandatory_tours)
# - annotate persons
persons = inject.get_table('persons').to_frame()
# need to reindex as we only handled persons with cdap_activity == 'M'
persons['mandatory_tour_frequency'] = choices.reindex(persons.index).fillna('').astype(str)
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=tracing.extend_trace_label(trace_label, 'annotate_persons'))
pipeline.replace_table("persons", persons)
tracing.print_summary('mandatory_tour_frequency', persons.mandatory_tour_frequency,
value_counts=True)
if trace_hh_id:
tracing.trace_df(mandatory_tours,
label="mandatory_tour_frequency.mandatory_tours",
warn_if_empty=True)
tracing.trace_df(persons,
label="mandatory_tour_frequency.persons",
warn_if_empty=True)
def mandatory_tour_frequency(persons_merged, chunk_size, trace_hh_id):
"""
This model predicts the frequency of making mandatory trips (see the
alternatives above) - these trips include work and school in some combination.
"""
trace_label = 'mandatory_tour_frequency'
model_settings = config.read_model_settings(
'mandatory_tour_frequency.yaml')
model_spec = simulate.read_model_spec(
file_name='mandatory_tour_frequency.csv')
alternatives = simulate.read_model_alts(
config.config_file_path('mandatory_tour_frequency_alternatives.csv'),
set_index='alt')
choosers = persons_merged.to_frame()
# filter based on results of CDAP
choosers = choosers[choosers.cdap_activity == 'M']
logger.info("Running mandatory_tour_frequency with %d persons",
len(choosers))
# - if no mandatory tours
if choosers.shape[0] == 0:
add_null_results(trace_label, model_settings)
return
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {}
expressions.assign_columns(df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choices = simulate.simple_simulate(
choosers=choosers,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='mandatory_tour_frequency')
# convert indexes to alternative names
choices = pd.Series(model_spec.columns[choices.values],
index=choices.index).reindex(
persons_merged.local.index)
# - create mandatory tours
"""
This reprocesses the choice of index of the mandatory tour frequency
alternatives into an actual dataframe of tours. Ending format is
the same as got non_mandatory_tours except trip types are "work" and "school"
"""
choosers['mandatory_tour_frequency'] = choices
mandatory_tours = process_mandatory_tours(
persons=choosers, mandatory_tour_frequency_alts=alternatives)
tours = pipeline.extend_table("tours", mandatory_tours)
tracing.register_traceable_table('tours', mandatory_tours)
pipeline.get_rn_generator().add_channel('tours', mandatory_tours)
# - annotate persons
persons = inject.get_table('persons').to_frame()
# need to reindex as we only handled persons with cdap_activity == 'M'
persons['mandatory_tour_frequency'] = choices.reindex(
persons.index).fillna('').astype(str)
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=tracing.extend_trace_label(trace_label,
'annotate_persons'))
pipeline.replace_table("persons", persons)
tracing.print_summary('mandatory_tour_frequency',
persons.mandatory_tour_frequency,
value_counts=True)
if trace_hh_id:
tracing.trace_df(mandatory_tours,
label="mandatory_tour_frequency.mandatory_tours",
warn_if_empty=True)
tracing.trace_df(persons,
label="mandatory_tour_frequency.persons",
warn_if_empty=True)
def destination_choice(non_mandatory_tours_merged, skim_dict,
destination_choice_spec, destination_choice_settings,
destination_size_terms, chunk_size, trace_hh_id):
"""
Given the tour generation from the above, each tour needs to have a
destination, so in this case tours are the choosers (with the associated
person that's making the tour)
"""
# choosers are tours - in a sense tours are choosing their destination
choosers = non_mandatory_tours_merged.to_frame()
alternatives = destination_size_terms.to_frame()
spec = destination_choice_spec.to_frame()
constants = config.get_model_constants(destination_choice_settings)
sample_size = destination_choice_settings["SAMPLE_SIZE"]
# create wrapper with keys for this lookup - in this case there is a TAZ in the choosers
# and a TAZ in the alternatives which get merged during interaction
# the skims will be available under the name "skims" for any @ expressions
skims = skim_dict.wrap("TAZ", "TAZ_r")
locals_d = {'skims': skims}
if constants is not None:
locals_d.update(constants)
logger.info("Running destination_choice with %d non_mandatory_tours" %
len(choosers.index))
choices_list = []
# segment by trip type and pick the right spec for each person type
for name, segment in choosers.groupby('tour_type'):
# FIXME - there are two options here escort with kids and without
kludge_name = name
if name == "escort":
logging.error(
"destination_choice escort not implemented - running shopping instead"
)
kludge_name = "shopping"
# the segment is now available to switch between size terms
locals_d['segment'] = kludge_name
# FIXME - no point in considering impossible alternatives
alternatives_segment = alternatives[alternatives[kludge_name] > 0]
logger.info("Running segment '%s' of %d tours %d alternatives" %
(name, len(segment), len(alternatives_segment)))
# name index so tracing knows how to slice
segment.index.name = 'tour_id'
choices = asim.interaction_simulate(segment,
alternatives_segment,
spec[[kludge_name]],
skims=skims,
locals_d=locals_d,
sample_size=sample_size,
chunk_size=chunk_size,
trace_label='destination.%s' %
name)
choices_list.append(choices)
choices = pd.concat(choices_list)
# FIXME - can there be null destinations?
if choices.isnull().any():
logger.error("destination_choice had %s null destinations" %
choices.isnull().sum())
assert choices.isnull().sum() == 0
tracing.print_summary('destination', choices, describe=True)
# every trip now has a destination which is the index from the
# alternatives table - in this case it's the destination taz
orca.add_column("non_mandatory_tours", "destination", choices)
if trace_hh_id:
tracing.trace_df(orca.get_table('non_mandatory_tours').to_frame(),
label="destination",
slicer='person_id',
index_label='tour',
columns=None,
warn_if_empty=True)
def tour_mode_choice_simulate(tours_merged, tour_mode_choice_spec,
tour_mode_choice_settings, skim_dict, skim_stack,
omx_file, trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = trace_hh_id and 'tour_mode_choice'
tours = tours_merged.to_frame()
nest_spec = config.get_logit_model_settings(tour_mode_choice_settings)
constants = config.get_model_constants(tour_mode_choice_settings)
logger.info("Running tour_mode_choice_simulate with %d tours" %
len(tours.index))
tracing.print_summary('tour_mode_choice_simulate tour_type',
tours.tour_type,
value_counts=True)
if trace_hh_id:
tracing.trace_df(tour_mode_choice_spec,
tracing.extend_trace_label(trace_label, 'spec'),
slicer='NONE',
transpose=False)
# setup skim keys
odt_skim_stack_wrapper = skim_stack.wrap(left_key='TAZ',
right_key='destination',
skim_key="out_period")
dot_skim_stack_wrapper = skim_stack.wrap(left_key='destination',
right_key='TAZ',
skim_key="in_period")
od_skims = skim_dict.wrap('TAZ', 'destination')
choices_list = []
for tour_type, segment in tours.groupby('tour_type'):
# if tour_type != 'work':
# continue
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" % (
tour_type,
len(segment.index),
))
# name index so tracing knows how to slice
segment.index.name = 'tour_id'
spec = get_segment_and_unstack(tour_mode_choice_spec, tour_type)
if trace_hh_id:
tracing.trace_df(spec,
tracing.extend_trace_label(
trace_label, 'spec.%s' % tour_type),
slicer='NONE',
transpose=False)
choices = _mode_choice_simulate(
segment,
skim_dict=skim_dict,
skim_stack=skim_stack,
odt_skim_stack_wrapper=odt_skim_stack_wrapper,
dot_skim_stack_wrapper=dot_skim_stack_wrapper,
od_skim_stack_wrapper=od_skims,
spec=spec,
constants=constants,
nest_spec=nest_spec,
trace_label=tracing.extend_trace_label(trace_label, tour_type),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices' %
tour_type,
choices,
value_counts=True)
choices_list.append(choices)
# FIXME - force garbage collection
mem = asim.memory_info()
logger.debug('memory_info tour_type %s, %s' % (tour_type, mem))
choices = pd.concat(choices_list)
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices,
value_counts=True)
orca.add_column("tours", "mode", choices)
if trace_hh_id:
trace_columns = ['mode', 'person_id', 'tour_type', 'tour_num']
tracing.trace_df(orca.get_table('tours').to_frame(),
label=tracing.extend_trace_label(trace_label, 'mode'),
slicer='tour_id',
index_label='tour_id',
columns=trace_columns,
warn_if_empty=True)
# FIXME - this forces garbage collection
asim.memory_info()
def atwork_subtour_mode_choice(tours, persons_merged, skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d subtours" %
(trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type,
value_counts=True)
# setup skim keys
orig_col_name = 'workplace_taz'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices = run_tour_mode_choice_simulate(
subtours_merged,
spec,
tour_purpose='atwork',
model_settings=model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
tracing.print_summary('%s choices' % trace_label,
choices,
value_counts=True)
assign_in_place(tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(
trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def trip_mode_choice(
trips,
tours_merged,
network_los,
chunk_size, trace_hh_id):
"""
Trip mode choice - compute trip_mode (same values as for tour_mode) for each trip.
Modes for each primary tour putpose are calculated separately because they have different
coefficient values (stored in trip_mode_choice_coeffs.csv coefficient file.)
Adds trip_mode column to trip table
"""
trace_label = 'trip_mode_choice'
model_settings = config.read_model_settings('trip_mode_choice.yaml')
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'trip_mode'
model_spec = \
simulate.read_model_spec(file_name=model_settings['SPEC'])
omnibus_coefficients = \
assign.read_constant_spec(config.config_file_path(model_settings['COEFFICIENTS']))
trips_df = trips.to_frame()
logger.info("Running %s with %d trips", trace_label, trips_df.shape[0])
tours_merged = tours_merged.to_frame()
tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]
nest_spec = config.get_logit_model_settings(model_settings)
tracing.print_summary('primary_purpose',
trips_df.primary_purpose, value_counts=True)
# - trips_merged - merge trips and tours_merged
trips_merged = pd.merge(
trips_df,
tours_merged,
left_on='tour_id',
right_index=True,
how="left")
assert trips_merged.index.equals(trips.index)
# setup skim keys
assert ('trip_period' not in trips_merged)
trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)
orig_col = 'origin'
dest_col = 'destination'
constants = {}
constants.update(config.get_model_constants(model_settings))
constants.update({
'ORIGIN': orig_col,
'DESTINATION': dest_col
})
skim_dict = network_los.get_default_skim_dict()
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,
dim3_key='trip_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,
dim3_key='trip_period')
od_skim_wrapper = skim_dict.wrap('origin', 'destination')
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_wrapper,
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col, dest_key=dest_col,
tod_key='trip_period', segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label, tag='tvpb_logsum_odt')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
# 'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
choices_list = []
for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):
segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)
logger.info("trip_mode_choice tour_type '%s' (%s trips)" %
(primary_purpose, len(trips_segment.index), ))
# name index so tracing knows how to slice
assert trips_segment.index.name == 'trip_id'
if network_los.zone_system == los.THREE_ZONE:
tvpb_logsum_odt.extend_trace_label(primary_purpose)
# tvpb_logsum_dot.extend_trace_label(primary_purpose)
locals_dict = assign.evaluate_constants(omnibus_coefficients[primary_purpose],
constants=constants)
locals_dict.update(constants)
expressions.annotate_preprocessors(
trips_segment, locals_dict, skims,
model_settings, segment_trace_label)
locals_dict.update(skims)
choices = mode_choice_simulate(
choosers=trips_segment,
spec=model_spec,
nest_spec=nest_spec,
skims=skims,
locals_d=locals_dict,
chunk_size=chunk_size,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
trace_label=trace_label,
trace_choice_name='trip_mode_choice')
if trace_hh_id:
# trace the coefficients
tracing.trace_df(pd.Series(locals_dict),
label=tracing.extend_trace_label(segment_trace_label, 'constants'),
transpose=False,
slicer='NONE')
# so we can trace with annotations
assign_in_place(trips_segment, choices)
tracing.trace_df(trips_segment,
label=tracing.extend_trace_label(segment_trace_label, 'trip_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices_df = pd.concat(choices_list)
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_type in tvpb_mode_path_types.items():
skim_cache = tvpb_logsum_odt.cache[path_type]
print(f"mode {mode} path_type {path_type}")
for c in skim_cache:
dest_col = c
if dest_col not in choices_df:
choices_df[dest_col] = np.nan
choices_df[dest_col].where(choices_df[mode_column_name] != mode, skim_cache[c], inplace=True)
# update trips table with choices (and otionally logssums)
trips_df = trips.to_frame()
assign_in_place(trips_df, choices_df)
tracing.print_summary('tour_modes',
trips_merged.tour_mode, value_counts=True)
tracing.print_summary('trip_mode_choice choices',
trips_df[mode_column_name], value_counts=True)
assert not trips_df[mode_column_name].isnull().any()
pipeline.replace_table("trips", trips_df)
if trace_hh_id:
tracing.trace_df(trips_df,
label=tracing.extend_trace_label(trace_label, 'trip_mode'),
slicer='trip_id',
index_label='trip_id',
warn_if_empty=True)
def joint_tour_composition(tours, households, persons, chunk_size,
trace_hh_id):
"""
This model predicts the makeup of the travel party (adults, children, or mixed).
"""
trace_label = 'joint_tour_composition'
model_settings_file_name = 'joint_tour_composition.yaml'
tours = tours.to_frame()
joint_tours = tours[tours.tour_category == 'joint']
# - if no joint tours
if joint_tours.shape[0] == 0:
add_null_results(trace_label, tours)
return
model_settings = config.read_model_settings(model_settings_file_name)
estimator = estimation.manager.begin_estimation('joint_tour_composition')
# - only interested in households with joint_tours
households = households.to_frame()
households = households[households.num_hh_joint_tours > 0]
persons = persons.to_frame()
persons = persons[persons.household_id.isin(households.index)]
logger.info("Running joint_tour_composition with %d joint tours" %
joint_tours.shape[0])
# - run preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {
'persons': persons,
'hh_time_window_overlap': hh_time_window_overlap
}
expressions.assign_columns(df=households,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
joint_tours_merged = pd.merge(joint_tours,
households,
left_on='household_id',
right_index=True,
how='left')
# - simple_simulate
model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])
coefficients_df = simulate.read_model_coefficients(model_settings)
model_spec = simulate.eval_coefficients(model_spec, coefficients_df,
estimator)
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
if estimator:
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
estimator.write_coefficients(coefficients_df, model_settings)
estimator.write_choosers(joint_tours_merged)
choices = simulate.simple_simulate(choosers=joint_tours_merged,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='composition',
estimator=estimator)
# convert indexes to alternative names
choices = pd.Series(model_spec.columns[choices.values],
index=choices.index)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(choices, 'tours', 'composition')
estimator.write_override_choices(choices)
estimator.end_estimation()
# add composition column to tours for tracing
joint_tours['composition'] = choices
# reindex since we ran model on a subset of households
tours['composition'] = choices.reindex(tours.index).fillna('').astype(str)
pipeline.replace_table("tours", tours)
tracing.print_summary('joint_tour_composition',
joint_tours.composition,
value_counts=True)
if trace_hh_id:
tracing.trace_df(joint_tours,
label="joint_tour_composition.joint_tours",
slicer='household_id')
def joint_tour_composition(
tours, households, persons,
chunk_size,
trace_hh_id):
"""
This model predicts the makeup of the travel party (adults, children, or mixed).
"""
trace_label = 'joint_tour_composition'
model_settings = config.read_model_settings('joint_tour_composition.yaml')
model_spec = simulate.read_model_spec(file_name='joint_tour_composition.csv')
tours = tours.to_frame()
joint_tours = tours[tours.tour_category == 'joint']
# - if no joint tours
if joint_tours.shape[0] == 0:
add_null_results(trace_label, tours)
return
# - only interested in households with joint_tours
households = households.to_frame()
households = households[households.num_hh_joint_tours > 0]
persons = persons.to_frame()
persons = persons[persons.household_id.isin(households.index)]
logger.info("Running joint_tour_composition with %d joint tours" % joint_tours.shape[0])
# - run preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {
'persons': persons,
'hh_time_window_overlap': hh_time_window_overlap
}
expressions.assign_columns(
df=households,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
joint_tours_merged = pd.merge(joint_tours, households,
left_on='household_id', right_index=True, how='left')
# - simple_simulate
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
choices = simulate.simple_simulate(
choosers=joint_tours_merged,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='composition')
# convert indexes to alternative names
choices = pd.Series(model_spec.columns[choices.values], index=choices.index)
# add composition column to tours for tracing
joint_tours['composition'] = choices
# reindex since we ran model on a subset of households
tours['composition'] = choices.reindex(tours.index).fillna('').astype(str)
pipeline.replace_table("tours", tours)
tracing.print_summary('joint_tour_composition', joint_tours.composition,
value_counts=True)
if trace_hh_id:
tracing.trace_df(joint_tours,
label="joint_tour_composition.joint_tours",
slicer='household_id')
def stop_frequency(
tours, tours_merged,
stop_frequency_alts,
skim_dict,
chunk_size,
trace_hh_id):
"""
stop frequency model
For each tour, shoose a number of intermediate inbound stops and outbound stops.
Create a trip table with inbound and outbound trips.
Thus, a tour with stop_frequency '2out_0in' will have two outbound and zero inbound stops,
and four corresponding trips: three outbound, and one inbound.
Adds stop_frequency str column to trips, with fields
creates trips table with columns:
::
- person_id
- household_id
- tour_id
- primary_purpose
- atwork
- trip_num
- outbound
- trip_count
"""
trace_label = 'stop_frequency'
model_settings = config.read_model_settings('stop_frequency.yaml')
tours = tours.to_frame()
tours_merged = tours_merged.to_frame()
assert not tours_merged.household_id.isnull().any()
assert not (tours_merged.origin == -1).any()
assert not (tours_merged.destination == -1).any()
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
# - run preprocessor to annotate tours_merged
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
# hack: preprocessor adds origin column in place if it does not exist already
od_skim_stack_wrapper = skim_dict.wrap('origin', 'destination')
skims = [od_skim_stack_wrapper]
locals_dict = {
"od_skims": od_skim_stack_wrapper
}
if constants is not None:
locals_dict.update(constants)
simulate.set_skim_wrapper_targets(tours_merged, skims)
# this should be pre-slice as some expressions may count tours by type
annotations = expressions.compute_columns(
df=tours_merged,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
assign_in_place(tours_merged, annotations)
tracing.print_summary('stop_frequency segments',
tours_merged.primary_purpose, value_counts=True)
choices_list = []
for segment_type, choosers in tours_merged.groupby('primary_purpose'):
logging.info("%s running segment %s with %s chooser rows" %
(trace_label, segment_type, choosers.shape[0]))
spec = simulate.read_model_spec(file_name='stop_frequency_%s.csv' % segment_type)
assert spec is not None, "spec for segment_type %s not found" % segment_type
choices = simulate.simple_simulate(
choosers=choosers,
spec=spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, segment_type),
trace_choice_name='stops')
# convert indexes to alternative names
choices = pd.Series(spec.columns[choices.values], index=choices.index)
choices_list.append(choices)
choices = pd.concat(choices_list)
tracing.print_summary('stop_frequency', choices, value_counts=True)
# add stop_frequency choices to tours table
assign_in_place(tours, choices.to_frame('stop_frequency'))
if 'primary_purpose' not in tours.columns:
assign_in_place(tours, tours_merged[['primary_purpose']])
pipeline.replace_table("tours", tours)
# create trips table
trips = process_trips(tours, stop_frequency_alts)
trips = pipeline.extend_table("trips", trips)
tracing.register_traceable_table('trips', trips)
pipeline.get_rn_generator().add_channel('trips', trips)
if trace_hh_id:
tracing.trace_df(tours,
label="stop_frequency.tours",
slicer='person_id',
columns=None)
tracing.trace_df(trips,
label="stop_frequency.trips",
slicer='person_id',
columns=None)
tracing.trace_df(annotations,
label="stop_frequency.annotations",
columns=None)
tracing.trace_df(tours_merged,
label="stop_frequency.tours_merged",
slicer='person_id',
columns=None)
def cdap_simulate(persons_merged, persons, households,
cdap_indiv_spec,
cdap_interaction_coefficients,
cdap_fixed_relative_proportions,
chunk_size, trace_hh_id):
"""
CDAP stands for Coordinated Daily Activity Pattern, which is a choice of
high-level activity pattern for each person, in a coordinated way with other
members of a person's household.
Because Python requires vectorization of computation, there are some specialized
routines in the cdap directory of activitysim for this purpose. This module
simply applies those utilities using the simulation framework.
"""
trace_label = 'cdap'
model_settings = config.read_model_settings('cdap.yaml')
persons_merged = persons_merged.to_frame()
constants = config.get_model_constants(model_settings)
cdap_interaction_coefficients = \
cdap.preprocess_interaction_coefficients(cdap_interaction_coefficients)
# specs are built just-in-time on demand and cached as injectables
# prebuilding here allows us to write them to the output directory
# (also when multiprocessing locutor might not see all household sizes)
logger.info("Pre-building cdap specs")
for hhsize in range(2, cdap.MAX_HHSIZE + 1):
spec = cdap.build_cdap_spec(cdap_interaction_coefficients, hhsize, cache=True)
if inject.get_injectable('locutor', False):
spec.to_csv(config.output_file_path('cdap_spec_%s.csv' % hhsize), index=True)
logger.info("Running cdap_simulate with %d persons", len(persons_merged.index))
choices = cdap.run_cdap(
persons=persons_merged,
cdap_indiv_spec=cdap_indiv_spec,
cdap_interaction_coefficients=cdap_interaction_coefficients,
cdap_fixed_relative_proportions=cdap_fixed_relative_proportions,
locals_d=constants,
chunk_size=chunk_size,
trace_hh_id=trace_hh_id,
trace_label=trace_label)
# - assign results to persons table and annotate
persons = persons.to_frame()
choices = choices.reindex(persons.index)
persons['cdap_activity'] = choices.cdap_activity
persons['cdap_rank'] = choices.cdap_rank
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=tracing.extend_trace_label(trace_label, 'annotate_persons'))
pipeline.replace_table("persons", persons)
# - annotate households table
households = households.to_frame()
expressions.assign_columns(
df=households,
model_settings=model_settings.get('annotate_households'),
trace_label=tracing.extend_trace_label(trace_label, 'annotate_households'))
pipeline.replace_table("households", households)
tracing.print_summary('cdap_activity', persons.cdap_activity, value_counts=True)
logger.info("cdap crosstabs:\n%s" %
pd.crosstab(persons.ptype, persons.cdap_activity, margins=True))
if trace_hh_id:
tracing.trace_df(inject.get_table('persons_merged').to_frame(),
label="cdap",
columns=['ptype', 'cdap_rank', 'cdap_activity'],
warn_if_empty=True)
def joint_tour_frequency(households, persons, chunk_size, trace_hh_id):
"""
This model predicts the frequency of making fully joint trips (see the
alternatives above).
"""
trace_label = 'joint_tour_frequency'
model_settings_file_name = 'joint_tour_frequency.yaml'
estimator = estimation.manager.begin_estimation('joint_tour_frequency')
model_settings = config.read_model_settings(model_settings_file_name)
alternatives = simulate.read_model_alts(
'joint_tour_frequency_alternatives.csv', set_index='alt')
# - only interested in households with more than one cdap travel_active person and
# - at least one non-preschooler
households = households.to_frame()
multi_person_households = households[
households.participates_in_jtf_model].copy()
# - only interested in persons in multi_person_households
# FIXME - gratuitous pathological efficiency move, just let yaml specify persons?
persons = persons.to_frame()
persons = persons[persons.household_id.isin(multi_person_households.index)]
logger.info(
"Running joint_tour_frequency with %d multi-person households" %
multi_person_households.shape[0])
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {
'persons': persons,
'hh_time_window_overlap': hh_time_window_overlap
}
expressions.assign_columns(df=multi_person_households,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])
coefficients_df = simulate.read_model_coefficients(model_settings)
model_spec = simulate.eval_coefficients(model_spec, coefficients_df,
estimator)
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
if estimator:
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
estimator.write_coefficients(coefficients_df, model_settings)
estimator.write_choosers(multi_person_households)
choices = simulate.simple_simulate(
choosers=multi_person_households,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='joint_tour_frequency',
estimator=estimator)
# convert indexes to alternative names
choices = pd.Series(model_spec.columns[choices.values],
index=choices.index)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(choices, 'households',
'joint_tour_frequency')
estimator.write_override_choices(choices)
estimator.end_estimation()
# - create joint_tours based on joint_tour_frequency choices
# - we need a person_id in order to generate the tour index (and for register_traceable_table)
# - but we don't know the tour participants yet
# - so we arbitrarily choose the first person in the household
# - to be point person for the purpose of generating an index and setting origin
temp_point_persons = persons.loc[persons.PNUM == 1]
temp_point_persons['person_id'] = temp_point_persons.index
temp_point_persons = temp_point_persons.set_index('household_id')
temp_point_persons = temp_point_persons[['person_id', 'home_zone_id']]
joint_tours = \
process_joint_tours(choices, alternatives, temp_point_persons)
tours = pipeline.extend_table("tours", joint_tours)
tracing.register_traceable_table('tours', joint_tours)
pipeline.get_rn_generator().add_channel('tours', joint_tours)
# - annotate households
# we expect there to be an alt with no tours - which we can use to backfill non-travelers
no_tours_alt = (alternatives.sum(axis=1) == 0).index[0]
households['joint_tour_frequency'] = choices.reindex(
households.index).fillna(no_tours_alt).astype(str)
households['num_hh_joint_tours'] = joint_tours.groupby('household_id').size().\
reindex(households.index).fillna(0).astype(np.int8)
pipeline.replace_table("households", households)
tracing.print_summary('joint_tour_frequency',
households.joint_tour_frequency,
value_counts=True)
if trace_hh_id:
tracing.trace_df(households, label="joint_tour_frequency.households")
tracing.trace_df(joint_tours,
label="joint_tour_frequency.joint_tours",
slicer='household_id')
if estimator:
survey_tours = estimation.manager.get_survey_table('tours')
survey_tours = survey_tours[survey_tours.tour_category == 'joint']
print(f"len(survey_tours) {len(survey_tours)}")
print(f"len(joint_tours) {len(joint_tours)}")
different = False
survey_tours_not_in_tours = survey_tours[~survey_tours.index.
isin(joint_tours.index)]
if len(survey_tours_not_in_tours) > 0:
print(f"survey_tours_not_in_tours\n{survey_tours_not_in_tours}")
different = True
tours_not_in_survey_tours = joint_tours[~joint_tours.index.
isin(survey_tours.index)]
if len(survey_tours_not_in_tours) > 0:
print(f"tours_not_in_survey_tours\n{tours_not_in_survey_tours}")
different = True
assert not different
def stop_frequency(tours, tours_merged, stop_frequency_alts, skim_dict,
chunk_size, trace_hh_id):
"""
stop frequency model
For each tour, shoose a number of intermediate inbound stops and outbound stops.
Create a trip table with inbound and outbound trips.
Thus, a tour with stop_frequency '2out_0in' will have two outbound and zero inbound stops,
and four corresponding trips: three outbound, and one inbound.
Adds stop_frequency str column to trips, with fields
creates trips table with columns:
::
- person_id
- household_id
- tour_id
- primary_purpose
- atwork
- trip_num
- outbound
- trip_count
"""
trace_label = 'stop_frequency'
model_settings = config.read_model_settings('stop_frequency.yaml')
tours = tours.to_frame()
tours_merged = tours_merged.to_frame()
assert not tours_merged.household_id.isnull().any()
assert not (tours_merged.origin == -1).any()
assert not (tours_merged.destination == -1).any()
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
# - run preprocessor to annotate tours_merged
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
# hack: preprocessor adds origin column in place if it does not exist already
od_skim_stack_wrapper = skim_dict.wrap('origin', 'destination')
skims = [od_skim_stack_wrapper]
locals_dict = {"od_skims": od_skim_stack_wrapper}
if constants is not None:
locals_dict.update(constants)
simulate.set_skim_wrapper_targets(tours_merged, skims)
# this should be pre-slice as some expressions may count tours by type
annotations = expressions.compute_columns(
df=tours_merged,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
assign_in_place(tours_merged, annotations)
tracing.print_summary('stop_frequency segments',
tours_merged.primary_purpose,
value_counts=True)
choices_list = []
for segment_type, choosers in tours_merged.groupby('primary_purpose'):
logging.info("%s running segment %s with %s chooser rows" %
(trace_label, segment_type, choosers.shape[0]))
spec = simulate.read_model_spec(file_name='stop_frequency_%s.csv' %
segment_type)
assert spec is not None, "spec for segment_type %s not found" % segment_type
choices = simulate.simple_simulate(
choosers=choosers,
spec=spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, segment_type),
trace_choice_name='stops')
# convert indexes to alternative names
choices = pd.Series(spec.columns[choices.values], index=choices.index)
choices_list.append(choices)
choices = pd.concat(choices_list)
tracing.print_summary('stop_frequency', choices, value_counts=True)
# add stop_frequency choices to tours table
assign_in_place(tours, choices.to_frame('stop_frequency'))
if 'primary_purpose' not in tours.columns:
assign_in_place(tours, tours_merged[['primary_purpose']])
pipeline.replace_table("tours", tours)
# create trips table
trips = process_trips(tours, stop_frequency_alts)
trips = pipeline.extend_table("trips", trips)
tracing.register_traceable_table('trips', trips)
pipeline.get_rn_generator().add_channel('trips', trips)
if trace_hh_id:
tracing.trace_df(tours,
label="stop_frequency.tours",
slicer='person_id',
columns=None)
tracing.trace_df(trips,
label="stop_frequency.trips",
slicer='person_id',
columns=None)
tracing.trace_df(annotations,
label="stop_frequency.annotations",
columns=None)
tracing.trace_df(tours_merged,
label="stop_frequency.tours_merged",
slicer='person_id',
columns=None)
def atwork_subtour_mode_choice(tours, persons_merged, network_los, chunk_size,
trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode'
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
logger.info("Running %s with %d subtours" %
(trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type,
value_counts=True)
constants = {}
constants.update(config.get_model_constants(model_settings))
skim_dict = network_los.get_default_skim_dict()
# setup skim keys
orig_col_name = 'workplace_zone_id'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='out_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='in_period')
odr_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='in_period')
dor_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims": odr_skim_stack_wrapper,
"dor_skims": dor_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col_name,
dest_key=dest_col_name,
tod_key='out_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_odt')
tvpb_logsum_dot = tvpb.wrap_logsum(orig_key=dest_col_name,
dest_key=orig_col_name,
tod_key='in_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_dot')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(
network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
estimator = estimation.manager.begin_estimation(
'atwork_subtour_mode_choice')
if estimator:
estimator.write_coefficients(
simulate.read_model_coefficients(model_settings))
estimator.write_coefficients_template(
simulate.read_model_coefficient_template(model_settings))
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# FIXME run_tour_mode_choice_simulate writes choosers post-annotation
choices_df = run_tour_mode_choice_simulate(
subtours_merged,
tour_purpose='atwork',
model_settings=model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
network_los=network_los,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_types in tvpb_mode_path_types.items():
for direction, skim in zip(['od', 'do'],
[tvpb_logsum_odt, tvpb_logsum_dot]):
path_type = path_types[direction]
skim_cache = skim.cache[path_type]
print(
f"mode {mode} direction {direction} path_type {path_type}")
for c in skim_cache:
dest_col = f'{direction}_{c}'
if dest_col not in choices_df:
choices_df[
dest_col] = 0 if pd.api.types.is_numeric_dtype(
skim_cache[c]) else ''
choices_df[dest_col].where(choices_df.tour_mode != mode,
skim_cache[c],
inplace=True)
if estimator:
estimator.write_choices(choices_df[mode_column_name])
choices_df[mode_column_name] = \
estimator.get_survey_values(choices_df[mode_column_name], 'tours', mode_column_name)
estimator.write_override_choices(choices_df[mode_column_name])
estimator.end_estimation()
tracing.print_summary('%s choices' % trace_label,
choices_df[mode_column_name],
value_counts=True)
assign_in_place(tours, choices_df)
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def atwork_subtour_destination(
tours,
persons_merged,
network_los,
chunk_size, trace_hh_id):
trace_label = 'atwork_subtour_destination'
model_settings_file_name = 'atwork_subtour_destination.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
future_settings = {
'SIZE_TERM_SELECTOR': 'atwork',
'SEGMENTS': ['atwork'],
'ORIG_ZONE_ID': 'workplace_zone_id'
}
model_settings = config.future_model_settings(model_settings_file_name, model_settings, future_settings)
destination_column_name = 'destination'
logsum_column_name = model_settings.get('DEST_CHOICE_LOGSUM_COLUMN_NAME')
want_logsums = logsum_column_name is not None
sample_table_name = model_settings.get('DEST_CHOICE_SAMPLE_TABLE_NAME')
want_sample_table = config.setting('want_dest_choice_sample_tables') and sample_table_name is not None
persons_merged = persons_merged.to_frame()
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results('atwork_subtour_destination')
return
estimator = estimation.manager.begin_estimation('atwork_subtour_destination')
if estimator:
estimator.write_coefficients(model_settings=model_settings)
# estimator.write_spec(model_settings, tag='SAMPLE_SPEC')
estimator.write_spec(model_settings, tag='SPEC')
estimator.set_alt_id(model_settings["ALT_DEST_COL_NAME"])
estimator.write_table(inject.get_injectable('size_terms'), 'size_terms', append=False)
estimator.write_table(inject.get_table('land_use').to_frame(), 'landuse', append=False)
estimator.write_model_settings(model_settings, model_settings_file_name)
choices_df, save_sample_df = tour_destination.run_tour_destination(
subtours,
persons_merged,
want_logsums,
want_sample_table,
model_settings,
network_los,
estimator,
chunk_size, trace_hh_id, trace_label)
if estimator:
estimator.write_choices(choices_df['choice'])
choices_df['choice'] = estimator.get_survey_values(choices_df['choice'], 'tours', 'destination')
estimator.write_override_choices(choices_df['choice'])
estimator.end_estimation()
subtours[destination_column_name] = choices_df['choice']
assign_in_place(tours, subtours[[destination_column_name]])
if want_logsums:
subtours[logsum_column_name] = choices_df['logsum']
assign_in_place(tours, subtours[[logsum_column_name]])
pipeline.replace_table("tours", tours)
tracing.print_summary(destination_column_name,
subtours[destination_column_name],
describe=True)
if want_sample_table:
assert len(save_sample_df.index.get_level_values(0).unique()) == len(choices_df)
# save_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'], append=True, inplace=True)
pipeline.extend_table(sample_table_name, save_sample_df)
if trace_hh_id:
tracing.trace_df(tours,
label='atwork_subtour_destination',
columns=['destination'])
def atwork_subtour_mode_choice(
tours,
persons_merged,
skim_dict, skim_stack,
chunk_size,
trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d subtours" % (trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type, value_counts=True)
# setup skim keys
orig_col_name = 'workplace_taz'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name, right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name, right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices = run_tour_mode_choice_simulate(
subtours_merged,
spec, tour_purpose='atwork', model_settings=model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
tracing.print_summary('%s choices' % trace_label, choices, value_counts=True)
assign_in_place(tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def cdap_simulate(persons_merged, persons, households, chunk_size,
trace_hh_id):
"""
CDAP stands for Coordinated Daily Activity Pattern, which is a choice of
high-level activity pattern for each person, in a coordinated way with other
members of a person's household.
Because Python requires vectorization of computation, there are some specialized
routines in the cdap directory of activitysim for this purpose. This module
simply applies those utilities using the simulation framework.
"""
trace_label = 'cdap'
model_settings = config.read_model_settings('cdap.yaml')
cdap_indiv_spec = simulate.read_model_spec(
file_name=model_settings['INDIV_AND_HHSIZE1_SPEC'])
# Rules and coefficients for generating interaction specs for different household sizes
cdap_interaction_coefficients = \
pd.read_csv(config.config_file_path('cdap_interaction_coefficients.csv'), comment='#')
"""
spec to compute/specify the relative proportions of each activity (M, N, H)
that should be used to choose activities for additional household members not handled by CDAP
This spec is handled much like an activitysim logit utility spec,
EXCEPT that the values computed are relative proportions, not utilities
(i.e. values are not exponentiated before being normalized to probabilities summing to 1.0)
"""
cdap_fixed_relative_proportions = \
simulate.read_model_spec(file_name=model_settings['FIXED_RELATIVE_PROPORTIONS_SPEC'])
persons_merged = persons_merged.to_frame()
constants = config.get_model_constants(model_settings)
cdap_interaction_coefficients = \
cdap.preprocess_interaction_coefficients(cdap_interaction_coefficients)
# specs are built just-in-time on demand and cached as injectables
# prebuilding here allows us to write them to the output directory
# (also when multiprocessing locutor might not see all household sizes)
logger.info("Pre-building cdap specs")
for hhsize in range(2, cdap.MAX_HHSIZE + 1):
spec = cdap.build_cdap_spec(cdap_interaction_coefficients,
hhsize,
cache=True)
if inject.get_injectable('locutor', False):
spec.to_csv(config.output_file_path('cdap_spec_%s.csv' % hhsize),
index=True)
estimator = estimation.manager.begin_estimation('cdap')
if estimator:
estimator.write_model_settings(model_settings, 'cdap.yaml')
estimator.write_spec(model_settings, tag='INDIV_AND_HHSIZE1_SPEC')
estimator.write_spec(model_settings=model_settings,
tag='FIXED_RELATIVE_PROPORTIONS_SPEC')
estimator.write_table(cdap_interaction_coefficients,
'interaction_coefficients',
index=False,
append=False)
estimator.write_choosers(persons_merged)
for hhsize in range(2, cdap.MAX_HHSIZE + 1):
spec = cdap.get_cached_spec(hhsize)
estimator.write_table(spec, 'spec_%s' % hhsize, append=False)
logger.info("Running cdap_simulate with %d persons",
len(persons_merged.index))
choices = cdap.run_cdap(
persons=persons_merged,
cdap_indiv_spec=cdap_indiv_spec,
cdap_interaction_coefficients=cdap_interaction_coefficients,
cdap_fixed_relative_proportions=cdap_fixed_relative_proportions,
locals_d=constants,
chunk_size=chunk_size,
trace_hh_id=trace_hh_id,
trace_label=trace_label)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(choices, 'persons',
'cdap_activity')
estimator.write_override_choices(choices)
estimator.end_estimation()
# - assign results to persons table and annotate
persons = persons.to_frame()
choices = choices.reindex(persons.index)
persons['cdap_activity'] = choices
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=tracing.extend_trace_label(trace_label,
'annotate_persons'))
pipeline.replace_table("persons", persons)
# - annotate households table
households = households.to_frame()
expressions.assign_columns(
df=households,
model_settings=model_settings.get('annotate_households'),
trace_label=tracing.extend_trace_label(trace_label,
'annotate_households'))
pipeline.replace_table("households", households)
tracing.print_summary('cdap_activity',
persons.cdap_activity,
value_counts=True)
logger.info(
"cdap crosstabs:\n%s" %
pd.crosstab(persons.ptype, persons.cdap_activity, margins=True))
def trip_mode_choice(
trips,
tours_merged,
skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
Trip mode choice - compute trip_mode (same values as for tour_mode) for each trip.
Modes for each primary tour putpose are calculated separately because they have different
coefficient values (stored in trip_mode_choice_coeffs.csv coefficient file.)
Adds trip_mode column to trip table
"""
trace_label = 'trip_mode_choice'
model_settings = config.read_model_settings('trip_mode_choice.yaml')
model_spec = \
simulate.read_model_spec(file_name=model_settings['SPEC'])
omnibus_coefficients = \
assign.read_constant_spec(config.config_file_path(model_settings['COEFFS']))
trips_df = trips.to_frame()
logger.info("Running %s with %d trips", trace_label, trips_df.shape[0])
tours_merged = tours_merged.to_frame()
tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]
nest_spec = config.get_logit_model_settings(model_settings)
tracing.print_summary('primary_purpose',
trips_df.primary_purpose, value_counts=True)
# - trips_merged - merge trips and tours_merged
trips_merged = pd.merge(
trips_df,
tours_merged,
left_on='tour_id',
right_index=True,
how="left")
assert trips_merged.index.equals(trips.index)
# setup skim keys
assert ('trip_period' not in trips_merged)
trips_merged['trip_period'] = skim_time_period_label(trips_merged.depart)
orig_col = 'origin'
dest_col = 'destination'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col, right_key=dest_col,
skim_key='trip_period')
od_skim_wrapper = skim_dict.wrap('origin', 'destination')
skims = {
"odt_skims": odt_skim_stack_wrapper,
"od_skims": od_skim_wrapper,
}
constants = config.get_model_constants(model_settings)
constants.update({
'ORIGIN': orig_col,
'DESTINATION': dest_col
})
choices_list = []
for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):
segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)
logger.info("trip_mode_choice tour_type '%s' (%s trips)" %
(primary_purpose, len(trips_segment.index), ))
# name index so tracing knows how to slice
assert trips_segment.index.name == 'trip_id'
locals_dict = assign.evaluate_constants(omnibus_coefficients[primary_purpose],
constants=constants)
locals_dict.update(constants)
annotate_preprocessors(
trips_segment, locals_dict, skims,
model_settings, segment_trace_label)
locals_dict.update(skims)
choices = simulate.simple_simulate(
choosers=trips_segment,
spec=model_spec,
nest_spec=nest_spec,
skims=skims,
locals_d=locals_dict,
chunk_size=chunk_size,
trace_label=segment_trace_label,
trace_choice_name='trip_mode_choice')
alts = model_spec.columns
choices = choices.map(dict(list(zip(list(range(len(alts))), alts))))
# tracing.print_summary('trip_mode_choice %s choices' % primary_purpose,
# choices, value_counts=True)
if trace_hh_id:
# trace the coefficients
tracing.trace_df(pd.Series(locals_dict),
label=tracing.extend_trace_label(segment_trace_label, 'constants'),
transpose=False,
slicer='NONE')
# so we can trace with annotations
trips_segment['trip_mode'] = choices
tracing.trace_df(trips_segment,
label=tracing.extend_trace_label(segment_trace_label, 'trip_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices = pd.concat(choices_list)
trips_df = trips.to_frame()
trips_df['trip_mode'] = choices
tracing.print_summary('tour_modes',
trips_merged.tour_mode, value_counts=True)
tracing.print_summary('trip_mode_choice choices',
choices, value_counts=True)
assert not trips_df.trip_mode.isnull().any()
pipeline.replace_table("trips", trips_df)
if trace_hh_id:
tracing.trace_df(trips_df,
label=tracing.extend_trace_label(trace_label, 'trip_mode'),
slicer='trip_id',
index_label='trip_id',
warn_if_empty=True)
def iterate_location_choice(
model_settings,
persons_merged, persons, households,
skim_dict, skim_stack,
chunk_size, trace_hh_id, locutor,
trace_label):
"""
iterate run_location_choice updating shadow pricing until convergence criteria satisfied
or max_iterations reached.
(If use_shadow_pricing not enabled, then just iterate once)
Parameters
----------
model_settings : dict
persons_merged : injected table
persons : injected table
skim_dict : skim.SkimDict
skim_stack : skim.SkimStack
chunk_size : int
trace_hh_id : int
locutor : bool
whether this process is the privileged logger of shadow_pricing when multiprocessing
trace_label : str
Returns
-------
adds choice column model_settings['DEST_CHOICE_COLUMN_NAME'] and annotations to persons table
"""
# column containing segment id
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# boolean to filter out persons not needing location modeling (e.g. is_worker, is_student)
chooser_filter_column = model_settings['CHOOSER_FILTER_COLUMN_NAME']
persons_merged_df = persons_merged.to_frame()
persons_merged_df = persons_merged_df[persons_merged[chooser_filter_column]]
spc = shadow_pricing.load_shadow_price_calculator(model_settings)
max_iterations = spc.max_iterations
logging.debug("%s max_iterations: %s" % (trace_label, max_iterations))
choices = None
for iteration in range(1, max_iterations + 1):
if spc.use_shadow_pricing and iteration > 1:
spc.update_shadow_prices()
choices = run_location_choice(
persons_merged_df,
skim_dict, skim_stack,
spc,
model_settings,
chunk_size, trace_hh_id,
trace_label=tracing.extend_trace_label(trace_label, 'i%s' % iteration))
choices_df = choices.to_frame('dest_choice')
choices_df['segment_id'] = \
persons_merged_df[chooser_segment_column].reindex(choices_df.index)
spc.set_choices(choices_df)
if locutor:
spc.write_trace_files(iteration)
if spc.use_shadow_pricing and spc.check_fit(iteration):
logging.info("%s converged after iteration %s" % (trace_label, iteration,))
break
# - shadow price table
if locutor:
if spc.use_shadow_pricing and 'SHADOW_PRICE_TABLE' in model_settings:
inject.add_table(model_settings['SHADOW_PRICE_TABLE'], spc.shadow_prices)
if 'MODELED_SIZE_TABLE' in model_settings:
inject.add_table(model_settings['MODELED_SIZE_TABLE'], spc.modeled_size)
dest_choice_column_name = model_settings['DEST_CHOICE_COLUMN_NAME']
tracing.print_summary(dest_choice_column_name, choices, value_counts=True)
persons_df = persons.to_frame()
# We only chose school locations for the subset of persons who go to school
# so we backfill the empty choices with -1 to code as no school location
NO_DEST_TAZ = -1
persons_df[dest_choice_column_name] = \
choices.reindex(persons_df.index).fillna(NO_DEST_TAZ).astype(int)
# - annotate persons table
if 'annotate_persons' in model_settings:
expressions.assign_columns(
df=persons_df,
model_settings=model_settings.get('annotate_persons'),
trace_label=tracing.extend_trace_label(trace_label, 'annotate_persons'))
pipeline.replace_table("persons", persons_df)
if trace_hh_id:
tracing.trace_df(persons_df,
label=trace_label,
warn_if_empty=True)
# - annotate households table
if 'annotate_households' in model_settings:
households_df = households.to_frame()
expressions.assign_columns(
df=households_df,
model_settings=model_settings.get('annotate_households'),
trace_label=tracing.extend_trace_label(trace_label, 'annotate_households'))
pipeline.replace_table("households", households_df)
if trace_hh_id:
tracing.trace_df(households_df,
label=trace_label,
warn_if_empty=True)
return persons_df
def workplace_location_simulate(persons_merged, workplace_location_sample,
workplace_location_spec,
workplace_location_settings, skim_dict,
destination_size_terms, chunk_size,
trace_hh_id):
"""
Workplace location model on workplace_location_sample annotated with mode_choice logsum
to select a work_taz from sample alternatives
"""
# for now I'm going to generate a workplace location for everyone -
# presumably it will not get used in downstream models for everyone -
# it should depend on CDAP and mandatory tour generation as to whether
# it gets used
choosers = persons_merged.to_frame()
alt_col_name = workplace_location_settings["ALT_COL_NAME"]
# alternatives are pre-sampled and annotated with logsums and pick_count
# but we have to merge additional alt columns into alt sample list
workplace_location_sample = workplace_location_sample.to_frame()
destination_size_terms = destination_size_terms.to_frame()
alternatives = \
pd.merge(workplace_location_sample, destination_size_terms,
left_on=alt_col_name, right_index=True, how="left")
tracing.dump_df(DUMP, alternatives, 'workplace_location_simulate',
'alternatives')
constants = config.get_model_constants(workplace_location_settings)
sample_pool_size = len(destination_size_terms.index)
logger.info("Running workplace_location_simulate with %d persons" %
len(choosers))
# create wrapper with keys for this lookup - in this case there is a TAZ in the choosers
# and a TAZ in the alternatives which get merged during interaction
# the skims will be available under the name "skims" for any @ expressions
skims = skim_dict.wrap("TAZ", alt_col_name)
locals_d = {'skims': skims, 'sample_pool_size': float(sample_pool_size)}
if constants is not None:
locals_d.update(constants)
# FIXME - MEMORY HACK - only include columns actually used in spec
chooser_columns = workplace_location_settings['SIMULATE_CHOOSER_COLUMNS']
choosers = choosers[chooser_columns]
tracing.dump_df(DUMP, choosers, 'workplace_location_simulate', 'choosers')
choices = interaction_sample_simulate(
choosers,
alternatives,
spec=workplace_location_spec,
choice_column=alt_col_name,
skims=skims,
locals_d=locals_d,
chunk_size=chunk_size,
trace_label=trace_hh_id and 'workplace_location',
trace_choice_name='workplace_location')
# FIXME - no need to reindex since we didn't slice choosers
# choices = choices.reindex(persons_merged.index)
tracing.print_summary('workplace_taz', choices, describe=True)
orca.add_column("persons", "workplace_taz", choices)
pipeline.add_dependent_columns("persons", "persons_workplace")
if trace_hh_id:
trace_columns = ['workplace_taz'
] + orca.get_table('persons_workplace').columns
tracing.trace_df(orca.get_table('persons_merged').to_frame(),
label="workplace_location",
columns=trace_columns,
warn_if_empty=True)
def workplace_location_simulate(persons_merged, workplace_location_spec,
workplace_location_settings, skim_dict,
destination_size_terms, chunk_size,
trace_hh_id):
"""
The workplace location model predicts the zones in which various people will
work.
"""
# for now I'm going to generate a workplace location for everyone -
# presumably it will not get used in downstream models for everyone -
# it should depend on CDAP and mandatory tour generation as to whether
# it gets used
choosers = persons_merged.to_frame()
alternatives = destination_size_terms.to_frame()
constants = config.get_model_constants(workplace_location_settings)
sample_size = workplace_location_settings["SAMPLE_SIZE"]
logger.info("Running workplace_location_simulate with %d persons" %
len(choosers))
# create wrapper with keys for this lookup - in this case there is a TAZ in the choosers
# and a TAZ in the alternatives which get merged during interaction
# the skims will be available under the name "skims" for any @ expressions
skims = skim_dict.wrap("TAZ", "TAZ_r")
locals_d = {'skims': skims}
if constants is not None:
locals_d.update(constants)
# FIXME - HACK - only include columns actually used in spec (which we pathologically know)
choosers = choosers[["income_segment", "TAZ", "mode_choice_logsums"]]
choices = asim.interaction_simulate(choosers,
alternatives,
spec=workplace_location_spec,
skims=skims,
locals_d=locals_d,
sample_size=sample_size,
chunk_size=chunk_size,
trace_label=trace_hh_id
and 'workplace_location',
trace_choice_name='workplace_location')
# FIXME - no need to reindex since we didn't slice choosers
# choices = choices.reindex(persons_merged.index)
tracing.print_summary('workplace_taz', choices, describe=True)
orca.add_column("persons", "workplace_taz", choices)
pipeline.add_dependent_columns("persons", "persons_workplace")
# FIXME - test prng repeatability
r = pipeline.get_rn_generator().random_for_df(choices)
orca.add_column("persons", "work_location_rand",
[item for sublist in r for item in sublist])
if trace_hh_id:
trace_columns = ['workplace_taz'
] + orca.get_table('persons_workplace').columns
tracing.trace_df(orca.get_table('persons_merged').to_frame(),
label="workplace_location",
columns=trace_columns,
warn_if_empty=True)
def tour_mode_choice_simulate(tours, persons_merged,
skim_dict, skim_stack,
chunk_size,
trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
persons_merged = persons_merged.to_frame()
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d tours" % (trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type, value_counts=True)
primary_tours_merged = pd.merge(primary_tours, persons_merged, left_on='person_id',
right_index=True, how='left', suffixes=('', '_r'))
# setup skim keys
orig_col_name = 'TAZ'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name, right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name, right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices_list = []
for tour_type, segment in primary_tours_merged.groupby('tour_type'):
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" %
(tour_type, len(segment.index), ))
# name index so tracing knows how to slice
assert segment.index.name == 'tour_id'
choices = run_tour_mode_choice_simulate(
segment,
spec, tour_type, model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, tour_type),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices' % tour_type,
choices, value_counts=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices = pd.concat(choices_list)
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices, value_counts=True)
# so we can trace with annotations
primary_tours['tour_mode'] = choices
# but only keep mode choice col
all_tours = tours.to_frame()
# uncomment to save annotations to table
# assign_in_place(all_tours, annotations)
assign_in_place(all_tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
def trip_mode_choice_simulate(trips_merged, trip_mode_choice_spec,
trip_mode_choice_settings, skim_dict, skim_stack,
omx_file, trace_hh_id):
"""
Trip mode choice simulate
"""
trips = trips_merged.to_frame()
nest_spec = config.get_logit_model_settings(trip_mode_choice_settings)
constants = config.get_model_constants(trip_mode_choice_settings)
logger.info("Running trip_mode_choice_simulate with %d trips" % len(trips))
odt_skim_stack_wrapper = skim_stack.wrap(left_key='OTAZ',
right_key='DTAZ',
skim_key="start_period")
od_skims = skim_dict.wrap('OTAZ', 'DTAZ')
choices_list = []
# loop by tour_type in order to easily query the expression coefficient file
for tour_type, segment in trips.groupby('tour_type'):
logger.info("running %s tour_type '%s'" % (
len(segment.index),
tour_type,
))
# name index so tracing knows how to slice
segment.index.name = 'trip_id'
# FIXME - check that destination is not null
trace_label = trace_hh_id and ('trip_mode_choice_%s' % tour_type)
choices = _mode_choice_simulate(
segment,
skim_dict=skim_dict,
skim_stack=skim_stack,
odt_skim_stack_wrapper=odt_skim_stack_wrapper,
dot_skim_stack_wrapper=None,
od_skim_stack_wrapper=od_skims,
spec=get_segment_and_unstack(trip_mode_choice_spec, tour_type),
constants=constants,
nest_spec=nest_spec,
trace_label=trace_label,
trace_choice_name='trip_mode_choice')
# FIXME - no point in printing verbose value_counts now that we have tracing?
tracing.print_summary('trip_mode_choice_simulate %s choices' %
tour_type,
choices,
value_counts=True)
choices_list.append(choices)
# FIXME - force garbage collection
mem = asim.memory_info()
logger.debug('memory_info tour_type %s, %s' % (tour_type, mem))
choices = pd.concat(choices_list)
tracing.print_summary('trip_mode_choice_simulate all tour type choices',
choices,
value_counts=True)
# FIXME - is this a NOP if trips table doesn't exist
orca.add_column("trips", "trip_mode", choices)
if trace_hh_id:
tracing.trace_df(orca.get_table('trips').to_frame(),
label="trip_mode",
slicer='trip_id',
index_label='trip_id',
warn_if_empty=True)
# FIXME - this forces garbage collection
asim.memory_info()
def tour_mode_choice_simulate(tours, persons_merged, network_los, chunk_size,
trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode'
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
logger.info("Running %s with %d tours" %
(trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type,
value_counts=True)
persons_merged = persons_merged.to_frame()
primary_tours_merged = pd.merge(primary_tours,
persons_merged,
left_on='person_id',
right_index=True,
how='left',
suffixes=('', '_r'))
constants = {}
# model_constants can appear in expressions
constants.update(config.get_model_constants(model_settings))
skim_dict = network_los.get_default_skim_dict()
# setup skim keys
orig_col_name = 'home_zone_id'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='out_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='in_period')
odr_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='in_period')
dor_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims":
odr_skim_stack_wrapper, # dot return skims for e.g. TNC bridge return fare
"dor_skims":
dor_skim_stack_wrapper, # odt return skims for e.g. TNC bridge return fare
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col_name,
dest_key=dest_col_name,
tod_key='out_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_odt')
tvpb_logsum_dot = tvpb.wrap_logsum(orig_key=dest_col_name,
dest_key=orig_col_name,
tod_key='in_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_dot')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(
network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
estimator = estimation.manager.begin_estimation('tour_mode_choice')
if estimator:
estimator.write_coefficients(model_settings=model_settings)
estimator.write_coefficients_template(model_settings=model_settings)
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# (run_tour_mode_choice_simulate writes choosers post-annotation)
# FIXME should normalize handling of tour_type and tour_purpose
# mtctm1 school tour_type includes univ, which has different coefficients from elementary and HS
# we should either add this column when tours created or add univ to tour_types
not_university = (primary_tours_merged.tour_type !=
'school') | ~primary_tours_merged.is_university
primary_tours_merged['tour_purpose'] = \
primary_tours_merged.tour_type.where(not_university, 'univ')
choices_list = []
for tour_purpose, tours_segment in primary_tours_merged.groupby(
'tour_purpose'):
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" % (
tour_purpose,
len(tours_segment.index),
))
if network_los.zone_system == los.THREE_ZONE:
tvpb_logsum_odt.extend_trace_label(tour_purpose)
tvpb_logsum_dot.extend_trace_label(tour_purpose)
# name index so tracing knows how to slice
assert tours_segment.index.name == 'tour_id'
choices_df = run_tour_mode_choice_simulate(
tours_segment,
tour_purpose,
model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
network_los=network_los,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, tour_purpose),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices_df' %
tour_purpose,
choices_df.tour_mode,
value_counts=True)
choices_list.append(choices_df)
choices_df = pd.concat(choices_list)
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_types in tvpb_mode_path_types.items():
for direction, skim in zip(['od', 'do'],
[tvpb_logsum_odt, tvpb_logsum_dot]):
path_type = path_types[direction]
skim_cache = skim.cache[path_type]
print(
f"mode {mode} direction {direction} path_type {path_type}")
for c in skim_cache:
dest_col = f'{direction}_{c}'
if dest_col not in choices_df:
choices_df[
dest_col] = np.nan if pd.api.types.is_numeric_dtype(
skim_cache[c]) else ''
choices_df[dest_col].where(choices_df.tour_mode != mode,
skim_cache[c],
inplace=True)
if estimator:
estimator.write_choices(choices_df.tour_mode)
choices_df.tour_mode = estimator.get_survey_values(
choices_df.tour_mode, 'tours', 'tour_mode')
estimator.write_override_choices(choices_df.tour_mode)
estimator.end_estimation()
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices_df.tour_mode,
value_counts=True)
# so we can trace with annotations
assign_in_place(primary_tours, choices_df)
# update tours table with mode choice (and optionally logsums)
all_tours = tours.to_frame()
assign_in_place(all_tours, choices_df)
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
def non_mandatory_tour_frequency(persons, persons_merged, chunk_size,
trace_hh_id):
"""
This model predicts the frequency of making non-mandatory trips
(alternatives for this model come from a separate csv file which is
configured by the user) - these trips include escort, shopping, othmaint,
othdiscr, eatout, and social trips in various combination.
"""
trace_label = 'non_mandatory_tour_frequency'
model_settings_file_name = 'non_mandatory_tour_frequency.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
# FIXME kind of tacky both that we know to add this here and del it below
# 'tot_tours' is used in model_spec expressions
alternatives = simulate.read_model_alts(
'non_mandatory_tour_frequency_alternatives.csv', set_index=None)
alternatives['tot_tours'] = alternatives.sum(axis=1)
# filter based on results of CDAP
choosers = persons_merged.to_frame()
choosers = choosers[choosers.cdap_activity.isin(['M', 'N'])]
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {'person_max_window': person_max_window}
expressions.assign_columns(df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
logger.info("Running non_mandatory_tour_frequency with %d persons",
len(choosers))
constants = config.get_model_constants(model_settings)
model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])
spec_segments = model_settings.get('SPEC_SEGMENTS', {})
# segment by person type and pick the right spec for each person type
choices_list = []
for segment_settings in spec_segments:
segment_name = segment_settings['NAME']
ptype = segment_settings['PTYPE']
# pick the spec column for the segment
segment_spec = model_spec[[segment_name]]
chooser_segment = choosers[choosers.ptype == ptype]
logger.info("Running segment '%s' of size %d", segment_name,
len(chooser_segment))
if len(chooser_segment) == 0:
# skip empty segments
continue
estimator = \
estimation.manager.begin_estimation(model_name=segment_name, bundle_name='non_mandatory_tour_frequency')
coefficients_df = simulate.read_model_coefficients(segment_settings)
segment_spec = simulate.eval_coefficients(segment_spec,
coefficients_df, estimator)
if estimator:
estimator.write_spec(model_settings, bundle_directory=True)
estimator.write_model_settings(model_settings,
model_settings_file_name,
bundle_directory=True)
# preserving coefficients file name makes bringing back updated coefficients more straightforward
estimator.write_coefficients(coefficients_df, segment_settings)
estimator.write_choosers(chooser_segment)
estimator.write_alternatives(alternatives, bundle_directory=True)
# FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column
# shuold we do it here or have interaction_simulate do it?
# chooser index must be duplicated in column or it will be omitted from interaction_dataset
# estimation requires that chooser_id is either in index or a column of interaction_dataset
# so it can be reformatted (melted) and indexed by chooser_id and alt_id
assert chooser_segment.index.name == 'person_id'
assert 'person_id' not in chooser_segment.columns
chooser_segment['person_id'] = chooser_segment.index
# FIXME set_alt_id - do we need this for interaction_simulate estimation bundle tables?
estimator.set_alt_id('alt_id')
estimator.set_chooser_id(chooser_segment.index.name)
choices = interaction_simulate(
chooser_segment,
alternatives,
spec=segment_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label='non_mandatory_tour_frequency.%s' % segment_name,
trace_choice_name='non_mandatory_tour_frequency',
estimator=estimator)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(
choices, 'persons', 'non_mandatory_tour_frequency')
estimator.write_override_choices(choices)
estimator.end_estimation()
choices_list.append(choices)
# FIXME - force garbage collection?
force_garbage_collect()
del alternatives['tot_tours'] # del tot_tours column we added above
# The choice value 'non_mandatory_tour_frequency' assigned by interaction_simulate
# is the index value of the chosen alternative in the alternatives table.
choices = pd.concat(choices_list).sort_index()
# add non_mandatory_tour_frequency column to persons
persons = persons.to_frame()
# we expect there to be an alt with no tours - which we can use to backfill non-travelers
no_tours_alt = (alternatives.sum(axis=1) == 0).index[0]
# need to reindex as we only handled persons with cdap_activity in ['M', 'N']
persons['non_mandatory_tour_frequency'] = \
choices.reindex(persons.index).fillna(no_tours_alt).astype(np.int8)
"""
We have now generated non-mandatory tour frequencies, but they are attributes of the person table
Now we create a "tours" table which has one row per tour that has been generated
(and the person id it is associated with)
But before we do that, we run an additional probablilistic step to extend/increase tour counts
beyond the strict limits of the tour_frequency alternatives chosen above (which are currently limited
to at most 2 escort tours and 1 each of shopping, othmaint, othdiscr, eatout, and social tours)
The choice value 'non_mandatory_tour_frequency' assigned by interaction_simulate is simply the
index value of the chosen alternative in the alternatives table.
get counts of each of the tour type alternatives (so we can extend)
escort shopping othmaint othdiscr eatout social
parent_id
2588676 2 0 0 1 1 0
2588677 0 1 0 1 0 0
"""
# counts of each of the tour type alternatives (so we can extend)
modeled_tour_counts = alternatives.loc[choices]
modeled_tour_counts.index = choices.index # assign person ids to the index
# - extend_tour_counts - probabalistic
extended_tour_counts = \
extend_tour_counts(choosers, modeled_tour_counts.copy(), alternatives,
trace_hh_id, tracing.extend_trace_label(trace_label, 'extend_tour_counts'))
num_modeled_tours = modeled_tour_counts.sum().sum()
num_extended_tours = extended_tour_counts.sum().sum()
logger.info("extend_tour_counts increased tour count by %s from %s to %s" %
(num_extended_tours - num_modeled_tours, num_modeled_tours,
num_extended_tours))
"""
create the non_mandatory tours based on extended_tour_counts
"""
if estimator:
override_tour_counts = \
estimation.manager.get_survey_values(extended_tour_counts,
table_name='persons',
column_names=['_%s' % c for c in extended_tour_counts.columns])
override_tour_counts = \
override_tour_counts.rename(columns={('_%s' % c): c for c in extended_tour_counts.columns})
logger.info(
"estimation get_survey_values override_tour_counts %s changed cells"
% (override_tour_counts != extended_tour_counts).sum().sum())
extended_tour_counts = override_tour_counts
"""
create the non_mandatory tours based on extended_tour_counts
"""
non_mandatory_tours = process_non_mandatory_tours(persons,
extended_tour_counts)
assert len(non_mandatory_tours) == extended_tour_counts.sum().sum()
if estimator:
# make sure they created the right tours
survey_tours = estimation.manager.get_survey_table(
'tours').sort_index()
non_mandatory_survey_tours = survey_tours[survey_tours.tour_category ==
'non_mandatory']
assert len(non_mandatory_survey_tours) == len(non_mandatory_tours)
assert non_mandatory_survey_tours.index.equals(
non_mandatory_tours.sort_index().index)
# make sure they created tours with the expected tour_ids
columns = ['person_id', 'household_id', 'tour_type', 'tour_category']
survey_tours = \
estimation.manager.get_survey_values(non_mandatory_tours,
table_name='tours',
column_names=columns)
tours_differ = (non_mandatory_tours[columns] !=
survey_tours[columns]).any(axis=1)
if tours_differ.any():
print("tours_differ\n%s" % tours_differ)
print("%s of %s tours differ" %
(tours_differ.sum(), len(tours_differ)))
print("differing survey_tours\n%s" % survey_tours[tours_differ])
print("differing modeled_tours\n%s" %
non_mandatory_tours[columns][tours_differ])
assert (not tours_differ.any())
pipeline.extend_table("tours", non_mandatory_tours)
tracing.register_traceable_table('tours', non_mandatory_tours)
pipeline.get_rn_generator().add_channel('tours', non_mandatory_tours)
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=trace_label)
pipeline.replace_table("persons", persons)
tracing.print_summary('non_mandatory_tour_frequency',
persons.non_mandatory_tour_frequency,
value_counts=True)
if trace_hh_id:
tracing.trace_df(
non_mandatory_tours,
label="non_mandatory_tour_frequency.non_mandatory_tours",
warn_if_empty=True)
tracing.trace_df(choosers,
label="non_mandatory_tour_frequency.choosers",
warn_if_empty=True)
tracing.trace_df(
persons,
label="non_mandatory_tour_frequency.annotated_persons",
warn_if_empty=True)
def atwork_subtour_frequency(tours, persons_merged, chunk_size, trace_hh_id):
"""
This model predicts the frequency of making at-work subtour tours
(alternatives for this model come from a separate csv file which is
configured by the user).
"""
trace_label = 'atwork_subtour_frequency'
model_settings = config.read_model_settings(
'atwork_subtour_frequency.yaml')
model_spec = simulate.read_model_spec(
file_name='atwork_subtour_frequency.csv')
alternatives = simulate.read_model_alts(
config.config_file_path('atwork_subtour_frequency_alternatives.csv'),
set_index='alt')
tours = tours.to_frame()
persons_merged = persons_merged.to_frame()
work_tours = tours[tours.tour_type == 'work']
# - if no work_tours
if len(work_tours) == 0:
add_null_results(trace_label, tours)
return
# merge persons into work_tours
work_tours = pd.merge(work_tours,
persons_merged,
left_on='person_id',
right_index=True)
logger.info("Running atwork_subtour_frequency with %d work tours",
len(work_tours))
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
assign_columns(df=work_tours,
model_settings=preprocessor_settings,
trace_label=trace_label)
choices = simulate.simple_simulate(
choosers=work_tours,
spec=model_spec,
nest_spec=nest_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='atwork_subtour_frequency')
# convert indexes to alternative names
choices = pd.Series(model_spec.columns[choices.values],
index=choices.index)
tracing.print_summary('atwork_subtour_frequency',
choices,
value_counts=True)
# add atwork_subtour_frequency column to tours
# reindex since we are working with a subset of tours
tours['atwork_subtour_frequency'] = choices.reindex(tours.index)
pipeline.replace_table("tours", tours)
# - create atwork_subtours based on atwork_subtour_frequency choice names
work_tours = tours[tours.tour_type == 'work']
assert not work_tours.atwork_subtour_frequency.isnull().any()
subtours = process_atwork_subtours(work_tours, alternatives)
tours = pipeline.extend_table("tours", subtours)
tracing.register_traceable_table('tours', subtours)
pipeline.get_rn_generator().add_channel('tours', subtours)
if trace_hh_id:
tracing.trace_df(tours, label='atwork_subtour_frequency.tours')
def school_location_simulate(persons_merged, school_location_sample,
school_location_spec, school_location_settings,
skim_dict, destination_size_terms, chunk_size,
trace_hh_id):
"""
School location model on school_location_sample annotated with mode_choice logsum
to select a school_taz from sample alternatives
"""
choosers = persons_merged.to_frame()
school_location_sample = school_location_sample.to_frame()
destination_size_terms = destination_size_terms.to_frame()
trace_label = 'school_location_simulate'
alt_col_name = school_location_settings["ALT_COL_NAME"]
constants = config.get_model_constants(school_location_settings)
# create wrapper with keys for this lookup - in this case there is a TAZ in the choosers
# and a TAZ in the alternatives which get merged during interaction
# the skims will be available under the name "skims" for any @ expressions
skims = skim_dict.wrap("TAZ", alt_col_name)
locals_d = {
'skims': skims,
}
if constants is not None:
locals_d.update(constants)
# FIXME - MEMORY HACK - only include columns actually used in spec
chooser_columns = school_location_settings['SIMULATE_CHOOSER_COLUMNS']
choosers = choosers[chooser_columns]
tracing.dump_df(DUMP, choosers, 'school_location_simulate', 'choosers')
choices_list = []
for school_type in ['university', 'highschool', 'gradeschool']:
locals_d['segment'] = school_type
choosers_segment = choosers[choosers["is_" + school_type]]
alts_segment = school_location_sample[
school_location_sample['school_type'] == school_type]
# alternatives are pre-sampled and annotated with logsums and pick_count
# but we have to merge additional alt columns into alt sample list
alts_segment = \
pd.merge(alts_segment, destination_size_terms,
left_on=alt_col_name, right_index=True, how="left")
tracing.dump_df(DUMP, alts_segment, trace_label,
'%s_alternatives' % school_type)
choices = interaction_sample_simulate(
choosers_segment,
alts_segment,
spec=school_location_spec[[school_type]],
choice_column=alt_col_name,
skims=skims,
locals_d=locals_d,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, school_type),
trace_choice_name='school_location')
choices_list.append(choices)
choices = pd.concat(choices_list)
# We only chose school locations for the subset of persons who go to school
# so we backfill the empty choices with -1 to code as no school location
choices = choices.reindex(persons_merged.index).fillna(-1).astype(int)
tracing.dump_df(DUMP, choices, trace_label, 'choices')
tracing.print_summary('school_taz', choices, describe=True)
inject.add_column("persons", "school_taz", choices)
pipeline.add_dependent_columns("persons", "persons_school")
if trace_hh_id:
trace_columns = ['school_taz'
] + inject.get_table('persons_school').columns
tracing.trace_df(inject.get_table('persons_merged').to_frame(),
label="school_location",
columns=trace_columns,
warn_if_empty=True)
def atwork_subtour_mode_choice_simulate(tours, persons_merged,
tour_mode_choice_spec,
tour_mode_choice_settings, skim_dict,
skim_stack, trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'subtour']
# merge persons into tours
choosers = pd.merge(subtours,
persons_merged.to_frame(),
left_on='person_id',
right_index=True)
nest_spec = config.get_logit_model_settings(tour_mode_choice_settings)
constants = config.get_model_constants(tour_mode_choice_settings)
logger.info("Running %s with %d subtours" %
(trace_label, len(subtours.index)))
tracing.print_summary('%s tour_type' % trace_label,
subtours.tour_type,
value_counts=True)
if trace_hh_id:
tracing.trace_df(tour_mode_choice_spec,
tracing.extend_trace_label(trace_label, 'spec'),
slicer='NONE',
transpose=False)
# setup skim keys
odt_skim_stack_wrapper = skim_stack.wrap(left_key='workplace_taz',
right_key='destination',
skim_key="out_period")
dot_skim_stack_wrapper = skim_stack.wrap(left_key='destination',
right_key='workplace_taz',
skim_key="in_period")
od_skims = skim_dict.wrap('workplace_taz', 'destination')
spec = get_segment_and_unstack(tour_mode_choice_spec, segment='workbased')
if trace_hh_id:
tracing.trace_df(spec,
tracing.extend_trace_label(trace_label, 'spec'),
slicer='NONE',
transpose=False)
choices = _mode_choice_simulate(
choosers,
odt_skim_stack_wrapper=odt_skim_stack_wrapper,
dot_skim_stack_wrapper=dot_skim_stack_wrapper,
od_skim_stack_wrapper=od_skims,
spec=spec,
constants=constants,
nest_spec=nest_spec,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
tracing.print_summary('%s choices' % trace_label,
choices,
value_counts=True)
subtours['destination'] = choices
assign_in_place(tours, subtours[['destination']])
if trace_hh_id:
trace_columns = [
'mode', 'person_id', 'tour_type', 'tour_num', 'parent_tour_id'
]
tracing.trace_df(subtours,
label=tracing.extend_trace_label(trace_label, 'mode'),
slicer='tour_id',
index_label='tour_id',
columns=trace_columns,
warn_if_empty=True)
# FIXME - this forces garbage collection
memory_info()
