def schedule_tours(tours, persons_merged, alts, spec, logsum_tour_purpose,
model_settings, timetable, timetable_window_id_col,
previous_tour, tour_owner_id_col, estimator, chunk_size,
tour_trace_label):
"""
chunking wrapper for _schedule_tours
While interaction_sample_simulate provides chunking support, the merged tours, persons
dataframe and the tdd_interaction_dataset are very big, so we want to create them inside
the chunking loop to minimize memory footprint. So we implement the chunking loop here,
and pass a chunk_size of 0 to interaction_sample_simulate to disable its chunking support.
"""
if not tours.index.is_monotonic_increasing:
logger.info(
"schedule_tours %s tours not monotonic_increasing - sorting df")
tours = tours.sort_index()
logger.info("%s schedule_tours running %d tour choices" %
(tour_trace_label, len(tours)))
# no more than one tour per timetable_window per call
if timetable_window_id_col is None:
assert not tours.index.duplicated().any()
else:
assert not tours[timetable_window_id_col].duplicated().any()
if 'LOGSUM_SETTINGS' in model_settings:
# we need skims to calculate tvpb skim overhead in 3_ZONE systems for use by calc_rows_per_chunk
skims = skims_for_logsums(logsum_tour_purpose, model_settings,
tour_trace_label)
else:
skims = None
row_size = chunk_size and \
tour_scheduling_calc_row_size(tours, persons_merged, alts, skims, spec, model_settings, tour_trace_label)
result_list = []
for i, chooser_chunk, chunk_trace_label \
in chunk.adaptive_chunked_choosers(tours, chunk_size, row_size, tour_trace_label):
choices = _schedule_tours(chooser_chunk,
persons_merged,
alts,
spec,
logsum_tour_purpose,
model_settings,
skims,
timetable,
timetable_window_id_col,
previous_tour,
tour_owner_id_col,
estimator,
tour_trace_label=chunk_trace_label)
result_list.append(choices)
mem.force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert len(choices.index == len(tours.index))
return choices
def trip_mode_choice(
trips,
tours_merged,
network_los,
chunk_size, trace_hh_id):
"""
Trip mode choice - compute trip_mode (same values as for tour_mode) for each trip.
Modes for each primary tour putpose are calculated separately because they have different
coefficient values (stored in trip_mode_choice_coeffs.csv coefficient file.)
Adds trip_mode column to trip table
"""
trace_label = 'trip_mode_choice'
model_settings = config.read_model_settings('trip_mode_choice.yaml')
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'trip_mode'
model_spec = \
simulate.read_model_spec(file_name=model_settings['SPEC'])
omnibus_coefficients = \
assign.read_constant_spec(config.config_file_path(model_settings['COEFFICIENTS']))
trips_df = trips.to_frame()
logger.info("Running %s with %d trips", trace_label, trips_df.shape[0])
tours_merged = tours_merged.to_frame()
tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]
nest_spec = config.get_logit_model_settings(model_settings)
tracing.print_summary('primary_purpose',
trips_df.primary_purpose, value_counts=True)
# - trips_merged - merge trips and tours_merged
trips_merged = pd.merge(
trips_df,
tours_merged,
left_on='tour_id',
right_index=True,
how="left")
assert trips_merged.index.equals(trips.index)
# setup skim keys
assert ('trip_period' not in trips_merged)
trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)
orig_col = 'origin'
dest_col = 'destination'
constants = {}
constants.update(config.get_model_constants(model_settings))
constants.update({
'ORIGIN': orig_col,
'DESTINATION': dest_col
})
skim_dict = network_los.get_default_skim_dict()
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,
dim3_key='trip_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,
dim3_key='trip_period')
od_skim_wrapper = skim_dict.wrap('origin', 'destination')
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_wrapper,
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col, dest_key=dest_col,
tod_key='trip_period', segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label, tag='tvpb_logsum_odt')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
# 'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
choices_list = []
for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):
segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)
logger.info("trip_mode_choice tour_type '%s' (%s trips)" %
(primary_purpose, len(trips_segment.index), ))
# name index so tracing knows how to slice
assert trips_segment.index.name == 'trip_id'
if network_los.zone_system == los.THREE_ZONE:
tvpb_logsum_odt.extend_trace_label(primary_purpose)
# tvpb_logsum_dot.extend_trace_label(primary_purpose)
locals_dict = assign.evaluate_constants(omnibus_coefficients[primary_purpose],
constants=constants)
locals_dict.update(constants)
expressions.annotate_preprocessors(
trips_segment, locals_dict, skims,
model_settings, segment_trace_label)
locals_dict.update(skims)
choices = mode_choice_simulate(
choosers=trips_segment,
spec=model_spec,
nest_spec=nest_spec,
skims=skims,
locals_d=locals_dict,
chunk_size=chunk_size,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
trace_label=trace_label,
trace_choice_name='trip_mode_choice')
if trace_hh_id:
# trace the coefficients
tracing.trace_df(pd.Series(locals_dict),
label=tracing.extend_trace_label(segment_trace_label, 'constants'),
transpose=False,
slicer='NONE')
# so we can trace with annotations
assign_in_place(trips_segment, choices)
tracing.trace_df(trips_segment,
label=tracing.extend_trace_label(segment_trace_label, 'trip_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices_df = pd.concat(choices_list)
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_type in tvpb_mode_path_types.items():
skim_cache = tvpb_logsum_odt.cache[path_type]
print(f"mode {mode} path_type {path_type}")
for c in skim_cache:
dest_col = c
if dest_col not in choices_df:
choices_df[dest_col] = np.nan
choices_df[dest_col].where(choices_df[mode_column_name] != mode, skim_cache[c], inplace=True)
# update trips table with choices (and otionally logssums)
trips_df = trips.to_frame()
assign_in_place(trips_df, choices_df)
tracing.print_summary('tour_modes',
trips_merged.tour_mode, value_counts=True)
tracing.print_summary('trip_mode_choice choices',
trips_df[mode_column_name], value_counts=True)
assert not trips_df[mode_column_name].isnull().any()
pipeline.replace_table("trips", trips_df)
if trace_hh_id:
tracing.trace_df(trips_df,
label=tracing.extend_trace_label(trace_label, 'trip_mode'),
slicer='trip_id',
index_label='trip_id',
warn_if_empty=True)
def atwork_subtour_mode_choice(tours, persons_merged, skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d subtours" %
(trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type,
value_counts=True)
# setup skim keys
orig_col_name = 'workplace_taz'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices = run_tour_mode_choice_simulate(
subtours_merged,
spec,
tour_purpose='atwork',
model_settings=model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
tracing.print_summary('%s choices' % trace_label,
choices,
value_counts=True)
assign_in_place(tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(
trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def non_mandatory_tour_frequency(persons, persons_merged, chunk_size,
trace_hh_id):
"""
This model predicts the frequency of making non-mandatory trips
(alternatives for this model come from a separate csv file which is
configured by the user) - these trips include escort, shopping, othmaint,
othdiscr, eatout, and social trips in various combination.
"""
trace_label = 'non_mandatory_tour_frequency'
model_settings_file_name = 'non_mandatory_tour_frequency.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
# FIXME kind of tacky both that we know to add this here and del it below
# 'tot_tours' is used in model_spec expressions
alternatives = simulate.read_model_alts(
'non_mandatory_tour_frequency_alternatives.csv', set_index=None)
alternatives['tot_tours'] = alternatives.sum(axis=1)
# filter based on results of CDAP
choosers = persons_merged.to_frame()
choosers = choosers[choosers.cdap_activity.isin(['M', 'N'])]
# - preprocessor
preprocessor_settings = model_settings.get('preprocessor', None)
if preprocessor_settings:
locals_dict = {'person_max_window': person_max_window}
expressions.assign_columns(df=choosers,
model_settings=preprocessor_settings,
locals_dict=locals_dict,
trace_label=trace_label)
logger.info("Running non_mandatory_tour_frequency with %d persons",
len(choosers))
constants = config.get_model_constants(model_settings)
model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])
spec_segments = model_settings.get('SPEC_SEGMENTS', {})
# segment by person type and pick the right spec for each person type
choices_list = []
for segment_settings in spec_segments:
segment_name = segment_settings['NAME']
ptype = segment_settings['PTYPE']
# pick the spec column for the segment
segment_spec = model_spec[[segment_name]]
chooser_segment = choosers[choosers.ptype == ptype]
logger.info("Running segment '%s' of size %d", segment_name,
len(chooser_segment))
if len(chooser_segment) == 0:
# skip empty segments
continue
estimator = \
estimation.manager.begin_estimation(model_name=segment_name, bundle_name='non_mandatory_tour_frequency')
coefficients_df = simulate.read_model_coefficients(segment_settings)
segment_spec = simulate.eval_coefficients(segment_spec,
coefficients_df, estimator)
if estimator:
estimator.write_spec(model_settings, bundle_directory=True)
estimator.write_model_settings(model_settings,
model_settings_file_name,
bundle_directory=True)
# preserving coefficients file name makes bringing back updated coefficients more straightforward
estimator.write_coefficients(coefficients_df, segment_settings)
estimator.write_choosers(chooser_segment)
estimator.write_alternatives(alternatives, bundle_directory=True)
# FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column
# shuold we do it here or have interaction_simulate do it?
# chooser index must be duplicated in column or it will be omitted from interaction_dataset
# estimation requires that chooser_id is either in index or a column of interaction_dataset
# so it can be reformatted (melted) and indexed by chooser_id and alt_id
assert chooser_segment.index.name == 'person_id'
assert 'person_id' not in chooser_segment.columns
chooser_segment['person_id'] = chooser_segment.index
# FIXME set_alt_id - do we need this for interaction_simulate estimation bundle tables?
estimator.set_alt_id('alt_id')
estimator.set_chooser_id(chooser_segment.index.name)
choices = interaction_simulate(
chooser_segment,
alternatives,
spec=segment_spec,
locals_d=constants,
chunk_size=chunk_size,
trace_label='non_mandatory_tour_frequency.%s' % segment_name,
trace_choice_name='non_mandatory_tour_frequency',
estimator=estimator)
if estimator:
estimator.write_choices(choices)
choices = estimator.get_survey_values(
choices, 'persons', 'non_mandatory_tour_frequency')
estimator.write_override_choices(choices)
estimator.end_estimation()
choices_list.append(choices)
# FIXME - force garbage collection?
force_garbage_collect()
del alternatives['tot_tours'] # del tot_tours column we added above
# The choice value 'non_mandatory_tour_frequency' assigned by interaction_simulate
# is the index value of the chosen alternative in the alternatives table.
choices = pd.concat(choices_list).sort_index()
# add non_mandatory_tour_frequency column to persons
persons = persons.to_frame()
# we expect there to be an alt with no tours - which we can use to backfill non-travelers
no_tours_alt = (alternatives.sum(axis=1) == 0).index[0]
# need to reindex as we only handled persons with cdap_activity in ['M', 'N']
persons['non_mandatory_tour_frequency'] = \
choices.reindex(persons.index).fillna(no_tours_alt).astype(np.int8)
"""
We have now generated non-mandatory tour frequencies, but they are attributes of the person table
Now we create a "tours" table which has one row per tour that has been generated
(and the person id it is associated with)
But before we do that, we run an additional probablilistic step to extend/increase tour counts
beyond the strict limits of the tour_frequency alternatives chosen above (which are currently limited
to at most 2 escort tours and 1 each of shopping, othmaint, othdiscr, eatout, and social tours)
The choice value 'non_mandatory_tour_frequency' assigned by interaction_simulate is simply the
index value of the chosen alternative in the alternatives table.
get counts of each of the tour type alternatives (so we can extend)
escort shopping othmaint othdiscr eatout social
parent_id
2588676 2 0 0 1 1 0
2588677 0 1 0 1 0 0
"""
# counts of each of the tour type alternatives (so we can extend)
modeled_tour_counts = alternatives.loc[choices]
modeled_tour_counts.index = choices.index # assign person ids to the index
# - extend_tour_counts - probabalistic
extended_tour_counts = \
extend_tour_counts(choosers, modeled_tour_counts.copy(), alternatives,
trace_hh_id, tracing.extend_trace_label(trace_label, 'extend_tour_counts'))
num_modeled_tours = modeled_tour_counts.sum().sum()
num_extended_tours = extended_tour_counts.sum().sum()
logger.info("extend_tour_counts increased tour count by %s from %s to %s" %
(num_extended_tours - num_modeled_tours, num_modeled_tours,
num_extended_tours))
"""
create the non_mandatory tours based on extended_tour_counts
"""
if estimator:
override_tour_counts = \
estimation.manager.get_survey_values(extended_tour_counts,
table_name='persons',
column_names=['_%s' % c for c in extended_tour_counts.columns])
override_tour_counts = \
override_tour_counts.rename(columns={('_%s' % c): c for c in extended_tour_counts.columns})
logger.info(
"estimation get_survey_values override_tour_counts %s changed cells"
% (override_tour_counts != extended_tour_counts).sum().sum())
extended_tour_counts = override_tour_counts
"""
create the non_mandatory tours based on extended_tour_counts
"""
non_mandatory_tours = process_non_mandatory_tours(persons,
extended_tour_counts)
assert len(non_mandatory_tours) == extended_tour_counts.sum().sum()
if estimator:
# make sure they created the right tours
survey_tours = estimation.manager.get_survey_table(
'tours').sort_index()
non_mandatory_survey_tours = survey_tours[survey_tours.tour_category ==
'non_mandatory']
assert len(non_mandatory_survey_tours) == len(non_mandatory_tours)
assert non_mandatory_survey_tours.index.equals(
non_mandatory_tours.sort_index().index)
# make sure they created tours with the expected tour_ids
columns = ['person_id', 'household_id', 'tour_type', 'tour_category']
survey_tours = \
estimation.manager.get_survey_values(non_mandatory_tours,
table_name='tours',
column_names=columns)
tours_differ = (non_mandatory_tours[columns] !=
survey_tours[columns]).any(axis=1)
if tours_differ.any():
print("tours_differ\n%s" % tours_differ)
print("%s of %s tours differ" %
(tours_differ.sum(), len(tours_differ)))
print("differing survey_tours\n%s" % survey_tours[tours_differ])
print("differing modeled_tours\n%s" %
non_mandatory_tours[columns][tours_differ])
assert (not tours_differ.any())
pipeline.extend_table("tours", non_mandatory_tours)
tracing.register_traceable_table('tours', non_mandatory_tours)
pipeline.get_rn_generator().add_channel('tours', non_mandatory_tours)
expressions.assign_columns(
df=persons,
model_settings=model_settings.get('annotate_persons'),
trace_label=trace_label)
pipeline.replace_table("persons", persons)
tracing.print_summary('non_mandatory_tour_frequency',
persons.non_mandatory_tour_frequency,
value_counts=True)
if trace_hh_id:
tracing.trace_df(
non_mandatory_tours,
label="non_mandatory_tour_frequency.non_mandatory_tours",
warn_if_empty=True)
tracing.trace_df(choosers,
label="non_mandatory_tour_frequency.choosers",
warn_if_empty=True)
tracing.trace_df(
persons,
label="non_mandatory_tour_frequency.annotated_persons",
warn_if_empty=True)
def atwork_subtour_mode_choice(
tours,
persons_merged,
skim_dict, skim_stack,
chunk_size,
trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d subtours" % (trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type, value_counts=True)
# setup skim keys
orig_col_name = 'workplace_taz'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name, right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name, right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices = run_tour_mode_choice_simulate(
subtours_merged,
spec, tour_purpose='atwork', model_settings=model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
tracing.print_summary('%s choices' % trace_label, choices, value_counts=True)
assign_in_place(tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def interaction_sample_simulate(
choosers, alternatives, spec, choice_column,
allow_zero_probs=False, zero_prob_choice_val=None,
skims=None, locals_d=None, chunk_size=0,
trace_label=None, trace_choice_name=None):
"""
Run a simulation in the situation in which alternatives must
be merged with choosers because there are interaction terms or
because alternatives are being sampled.
optionally (if chunk_size > 0) iterates over choosers in chunk_size chunks
Parameters
----------
choosers : pandas.DataFrame
DataFrame of choosers
alternatives : pandas.DataFrame
DataFrame of alternatives - will be merged with choosers
index domain same as choosers, but repeated for each alternative
spec : pandas.DataFrame
A Pandas DataFrame that gives the specification of the variables to
compute and the coefficients for each variable.
Variable specifications must be in the table index and the
table should have only one column of coefficients.
skims : Skims object
The skims object is used to contain multiple matrices of
origin-destination impedances. Make sure to also add it to the
locals_d below in order to access it in expressions. The *only* job
of this method in regards to skims is to call set_df with the
dataframe that comes back from interacting choosers with
alternatives. See the skims module for more documentation on how
the skims object is intended to be used.
locals_d : Dict
This is a dictionary of local variables that will be the environment
for an evaluation of an expression that begins with @
chunk_size : int
if chunk_size > 0 iterates over choosers in chunk_size chunks
trace_label: str
This is the label to be used for trace log file entries and dump file names
when household tracing enabled. No tracing occurs if label is empty or None.
trace_choice_name: str
This is the column label to be used in trace file csv dump of choices
Returns
-------
choices : pandas.Series
A series where index should match the index of the choosers DataFrame
and values will match the index of the alternatives DataFrame -
choices are simulated in the standard Monte Carlo fashion
"""
trace_label = tracing.extend_trace_label(trace_label, 'interaction_sample_simulate')
rows_per_chunk, effective_chunk_size = \
calc_rows_per_chunk(chunk_size, choosers, alternatives, spec=spec, trace_label=trace_label)
result_list = []
for i, num_chunks, chooser_chunk, alternative_chunk \
in chunk.chunked_choosers_and_alts(choosers, alternatives, rows_per_chunk):
logger.info("Running chunk %s of %s size %d" % (i, num_chunks, len(chooser_chunk)))
chunk_trace_label = tracing.extend_trace_label(trace_label, 'chunk_%s' % i) \
if num_chunks > 1 else trace_label
chunk.log_open(chunk_trace_label, chunk_size, effective_chunk_size)
choices = _interaction_sample_simulate(
chooser_chunk, alternative_chunk, spec, choice_column,
allow_zero_probs, zero_prob_choice_val,
skims, locals_d,
chunk_trace_label, trace_choice_name)
chunk.log_close(chunk_trace_label)
result_list.append(choices)
force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert len(choices.index == len(choosers.index))
return choices
def run_location_choice(
persons_merged_df,
skim_dict, skim_stack,
spc,
model_settings,
chunk_size, trace_hh_id, trace_label
):
"""
Run the three-part location choice algorithm to generate a location choice for each chooser
Handle the various segments separately and in turn for simplicity of expression files
Parameters
----------
persons_merged_df : pandas.DataFrame
persons table merged with households and land_use
skim_dict : skim.SkimDict
skim_stack : skim.SkimStack
spc : ShadowPriceCalculator
to get size terms
model_settings : dict
chunk_size : int
trace_hh_id : int
trace_label : str
Returns
-------
pandas.Series
location choices (zone ids) indexed by persons_merged_df.index
"""
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segment_ids = model_settings['SEGMENT_IDS']
choices_list = []
for segment_name, segment_id in iteritems(segment_ids):
choosers = persons_merged_df[persons_merged_df[chooser_segment_column] == segment_id]
# size_term and shadow price adjustment - one row per zone
dest_size_terms = spc.dest_size_terms(segment_name)
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label, segment_name)
continue
# - location_sample
location_sample_df = \
run_location_sample(
segment_name,
choosers,
skim_dict,
dest_size_terms,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - location_logsums
location_sample_df = \
run_location_logsums(
segment_name,
choosers,
skim_dict, skim_stack,
location_sample_df,
model_settings,
chunk_size,
trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - location_simulate
choices = \
run_location_simulate(
segment_name,
choosers,
location_sample_df,
skim_dict,
dest_size_terms,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
choices_list.append(choices)
# FIXME - want to do this here?
del location_sample_df
force_garbage_collect()
return pd.concat(choices_list) if len(choices_list) > 0 else pd.Series()
def tour_mode_choice_simulate(tours, persons_merged, network_los, chunk_size,
trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode'
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
logger.info("Running %s with %d tours" %
(trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type,
value_counts=True)
persons_merged = persons_merged.to_frame()
primary_tours_merged = pd.merge(primary_tours,
persons_merged,
left_on='person_id',
right_index=True,
how='left',
suffixes=('', '_r'))
constants = {}
# model_constants can appear in expressions
constants.update(config.get_model_constants(model_settings))
skim_dict = network_los.get_default_skim_dict()
# setup skim keys
orig_col_name = 'home_zone_id'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='out_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='in_period')
odr_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='in_period')
dor_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims":
odr_skim_stack_wrapper, # dot return skims for e.g. TNC bridge return fare
"dor_skims":
dor_skim_stack_wrapper, # odt return skims for e.g. TNC bridge return fare
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col_name,
dest_key=dest_col_name,
tod_key='out_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_odt')
tvpb_logsum_dot = tvpb.wrap_logsum(orig_key=dest_col_name,
dest_key=orig_col_name,
tod_key='in_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_dot')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(
network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
estimator = estimation.manager.begin_estimation('tour_mode_choice')
if estimator:
estimator.write_coefficients(model_settings=model_settings)
estimator.write_coefficients_template(model_settings=model_settings)
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# (run_tour_mode_choice_simulate writes choosers post-annotation)
# FIXME should normalize handling of tour_type and tour_purpose
# mtctm1 school tour_type includes univ, which has different coefficients from elementary and HS
# we should either add this column when tours created or add univ to tour_types
not_university = (primary_tours_merged.tour_type !=
'school') | ~primary_tours_merged.is_university
primary_tours_merged['tour_purpose'] = \
primary_tours_merged.tour_type.where(not_university, 'univ')
choices_list = []
for tour_purpose, tours_segment in primary_tours_merged.groupby(
'tour_purpose'):
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" % (
tour_purpose,
len(tours_segment.index),
))
if network_los.zone_system == los.THREE_ZONE:
tvpb_logsum_odt.extend_trace_label(tour_purpose)
tvpb_logsum_dot.extend_trace_label(tour_purpose)
# name index so tracing knows how to slice
assert tours_segment.index.name == 'tour_id'
choices_df = run_tour_mode_choice_simulate(
tours_segment,
tour_purpose,
model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
network_los=network_los,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, tour_purpose),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices_df' %
tour_purpose,
choices_df.tour_mode,
value_counts=True)
choices_list.append(choices_df)
# FIXME - force garbage collection
force_garbage_collect()
choices_df = pd.concat(choices_list)
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_types in tvpb_mode_path_types.items():
for direction, skim in zip(['od', 'do'],
[tvpb_logsum_odt, tvpb_logsum_dot]):
path_type = path_types[direction]
skim_cache = skim.cache[path_type]
print(
f"mode {mode} direction {direction} path_type {path_type}")
for c in skim_cache:
dest_col = f'{direction}_{c}'
if dest_col not in choices_df:
choices_df[
dest_col] = np.nan if pd.api.types.is_numeric_dtype(
skim_cache[c]) else ''
choices_df[dest_col].where(choices_df.tour_mode != mode,
skim_cache[c],
inplace=True)
if estimator:
estimator.write_choices(choices_df.tour_mode)
choices_df.tour_mode = estimator.get_survey_values(
choices_df.tour_mode, 'tours', 'tour_mode')
estimator.write_override_choices(choices_df.tour_mode)
estimator.end_estimation()
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices_df.tour_mode,
value_counts=True)
# so we can trace with annotations
assign_in_place(primary_tours, choices_df)
# update tours table with mode choice (and optionally logsums)
all_tours = tours.to_frame()
assign_in_place(all_tours, choices_df)
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
def run_trip_scheduling_choice(spec, tours, skims, locals_dict, chunk_size,
trace_hh_id, trace_label):
NUM_TOUR_LEGS = 3
trace_label = tracing.extend_trace_label(trace_label,
'interaction_sample_simulate')
# FIXME: The duration, start, and end should be ints well before we get here...
tours[TOUR_DURATION_COLUMN] = tours[TOUR_DURATION_COLUMN].astype(np.int8)
# Setup boolean columns to make it easier to identify
# intermediate stops later in the model.
tours[HAS_OB_STOPS] = tours[NUM_OB_STOPS] >= 1
tours[HAS_IB_STOPS] = tours[NUM_IB_STOPS] >= 1
# Calculate a matrix with the appropriate alternative sizes
# based on the total tour duration. This is used to calculate
# chunk sizes.
max_duration = tours[TOUR_DURATION_COLUMN].max()
alt_sizes = generate_alternative_sizes(max_duration, NUM_TOUR_LEGS)
# Assert the number of tour leg schedule alternatives for each tour
tours[NUM_ALTERNATIVES] = 1
tours.loc[tours[HAS_OB_STOPS] != tours[HAS_IB_STOPS],
NUM_ALTERNATIVES] = tours[TOUR_DURATION_COLUMN] + 1
tours.loc[tours[HAS_OB_STOPS] & tours[HAS_IB_STOPS], NUM_ALTERNATIVES] = \
tours.apply(lambda x: alt_sizes[1, x.duration], axis=1)
# If no intermediate stops on the tour, then then main leg duration
# equals the tour duration and the intermediate durations are zero
tours.loc[~tours[HAS_OB_STOPS] & ~tours[HAS_IB_STOPS],
MAIN_LEG_DURATION] = tours[TOUR_DURATION_COLUMN]
tours.loc[~tours[HAS_OB_STOPS] & ~tours[HAS_IB_STOPS],
[IB_DURATION, OB_DURATION]] = 0
# We only need to determine schedules for tours with intermediate stops
indirect_tours = tours.loc[tours[HAS_OB_STOPS] | tours[HAS_IB_STOPS]]
# Crudely calculate a chunk size
# 5=number of columns in the alternatives (3 leg times + index)
tour_row_size = (
4 + len(tours.columns)) * indirect_tours[NUM_ALTERNATIVES].mean()
# rpc, effective_chunk_size = chunk.rows_per_chunk(chunk_size, tour_row_size, indirect_tours.shape[0], trace_label)
row_size = chunk_size and trip_schedule_calc_row_size(
indirect_tours, trace_label)
# Iterate through the chunks
result_list = []
for i, choosers, chunk_trace_label in \
chunk.adaptive_chunked_choosers(indirect_tours, chunk_size, row_size, trace_label):
# Sort the choosers and get the schedule alternatives
choosers = choosers.sort_index()
schedules = generate_schedule_alternatives(choosers).sort_index()
# Assuming we did the max_alt_size calculation correctly,
# we should get the same sizes here.
assert choosers[NUM_ALTERNATIVES].sum() == schedules.shape[0]
# Run the simulation
choices = _interaction_sample_simulate(
choosers=choosers,
alternatives=schedules,
spec=spec,
choice_column=SCHEDULE_ID,
allow_zero_probs=True,
zero_prob_choice_val=-999,
want_logsums=False,
skims=skims,
locals_d=locals_dict,
trace_label=chunk_trace_label,
trace_choice_name='trip_schedule_stage_1',
estimator=None)
assert len(choices.index) == len(choosers.index)
choices = schedules[schedules[SCHEDULE_ID].isin(choices)].drop(
columns='tour_id')
result_list.append(choices)
force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert len(choices.index) == len(indirect_tours.index)
# The choices here are only the indirect tours, so the durations
# need to be updated on the main tour dataframe.
tours.update(choices[[MAIN_LEG_DURATION, OB_DURATION, IB_DURATION]])
# Cleanup data types and drop temporary columns
tours[[MAIN_LEG_DURATION, OB_DURATION, IB_DURATION]] = \
tours[[MAIN_LEG_DURATION, OB_DURATION, IB_DURATION]].astype(np.int8)
tours = tours.drop(columns=TEMP_COLS)
return tours
def run_location_choice(persons_merged_df, network_los,
shadow_price_calculator, want_logsums,
want_sample_table, estimator, model_settings,
chunk_size, trace_hh_id, trace_label):
"""
Run the three-part location choice algorithm to generate a location choice for each chooser
Handle the various segments separately and in turn for simplicity of expression files
Parameters
----------
persons_merged_df : pandas.DataFrame
persons table merged with households and land_use
network_los : los.Network_LOS
shadow_price_calculator : ShadowPriceCalculator
to get size terms
want_logsums : boolean
want_sample_table : boolean
estimator: Estimator object
model_settings : dict
chunk_size : int
trace_hh_id : int
trace_label : str
Returns
-------
choices : pandas.DataFrame indexed by persons_merged_df.index
'choice' : location choices (zone ids)
'logsum' : float logsum of choice utilities across alternatives
logsums optional & only returned if DEST_CHOICE_LOGSUM_COLUMN_NAME specified in model_settings
"""
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segment_ids = model_settings['SEGMENT_IDS']
choices_list = []
sample_list = []
for segment_name, segment_id in segment_ids.items():
choosers = persons_merged_df[persons_merged_df[chooser_segment_column]
== segment_id]
# size_term and shadow price adjustment - one row per zone
dest_size_terms = shadow_price_calculator.dest_size_terms(segment_name)
assert dest_size_terms.index.is_monotonic_increasing, \
f"shadow_price_calculator.dest_size_terms({segment_name}) not monotonic_increasing"
if choosers.shape[0] == 0:
logger.info(
f"{trace_label} skipping segment {segment_name}: no choosers")
continue
# - location_sample
location_sample_df = \
run_location_sample(
segment_name,
choosers,
network_los,
dest_size_terms,
estimator,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - location_logsums
location_sample_df = \
run_location_logsums(
segment_name,
choosers,
network_los,
location_sample_df,
model_settings,
chunk_size,
trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - location_simulate
choices_df = \
run_location_simulate(
segment_name,
choosers,
location_sample_df,
network_los,
dest_size_terms,
want_logsums,
estimator,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
if estimator:
if trace_hh_id:
estimation_trace_label = \
tracing.extend_trace_label(trace_label, f'estimation.{segment_name}.modeled_choices')
tracing.trace_df(choices_df, label=estimation_trace_label)
estimator.write_choices(choices_df.choice)
choices_df.choice = estimator.get_survey_values(
choices_df.choice,
'persons',
column_names=model_settings['DEST_CHOICE_COLUMN_NAME'])
estimator.write_override_choices(choices_df.choice)
if want_logsums:
# if we override choices, we need to to replace choice logsum with ologsim for override location
# fortunately, as long as we aren't sampling dest alts, the logsum will be in location_sample_df
# if we start sampling dest alts, we will need code below to compute override location logsum
assert estimator.want_unsampled_alternatives
# merge mode_choice_logsum for the overridden location
# alt_logsums columns: ['person_id', 'choice', 'logsum']
alt_dest_col = model_settings['ALT_DEST_COL_NAME']
alt_logsums = \
location_sample_df[[alt_dest_col, ALT_LOGSUM]]\
.rename(columns={alt_dest_col: 'choice', ALT_LOGSUM: 'logsum'})\
.reset_index()
# choices_df columns: ['person_id', 'choice']
choices_df = choices_df[['choice']].reset_index()
# choices_df columns: ['person_id', 'choice', 'logsum']
choices_df = pd.merge(choices_df, alt_logsums,
how='left').set_index('person_id')
logger.debug(
f"{trace_label} segment {segment_name} estimation: override logsums"
)
if trace_hh_id:
estimation_trace_label = \
tracing.extend_trace_label(trace_label, f'estimation.{segment_name}.survey_choices')
tracing.trace_df(choices_df, estimation_trace_label)
choices_list.append(choices_df)
if want_sample_table:
# FIXME - sample_table
location_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'],
append=True,
inplace=True)
sample_list.append(location_sample_df)
# FIXME - want to do this here?
del location_sample_df
mem.force_garbage_collect()
if len(choices_list) > 0:
choices_df = pd.concat(choices_list)
else:
# this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
logger.warning("%s no choices", trace_label)
choices_df = pd.DataFrame(columns=['choice', 'logsum'])
if len(sample_list) > 0:
save_sample_df = pd.concat(sample_list)
else:
# this could happen either with small samples as above, or if no saved sample desired
save_sample_df = None
return choices_df, save_sample_df
def run_tour_destination(
tours,
persons_merged,
model_settings,
skim_dict,
skim_stack,
chunk_size, trace_hh_id, trace_label):
size_term_calculator = SizeTermCalculator(model_settings['SIZE_TERM_SELECTOR'])
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segments = model_settings['SEGMENTS']
# interaction_sample_simulate insists choosers appear in same order as alts
tours = tours.sort_index()
choices_list = []
for segment_name in segments:
choosers = tours[tours[chooser_segment_column] == segment_name]
# size_term segment is segment_name
segment_destination_size_terms = size_term_calculator.dest_size_terms_df(segment_name)
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label, segment_name)
continue
# - destination_sample
spec_segment_name = segment_name # spec_segment_name is segment_name
location_sample_df = \
run_destination_sample(
spec_segment_name,
choosers,
persons_merged,
model_settings,
skim_dict,
segment_destination_size_terms,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - destination_logsums
tour_purpose = segment_name # tour_purpose is segment_name
location_sample_df = \
run_destination_logsums(
tour_purpose,
persons_merged,
location_sample_df,
model_settings,
skim_dict, skim_stack,
chunk_size, trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - destination_simulate
spec_segment_name = segment_name # spec_segment_name is segment_name
choices = \
run_destination_simulate(
spec_segment_name,
choosers,
persons_merged,
location_sample_df,
model_settings,
skim_dict,
segment_destination_size_terms,
chunk_size,
tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
choices_list.append(choices)
# FIXME - want to do this here?
del location_sample_df
force_garbage_collect()
return pd.concat(choices_list) if len(choices_list) > 0 else pd.Series()
def tour_mode_choice_simulate(tours, persons_merged,
skim_dict, skim_stack,
chunk_size,
trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
persons_merged = persons_merged.to_frame()
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d tours" % (trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type, value_counts=True)
primary_tours_merged = pd.merge(primary_tours, persons_merged, left_on='person_id',
right_index=True, how='left', suffixes=('', '_r'))
# setup skim keys
orig_col_name = 'TAZ'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name, right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name, right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices_list = []
for tour_type, segment in primary_tours_merged.groupby('tour_type'):
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" %
(tour_type, len(segment.index), ))
# name index so tracing knows how to slice
assert segment.index.name == 'tour_id'
choices = run_tour_mode_choice_simulate(
segment,
spec, tour_type, model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, tour_type),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices' % tour_type,
choices, value_counts=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices = pd.concat(choices_list)
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices, value_counts=True)
# so we can trace with annotations
primary_tours['tour_mode'] = choices
# but only keep mode choice col
all_tours = tours.to_frame()
# uncomment to save annotations to table
# assign_in_place(all_tours, annotations)
assign_in_place(all_tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
def atwork_subtour_mode_choice(tours, persons_merged, skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode'
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d subtours" %
(trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type,
value_counts=True)
# setup skim keys
orig_col_name = 'workplace_taz'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='in_period')
odr_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='in_period')
dor_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims": odr_skim_stack_wrapper,
"dor_skims": dor_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
estimator = estimation.manager.begin_estimation(
'atwork_subtour_mode_choice')
if estimator:
estimator.write_coefficients(
simulate.read_model_coefficients(model_settings))
estimator.write_coefficients_template(
simulate.read_model_coefficient_template(model_settings))
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# FIXME run_tour_mode_choice_simulate writes choosers post-annotation
choices_df = run_tour_mode_choice_simulate(
subtours_merged,
tour_purpose='atwork',
model_settings=model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
if estimator:
estimator.write_choices(choices_df[mode_column_name])
choices_df[mode_column_name] = \
estimator.get_survey_values(choices_df[mode_column_name], 'tours', mode_column_name)
estimator.write_override_choices(choices_df[mode_column_name])
estimator.end_estimation()
tracing.print_summary('%s choices' % trace_label,
choices_df[mode_column_name],
value_counts=True)
assign_in_place(tours, choices_df)
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def trip_mode_choice(
trips,
tours_merged,
skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
Trip mode choice - compute trip_mode (same values as for tour_mode) for each trip.
Modes for each primary tour putpose are calculated separately because they have different
coefficient values (stored in trip_mode_choice_coeffs.csv coefficient file.)
Adds trip_mode column to trip table
"""
trace_label = 'trip_mode_choice'
model_settings = config.read_model_settings('trip_mode_choice.yaml')
model_spec = \
simulate.read_model_spec(file_name=model_settings['SPEC'])
omnibus_coefficients = \
assign.read_constant_spec(config.config_file_path(model_settings['COEFFS']))
trips_df = trips.to_frame()
logger.info("Running %s with %d trips", trace_label, trips_df.shape[0])
tours_merged = tours_merged.to_frame()
tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]
nest_spec = config.get_logit_model_settings(model_settings)
tracing.print_summary('primary_purpose',
trips_df.primary_purpose, value_counts=True)
# - trips_merged - merge trips and tours_merged
trips_merged = pd.merge(
trips_df,
tours_merged,
left_on='tour_id',
right_index=True,
how="left")
assert trips_merged.index.equals(trips.index)
# setup skim keys
assert ('trip_period' not in trips_merged)
trips_merged['trip_period'] = skim_time_period_label(trips_merged.depart)
orig_col = 'origin'
dest_col = 'destination'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col, right_key=dest_col,
skim_key='trip_period')
od_skim_wrapper = skim_dict.wrap('origin', 'destination')
skims = {
"odt_skims": odt_skim_stack_wrapper,
"od_skims": od_skim_wrapper,
}
constants = config.get_model_constants(model_settings)
constants.update({
'ORIGIN': orig_col,
'DESTINATION': dest_col
})
choices_list = []
for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):
segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)
logger.info("trip_mode_choice tour_type '%s' (%s trips)" %
(primary_purpose, len(trips_segment.index), ))
# name index so tracing knows how to slice
assert trips_segment.index.name == 'trip_id'
locals_dict = assign.evaluate_constants(omnibus_coefficients[primary_purpose],
constants=constants)
locals_dict.update(constants)
annotate_preprocessors(
trips_segment, locals_dict, skims,
model_settings, segment_trace_label)
locals_dict.update(skims)
choices = simulate.simple_simulate(
choosers=trips_segment,
spec=model_spec,
nest_spec=nest_spec,
skims=skims,
locals_d=locals_dict,
chunk_size=chunk_size,
trace_label=segment_trace_label,
trace_choice_name='trip_mode_choice')
alts = model_spec.columns
choices = choices.map(dict(list(zip(list(range(len(alts))), alts))))
# tracing.print_summary('trip_mode_choice %s choices' % primary_purpose,
# choices, value_counts=True)
if trace_hh_id:
# trace the coefficients
tracing.trace_df(pd.Series(locals_dict),
label=tracing.extend_trace_label(segment_trace_label, 'constants'),
transpose=False,
slicer='NONE')
# so we can trace with annotations
trips_segment['trip_mode'] = choices
tracing.trace_df(trips_segment,
label=tracing.extend_trace_label(segment_trace_label, 'trip_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices = pd.concat(choices_list)
trips_df = trips.to_frame()
trips_df['trip_mode'] = choices
tracing.print_summary('tour_modes',
trips_merged.tour_mode, value_counts=True)
tracing.print_summary('trip_mode_choice choices',
choices, value_counts=True)
assert not trips_df.trip_mode.isnull().any()
pipeline.replace_table("trips", trips_df)
if trace_hh_id:
tracing.trace_df(trips_df,
label=tracing.extend_trace_label(trace_label, 'trip_mode'),
slicer='trip_id',
index_label='trip_id',
warn_if_empty=True)
def run_joint_tour_destination(tours, persons_merged, households_merged,
want_logsums, want_sample_table, model_settings,
network_los, estimator, chunk_size, trace_hh_id,
trace_label):
size_term_calculator = tour_destination.SizeTermCalculator(
model_settings['SIZE_TERM_SELECTOR'])
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segments = model_settings['SEGMENTS']
# interaction_sample_simulate insists choosers appear in same order as alts
tours = tours.sort_index()
sample_list = []
choices_list = []
for segment_name in segments:
choosers = tours[tours[chooser_segment_column] == segment_name]
# size_term segment is segment_name
segment_destination_size_terms = size_term_calculator.dest_size_terms_df(
segment_name)
# FIXME - no point in considering impossible alternatives (where dest size term is zero)
segment_destination_size_terms = segment_destination_size_terms[
segment_destination_size_terms['size_term'] > 0]
logger.info(
"Running segment '%s' of %d joint_tours %d alternatives" %
(segment_name, len(choosers), len(segment_destination_size_terms)))
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label,
segment_name)
continue
# - destination_sample
spec_segment_name = segment_name # spec_segment_name is segment_name
location_sample_df = \
run_destination_sample(
spec_segment_name,
choosers,
households_merged,
model_settings,
network_los,
segment_destination_size_terms,
estimator,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - destination_logsums
tour_purpose = segment_name # tour_purpose is segment_name
location_sample_df = \
run_destination_logsums(
tour_purpose,
persons_merged,
location_sample_df,
model_settings,
network_los,
chunk_size, trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - destination_simulate
spec_segment_name = segment_name # spec_segment_name is segment_name
choices = \
run_destination_simulate(
spec_segment_name,
choosers,
persons_merged,
destination_sample=location_sample_df,
want_logsums=want_logsums,
model_settings=model_settings,
network_los=network_los,
destination_size_terms=segment_destination_size_terms,
estimator=estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
choices_list.append(choices)
if want_sample_table:
# FIXME - sample_table
location_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'],
append=True,
inplace=True)
sample_list.append(location_sample_df)
# FIXME - want to do this here?
del location_sample_df
force_garbage_collect()
if len(choices_list) > 0:
choices_df = pd.concat(choices_list)
else:
# this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
logger.warning("%s no choices", trace_label)
choices_df = pd.DataFrame(columns=['choice', 'logsum'])
if len(sample_list) > 0:
save_sample_df = pd.concat(sample_list)
else:
# this could happen either with small samples as above, or if no saved sample desired
save_sample_df = None
return choices_df, save_sample_df
def run_location_choice(
persons_merged_df,
network_los,
shadow_price_calculator,
want_logsums,
want_sample_table,
estimator,
model_settings,
chunk_size, trace_hh_id, trace_label
):
"""
Run the three-part location choice algorithm to generate a location choice for each chooser
Handle the various segments separately and in turn for simplicity of expression files
Parameters
----------
persons_merged_df : pandas.DataFrame
persons table merged with households and land_use
network_los : los.Network_LOS
shadow_price_calculator : ShadowPriceCalculator
to get size terms
want_logsums : boolean
want_sample_table : boolean
estimator: Estimator object
model_settings : dict
chunk_size : int
trace_hh_id : int
trace_label : str
Returns
-------
choices : pandas.DataFrame indexed by persons_merged_df.index
'choice' : location choices (zone ids)
'logsum' : float logsum of choice utilities across alternatives
logsums optional & only returned if DEST_CHOICE_LOGSUM_COLUMN_NAME specified in model_settings
"""
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segment_ids = model_settings['SEGMENT_IDS']
choices_list = []
sample_list = []
for segment_name, segment_id in segment_ids.items():
choosers = persons_merged_df[persons_merged_df[chooser_segment_column] == segment_id]
# size_term and shadow price adjustment - one row per zone
dest_size_terms = shadow_price_calculator.dest_size_terms(segment_name)
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label, segment_name)
continue
# - location_sample
location_sample_df = \
run_location_sample(
segment_name,
choosers,
network_los,
dest_size_terms,
estimator,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - location_logsums
location_sample_df = \
run_location_logsums(
segment_name,
choosers,
network_los,
location_sample_df,
model_settings,
chunk_size,
trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - location_simulate
choices_df = \
run_location_simulate(
segment_name,
choosers,
location_sample_df,
network_los,
dest_size_terms,
want_logsums,
estimator,
model_settings,
chunk_size,
tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
if estimator:
estimator.write_choices(choices_df.choice)
choices_df.choice = estimator.get_survey_values(choices_df.choice, 'persons',
column_names=model_settings['DEST_CHOICE_COLUMN_NAME'])
estimator.write_override_choices(choices_df.choice)
choices_list.append(choices_df)
if want_sample_table:
# FIXME - sample_table
location_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'],
append=True, inplace=True)
sample_list.append(location_sample_df)
# FIXME - want to do this here?
del location_sample_df
mem.force_garbage_collect()
if len(choices_list) > 0:
choices_df = pd.concat(choices_list)
else:
# this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
logger.warning("%s no choices", trace_label)
choices_df = pd.DataFrame(columns=['choice', 'logsum'])
if len(sample_list) > 0:
save_sample_df = pd.concat(sample_list)
else:
# this could happen either with small samples as above, or if no saved sample desired
save_sample_df = None
return choices_df, save_sample_df
def run_tour_destination(tours, persons_merged, want_logsums,
want_sample_table, model_settings, network_los,
estimator, chunk_size, trace_hh_id, trace_label):
size_term_calculator = SizeTermCalculator(
model_settings['SIZE_TERM_SELECTOR'])
# maps segment names to compact (integer) ids
segments = model_settings['SEGMENTS']
chooser_segment_column = model_settings.get('CHOOSER_SEGMENT_COLUMN_NAME',
None)
if chooser_segment_column is None:
assert len(segments) == 1, \
f"CHOOSER_SEGMENT_COLUMN_NAME not specified in model_settings to slice SEGMENTS: {segments}"
choices_list = []
sample_list = []
for segment_name in segments:
segment_trace_label = tracing.extend_trace_label(
trace_label, segment_name)
if chooser_segment_column is not None:
choosers = tours[tours[chooser_segment_column] == segment_name]
else:
choosers = tours.copy()
# Note: size_term_calculator omits zones with impossible alternatives (where dest size term is zero)
segment_destination_size_terms = size_term_calculator.dest_size_terms_df(
segment_name, segment_trace_label)
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label,
segment_name)
continue
# - destination_sample
spec_segment_name = segment_name # spec_segment_name is segment_name
location_sample_df = \
run_destination_sample(
spec_segment_name,
choosers,
persons_merged,
model_settings,
network_los,
segment_destination_size_terms,
estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(segment_trace_label, 'sample'))
# - destination_logsums
tour_purpose = segment_name # tour_purpose is segment_name
location_sample_df = \
run_destination_logsums(
tour_purpose,
persons_merged,
location_sample_df,
model_settings,
network_los,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(segment_trace_label, 'logsums'))
# - destination_simulate
spec_segment_name = segment_name # spec_segment_name is segment_name
choices = \
run_destination_simulate(
spec_segment_name,
choosers,
persons_merged,
destination_sample=location_sample_df,
want_logsums=want_logsums,
model_settings=model_settings,
network_los=network_los,
destination_size_terms=segment_destination_size_terms,
estimator=estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(segment_trace_label, 'simulate'))
choices_list.append(choices)
if want_sample_table:
# FIXME - sample_table
location_sample_df.set_index(model_settings['ALT_DEST_COL_NAME'],
append=True,
inplace=True)
sample_list.append(location_sample_df)
# FIXME - want to do this here?
del location_sample_df
force_garbage_collect()
if len(choices_list) > 0:
choices_df = pd.concat(choices_list)
else:
# this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
logger.warning("%s no choices", trace_label)
choices_df = pd.DataFrame(columns=['choice', 'logsum'])
if len(sample_list) > 0:
save_sample_df = pd.concat(sample_list)
else:
# this could happen either with small samples as above, or if no saved sample desired
save_sample_df = None
return choices_df, save_sample_df
def tour_mode_choice_simulate(tours, persons_merged, skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode' # FIXME - should be passed in?
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
persons_merged = persons_merged.to_frame()
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d tours" %
(trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type,
value_counts=True)
primary_tours_merged = pd.merge(primary_tours,
persons_merged,
left_on='person_id',
right_index=True,
how='left',
suffixes=('', '_r'))
# setup skim keys
orig_col_name = 'TAZ'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='in_period')
odr_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='in_period')
dor_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims": odr_skim_stack_wrapper,
"dor_skims": dor_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
estimator = estimation.manager.begin_estimation('tour_mode_choice')
if estimator:
estimator.write_coefficients(
simulate.read_model_coefficients(model_settings))
estimator.write_coefficients_template(
simulate.read_model_coefficient_template(model_settings))
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# FIXME run_tour_mode_choice_simulate writes choosers post-annotation
choices_list = []
primary_tours_merged['primary_purpose'] = \
primary_tours_merged.tour_type.where((primary_tours_merged.tour_type != 'school') |
~primary_tours_merged.is_university, 'univ')
for primary_purpose, tours_segment in primary_tours_merged.groupby(
'primary_purpose'):
logger.info(
"tour_mode_choice_simulate primary_purpose '%s' (%s tours)" % (
primary_purpose,
len(tours_segment.index),
))
# name index so tracing knows how to slice
assert tours_segment.index.name == 'tour_id'
choices_df = run_tour_mode_choice_simulate(
tours_segment,
primary_purpose,
model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label,
primary_purpose),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices_df' %
primary_purpose,
choices_df.tour_mode,
value_counts=True)
choices_list.append(choices_df)
# FIXME - force garbage collection
force_garbage_collect()
choices_df = pd.concat(choices_list)
if estimator:
estimator.write_choices(choices_df.tour_mode)
choices_df.tour_mode = estimator.get_survey_values(
choices_df.tour_mode, 'tours', 'tour_mode')
estimator.write_override_choices(choices_df.tour_mode)
estimator.end_estimation()
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices_df.tour_mode,
value_counts=True)
# so we can trace with annotations
assign_in_place(primary_tours, choices_df)
# but only keep mode choice col
all_tours = tours.to_frame()
assign_in_place(all_tours, choices_df)
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
def interaction_sample(choosers,
alternatives,
spec,
sample_size,
alt_col_name,
allow_zero_probs=False,
skims=None,
locals_d=None,
chunk_size=0,
trace_label=None):
"""
Run a simulation in the situation in which alternatives must
be merged with choosers because there are interaction terms or
because alternatives are being sampled.
optionally (if chunk_size > 0) iterates over choosers in chunk_size chunks
Parameters
----------
choosers : pandas.DataFrame
DataFrame of choosers
alternatives : pandas.DataFrame
DataFrame of alternatives - will be merged with choosers and sampled
spec : pandas.DataFrame
A Pandas DataFrame that gives the specification of the variables to
compute and the coefficients for each variable.
Variable specifications must be in the table index and the
table should have only one column of coefficients.
sample_size : int, optional
Sample alternatives with sample of given size. By default is None,
which does not sample alternatives.
alt_col_name: str
name to give the sampled_alternative column
skims : Skims object
The skims object is used to contain multiple matrices of
origin-destination impedances. Make sure to also add it to the
locals_d below in order to access it in expressions. The *only* job
of this method in regards to skims is to call set_df with the
dataframe that comes back from interacting choosers with
alternatives. See the skims module for more documentation on how
the skims object is intended to be used.
locals_d : Dict
This is a dictionary of local variables that will be the environment
for an evaluation of an expression that begins with @
chunk_size : int
if chunk_size > 0 iterates over choosers in chunk_size chunks
trace_label: str
This is the label to be used for trace log file entries and dump file names
when household tracing enabled. No tracing occurs if label is empty or None.
Returns
-------
choices_df : pandas.DataFrame
A DataFrame where index should match the index of the choosers DataFrame
(except with sample_size rows for each choser row, one row for each alt sample)
and columns alt_col_name, prob, rand, pick_count
<alt_col_name>:
alt identifier from alternatives[<alt_col_name>
prob: float
the probability of the chosen alternative
pick_count : int
number of duplicate picks for chooser, alt
"""
trace_label = tracing.extend_trace_label(trace_label, 'interaction_sample')
# we return alternatives ordered in (index, alt_col_name)
# if choosers index is not ordered, it is probably a mistake, since the alts wont line up
assert alt_col_name is not None
assert choosers.index.is_monotonic_increasing
sample_size = min(sample_size, len(alternatives.index))
rows_per_chunk, effective_chunk_size = \
calc_rows_per_chunk(chunk_size, choosers, alternatives, trace_label)
result_list = []
for i, num_chunks, chooser_chunk in chunk.chunked_choosers(
choosers, rows_per_chunk):
logger.info("Running chunk %s of %s size %d" %
(i, num_chunks, len(chooser_chunk)))
chunk_trace_label = tracing.extend_trace_label(trace_label, 'chunk_%s' % i) \
if num_chunks > 1 else trace_label
chunk.log_open(chunk_trace_label, chunk_size, effective_chunk_size)
choices = _interaction_sample(chooser_chunk, alternatives, spec,
sample_size, alt_col_name,
allow_zero_probs, skims, locals_d,
chunk_trace_label)
chunk.log_close(chunk_trace_label)
if choices.shape[0] > 0:
# might not be any if allow_zero_probs
result_list.append(choices)
force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert allow_zero_probs or (len(choosers.index) == len(
np.unique(choices.index.values)))
# keep alts in canonical order so choices based on their probs are stable across runs
choices = choices.sort_values(by=alt_col_name).sort_index(kind='mergesort')
return choices
def run_tour_destination(tours, persons_merged, model_settings, skim_dict,
skim_stack, chunk_size, trace_hh_id, trace_label):
size_term_calculator = SizeTermCalculator(
model_settings['SIZE_TERM_SELECTOR'])
chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']
# maps segment names to compact (integer) ids
segments = model_settings['SEGMENTS']
# interaction_sample_simulate insists choosers appear in same order as alts
tours = tours.sort_index()
choices_list = []
for segment_name in segments:
choosers = tours[tours[chooser_segment_column] == segment_name]
# size_term segment is segment_name
segment_destination_size_terms = size_term_calculator.dest_size_terms_df(
segment_name)
if choosers.shape[0] == 0:
logger.info("%s skipping segment %s: no choosers", trace_label,
segment_name)
continue
# - destination_sample
spec_segment_name = segment_name # spec_segment_name is segment_name
location_sample_df = \
run_destination_sample(
spec_segment_name,
choosers,
persons_merged,
model_settings,
skim_dict,
segment_destination_size_terms,
chunk_size,
tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name))
# - destination_logsums
tour_purpose = segment_name # tour_purpose is segment_name
location_sample_df = \
run_destination_logsums(
tour_purpose,
persons_merged,
location_sample_df,
model_settings,
skim_dict, skim_stack,
chunk_size, trace_hh_id,
tracing.extend_trace_label(trace_label, 'logsums.%s' % segment_name))
# - destination_simulate
spec_segment_name = segment_name # spec_segment_name is segment_name
choices = \
run_destination_simulate(
spec_segment_name,
choosers,
persons_merged,
location_sample_df,
model_settings,
skim_dict,
segment_destination_size_terms,
chunk_size,
tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name))
choices_list.append(choices)
# FIXME - want to do this here?
del location_sample_df
force_garbage_collect()
return pd.concat(choices_list) if len(choices_list) > 0 else pd.Series()
def schedule_tours(
tours, persons_merged, alts,
spec, logsum_tour_purpose,
model_settings,
timetable, timetable_window_id_col,
previous_tour, tour_owner_id_col,
chunk_size, tour_trace_label):
"""
chunking wrapper for _schedule_tours
While interaction_sample_simulate provides chunking support, the merged tours, persons
dataframe and the tdd_interaction_dataset are very big, so we want to create them inside
the chunking loop to minimize memory footprint. So we implement the chunking loop here,
and pass a chunk_size of 0 to interaction_sample_simulate to disable its chunking support.
"""
if not tours.index.is_monotonic_increasing:
logger.info("schedule_tours %s tours not monotonic_increasing - sorting df")
tours = tours.sort_index()
logger.info("%s schedule_tours running %d tour choices" % (tour_trace_label, len(tours)))
# no more than one tour per timetable_window per call
if timetable_window_id_col is None:
assert not tours.index.duplicated().any()
else:
assert not tours[timetable_window_id_col].duplicated().any()
rows_per_chunk, effective_chunk_size = \
calc_rows_per_chunk(chunk_size, tours, persons_merged, alts, trace_label=tour_trace_label)
result_list = []
for i, num_chunks, chooser_chunk \
in chunk.chunked_choosers(tours, rows_per_chunk):
logger.info("Running chunk %s of %s size %d" % (i, num_chunks, len(chooser_chunk)))
chunk_trace_label = tracing.extend_trace_label(tour_trace_label, 'chunk_%s' % i) \
if num_chunks > 1 else tour_trace_label
chunk.log_open(chunk_trace_label, chunk_size, effective_chunk_size)
choices = _schedule_tours(chooser_chunk, persons_merged,
alts, spec, logsum_tour_purpose,
model_settings,
timetable, timetable_window_id_col,
previous_tour, tour_owner_id_col,
tour_trace_label=chunk_trace_label)
chunk.log_close(chunk_trace_label)
result_list.append(choices)
mem.force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert len(choices.index == len(tours.index))
return choices
def schedule_tours(tours, persons_merged, alts, spec, logsum_tour_purpose,
model_settings, timetable, timetable_window_id_col,
previous_tour, tour_owner_id_col, estimator, chunk_size,
tour_trace_label):
"""
chunking wrapper for _schedule_tours
While interaction_sample_simulate provides chunking support, the merged tours, persons
dataframe and the tdd_interaction_dataset are very big, so we want to create them inside
the chunking loop to minimize memory footprint. So we implement the chunking loop here,
and pass a chunk_size of 0 to interaction_sample_simulate to disable its chunking support.
"""
if not tours.index.is_monotonic_increasing:
logger.info(
"schedule_tours %s tours not monotonic_increasing - sorting df")
tours = tours.sort_index()
logger.info("%s schedule_tours running %d tour choices" %
(tour_trace_label, len(tours)))
# no more than one tour per timetable_window per call
if timetable_window_id_col is None:
assert not tours.index.duplicated().any()
else:
assert not tours[timetable_window_id_col].duplicated().any()
rows_per_chunk, effective_chunk_size = \
calc_rows_per_chunk(chunk_size, tours, persons_merged, alts,
model_settings=model_settings, trace_label=tour_trace_label)
result_list = []
for i, num_chunks, chooser_chunk \
in chunk.chunked_choosers(tours, rows_per_chunk):
logger.info("Running chunk %s of %s size %d" %
(i, num_chunks, len(chooser_chunk)))
chunk_trace_label = tracing.extend_trace_label(tour_trace_label, 'chunk_%s' % i) \
if num_chunks > 1 else tour_trace_label
chunk.log_open(chunk_trace_label, chunk_size, effective_chunk_size)
choices = _schedule_tours(chooser_chunk,
persons_merged,
alts,
spec,
logsum_tour_purpose,
model_settings,
timetable,
timetable_window_id_col,
previous_tour,
tour_owner_id_col,
estimator,
tour_trace_label=chunk_trace_label)
chunk.log_close(chunk_trace_label)
result_list.append(choices)
mem.force_garbage_collect()
# FIXME: this will require 2X RAM
# if necessary, could append to hdf5 store on disk:
# http://pandas.pydata.org/pandas-docs/stable/io.html#id2
if len(result_list) > 1:
choices = pd.concat(result_list)
assert len(choices.index == len(tours.index))
return choices
def atwork_subtour_mode_choice(tours, persons_merged, network_los, chunk_size,
trace_hh_id):
"""
At-work subtour mode choice simulate
"""
trace_label = 'atwork_subtour_mode_choice'
model_settings_file_name = 'tour_mode_choice.yaml'
model_settings = config.read_model_settings(model_settings_file_name)
logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')
mode_column_name = 'tour_mode'
tours = tours.to_frame()
subtours = tours[tours.tour_category == 'atwork']
# - if no atwork subtours
if subtours.shape[0] == 0:
tracing.no_results(trace_label)
return
subtours_merged = \
pd.merge(subtours, persons_merged.to_frame(),
left_on='person_id', right_index=True, how='left')
logger.info("Running %s with %d subtours" %
(trace_label, subtours_merged.shape[0]))
tracing.print_summary('%s tour_type' % trace_label,
subtours_merged.tour_type,
value_counts=True)
constants = {}
constants.update(config.get_model_constants(model_settings))
skim_dict = network_los.get_default_skim_dict()
# setup skim keys
orig_col_name = 'workplace_zone_id'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='out_period')
dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='in_period')
odr_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col_name,
dest_key=dest_col_name,
dim3_key='in_period')
dor_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col_name,
dest_key=orig_col_name,
dim3_key='out_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"odr_skims": odr_skim_stack_wrapper,
"dor_skims": dor_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
if network_los.zone_system == los.THREE_ZONE:
# fixme - is this a lightweight object?
tvpb = network_los.tvpb
tvpb_logsum_odt = tvpb.wrap_logsum(orig_key=orig_col_name,
dest_key=dest_col_name,
tod_key='out_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_odt')
tvpb_logsum_dot = tvpb.wrap_logsum(orig_key=dest_col_name,
dest_key=orig_col_name,
tod_key='in_period',
segment_key='demographic_segment',
cache_choices=True,
trace_label=trace_label,
tag='tvpb_logsum_dot')
skims.update({
'tvpb_logsum_odt': tvpb_logsum_odt,
'tvpb_logsum_dot': tvpb_logsum_dot
})
# TVPB constants can appear in expressions
constants.update(
network_los.setting('TVPB_SETTINGS.tour_mode_choice.CONSTANTS'))
estimator = estimation.manager.begin_estimation(
'atwork_subtour_mode_choice')
if estimator:
estimator.write_coefficients(
simulate.read_model_coefficients(model_settings))
estimator.write_coefficients_template(
simulate.read_model_coefficient_template(model_settings))
estimator.write_spec(model_settings)
estimator.write_model_settings(model_settings,
model_settings_file_name)
# FIXME run_tour_mode_choice_simulate writes choosers post-annotation
choices_df = run_tour_mode_choice_simulate(
subtours_merged,
tour_purpose='atwork',
model_settings=model_settings,
mode_column_name=mode_column_name,
logsum_column_name=logsum_column_name,
network_los=network_los,
skims=skims,
constants=constants,
estimator=estimator,
chunk_size=chunk_size,
trace_label=trace_label,
trace_choice_name='tour_mode_choice')
# add cached tvpb_logsum tap choices for modes specified in tvpb_mode_path_types
if network_los.zone_system == los.THREE_ZONE:
tvpb_mode_path_types = model_settings.get('tvpb_mode_path_types')
for mode, path_types in tvpb_mode_path_types.items():
for direction, skim in zip(['od', 'do'],
[tvpb_logsum_odt, tvpb_logsum_dot]):
path_type = path_types[direction]
skim_cache = skim.cache[path_type]
print(
f"mode {mode} direction {direction} path_type {path_type}")
for c in skim_cache:
dest_col = f'{direction}_{c}'
if dest_col not in choices_df:
choices_df[
dest_col] = 0 if pd.api.types.is_numeric_dtype(
skim_cache[c]) else ''
choices_df[dest_col].where(choices_df.tour_mode != mode,
skim_cache[c],
inplace=True)
if estimator:
estimator.write_choices(choices_df[mode_column_name])
choices_df[mode_column_name] = \
estimator.get_survey_values(choices_df[mode_column_name], 'tours', mode_column_name)
estimator.write_override_choices(choices_df[mode_column_name])
estimator.end_estimation()
tracing.print_summary('%s choices' % trace_label,
choices_df[mode_column_name],
value_counts=True)
assign_in_place(tours, choices_df)
pipeline.replace_table("tours", tours)
if trace_hh_id:
tracing.trace_df(tours[tours.tour_category == 'atwork'],
label=tracing.extend_trace_label(
trace_label, mode_column_name),
slicer='tour_id',
index_label='tour_id')
force_garbage_collect()
def tour_mode_choice_simulate(tours, persons_merged, skim_dict, skim_stack,
chunk_size, trace_hh_id):
"""
Tour mode choice simulate
"""
trace_label = 'tour_mode_choice'
model_settings = config.read_model_settings('tour_mode_choice.yaml')
spec = tour_mode_choice_spec(model_settings)
primary_tours = tours.to_frame()
assert not (primary_tours.tour_category == 'atwork').any()
persons_merged = persons_merged.to_frame()
nest_spec = config.get_logit_model_settings(model_settings)
constants = config.get_model_constants(model_settings)
logger.info("Running %s with %d tours" %
(trace_label, primary_tours.shape[0]))
tracing.print_summary('tour_types',
primary_tours.tour_type,
value_counts=True)
primary_tours_merged = pd.merge(primary_tours,
persons_merged,
left_on='person_id',
right_index=True,
how='left',
suffixes=('', '_r'))
# setup skim keys
orig_col_name = 'TAZ'
dest_col_name = 'destination'
out_time_col_name = 'start'
in_time_col_name = 'end'
odt_skim_stack_wrapper = skim_stack.wrap(left_key=orig_col_name,
right_key=dest_col_name,
skim_key='out_period')
dot_skim_stack_wrapper = skim_stack.wrap(left_key=dest_col_name,
right_key=orig_col_name,
skim_key='in_period')
od_skim_stack_wrapper = skim_dict.wrap(orig_col_name, dest_col_name)
skims = {
"odt_skims": odt_skim_stack_wrapper,
"dot_skims": dot_skim_stack_wrapper,
"od_skims": od_skim_stack_wrapper,
'orig_col_name': orig_col_name,
'dest_col_name': dest_col_name,
'out_time_col_name': out_time_col_name,
'in_time_col_name': in_time_col_name
}
choices_list = []
for tour_type, segment in primary_tours_merged.groupby('tour_type'):
logger.info("tour_mode_choice_simulate tour_type '%s' (%s tours)" % (
tour_type,
len(segment.index),
))
# name index so tracing knows how to slice
assert segment.index.name == 'tour_id'
choices = run_tour_mode_choice_simulate(
segment,
spec,
tour_type,
model_settings,
skims=skims,
constants=constants,
nest_spec=nest_spec,
chunk_size=chunk_size,
trace_label=tracing.extend_trace_label(trace_label, tour_type),
trace_choice_name='tour_mode_choice')
tracing.print_summary('tour_mode_choice_simulate %s choices' %
tour_type,
choices,
value_counts=True)
choices_list.append(choices)
# FIXME - force garbage collection
force_garbage_collect()
choices = pd.concat(choices_list)
tracing.print_summary('tour_mode_choice_simulate all tour type choices',
choices,
value_counts=True)
# so we can trace with annotations
primary_tours['tour_mode'] = choices
# but only keep mode choice col
all_tours = tours.to_frame()
# uncomment to save annotations to table
# assign_in_place(all_tours, annotations)
assign_in_place(all_tours, choices.to_frame('tour_mode'))
pipeline.replace_table("tours", all_tours)
if trace_hh_id:
tracing.trace_df(primary_tours,
label=tracing.extend_trace_label(
trace_label, 'tour_mode'),
slicer='tour_id',
index_label='tour_id',
warn_if_empty=True)
