def reduce_memory(self, temp_df, ttime_df):
"""
:param temp_df:
:param ttime_df:
:return:
"""
# set dictionary of Dtypes to reduce size requirement of travel time dataframe
dtype_temp_df = {}
dtype_defs = common.dtype_defintions(
control_parameters.dirListing,
control_parameters.EarlyValidFiles.getJSONFileList())
dtype_temp_df = dtype_defs[
control_parameters.EarlyValidFiles.DTYPE_TRESO_TRIPS]
dtype_ttime_df = dtype_defs[
control_parameters.EarlyValidFiles.DTYPE_SOV_TIMES]
# save some memory
temp_df.drop(['is_virtual', 'trip_km'], axis=1, inplace=True)
for key, value in dtype_ttime_df.items():
ttime_df[key] = ttime_df[key].astype(value)
for key, value in dtype_temp_df.items():
temp_df[key] = temp_df[key].astype(value)
validation.logger.info(
"Dtypes for the trips file are reset to reduce the memory footprint"
)
return (temp_df, ttime_df)
def run(self, hbw_trip_bin, hbw_stn_bin, hbw_egg_bin, purpose, trips_vehtype):
# batch requisite json file for DTYPE
dataFrameDtype = common.dtype_defintions(control_parameters.dirListing,
EarlyValidFiles.getJSONFileList())
# lists of binary probabilities from MLOGIT
hbw_trip_bin = [] # get the list of probability chunks
hbw_stn_bin = []
hbw_egg_bin = []
# With the data batched in it is time to now create access and egress station files for the trip purposes.
# Need to first split the files by creating an acess and egress index
access_indx = list(range(0, 14)) + list(range(26, 38)) + list(range(51, 63)) + list(range(75, 77))
egress_indx = list(range(0, 2)) + list(range(14, 26)) + list(range(38, 51)) + list(range(63, 75))
for chunk_prob, chunk_station, chunk_egress in zip(hbw_trip_bin, hbw_stn_bin, hbw_egg_bin):
hbw_prob = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_prob),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_ELEMENTAL_PROB]]))
hbw_stn = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_station),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_STATION_CHOICE]]))
hbw_eg_prob = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_egress),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_EGRESS_PROB]]))
# for work get the columns that represent the access and egress stations as they are mixed in Bill's file
hbw_st_acc = hbw_stn.iloc[:, access_indx]
hbw_st_egg = hbw_stn.iloc[:, egress_indx]
# get the relevant columns to start the process
hbw_prob_l1 = hbw_prob.iloc[:, 3:]
hbw_prob_append = hbw_prob.iloc[:, 0:3]
# The structure of the egress probability file is such that it requires some adjustments. Specifically,
# the columns need to get appended as a row. Get the first two columns and then drop all the primary mode
# columns as we only need the access mode and its probabilities in the df for Cheval
# first_cols = [0, 1]
s_pos = list(range(2, hbw_eg_prob.shape[1], 3))
hbw_eg_prob.drop(hbw_eg_prob.columns[s_pos], axis=1, inplace=True)
# run for the vehicle segments
for veh_segment in range(0, 4):
hbw_mode = elemental_mode(hbw_prob_l1, veh_segment, hbw_prob_append, purpose, trips_vehtype)
hbw_mode = access_egress_station(hbw_st_acc, hbw_st_egg, hbw_mode)
hbw_mode = egress_prob(hbw_mode, hbw_eg_prob)
return hbw_mode
def __init__(self, seed):
self.seed = seed
# check if files exist. If not, then stop program
if not self.validation():
mprobe_valid.logger.info("validation failed")
exit(0)
else:
mprobe_valid.logger.info("validation success")
# batch requisite json file for DTYPE. This covers the household and trip dataframes only
self.dataFrameDtype = common.dtype_defintions(mprobe_valid.dirListing_abm,
mprobe_valid.EarlyValidFiles_MlogitProbe.getJSONFileList())
def __init__(self, seed):
self.seed = seed
# check if files exist. If not, then stop program
if not self.validation():
common.logger.info("validataion failed")
exit(0)
else:
common.logger.info("validataion success")
# batch requisite json file for DTYPE
self.dataFrameDtype = common.dtype_defintions(
control_parameters.dirListing, EarlyValidFiles.getJSONFileList())
def __init__(self, seed):
self.seed = seed
# check if files exist. If not, then stop program
if not self.validation():
common.logger.info("validataion failed")
exit(0)
else:
common.logger.info("validataion success")
# batch requisite json file for DTYPE
self.dataFrameDtype = common.dtype_defintions(control_parameters.dirListing,
EarlyValidFiles.getJSONFileList())
def __init__(self, seed):
self.seed = seed
# check if files exist. If not, then stop program
if not self.validation():
mprobe_valid.logger.info("validataion failed")
exit(0)
else:
mprobe_valid.logger.info("validataion success")
# batch requisite json file for DTYPE
self.dataFrameDtype = common.dtype_defintions(
mprobe_valid.dirListing_abm,
mprobe_valid.EarlyValidFiles_MlogitProbe.getJSONFileList())
def run(self, peak_offpeak, purpose, trips_vehtype):
#hbw_trip_bin, hbw_stn_bin, hbw_egg_bin =
# batch requisite json file for DTYPE
common.logger.info("Batch in the DTYPE definitions")
dataFrameDtype = common.dtype_defintions(
control_parameters.dirListing, EarlyValidFiles.getJSONFileList())
hbw_trip_bin, hbw_stn_bin, hbw_egg_bin = self.getPeak_OffPeakFile(
peak_offpeak, purpose)
# dictionary of dfs to collect chunk df
hbw_mode_allchunks = {}
# With the data batched in it is time to now create access and egress station files for the trip purposes.
# Need to first split the files by creating an acess and egress index
access_indx = list(range(0, 14)) + list(range(26, 38)) + list(
range(51, 63)) + list(range(75, 77))
egress_indx = list(range(0, 2)) + list(range(14, 26)) + list(
range(38, 51)) + list(range(63, 75))
for chunk_prob, chunk_station, chunk_egress in zip(
hbw_trip_bin, hbw_stn_bin, hbw_egg_bin):
hbw_prob = pd.DataFrame(
np.fromfile(os.path.join(control_parameters.dirListing,
chunk_prob),
dtype=[
tuple(t) for t in dataFrameDtype[
EarlyValidFiles.DTYPE_ELEMENTAL_PROB]
]))
hbw_stn = pd.DataFrame(
np.fromfile(os.path.join(control_parameters.dirListing,
chunk_station),
dtype=[
tuple(t) for t in dataFrameDtype[
EarlyValidFiles.DTYPE_STATION_CHOICE]
]))
hbw_eg_prob = pd.DataFrame(
np.fromfile(os.path.join(control_parameters.dirListing,
chunk_egress),
dtype=[
tuple(t) for t in dataFrameDtype[
EarlyValidFiles.DTYPE_EGRESS_PROB]
]))
# for work get the columns that represent the access and egress stations as they are mixed in Bill's file
hbw_st_acc = hbw_stn.iloc[:, access_indx]
hbw_st_egg = hbw_stn.iloc[:, egress_indx]
# get the relevant columns to start the process
hbw_prob_l1 = hbw_prob.iloc[:, 3:]
hbw_prob_append = hbw_prob.iloc[:, 0:3]
# The structure of the egress probability file is such that it requires some adjustments. Specifically,
# the columns need to get appended as a row. Get the first two columns and then drop all the primary mode
# columns as we only need the access mode and its probabilities in the df for Cheval
# first_cols = [0, 1]
s_pos = list(range(2, hbw_eg_prob.shape[1], 3))
hbw_eg_prob.drop(hbw_eg_prob.columns[s_pos], axis=1, inplace=True)
# run for the vehicle segments
hbw_modes = {}
common.logger.info(
"Start running vehicle segments for each chunk %s" %
chunk_prob)
for veh_segment in range(0, 4):
hbw_mode = self.elemental_mode(hbw_prob_l1, veh_segment,
hbw_prob_append, purpose,
trips_vehtype)
if len(hbw_mode) > 0:
hbw_mode = self.access_egress_station(
hbw_st_acc, hbw_st_egg, hbw_mode)
hbw_mode = self.egress_prob(hbw_mode, hbw_eg_prob)
hbw_modes[veh_segment] = hbw_mode
if len(hbw_modes) > 0:
hbw_mode_allchunks[chunk_prob] = pd.concat(hbw_modes.values(),
ignore_index=True)
return pd.concat(hbw_mode_allchunks.values(),
ignore_index=True)
# if the probability file and consequently the remaining files are empty then return an empty dataframe
hbw_mode_allchunks = pd.DataFrame()
return hbw_mode_allchunks
def __init__(self):
# batch requisite json file for DTYPE
mprobe_valid.logger.info("Get DTYPE definitions for the various files")
self.dataFrameDtype = common.dtype_defintions(mprobe_valid.dirListing_abm,
mprobe_valid.EarlyValidFiles_MlogitProbe.getJSONFileList())
def run(self, hbw_trip_bin, hbw_stn_bin, hbw_egg_bin, purpose, trips_vehtype):
# batch requisite json file for DTYPE
dataFrameDtype = common.dtype_defintions(control_parameters.dirListing,
EarlyValidFiles.getJSONFileList())
# lists of binary probabilities from MLOGIT
hbw_trip_bin = [] # get the list of probability chunks
hbw_stn_bin = []
hbw_egg_bin = []
# With the data batched in it is time to now create access and egress station files for the trip purposes.
# Need to first split the files by creating an acess and egress index
access_indx = list(range(0, 14)) + list(range(26, 38)) + list(range(
51, 63)) + list(range(75, 77))
egress_indx = list(range(0, 2)) + list(range(14, 26)) + list(range(
38, 51)) + list(range(63, 75))
for chunk_prob, chunk_station, chunk_egress in zip(hbw_trip_bin,
hbw_stn_bin,
hbw_egg_bin):
hbw_prob = pd.DataFrame(
np.fromfile(os.path.join(control_parameters.dirListing,
chunk_prob),
dtype=[
tuple(t) for t in dataFrameDtype[
EarlyValidFiles.DTYPE_ELEMENTAL_PROB]
]))
hbw_stn = pd.DataFrame(
np.fromfile(os.path.join(control_parameters.dirListing,
chunk_station),
dtype=[
tuple(t) for t in dataFrameDtype[
EarlyValidFiles.DTYPE_STATION_CHOICE]
]))
hbw_eg_prob = pd.DataFrame(
np.fromfile(
os.path.join(control_parameters.dirListing, chunk_egress),
dtype=[
tuple(t)
for t in dataFrameDtype[EarlyValidFiles.DTYPE_EGRESS_PROB]
]))
# for work get the columns that represent the access and egress stations as they are mixed in Bill's file
hbw_st_acc = hbw_stn.iloc[:, access_indx]
hbw_st_egg = hbw_stn.iloc[:, egress_indx]
# get the relevant columns to start the process
hbw_prob_l1 = hbw_prob.iloc[:, 3:]
hbw_prob_append = hbw_prob.iloc[:, 0:3]
# The structure of the egress probability file is such that it requires some adjustments. Specifically,
# the columns need to get appended as a row. Get the first two columns and then drop all the primary mode
# columns as we only need the access mode and its probabilities in the df for Cheval
# first_cols = [0, 1]
s_pos = list(range(2, hbw_eg_prob.shape[1], 3))
hbw_eg_prob.drop(hbw_eg_prob.columns[s_pos], axis=1, inplace=True)
# run for the vehicle segments
for veh_segment in range(0, 4):
hbw_mode = elemental_mode(hbw_prob_l1, veh_segment,
hbw_prob_append, purpose, trips_vehtype)
hbw_mode = access_egress_station(hbw_st_acc, hbw_st_egg, hbw_mode)
hbw_mode = egress_prob(hbw_mode, hbw_eg_prob)
return hbw_mode
def run(self, peak_offpeak, purpose, trips_vehtype):
#hbw_trip_bin, hbw_stn_bin, hbw_egg_bin =
# batch requisite json file for DTYPE
common.logger.info("Batch in the DTYPE definitions")
dataFrameDtype = common.dtype_defintions(control_parameters.dirListing,
EarlyValidFiles.getJSONFileList())
hbw_trip_bin, hbw_stn_bin, hbw_egg_bin = self._getPeak_OffPeakFile(peak_offpeak, purpose)
# dictionary of dfs to collect chunk df
hbw_mode_allchunks = {}
# With the data batched in it is time to now create access and egress station files for the trip purposes.
# Need to first split the files by creating an acess and egress index
access_indx = list(range(0, 14)) + list(range(26, 38)) + list(range(51, 63)) + list(range(75, 77))
egress_indx = list(range(0, 2)) + list(range(14, 26)) + list(range(38, 51)) + list(range(63, 75))
for chunk_prob, chunk_station, chunk_egress in zip(hbw_trip_bin, hbw_stn_bin, hbw_egg_bin):
hbw_prob = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_prob),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_ELEMENTAL_PROB]]))
hbw_stn = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_station),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_STATION_CHOICE]]))
hbw_eg_prob = pd.DataFrame(np.fromfile(os.path.join(control_parameters.dirListing, chunk_egress),
dtype=[tuple(t) for t in
dataFrameDtype[EarlyValidFiles.DTYPE_EGRESS_PROB]]))
# for work get the columns that represent the access and egress stations as they are mixed in Bill's file
hbw_st_acc = hbw_stn.iloc[:, access_indx]
hbw_st_egg = hbw_stn.iloc[:, egress_indx]
# get the relevant columns to start the process
hbw_prob_l1 = hbw_prob.iloc[:, 3:]
hbw_prob_append = hbw_prob.iloc[:, 0:3]
# The structure of the egress probability file is such that it requires some adjustments. Specifically,
# the columns need to get appended as a row. Get the first two columns and then drop all the primary mode
# columns as we only need the access mode and its probabilities in the df for Cheval
# first_cols = [0, 1]
s_pos = list(range(2, hbw_eg_prob.shape[1], 3))
hbw_eg_prob.drop(hbw_eg_prob.columns[s_pos], axis=1, inplace=True)
# run for the vehicle segments
hbw_modes = {}
common.logger.info("Start running vehicle segments for each chunk %s" %chunk_prob)
for veh_segment in range(0, 4):
hbw_mode = self.elemental_mode(hbw_prob_l1, veh_segment, hbw_prob_append, purpose, trips_vehtype)
if len(hbw_mode)>0:
hbw_mode = self.access_egress_station(hbw_st_acc, hbw_st_egg, hbw_mode)
hbw_mode = self.egress_prob(hbw_mode, hbw_eg_prob)
hbw_modes[veh_segment] = hbw_mode
if len(hbw_modes)>0:
hbw_mode_allchunks[chunk_prob] = pd.concat(hbw_modes.values(), ignore_index=True)
return pd.concat(hbw_mode_allchunks.values(), ignore_index=True)
# if the probability file and consequently the remaining files are empty then return an empty dataframe
hbw_mode_allchunks = pd.DataFrame()
return hbw_mode_allchunks