def compact_land_use(
mc_obj,
zone_factor_file,
):
''' shifts growth in trips according to zones and factors defined in file
:param mc_obj: the mode choice object containing parameters and inputs
:param zone_factor_file: file containing zones and growth factor
'''
modified_trips = mc_obj.config.scen_path + r"growth_shift_trip_tables.omx"
pre_MC_trip_file_2016 = mc_obj.config.data_path + r"..\2016\pre_MC_trip_6_purposes.omx"
pre_MC_trip_file_future = mc_obj.config.data_path + r".\pre_MC_trip_6_purposes.omx"
zone_factors = pd.read_csv(mc_obj.config.param_path +
zone_factor_file).sort_values('ID')
all_taz = pd.read_csv(mc_obj.data_paths['taz_file'])
taz_factors = all_taz[['ID',
'TOWN']].merge(zone_factors, on='ID',
how='left').fillna(1)[:md.max_zone]
# ==DEBUG===========================================================================
# margs_2016 = {}
# margs_2040_base = {}
# margs_2040_new = {}
#
# margs_2016['ID'] = taz_factors['ID']
# margs_2016['BOSTON_NB'] = taz_factors['BOSTON_NB']
# margs_2040_base['ID'] = taz_factors['ID']
# margs_2040_base['BOSTON_NB'] = taz_factors['BOSTON_NB']
# margs_2040_new['ID'] = taz_factors['ID']
# margs_2040_new['BOSTON_NB'] = taz_factors['BOSTON_NB']
# =============================================================================
with omx.open_file(pre_MC_trip_file_2016) as f1, omx.open_file(
pre_MC_trip_file_future) as f2, omx.open_file(modified_trips,
'w') as fout:
for name in f1.list_matrices():
tt_2016 = np.array(f1[name])[:md.max_zone, :md.max_zone]
tt_future = np.array(f2[name])[:md.max_zone, :md.max_zone]
# only apply to home-based productions
if (name[:3] != 'NHB'):
diff = np.maximum(tt_future - tt_2016, 0)
adj_diff = np.multiply(
diff.T, (np.array(taz_factors['POP_FACTOR']) - 1)).T
tt_newfuture = tt_future + adj_diff
else:
tt_newfuture = tt_future
# =DEBUG============================================================================
# margs_2016[name] = tt_2016.sum(axis = 1)
# margs_2040_base[name] = tt_future.sum(axis = 1)
# margs_2040_new[name] = tt_newfuture.sum(axis = 1)
# =============================================================================
fout[name] = tt_newfuture
mc_obj.pre_MC_trip_table = mtx.store_omx_as_dict(modified_trips)
def create_subset(dest_store, dest_skims, maxZone, households_sample_size=0):
dest_store_path = os.path.join(dest_data_dir, dest_store)
dest_skims_path = os.path.join(dest_data_dir, dest_skims)
print('land_use_taz')
df = pd.read_hdf(source_store, 'land_use_taz')
df = df[df.index <= maxZone]
df.to_hdf(dest_store_path, 'land_use_taz')
del df
print('households')
hh_df = pd.read_hdf(source_store, 'households')
hh_df = hh_df[hh_df.TAZ <= maxZone]
if households_sample_size:
hh_df = hh_df.take(np.random.choice(len(hh_df), size=households_sample_size, replace=False))
hh_df.to_hdf(dest_store_path, 'households')
print('persons')
per_df = pd.read_hdf(source_store, 'persons')
per_df = per_df[per_df.household_id.isin(hh_df.index)]
per_df.to_hdf(dest_store_path, 'persons')
# process all skims
skims = omx.open_file(source_skims)
skims_out = omx.open_file(dest_skims_path, 'w')
skimsToProcess = skims.list_matrices()
for skimName in skimsToProcess:
print(skimName)
skims_out[skimName] = skims[skimName][0:maxZone, 0:maxZone]
skims_out[skimName].attrs.TITLE = '' # remove funny character for OMX viewer
def crop_omx(omx_file_name, zones, num_outfiles=1):
skim_data_type = np.float32
omx_in = omx.open_file(input_path(f"{omx_file_name}.omx"))
print(f"omx_in shape {omx_in.shape()}")
offset_map = None
for mapping_name in omx_in.listMappings():
_offset_map = np.asanyarray(omx_in.mapentries(mapping_name))
if offset_map is not None or not (_offset_map == np.arange(
1,
len(_offset_map) + 1)).all():
assert offset_map is None or (offset_map == _offset_map).all()
offset_map = _offset_map
if offset_map is not None:
om = pd.Series(offset_map)
om = om[om.isin(zones.values)]
indexes = om.index.values
else:
indexes = zones.index.tolist(
) # index of TAZ in skim (zero-based, no mapping)
labels = zones.values # TAZ zone_ids in omx index order
# create
if num_outfiles == 1:
omx_out = [omx.open_file(output_path(f"{omx_file_name}.omx"), 'w')]
else:
omx_out = [
omx.open_file(output_path(f"{omx_file_name}{i + 1}.omx"), 'w')
for i in range(num_outfiles)
]
for omx_file in omx_out:
omx_file.create_mapping('ZONE', labels)
iskim = 0
for mat_name in omx_in.list_matrices():
# make sure we have a vanilla numpy array, not a CArray
m = np.asanyarray(omx_in[mat_name]).astype(skim_data_type)
m = m[indexes, :][:, indexes]
print(f"{mat_name} {m.shape}")
omx_file = omx_out[iskim % num_outfiles]
omx_file[mat_name] = m
iskim += 1
omx_in.close()
for omx_file in omx_out:
omx_file.close()
def slice_skims(source_data_dir, dest_data_dir, file_name, max_zone):
in_file_path = os.path.join(source_data_dir, file_name)
out_file_path = os.path.join(dest_data_dir, file_name)
# process all skims
skims = omx.open_file(in_file_path)
skims_out = omx.open_file(out_file_path, 'w')
skimsToProcess = skims.listMatrices()
for skimName in skimsToProcess:
print "slice_skims %s: %s" % (file_name, skimName)
skims_out[skimName] = skims[skimName][0:max_zone, 0:max_zone]
skims_out[skimName].attrs.TITLE = '' # remove funny character for OMX viewer
def test_highway():
from tm2py.controller import RunController
from tm2py.examples import get_example
import openmatrix as _omx
union_city_root = os.path.join(os.getcwd(), _EXAMPLES_DIR, "UnionCity")
get_example(example_name="UnionCity",
example_subdir=_EXAMPLES_DIR,
root_dir=os.getcwd())
controller = RunController([
os.path.join(_EXAMPLES_DIR, r"scenario_config.toml"),
os.path.join(_EXAMPLES_DIR, r"model_config.toml"),
],
run_dir=union_city_root)
controller.run()
root = os.path.join(controller.run_dir, r"skim_matrices\highway")
ref_root = os.path.join(controller.run_dir, r"ref_skim_matrices\highway")
open_files = []
file_names = [name for name in os.listdir(root) if name.endswith(".omx")]
different_skims = []
try:
for name in file_names:
skims = _omx.open_file(os.path.join(root, name))
open_files.append(skims)
ref_skims = _omx.open_file(os.path.join(ref_root, name))
open_files.append(ref_skims)
for key in skims.list_matrices():
data = skims[key].read()
ref_data = ref_skims[key].read()
if not (data == ref_data).all():
different_skims.append(key)
finally:
for f in open_files:
f.close()
assert (
len(different_skims) == 0
), f"there are {len(different_skims)} different skims: {','.join(different_skims)}"
count_different_lines = 0
with open(os.path.join(root, "HWYSKIM_MAZMAZ_DA.csv")) as data:
with open(os.path.join(ref_root, "HWYSKIM_MAZMAZ_DA.csv")) as ref_data:
for line in data:
ref_line = next(ref_data)
if ref_line != line:
count_different_lines += 1
assert (
count_different_lines == 0
), f"HWYSKIM_MAZMAZ_DA.csv differs on {count_different_lines} lines"
def test_contains():
with omx.open_file(TEST_FILE, 'w') as f:
add_m1_node(f)
nt.assert_in('m1', f)
# keep this here to be sure we're actually running
# File.__contains__
assert 'm1' in f
def test_set_get_del():
with omx.open_file(TEST_FILE, 'w') as f:
add_m1_node(f)
npt.assert_array_equal(f['m1'], ones5x5())
nt.assert_equal(f.shape(), (5, 5))
del f['m1']
nt.assert_not_in('m1', f)
def vmt_by_neighborhood(mc_obj, out_fn=None):
'''
Summarizes Truck VMT production and attraction by the 26 Boston neighborhoods.
:param mc_obj: mode choice module object as defined in the IPython notebook
:param out_fn: output csv filename; if None specified, in the output path defined in config.py
:param by: grouping used for the summary; if None specified, only aggregate production and attraction will be provided.
'''
trk_trip = omx.open_file(
mc_obj.config.data_path + mc_obj.data_paths['truck_trip_table'], 'r')
if out_fn is None:
out_fn = mc_obj.config.out_path + f'trk_vmt_by_neighborhood.csv'
vmt_master_table = pd.DataFrame(
columns=['Production', 'Attraction', 'truck'])
for truck in md.truck_categories:
if trk_trip[truck]:
trk_trip_table = np.array(trk_trip[truck])[0:md.max_zone,
0:md.max_zone]
vmt_table = __mt_prod_attr_nhood(mc_obj, trk_trip_table,
mc_obj.drive_skim_PK)
vmt_table['truck'] = truck
vmt_master_table = vmt_master_table.append(vmt_table, sort=True)
vmt_summary = pd.concat([
vmt_master_table.groupby(['truck', 'BOSTON_NB']).sum().loc[truck]
for truck in md.truck_categories
],
axis=1,
keys=md.truck_categories)
vmt_summary.to_csv(out_fn)
trk_trip.close()
def load(self, file_path: str):
"""
Loads matrix from disk. All cores and indices are load. First index is default
Parameters
----------
file_path: Path to AEM file on disk
------------------------------------------------------------
Example
>>> zones_in_the_model = 3317
>>> names_list = ['Car trips', 'pt trips', 'DRT trips', 'bike trips', 'walk trips']
>>> mat = AequilibraeMatrix()
>>> mat.create_empty(file_name='my/path/to/file', zones=zones_in_the_model, matrix_names= names_list)
>>> mat.close()
>>> mat2 = AequilibraeMatrix()
>>> mat2.load('my/path/to/file')
>>> mat2.zones
3317
"""
self.file_path = file_path
if os.path.splitext(file_path)[-1].upper() == ".OMX":
self.omx = True
self.omx_file = omx.open_file(file_path, "r")
self.__load_omx__()
else:
self.__load_aem__()
def to_omx(file: str, matrices: Dict[str, MATRIX_TYPES], zone_index: pd.Index = None, title: str = '',
descriptions: Dict[str, str] = None, attrs: Dict[str, dict] = None, mapping: str = 'zone_numbers'):
"""Creates a new (or overwrites an old) OMX file with a collection of matrices.
Args:
file: OMX to write.
matrices: Collection of matrices to write. MUST be a dict, to permit the encoding of matrix metadata,
and must contain the same types: all Numpy arrays, all Series, or all DataFrames. Checking is done to
ensure that all items have the same shape and labels.
zone_index: Override zone labels to use. Generally only useful if writing a dict of raw Numpy arrays.
title: The title saved in the OMX file.
descriptions: A dict of descriptions (one for each given matrix), or None to not use.
attrs: A dict of dicts (one for each given matrix), or None to not use
mapping: Name of the mapping internal to the OMX file
"""
matrices, zone_index = _prep_matrix_dict(matrices, zone_index)
if descriptions is None:
descriptions = {name: '' for name in matrices.keys()}
if attrs is None:
attrs = {name: None for name in matrices.keys()}
file = str(file) # Converts from Path
with omx.open_file(file, mode='w', title=title) as omx_file:
omx_file.create_mapping(mapping, zone_index.tolist())
for name, array in matrices.items():
description = descriptions[name]
attr = attrs[name]
omx_file.create_matrix(name, obj=np.ascontiguousarray(array), title=description, attrs=attr)
return
def load(self, file_path: str):
"""
Loads matrix from disk. All cores and indices are load. First index is default
Args:
file_path (:obj:`str`): Path to AEM or OMX file on disk
::
zones_in_the_model = 3317
names_list = ['Car trips', 'pt trips', 'DRT trips', 'bike trips', 'walk trips']
mat = AequilibraeMatrix()
mat.create_empty(file_name='my/path/to/file', zones=zones_in_the_model, matrix_names= names_list)
mat.close()
mat2 = AequilibraeMatrix()
mat2.load('my/path/to/file.omx')
mat2.zones
3317
"""
self.file_path = file_path
if os.path.splitext(file_path)[-1].upper() == ".OMX":
self.__omx = True
self.omx_file = omx.open_file(file_path, "a")
self.__load_omx__()
else:
self.__load_aem__()
def load_skims(omx_file_path, skim_info, skim_buffers):
skim_data = skim_data_from_buffers(skim_buffers, skim_info)
block_offsets = skim_info['block_offsets']
omx_keys = skim_info['omx_keys']
# read skims into skim_data
with omx.open_file(omx_file_path) as omx_file:
for skim_key, omx_key in iteritems(omx_keys):
omx_data = omx_file[omx_key]
assert np.issubdtype(omx_data.dtype, np.floating)
block, offset = block_offsets[skim_key]
block_data = skim_data[block]
logger.debug(
"load_skims load omx_key %s skim_key %s to block %s offset %s"
% (omx_key, skim_key, block, offset))
# this will trigger omx readslice to read and copy data to skim_data's buffer
a = block_data[:, :, offset]
a[:] = omx_data[:]
logger.info("load_skims loaded skims from %s" % (omx_file_path, ))
def load_skims(omx_file_path, skim_info, skim_buffers):
skim_data = skim_data_from_buffers(skim_buffers, skim_info)
block_offsets = skim_info['block_offsets']
omx_keys = skim_info['omx_keys']
# read skims into skim_data
with omx.open_file(omx_file_path) as omx_file:
for skim_key, omx_key in iteritems(omx_keys):
omx_data = omx_file[omx_key]
assert np.issubdtype(omx_data.dtype, np.floating)
block, offset = block_offsets[skim_key]
block_data = skim_data[block]
logger.debug("load_skims load omx_key %s skim_key %s to block %s offset %s" %
(omx_key, skim_key, block, offset))
# this will trigger omx readslice to read and copy data to skim_data's buffer
a = block_data[:, :, offset]
a[:] = omx_data[:]
logger.info("load_skims loaded skims from %s" % (omx_file_path, ))
def __init__(self,
omx_file_path,
name,
zone_index,
transpose=False,
cache_skims=True):
self.skims = {}
self.name = name
self.transpose = transpose # never used by any caller?
self.omx = omx.open_file(omx_file_path, 'r')
self.omx_shape = tuple([int(s) for s in self.omx.shape()])
# skims must be same shape as zone file
# (or we will need to be smarter about discontinuous zone ids)
self.length = zone_index.shape[0]
assert self.omx_shape[0] == self.length
assert self.omx_shape[1] == self.length
self.skim_dtype = np.float64
self.cache_skims = cache_skims
self.usage = {key: 0 for key in self.omx.listMatrices()}
logger.debug("omx file %s skim shape: %s number of skims: %s" %
(name, self.omx_shape, len(self.usage)))
def test_open_readonly_hdf5_file():
with tables.open_file(TEST_FILE, 'w'):
pass
assert os.path.isfile(TEST_FILE)
with omx.open_file(TEST_FILE, 'r'):
pass
def omx_file(data_dir, asim_settings):
print("opening omx file")
fname = os.path.join(data_dir, asim_settings["skims_file"])
file = omx.open_file(fname)
asim_utils.close_on_exit(file, fname)
return file
def land_use_growth_shift(config, factor=0.5):
'''apply land use growth shifts to tazs defined in Densified_TAZs.csv
:param config: the scenario object defined in the config.py - path to output trip table and zone files
:param factor: share of growth to be reallocated
'''
modified_2040 = config.scen_path + r"growth_shift_trip_tables.omx"
pre_MC_trip_file_2016 = config.data_path + r"..\2016\pre_MC_trip_6_purposes.omx"
pre_MC_trip_file_2040 = config.data_path + r"..\2040\pre_MC_trip_6_purposes.omx"
dense_taz = pd.read_csv(config.scen_path +
r"Densified_TAZs.csv").sort_values('ID_FOR_CS')[[
'ID_FOR_CS'
]]
dense_taz['dense'] = 1
all_taz = pd.read_csv(config.taz_file)
dense_taz_list = all_taz[['ID_FOR_CS']].merge(
dense_taz, on='ID_FOR_CS',
how='left').fillna(0).astype(bool)[:md.max_zone]
with omx.open_file(pre_MC_trip_file_2016) as f1, omx.open_file(
pre_MC_trip_file_2040) as f2, omx.open_file(modified_2040,
'w') as fout:
for name in f1.list_matrices():
tt_2016 = np.array(f1[name])[:md.max_zone, :md.max_zone]
tt_2040 = np.array(f2[name])[:md.max_zone, :md.max_zone]
diff = tt_2040 - tt_2016
prod_sum_2016 = tt_2016.sum(axis=1)
prod_sum_2040 = tt_2040.sum(axis=1)
prod_sum_diff = diff.sum(axis=1)
sum_to_transfer = prod_sum_diff[~dense_taz_list['dense'].
values].sum() * factor
prod_sum_diff[~dense_taz_list['dense'].values] = prod_sum_diff[
~dense_taz_list['dense'].values] * (1 - factor)
prod_sum_diff[dense_taz_list['dense'].values] = (
prod_sum_diff[dense_taz_list['dense'].values] +
sum_to_transfer * prod_sum_2040[dense_taz_list['dense'].values]
/ prod_sum_2040[dense_taz_list['dense'].values].sum())
# distribute prod_sum to each attraction zone
tt_new = pd.DataFrame(tt_2040).divide(
pd.Series(prod_sum_2040),
axis=0).fillna(0).multiply(prod_sum_2016 + prod_sum_diff,
axis=0).values
fout[name] = tt_new
def read_omx(file: str, matrices: Iterable[str] = None, mapping: str = None, raw: bool = False,
tall: bool = False, squeeze: bool = True) -> Union[MATRIX_TYPES, Dict[str, MATRIX_TYPES]]:
"""
Reads Open Matrix (OMX) files. An OMX file can contain multiple matrices, so this function
typically returns a Dict.
Args:
file: OMX file from which to read. Cannot be an open file handler.
matrices: List of matrices to read from the file. If None, all matrices will be read.
mapping: The zone number mapping to use, if known in advance. If None, and the OMX file only contains
one mapping, then that one is used. No mapping is read if raw is False.
raw: If True, matrices will be returned as raw Numpy arrays. Otherwise, Pandas objects are returned
tall: If True, matrices will be returned in 1D format (pd.Series if raw is False). Otherwise, a 2D object
is returned.
squeeze: If True, and the file contains exactly one matrix, return that matrix instead of a Dict.
Returns:
The matrix, or matrices contained in the OMX file.
"""
file = str(file)
with omx.open_file(file, mode='r') as omx_file:
if mapping is None and not raw:
all_mappings = omx_file.list_mappings()
assert len(all_mappings) == 1
mapping = all_mappings[0]
if matrices is None:
matrices = sorted(omx_file.list_matrices())
else:
matrices = sorted(matrices)
if not raw:
labels = pd.Index(omx_file.mapping(mapping).keys())
if tall:
labels = pd.MultiIndex.from_product([labels, labels], names=['o', 'd'])
return_value = {}
for matrix_name in matrices:
wrapper = omx_file[matrix_name]
matrix = wrapper.read()
if tall:
n = matrix.shape[0] * matrix.shape[1]
matrix.shape = n
if not raw:
if tall: matrix = pd.Series(matrix, index=labels)
else: matrix = pd.DataFrame(matrix, index=labels, columns=labels)
matrix.name = matrix_name
return_value[matrix_name] = matrix
if len(matrices) == 1 and squeeze:
return return_value[matrices[0]]
return return_value
def omx_file(data_dir, settings):
logger.debug("opening omx file")
fname = os.path.join(data_dir, settings["skims_file"])
file = omx.open_file(fname)
pipeline.close_on_exit(file, fname)
return file
def test_get_length_of_file():
with omx.open_file(TEST_FILE, 'w') as f:
f['m1'] = np.ones((5, 5))
f['m2'] = np.ones((5, 5))
f['m3'] = np.ones((5, 5))
f['m4'] = np.ones((5, 5))
f['m5'] = np.ones((5, 5))
nt.assert_equal(len(f), 5)
nt.assert_equal(len(f.list_matrices()), 5)
def test_add_numpy_matrix_using_brackets():
with omx.open_file(TEST_FILE, 'w') as f:
f['m1'] = ones5x5()
npt.assert_array_equal(f['m1'], ones5x5())
nt.assert_equal(f.shape(), (5, 5))
# test check for shape matching
with nt.assert_raises(omx.Exceptions.ShapeError):
f.create_matrix('m2', obj=np.ones((8, 8)))
def regress():
persons_df = pipeline.get_table('persons')
persons_df = persons_df[persons_df.household_id == HH_ID]
print("persons_df\n%s" % persons_df[['value_of_time', 'distance_to_work']])
"""
persons_df
person_id value_of_time distance_to_work
person_id
3249922 23.349532 0.62
3249923 23.349532 0.62
"""
tours_df = pipeline.get_table('tours')
regress_tour_modes(tours_df)
assert tours_df.shape[0] > 0
assert not tours_df.tour_mode.isnull().any()
# optional logsum column was added to all tours except mandatory
assert 'destination_logsum' in tours_df
if (tours_df.destination_logsum.isnull() !=
(tours_df.tour_category == 'mandatory')).any():
print(tours_df[(tours_df.destination_logsum.isnull() !=
(tours_df.tour_category == 'mandatory'))])
assert (tours_df.destination_logsum.isnull() == (
tours_df.tour_category == 'mandatory')).all()
# mode choice logsum calculated for all tours
assert 'mode_choice_logsum' in tours_df
assert not tours_df.mode_choice_logsum.isnull().any()
trips_df = pipeline.get_table('trips')
assert trips_df.shape[0] > 0
assert not trips_df.purpose.isnull().any()
assert not trips_df.depart.isnull().any()
assert not trips_df.trip_mode.isnull().any()
# mode_choice_logsum calculated for all trips
assert not trips_df.mode_choice_logsum.isnull().any()
# should be at least two tours per trip
assert trips_df.shape[0] >= 2 * tours_df.shape[0]
# write_trip_matrices
trip_matrices_file = config.output_file_path('trips_md.omx')
assert os.path.exists(trip_matrices_file)
trip_matrices = omx.open_file(trip_matrices_file)
assert trip_matrices.shape() == (25, 25)
assert 'WALK_MD' in trip_matrices.list_matrices()
walk_trips = np.array(trip_matrices['WALK_MD'])
assert walk_trips.dtype == np.dtype('float64')
trip_matrices.close()
def compile_omx_files(manifest, dest_file_name, source_data_dir):
with omx.open_file(dest_file_name, 'a') as dest_omx:
for row in manifest.itertuples(index=True):
source_file_name = os.path.join(source_data_dir,
row.source_file_name)
if row.skim_key2:
dest_key = row.skim_key1 + '__' + row.skim_key2
else:
dest_key = row.skim_key1
print(
"Reading '%s' from '%s' in %s' % (dest_key, row.source_key, source_file_name)"
)
with omx.open_file(source_file_name, 'r') as source_omx:
if row.source_key not in source_omx.list_matrices():
print(
"Source matrix with key '%s' not found in file '%s'" %
(
row.source_key,
source_file_name,
))
print(source_omx.list_matrices())
raise RuntimeError(
"Source matrix with key '%s' not found in file '%s'" %
(row.source_key, dest_omx))
data = source_omx[row.source_key]
if dest_key in dest_omx.list_matrices():
print("deleting existing dest key '%s'" % (dest_key, ))
dest_omx.removeNode(dest_omx.root.data, dest_key)
data = np.delete(data,
np.r_[135:140, 420:422, 1781:1788, 2873:2881],
axis=0)
data = np.delete(data,
np.r_[135:140, 420:422, 1781:1788, 2873:2881],
axis=1)
dest_omx[dest_key] = data
def open_skimfile(self):
self.skimfile_path = tkinter.filedialog.askopenfilename()
if self.skimfile_path:
self.skimfile_lbl["text"] = os.path.basename(self.skimfile_path)
self.skims = omx.open_file(self.skimfile_path, "r")
mtxs = self.skims.list_matrices()
self.skim_selector.set_menu(mtxs[0], *mtxs)
self.redraw_map()
def test_skim_writing(skim, tmp_path):
tmp_dir = tmp_path / str(uuid.uuid4())
os.mkdir(tmp_dir)
skim.write_to_file(file_root=tmp_dir, attributes=test_additional_attribs)
with omx.open_file(tmp_dir / "skimmy.omx", 'r') as f:
assert np.alltrue(np.array(f) == test_matrix)
assert f[test_name].attrs.num_zones == test_matrix.shape[0]
assert f[test_name].attrs.name == test_name
assert f[test_name].attrs.index_to_zone_ids == index_to_zone_ids
assert f[test_name].attrs.foo == "bar"
def get_omx_matrix(matrix_dir, omx_file_name, omx_key, close_after_read=True):
if not omx_file_name:
return 0.0
# print "reading %s / %s '%s'" % (matrix_dir, omx_file_name, omx_key)
omx_file_name = os.path.join(matrix_dir, omx_file_name)
omx_file = omx.open_file(omx_file_name, 'r')
matrix = omx_file[omx_key][:, :]
if close_after_read:
# print "closing %s / %s '%s'" % (matrix_dir, omx_file_name, omx_key)
omx_file.close()
return matrix
def test_len_list_iter():
names = ['m{}'.format(x) for x in range(5)]
with omx.open_file(TEST_FILE, 'w') as f:
for m in names:
f[m] = ones5x5()
for mat in f:
npt.assert_array_equal(mat, ones5x5())
nt.assert_equal(len(f), len(names))
nt.assert_equal(f.list_matrices(), names)
def test_export_to_omx(self):
self.new_matrix.export(self.omx_export_name)
omxfile = omx.open_file(self.omx_export_name, "r")
# Check if matrices values are compatible
for m in self.new_matrix.names:
sm = np.nansum(self.new_matrix.matrix[m])
sm2 = np.nansum(np.array(omxfile[m]))
self.assertEqual(sm, sm2, "Matrix {} was exported with the wrong value".format(m))
del omxfile
def cvt_to_omx(txt_file_name, omx_file_name, omx_key):
data = np.loadtxt(txt_file_name, dtype=np.float, delimiter=',')
print "---"
print "cvt_to_omx from", txt_file_name
print "cvt_to_omx to %s / %s" % (omx_file_name, omx_key)
# print m
omx_file = omx.open_file(omx_file_name, 'a')
omx_setMatrix(omx_file, omx_key, data)
omx_file.close()
def write_matrices(aggregate_trips, zone_index, orig_index, dest_index,
model_settings):
"""
Write aggregated trips to OMX format.
The MATRICES setting lists the new OMX files to write.
Each file can contain any number of 'tables', each specified by a
table key ('name') and a trips table column ('data_field') to use
for aggregated counts.
Any data type may be used for columns added in the annotation phase,
but the table 'data_field's must be summable types: ints, floats, bools.
"""
matrix_settings = model_settings.get('MATRICES')
if not matrix_settings:
logger.error('Missing MATRICES setting in write_trip_matrices.yaml')
for matrix in matrix_settings:
filename = matrix.get('file_name')
filepath = config.output_file_path(filename)
logger.info('opening %s' % filepath)
file = omx.open_file(filepath, 'w') # possibly overwrite existing file
table_settings = matrix.get('tables')
for table in table_settings:
table_name = table.get('name')
col = table.get('data_field')
if col not in aggregate_trips:
logger.error(
f'missing {col} column in aggregate_trips DataFrame')
return
hh_weight_col = model_settings.get('HH_EXPANSION_WEIGHT_COL')
if hh_weight_col:
aggregate_trips[col] = aggregate_trips[col] / aggregate_trips[
hh_weight_col]
data = np.zeros((len(zone_index), len(zone_index)))
data[orig_index, dest_index] = aggregate_trips[col]
logger.info('writing %s' % table_name)
file[table_name] = data # write to file
# include the index-to-zone map in the file
logger.info('adding %s mapping for %s zones to %s' %
(zone_index.name, zone_index.size, filename))
file.create_mapping(zone_index.name, zone_index.to_numpy())
logger.info('closing %s' % filepath)
file.close()
def inter_extrap_trucktriptable(config, year=2030):
'''interpolate or extrapolate truck trip tables
:param config: the scenario object defined in the config.py - path to output trip table
:param year: year of interpolated or extrapolated trip table
'''
new_table = config.data_path + r"..\\" + str(year) + "\\truck_trips.omx"
pre_MC_trip_file_2016 = config.data_path + r"..\2016\\truck_trips.omx"
pre_MC_trip_file_2040 = config.data_path + r"..\2040\\truck_trips.omx"
with omx.open_file(pre_MC_trip_file_2016) as f1, omx.open_file(
pre_MC_trip_file_2040) as f2, omx.open_file(new_table,
'w') as fout:
for name in f1.list_matrices():
tt_2016 = np.array(f1[name])[:md.max_zone, :md.max_zone]
tt_2040 = np.array(f2[name])[:md.max_zone, :md.max_zone]
diff = tt_2040 - tt_2016
yearly_diff = diff / 24 # annual growth of trips
growth_years = year - 2016
tt_new = tt_2016 + (yearly_diff * growth_years)
tt_new = tt_new.clip(min=0) # don't let trips go negative
fout[name] = tt_new
def tap_skim_dict(data_dir, settings):
logger.info("loading tap_skim_dict")
cache_skim_key_values = settings['skim_time_periods']['labels']
skim_dict = askim.SkimDict()
for skims_file in settings["tap_skims_files"]:
skims_file_path = config.data_file_path(skims_file)
with omx.open_file(skims_file_path) as omx_file:
add_to_skim_dict(skim_dict, omx_file, cache_skim_key_values)
return skim_dict
def omx_getMatrix(omx_file_name, omx_key):
with omx.open_file(omx_file_name, 'r') as omx_file:
if omx_key not in omx_file.list_matrices():
print "Source matrix with key '%s' not found in file '%s" % (omx_key, omx_file,)
print omx_file.list_matrices()
raise RuntimeError("Source matrix with key '%s' not found in file '%s"
% (omx_key, omx_file,))
data = omx_file[omx_key]
return data
def get_skim_info(omx_file_path, tags_to_load=None):
# this is sys.maxint for p2.7 but no limit for p3
# windows sys.maxint = 2147483647
MAX_BLOCK_BYTES = sys.maxint - 1 if sys.version_info < (3,) else sys.maxsize - 1
# Note: we load all skims except those with key2 not in tags_to_load
# Note: we require all skims to be of same dtype so they can share buffer - is that ok?
# fixme is it ok to require skims be all the same type? if so, is this the right choice?
skim_dtype = np.float32
omx_name = os.path.splitext(os.path.basename(omx_file_path))[0]
with omx.open_file(omx_file_path) as omx_file:
# omx_shape = tuple(map(int, tuple(omx_file.shape()))) # sometimes omx shape are floats!
# fixme call to omx_file.shape() failing in windows p3.5
omx_shape = omx_file.shape()
omx_shape = (int(omx_shape[0]), int(omx_shape[1])) # sometimes omx shape are floats!
omx_skim_names = omx_file.listMatrices()
# - omx_keys dict maps skim key to omx_key
# DISTWALK: DISTWALK
# ('DRV_COM_WLK_BOARDS', 'AM'): DRV_COM_WLK_BOARDS__AM, ...
omx_keys = OrderedDict()
for skim_name in omx_skim_names:
key1, sep, key2 = skim_name.partition('__')
# - ignore composite tags not in tags_to_load
if tags_to_load and sep and key2 not in tags_to_load:
continue
skim_key = (key1, key2) if sep else key1
omx_keys[skim_key] = skim_name
num_skims = len(omx_keys)
skim_data_shape = omx_shape + (num_skims, )
# - key1_subkeys dict maps key1 to dict of subkeys with that key1
# DIST: {'DIST': 0}
# DRV_COM_WLK_BOARDS: {'MD': 1, 'AM': 0, 'PM': 2}, ...
key1_subkeys = OrderedDict()
for skim_key, omx_key in iteritems(omx_keys):
if isinstance(skim_key, tuple):
key1, key2 = skim_key
else:
key1 = key2 = skim_key
key2_dict = key1_subkeys.setdefault(key1, {})
key2_dict[key2] = len(key2_dict)
# - blocks dict maps block name to blocksize (number of subkey skims in block)
# skims_0: 198,
# skims_1: 198, ...
# - key1_block_offsets dict maps key1 to (block, offset) of first skim with that key1
# DISTWALK: (0, 2),
# DRV_COM_WLK_BOARDS: (0, 3), ...
if MAX_BLOCK_BYTES:
max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
max_skims_per_block = max_block_items // np.prod(omx_shape)
else:
max_skims_per_block = num_skims
def block_name(block):
return "skim_%s_%s" % (omx_name, block)
key1_block_offsets = OrderedDict()
blocks = OrderedDict()
block = offset = 0
for key1, v in iteritems(key1_subkeys):
num_subkeys = len(v)
if offset + num_subkeys > max_skims_per_block: # next block
blocks[block_name(block)] = offset
block += 1
offset = 0
key1_block_offsets[key1] = (block, offset)
offset += num_subkeys
blocks[block_name(block)] = offset # last block
# - block_offsets dict maps skim_key to (block, offset) of omx matrix
# DIST: (0, 0),
# ('DRV_COM_WLK_BOARDS', 'AM'): (0, 3),
# ('DRV_COM_WLK_BOARDS', 'MD') (0, 4), ...
block_offsets = OrderedDict()
for skim_key in omx_keys:
if isinstance(skim_key, tuple):
key1, key2 = skim_key
else:
key1 = key2 = skim_key
block, key1_offset = key1_block_offsets[key1]
key2_relative_offset = key1_subkeys.get(key1).get(key2)
block_offsets[skim_key] = (block, key1_offset + key2_relative_offset)
logger.debug("get_skim_info from %s" % (omx_file_path, ))
logger.debug("get_skim_info skim_dtype %s omx_shape %s num_skims %s num_blocks %s" %
(skim_dtype, omx_shape, num_skims, len(blocks)))
skim_info = {
'omx_name': omx_name,
'omx_shape': omx_shape,
'num_skims': num_skims,
'dtype': skim_dtype,
'omx_keys': omx_keys,
'key1_block_offsets': key1_block_offsets,
'block_offsets': block_offsets,
'blocks': blocks,
}
return skim_info
walkMgraEquivMinutes.rename(
columns={'i': 'OMAZ', 'j': 'DMAZ', 'percieved': 'walk_perceived',
'actual': 'walk_actual', 'gain': 'walk_gain'},
inplace=True)
mgra13_based_input2012 = pd.read_csv(folder + "mgra13_based_input2012.csv")
mgra13_based_input2012.rename(columns={'mgra': 'MAZ', 'taz': 'TAZ'}, inplace=True)
Accessam = pd.read_csv(folder + "Accessam.csv")
Taps = pd.read_csv(folder + "Taps.csv")
Tap_ptype = pd.read_csv(folder + "Tap_ptype.csv")
Zone_term = pd.read_csv(folder + "Zone_term.csv")
Zone_park = pd.read_csv(folder + "Zone_park.csv")
# read taz and tap skim to get zone ids
with openmatrix.open_file(folder + "impdan_AM.omx") as taz_skim:
taz_numbers = taz_skim.mapentries("Origin")
with openmatrix.open_file(folder + "implocl_AMo.omx") as tap_skim:
tap_numbers = tap_skim.mapentries("Rows")
# TAZ
TAZ = pd.DataFrame({"offset": range(len(taz_numbers)), "TAZ": taz_numbers})
assert len(np.intersect1d(TAZ.TAZ, Zone_term.TAZ)) == len(Zone_term.TAZ)
TAZ = TAZ.merge(Zone_term, how="left")
assert len(np.intersect1d(TAZ.index, Zone_park.TAZ)) == len(Zone_park.TAZ)
TAZ = TAZ.merge(Zone_park, how="left")
TAZ.set_index("TAZ", drop=True, inplace=True, verify_integrity=True)
# MAZ
MAZ = mgra13_based_input2012
raise RuntimeError("Source matrix with key '%s' not found in file '%s"
% (omx_key, omx_file,))
data = omx_file[omx_key]
return data
manifest_dir = '.'
source_data_dir = '.'
dest_data_dir = '.'
manifest_file_name = os.path.join(manifest_dir, 'skim_manifest.csv')
dest_file_name = os.path.join(dest_data_dir, 'skims.omx')
with omx.open_file(dest_file_name, 'a') as dest_omx:
manifest = read_manifest(manifest_file_name)
for row in manifest.itertuples(index=True):
source_file_name = os.path.join(source_data_dir, row.source_file_name)
if row.skim_key2:
dest_key = row.skim_key1 + '__' + row.skim_key2
else:
dest_key = row.skim_key1
print "Reading '%s' from '%s' in %s" % (dest_key, row.source_key, source_file_name)
with omx.open_file(source_file_name, 'r') as source_omx:
#############################################################
ptypes = ["", "Full-time worker", "Part-time worker", "University student", "Non-worker",
"Retired", "Student of driving age", "Student of non-driving age",
"Child too young for school"]
mode_labels = ["", "DRIVEALONEFREE", "DRIVEALONEPAY", "SHARED2FREE", "SHARED2PAY", "SHARED3FREE",
"SHARED3PAY", "WALK", "BIKE", "WALK_LOC", "WALK_LRF", "WALK_EXP", "WALK_HVY",
"WALK_COM", "DRIVE_LOC", "DRIVE_LRF", "DRIVE_EXP", "DRIVE_HVY", "DRIVE_COM"]
#############################################################
# DISTANCE SKIM
#############################################################
# read distance matrix (DIST)
distmat = omx.open_file(distance_matrix_filename)["DIST"][:]
#############################################################
# EXPORT TABLES
#############################################################
# write tables for verification
tazs = pd.read_hdf(pipeline_filename, "land_use/initialize_landuse")
tazs["zone"] = tazs.index
tazs.to_csv(asim_zones_filename, index=False)
access = pd.read_hdf(pipeline_filename, "accessibility/compute_accessibility")
access.to_csv(asim_access_filename, index=False)
hh = pd.read_hdf(pipeline_filename, "households/joint_tour_frequency")
hh["household_id"] = hh.index