def create_subset(data_file_out, skims_file_out, maxZone, households_sample_size=0):
# process all data tables
print 'skims/accessibility'
df = pd.read_hdf(data_file, 'skims/accessibility')
df = df[df.index <= maxZone]
df.to_hdf(data_file_out, 'skims/accessibility')
print 'land_use/taz_data'
df = pd.read_hdf(data_file, 'land_use/taz_data')
df = df[df.index <= maxZone]
df.to_hdf(data_file_out, 'land_use/taz_data')
print 'households'
df = pd.read_hdf(data_file, 'households')
df = df[df.TAZ <= maxZone]
if households_sample_size:
df = df.take(np.random.choice(len(df), size=households_sample_size, replace=False))
df.to_hdf(data_file_out, 'households')
print 'persons'
per = pd.read_hdf(data_file, 'persons')
per = per[per.household_id.isin(df.index)]
per.to_hdf(data_file_out, 'persons')
# process all skims
skims = omx.openFile(skims_file)
skims_out = omx.openFile(skims_file_out, 'a')
skimsToProcess = skims.listMatrices()
for skimName in skimsToProcess:
print skimName
skims_out[skimName] = skims[skimName][0:maxZone, 0:maxZone]
skims_out[skimName].attrs.TITLE = '' # remove funny character for OMX viewer
def omx_file(request):
omx_file = omx.openFile(os.path.join(os.path.dirname(__file__), "nonmotskm.omx"))
def fin():
omx_file.close()
request.addfinalizer(fin)
return omx_file
def omx_getMatrix(omx_file_name, omx_key):
with omx.openFile(omx_file_name, 'r') as omx_file:
if omx_key not in omx_file.listMatrices():
print "Source matrix with key '%s' not found in file '%s" % (omx_key, omx_file,)
print omx_file.listMatrices()
raise RuntimeError("Source matrix with key '%s' not found in file '%s"
% (omx_key, omx_file,))
data = omx_file[omx_key]
return data
def test_full_run(store):
orca.add_injectable("configs_dir",
os.path.join(os.path.dirname(__file__), '..', '..',
'..', 'example'))
tmp_name = tempfile.NamedTemporaryFile(suffix='.omx').name
tmp = omx.openFile(tmp_name, 'w')
tmp['DIST'] = np.ones((1454, 1454))
orca.add_injectable("omx_file", tmp)
orca.add_injectable("store", store)
orca.add_injectable("nonmotskm_matrix", np.ones((1454, 1454)))
orca.add_injectable("set_random_seed", set_random_seed)
# grab some of the tables
orca.get_table("land_use").to_frame().info()
orca.get_table("households").to_frame().info()
orca.get_table("persons").to_frame().info()
assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
# run the models in the expected order
orca.run(["school_location_simulate"])
orca.run(["workplace_location_simulate"])
orca.run(["auto_ownership_simulate"])
orca.run(["cdap_simulate"])
orca.run(['mandatory_tour_frequency'])
orca.get_table("mandatory_tours").tour_type.value_counts()
orca.run(['non_mandatory_tour_frequency'])
orca.get_table("non_mandatory_tours").tour_type.value_counts()
orca.run(["destination_choice"])
orca.run(["mandatory_scheduling"])
orca.run(["non_mandatory_scheduling"])
orca.run(["mode_choice_simulate"])
orca.clear_cache()
tmp.close()
os.remove(tmp_name)
def nonmotskm_omx(data_dir):
return omx.openFile(os.path.join(data_dir, 'data', "nonmotskm.omx"))
def omx_file(data_dir):
logger.debug("opening omx file")
return omx.openFile(os.path.join(data_dir, 'data', "nonmotskm.omx"))
raise RuntimeError("Source matrix with key '%s' not found in file '%s"
% (omx_key, omx_file,))
data = omx_file[omx_key]
return data
manifest_dir = '.'
source_data_dir = './source_skims'
dest_data_dir = './data'
manifest_file_name = os.path.join(manifest_dir, 'skim_manifest.csv')
dest_file_name = os.path.join(dest_data_dir, 'skims.omx')
with omx.openFile(dest_file_name, 'a') as dest_omx:
manifest = read_manifest(manifest_file_name)
for row in manifest.itertuples(index=True):
source_file_name = os.path.join(source_data_dir, row.source_file_name)
if row.skim_key2:
dest_key = row.skim_key1 + '__' + row.skim_key2
else:
dest_key = row.skim_key1
print "Reading '%s' from '%s' in %s" % (dest_key, row.source_key, source_file_name)
with omx.openFile(source_file_name, 'r') as source_omx:
def omx_file(data_dir):
return omx.openFile(os.path.join(data_dir, 'data', "nonmotskm.omx"))
def omx_file(data_dir, settings):
logger.debug("opening omx file")
return omx.openFile(os.path.join(data_dir, settings["skims_file"]))