def skim_dict(data_dir, settings):
omx_file_path = config.data_file_path(settings["skims_file"])
tags_to_load = settings['skim_time_periods']['labels']
logger.info("loading skim_dict from %s" % (omx_file_path, ))
# select the skims to load
skim_info = get_skim_info(omx_file_path, tags_to_load)
logger.debug("omx_shape %s skim_dtype %s" %
(skim_info['omx_shape'], skim_info['dtype']))
skim_buffers = inject.get_injectable('data_buffers', None)
if skim_buffers:
logger.info('Using existing skim_buffers for skims')
else:
skim_buffers = buffers_for_skims(skim_info, shared=False)
load_skims(omx_file_path, skim_info, skim_buffers)
skim_data = skim_data_from_buffers(skim_buffers, skim_info)
block_names = list(skim_info['blocks'].keys())
for i in range(len(skim_data)):
block_name = block_names[i]
block_data = skim_data[i]
logger.info(
"block_name %s bytes %s (%s)" %
(block_name, block_data.nbytes, util.GB(block_data.nbytes)))
# create skim dict
skim_dict = skim.SkimDict(skim_data, skim_info)
skim_dict.offset_mapper.set_offset_int(-1)
return skim_dict
def tap_skim_dict(data_dir, settings):
logger.info("loading tap_skim_dict")
cache_skim_key_values = settings['skim_time_periods']['labels']
skim_dict = askim.SkimDict()
for skims_file in settings["tap_skims_files"]:
skims_file_path = config.data_file_path(skims_file)
with omx.open_file(skims_file_path) as omx_file:
add_to_skim_dict(skim_dict, omx_file, cache_skim_key_values)
return skim_dict
def taz_skim_dict(data_dir, settings):
logger.info("loading taz_skim_dict")
skims_file = os.path.join(data_dir, settings["taz_skims_file"])
cache_skim_key_values = settings['time_periods']['labels']
skim_dict = askim.SkimDict()
with omx.open_file(skims_file) as omx_file:
add_to_skim_dict(skim_dict, omx_file, cache_skim_key_values)
return skim_dict
def skim_dict(omx_file, cache_skim_key_values):
logger.info("skims injectable loading skims")
skim_dict = askim.SkimDict()
skim_dict.offset_mapper.set_offset_int(-1)
skims_in_omx = omx_file.listMatrices()
for skim_name in skims_in_omx:
key, sep, key2 = skim_name.partition('__')
skim_data = omx_file[skim_name]
if not sep:
# no separator - this is a simple 2d skim - we load them all
skim_dict.set(key, skim_data)
else:
# there may be more time periods in the skim than are used by the model
# cache_skim_key_values is a list of time periods (frem settings) that are used
# FIXME - assumes that the only types of key2 are time_periods
if key2 in cache_skim_key_values:
skim_dict.set((key, key2), skim_data)
return skim_dict