def get_output_dataset_and_handler(output_packet_shape, **dataset_args):
dataset = ds.NumpyDataset(dataset_args['name'],
output_packet_shape,
item_types=dataset_args['item_types'],
dtype=dataset_args['dtype'])
output_handler = fs_io.DatasetFsPersistencyHandler(
save_dir=dataset_args['outdir'])
return dataset, output_handler
def main(**settings):
# load dataset
name, srcdir = settings['name'], settings['srcdir']
item_types = settings['item_types']
input_handler = io_utils.DatasetFsPersistencyHandler(load_dir=srcdir)
dataset = input_handler.load_dataset(name, item_types=item_types)
# create dataset splitter
fraction = settings['test_items_fraction']
num_items = settings['test_items_count']
splitter = netutils.DatasetSplitter('RANDOM',
items_fraction=fraction,
num_items=num_items)
# import network
net_module_name = settings['network']
model = netutils.import_model(net_module_name, dataset.item_shapes,
**settings)
graph = model.network_graph
# prepare network trainer
num_epochs = settings['num_epochs']
trainer = train_utils.TfModelTrainer(
splitter.get_data_and_targets(dataset), **settings)
# main loop
weights = {
layer: model.get_layer_weights(layer)
for layer in graph.trainable_layers
}
biases = {
layer: model.get_layer_biases(layer)
for layer in graph.trainable_layers
}
run_id = 'cval_{}'.format(netutils.get_default_run_id(net_module_name))
num_crossvals = settings['num_crossvals']
for run_idx in range(num_crossvals):
print('Starting run {}'.format(run_idx + 1))
data_dict = splitter.get_data_and_targets(dataset,
dict_format='PER_SET')
tr, te = data_dict['train'], data_dict['test']
inputs_dict = {
'train_data': netutils.convert_to_model_inputs_dict(model, tr),
'train_targets': netutils.convert_to_model_outputs_dict(model, tr),
'test_data': netutils.convert_to_model_inputs_dict(model, te),
'test_targets': netutils.convert_to_model_outputs_dict(model, te),
}
trainer.train_model(model, data_dict=inputs_dict, run_id=run_id)
# restore initial weights and biases
for layer in graph.trainable_layers:
model.set_layer_weights(layer, weights[layer])
model.set_layer_biases(layer, biases[layer])
def main(**settings):
srcdir, outdir = settings['srcdir'], settings['outdir']
name, outname = args['name'], args['outname']
if outname is None:
outname = name
if outdir is None:
outdir = srcdir
io_handler = io_utils.DatasetFsPersistencyHandler(load_dir=srcdir,
save_dir=outdir)
items_slice = args['items_slice']
old_dataset = io_handler.load_dataset(name)
new_dataset = ds.NumpyDataset(outname, old_dataset.accepted_packet_shape,
item_types=old_dataset.item_types,
dtype=old_dataset.dtype)
new_dataset.merge_with(old_dataset, items_slice)
io_handler.save_dataset(new_dataset)
def main(**settings):
logdir = settings['logdir']
# disable CUDA device access
if settings['usecpu']:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
# load dataset
name, srcdir = settings['name'], settings['srcdir']
item_types = settings['item_types']
input_handler = dset_io.DatasetFsPersistencyHandler(load_dir=srcdir)
dataset = input_handler.load_dataset(name, item_types=item_types)
items_slice = settings.get('items_slice', slice(0, None))
data = dataset.get_data_as_dict(items_slice)
# import network model
net_module_name = settings['network']
model = netutils.import_model(net_module_name, dataset.item_shapes,
**settings)
graph = model.network_graph
for data_path_layers in graph.data_paths.values():
model.enable_hidden_layer_output(data_path_layers)
# feed data to network model
inputs = netutils.convert_dataset_items_to_model_inputs(model, data)
activations = model.get_hidden_layer_activations(inputs)
input_layers, hidden_layers = graph.input_layers, graph.hidden_layers
#data_paths = model.network_graph.data_paths
for layer_name, layer_activations in activations.items():
out_dir = os.path.join(logdir, layer_name)
os.makedirs(out_dir, exist_ok=True)
layer = hidden_layers[layer_name]['layer']
if len(layer.shape[1:]) == 3:
fig_creator = acviz.visualize_3d_activations
elif len(layer.shape[1:]) == 1:
fig_creator = acviz.visualize_1d_activations
print('creating activation figures for layer {}'.format(layer_name))
for idx in range(len(layer_activations)):
fig = fig_creator(layer_activations[idx], layer_name)
savefile = os.path.join(
out_dir,
'layer_{}_item_{}.svg'.format(layer_name,
items_slice.start + idx))
acviz.save_figure(fig, savefile)
import cmdint.cmd_interface_checker as cmd
# command line argument parsing
cmd_int = cmd.CmdInterface()
args = cmd_int.get_cmd_args(sys.argv[1:])
name, srcdir = args.name, args.srcdir
item_types = args.item_types
# do not use the GPU
if args.usecpu:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
# load input dataset
input_handler = io_utils.DatasetFsPersistencyHandler(load_dir=srcdir)
dataset = input_handler.load_dataset(name, item_types=item_types)
#load trained network model
logdir = cutils.get_config_for_module("model_checker")['default']['logdir']
if not os.path.exists(logdir):
os.mkdir(logdir)
network_module_name, model_file = args.network, args.model_file
network_module_name = "net.samples." + network_module_name
run_id = netutils.get_default_run_id(network_module_name)
tb_dir = os.path.join(logdir, run_id)
os.mkdir(tb_dir)
model = netutils.import_model(network_module_name,
dataset.item_shapes,
model_file=model_file,
tb_dir=tb_dir)
def setUpClass(cls):
super(TestDatasetFsPersistencyManager, cls).setUpClass()
cls.meta_order = list(cls.metafields)
cls.meta_order.sort()
n_data = cls.n_packets
item_types = cls.item_types
item_types['gtux'], item_types['gtuy'] = False, False
items = cls.items
items['gtux'], items['gtuy'] = False, False
# a mock dataset to use (also serves as value container)
m_dataset = mock.create_autospec(ds.NumpyDataset)
m_dataset.name = 'test'
m_dataset.dtype = 'float32'
m_dataset.num_data = n_data
m_dataset.accepted_packet_shape = cls.packet_shape
m_dataset.item_types = item_types
m_dataset.item_shapes = dat.get_data_item_shapes(
m_dataset.accepted_packet_shape, item_types)
m_dataset.metadata_fields = cls.metafields
m_dataset.get_data_as_dict.return_value = items
m_dataset.get_data_as_arraylike.return_value = tuple(
items[k] for k in cons.ALL_ITEM_TYPES if not item_types[k])
m_dataset.get_targets.return_value = cls.mock_targets
m_dataset.get_metadata.return_value = cls.mock_meta
cls.m_dataset = m_dataset
cls.configfile_contents = (
'[general]{}'.format(os.linesep) +
'num_data = {}{}'.format(n_data, os.linesep) +
'metafields = {}{}'.format(cls.metafields, os.linesep) +
'dtype = {}{}'.format(m_dataset.dtype, os.linesep) +
'{}[packet_shape]{}'.format(os.linesep, os.linesep) +
'num_frames = {}{}'.format(cls.n_f, os.linesep) +
'frame_height = {}{}'.format(cls.f_h, os.linesep) +
'frame_width = {}{}'.format(cls.f_w, os.linesep) +
'{}[item_types]{}'.format(os.linesep, os.linesep) +
''.join('{} = {}{}'.format(k, item_types[k], os.linesep)
for k in cons.ALL_ITEM_TYPES) +
os.linesep
)
cls.loaddir, cls.savedir = '/dsets', '/test'
suffixes = {k: '_{}_test'.format(k) for k in cons.ALL_ITEM_TYPES}
cls.datafiles = {k: '{}{}.npy'.format(m_dataset.name, suffixes[k])
for k in cons.ALL_ITEM_TYPES}
conf_suffix = '_config_test'
cls.configfile = '{}{}.ini'.format(m_dataset.name, conf_suffix)
with mock.patch('os.path.isdir', return_value=True),\
mock.patch('os.path.exists', return_value=True):
cls.handler = fs_io.DatasetFsPersistencyHandler(
cls.loaddir,
cls.savedir,
configfile_suffix=conf_suffix,
data_handler = mock.create_autospec(
data_io.NumpyDataPersistencyHandler
),
targets_handler=mock.create_autospec(
targets_io.NumpyTargetsPersistencyHandler
),
metadata_handler=mock.create_autospec(
meta_io.TSVMetadataPersistencyHandler
)
)
})
# main loop
for handler in iteration_handlers:
start, stop = handler['start'], handler['stop']
packet_handler = handler['packet_handler']
target = handler['target']
# idx serves as both an index into targets and data, as well as
# shower angle in xy projection
for idx in range(start, stop):
packet, meta = packet_handler(idx)
dataset.add_data_item(packet, target, meta)
return dataset
if __name__ == '__main__':
import cmdint.cmd_interface_generator as cmd
# command line parsing
ui = cmd.CmdInterface()
args = ui.get_cmd_args(sys.argv[1:])
print(args)
data_generator = SimulatedDataGenerator(args.shower_template,
args.bg_template)
handler = io_utils.DatasetFsPersistencyHandler(save_dir=args.outdir)
dataset = data_generator.create_dataset(args.name,
args.num_data,
item_types=args.item_types,
dtype=args.dtype)
handler.save_dataset(dataset, metafields_order=cons.SYNTH_METADATA)