def __init__(self, mode, distributed, dataset, data_format, seed=None): if mode not in ['train', 'inference', 'iotest']: raise Exception("Larcv Fetcher can't handle mode ", mode) random_access_mode = dataset.access_mode if random_access_mode != "serial_access" and mode == "inference": logger.warn("Using random blocks in inference - possible bug!") if distributed: from larcv import distributed_queue_interface self._larcv_interface = distributed_queue_interface.queue_interface( random_access_mode=random_access_mode, seed=seed) else: from larcv import queueloader self._larcv_interface = queueloader.queue_interface( random_access_mode=random_access_mode, seed=seed) self.mode = mode self.image_mode = data_format self.input_dimension = dataset.dimension self.distributed = distributed self.writer = None
def __init__(self, mode, distributed, image_mode, label_mode, input_dimension, seed=None): if mode not in ['train', 'inference', 'iotest']: raise Exception("Larcv Fetcher can't handle mode ", mode) if mode == "inference": random_access_mode = "serial_access" else: random_access_mode = "random_blocks" if distributed: from larcv import distributed_queue_interface self._larcv_interface = distributed_queue_interface.queue_interface( random_access_mode=random_access_mode, seed=seed) else: from larcv import queueloader self._larcv_interface = queueloader.queue_interface( random_access_mode=random_access_mode, seed=seed) self.mode = mode self.image_mode = image_mode self.label_mode = label_mode self.input_dimension = input_dimension self.writer = None
def create_interface_object(args): config = build_config_file(args) if args.distributed: if args.io_mode == 'queue': larcv_interface = distributed_queue_interface.queue_interface( random_access_mode=args.event_order) else: larcv_interface = distributed_larcv_interface.thread_interface() else: if args.io_mode == 'queue': larcv_interface = queueloader.queue_interface( random_access_mode=args.event_order) else: larcv_interface = threadloader.thread_interface() # Generate a named temp file: main_file = tempfile.NamedTemporaryFile(mode='w', delete=False) main_file.write(config) main_file.close() # Prepare data managers: io_config = { 'filler_name': 'IOTest', 'filler_cfg': main_file.name, 'verbosity': 5, 'make_copy': True } # By default, fetching data and label as the keywords from the file: data_keys = OrderedDict({'image': 'data', 'label': 'label'}) if args.distributed: if args.io_mode == 'queue': larcv_interface.prepare_manager('primary', io_config, COMM.Get_size() * args.local_batch_size, data_keys, color=0) else: larcv_interface.prepare_manager( 'primary', io_config, COMM.Get_size() * args.local_batch_size, data_keys) else: if args.io_mode == 'queue': larcv_interface.prepare_manager('primary', io_config, args.local_batch_size, data_keys) else: larcv_interface.prepare_manager('primary', io_config, args.local_batch_size, data_keys) return larcv_interface
def __init__(self): # Rely on the base class for most standard parameters, only # search for parameters relevant for distributed computing here # Put the IO rank as the last rank in the COMM, since rank 0 does tf saves root_rank = hvd.size() - 1 if FLAGS.COMPUTE_MODE == "GPU": os.environ['CUDA_VISIBLE_DEVICES'] = str(hvd.local_rank()) self._larcv_interface = queue_interface() self._iteration = 0 self._rank = hvd.rank() self._cleanup = []
def __init__(self, distributed, dataset, seed=0): random_access_mode = dataset.access_mode if distributed: from larcv import distributed_queue_interface self._larcv_interface = distributed_queue_interface.queue_interface( random_access_mode=random_access_mode.name, seed=seed) else: from larcv import queueloader self._larcv_interface = queueloader.queue_interface( random_access_mode=random_access_mode.name, seed=seed) self.distributed = distributed self.dataset = dataset self.writer = None
def __init__(self, mode, distributed, downsample, dataformat, synthetic, sparse, seed=None): if mode not in ['train', 'inference', 'iotest']: raise Exception("Larcv Fetcher can't handle mode ", mode) if not synthetic: if distributed: from larcv import distributed_queue_interface self._larcv_interface = distributed_queue_interface.queue_interface( ) else: from larcv import queueloader if mode == "inference": self._larcv_interface = queueloader.queue_interface( random_access_mode="serial_access", seed=seed) elif mode == "train" or mode == "iotest": self._larcv_interface = queueloader.queue_interface( random_access_mode="random_blocks", seed=seed) else: # Must be synthetic self._larcv_interface = None self.mode = mode self.downsample = downsample self.dataformat = dataformat self.synthetic = synthetic self.sparse = sparse self.writer = None # Compute the realized image shape: self.full_image_shape = [ self.FULL_RESOLUTION_H, self.FULL_RESOLUTION_W ] self.ds = 2**downsample self.image_shape = [int(i / self.ds) for i in self.full_image_shape]
def load_data(): ''' initialize data loading ''' # config files main_fname = os.environ['HOME'] + '/bb2nu_rl_dnn/larcvconfig_train_lr.txt' aux_fname = os.environ['HOME'] + '/bb2nu_rl_dnn/larcvconfig_test_lr.txt' # initilize io root_rank = hvd.size() - 1 #_larcv_interface = queue_interface( random_access_mode="serial_access" ) #_larcv_interface = queue_interface( random_access_mode="random_events" ) _larcv_interface = queue_interface(random_access_mode="random_blocks") # Prepare data managers: io_config = { 'filler_name': 'TrainIO', 'filler_cfg': main_fname, 'verbosity': 1, 'make_copy': True } aux_io_config = { 'filler_name': 'TestIO', 'filler_cfg': aux_fname, 'verbosity': 1, 'make_copy': True } # Build up the data_keys: data_keys = OrderedDict() data_keys['image'] = 'data' data_keys['label'] = 'label' aux_data_keys = OrderedDict() aux_data_keys['image'] = 'test_data' aux_data_keys['label'] = 'test_label' _larcv_interface.prepare_manager('train', io_config, global_batch_size, data_keys, color=0) _larcv_interface.prepare_manager('test', aux_io_config, global_batch_size, aux_data_keys, color=0) return _larcv_interface
def __init__(self): # Rely on the base class for most standard parameters, only # search for parameters relevant for distributed computing here # Put the IO rank as the last rank in the COMM, since rank 0 does tf saves root_rank = hvd.size() - 1 if self.args.COMPUTE_MODE == "GPU": os.environ['CUDA_VISIBLE_DEVICES'] = str(hvd.local_rank()) self._larcv_interface = queue_interface()#read_option='read_from_single_local_rank') self._iteration = 0 self._rank = hvd.rank() self._cleanup = [] self._global_step = torch.as_tensor(-1) if self._rank == 0: self.args.dump_config()
def __init__(self, distributed, seed=None, inference=False): self._cleanup = [] self._eventID_labels = {} self._eventID_energies = {} if inference: random_access_mode = "serial_access" else: random_access_mode = "random_blocks" self._color = None if distributed: from larcv import distributed_queue_interface self._larcv_interface = distributed_queue_interface.queue_interface(random_access_mode=random_access_mode) self._color = 0 else: from larcv import queueloader self._larcv_interface = queueloader.queue_interface(random_access_mode=random_access_mode, seed=None) self.inference = inference