def _init_single_source_dataset(self, dataset_name, split): dataset_specs = self._get_dataset_spec(dataset_name) self.specs_dict[split] = dataset_specs multi_source_pipeline = pipeline.make_one_source_batch_pipeline( dataset_spec=dataset_specs, batch_size=self.batch_size, split=split, image_size=84) iterator = multi_source_pipeline.make_one_shot_iterator() return iterator.get_next()
def generate_meta_train_batch_pipeline(self): """ Creates a batch generator for examples coming from the meta-train split. Also creates an episode generator for examples coming from the meta-valid split. Indeed, the way data comes from the meta-valid split should match its meta-test split counter-part. The meta-valid data generator will use the episode_config for the episodes description if its own configuration is not provided. ---------------------------------------------------------------------- Details of some arguments inside the function : The following arguments are ignored since we are using fixed episode description : - max_ways_upper_bound, max_num_query, max_support_{}, min/max_log_weight dag_ontology : only relevant for ImageNet dataset. Ignored. bilevel_ontology : Whether to ignore Omniglot's DAG ontology when sampling classes from it. Ignored. min_examples : 0 means that no class is discarded from having a too small number of examples. """ self.meta_train_pipeline = pipeline.make_one_source_batch_pipeline( dataset_spec=self.dataset_spec, split=learning_spec.Split.TRAIN, batch_size= self.batch_size, pool=None, shuffle_buffer_size=None, read_buffer_size_bytes=None, num_prefetch=0, image_size=self.image_size_batch, num_to_take=None ) self.meta_valid_pipeline = pipeline.make_one_source_episode_pipeline( dataset_spec=self.dataset_spec, use_dag_ontology=False, use_bilevel_ontology=False, split=learning_spec.Split.VALID, episode_descr_config=self.fixed_ways_shots_valid, pool=None, shuffle_buffer_size=None, read_buffer_size_bytes=None, num_prefetch=0, image_size=self.valid_episode_config[0], num_to_take=None ) logging.info('Meta-valid episode config : {}'.format( self.valid_episode_config)) logging.info('Meta-train batch config : [{}, {}]'.format( self.batch_size, self.image_size_batch))
def build_batch(self, split): """Builds a Batch object containing the next data for "split". Args: split: A string, either 'train', 'valid', or 'test'. Returns: An EpisodeDataset. """ shuffle_buffer_size = self.data_config.shuffle_buffer_size read_buffer_size_bytes = self.data_config.read_buffer_size_bytes _, image_shape, dataset_spec_list, _, _ = self.benchmark_spec dataset_split, batch_size = self.split_episode_or_batch_specs[split] for dataset_spec in dataset_spec_list: if dataset_spec.name in DATASETS_WITH_EXAMPLE_SPLITS: raise ValueError( 'Batch pipeline is used only at meta-train time, and does not ' 'handle datasets with example splits, which should only be used ' 'at meta-test (evaluation) time.') # TODO(lamblinp): pass specs directly to the pipeline builder. # TODO(lamblinp): move the special case directly in make_..._pipeline if len(dataset_spec_list) == 1: data_pipeline = pipeline.make_one_source_batch_pipeline( dataset_spec_list[0], split=dataset_split, batch_size=batch_size, shuffle_buffer_size=shuffle_buffer_size, read_buffer_size_bytes=read_buffer_size_bytes, image_size=image_shape[0]) else: data_pipeline = pipeline.make_multisource_batch_pipeline( dataset_spec_list, split=dataset_split, batch_size=batch_size, shuffle_buffer_size=shuffle_buffer_size, read_buffer_size_bytes=read_buffer_size_bytes, image_size=image_shape[0]) data_pipeline = apply_dataset_options(data_pipeline) iterator = data_pipeline.make_one_shot_iterator() images, class_ids = iterator.get_next() return providers.Batch(images=images, labels=class_ids)