def common_config(manifest_file, manifest_root, batch_size): ''' Common configuration file for aeon loader manifest_file(str): Name of the manifest file manifest_root(str): Path for the manifest file batch_size(int): Batch size used for training ''' cache_root = get_data_cache_or_nothing('ucf-cache/') video_config = { 'type': "video", 'max_frame_count': 16, 'frame': { 'height': 112, 'width': 112 } } label_config = {'type': "label", 'binary': True} augmentation_config = {'type': 'image', 'scale': [0.875, 0.875]} configs = { 'manifest_filename': manifest_file, 'manifest_root': manifest_root, 'batch_size': batch_size, 'block_size': 5000, 'augmentation': [augmentation_config], 'cache_directory': cache_root, 'etl': [video_config, label_config] } return configs
def common_config(manifest_file, batch_size): cache_root = get_data_cache_or_nothing('mnist-cache/') return {'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'minibatch_size': batch_size, 'macrobatch_size': 25000, 'cache_directory': cache_root, 'type': 'image,label', 'image': {'height': 28, 'width': 28, 'channels': 1}, 'label': {'binary': False}}
def common_config(manifest_file, batch_size, subset_pct): cache_root = get_data_cache_or_nothing('lsun-cache/') image_config = {"type": "image", "height": 64, "width": 64} label_config = {"type": "label", "binary": False} return {'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'subset_fraction': float(subset_pct / 100.0), 'batch_size': batch_size, 'cache_directory': cache_root, 'etl': [image_config, label_config]}
def common_config(manifest_file, batch_size): cache_root = get_data_cache_or_nothing('cifar10-cache/') image_config = {"type": "image", "height": 32, "width": 32} label_config = {"type": "label", "binary": False} augmentation = {"type": "image", "scale": [0.8, 0.8]} return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] }
def common_config(manifest_file, batch_size): cache_root = get_data_cache_or_nothing('cifar10-cache/') return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'minibatch_size': batch_size, 'macrobatch_size': 5000, 'type': 'image,label', 'cache_directory': cache_root, 'image': { 'height': 32, 'width': 32, 'scale': [0.8, 0.8] }, 'label': { 'binary': False } }
def common_config(manifest_file, batch_size, dataset=dataset): if (dataset == "i1k"): cache_root = get_data_cache_or_nothing("i1k-cache/") image_config = {"type": "image", "height": 299, "width": 299} label_config = {"type": "label", "binary": False} augmentation = {"type": "image", "flip_enable": True} return { 'manifest_filename': manifest_file, 'manifest_root': datadir, 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } print("Imagenet") else: raise NotImplementedError("Only Imagenet 1K is supported")
def common_config(manifest_file, batch_size, valid_set=False): if (dataset == "cifar10"): # Define Cache cache_root = get_data_cache_or_nothing('cifar10-cache/') # Define image properties image_config = { "type": "image", "channels": 3, "height": 32, "width": 32 } # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "padding": 4, "crop_enable": False, "flip_enable": True } # Don't enable augmentations if it is test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } elif (dataset == "i1k"): # Define cache cache_root = get_data_cache_or_nothing("i1k-cache/") # Define image properties image_config = { "type": "image", "channels": 3, "height": 224, "width": 224 } # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "center": False, "flip_enable": True, "scale": [0.08, 1.0], "do_area_scale": True, "horizontal_distortion": [0.75, 1.33], "lighting": [0.0, 0.01], "contrast": [0.9, 1.1], "brightness": [0.9, 1.1], "saturation": [0.9, 1.1] } # Dont do augemtations on test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } # Do augmentations on training set return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } elif (dataset == "cifar100"): # Define Cache cache_root = get_data_cache_or_nothing('cifar100-cache/') # Define image properties image_config = { "type": "image", "channels": 3, "height": 32, "width": 32 } # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "padding": 4, "crop_enable": False, "flip_enable": True } # Don't enable augmentations if it is test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } elif (dataset == "i1k100"): # Define cache cache_root = get_data_cache_or_nothing("i1k-cache/") # Define image properties image_config = { "type": "image", "channels": 3, "height": 224, "width": 224 } # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "center": False, "flip_enable": True, "scale": [0.08, 1.0], "do_area_scale": True, "horizontal_distortion": [0.75, 1.33], "lighting": [0.0, 0.01], "contrast": [0.9, 1.1], "brightness": [0.9, 1.1], "saturation": [0.9, 1.1] } # Dont do augemtations on test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } # Do augmentations on training set return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } else: raise NameError("Unkown dataset.Choose correct dataset")
def make_aeon_dataloader(manifest_filename, audio_length, transcript_length, sample_freq_hz=16000, frame_length=.025, frame_stride=.01, feature_type='mfsc', num_filters=13, alphabet="_'ABCDEFGHIJKLMNOPQRSTUVWXYZ ", batch_size=32, cache_root=None, num_batches=None, single_iteration=False, seed=None): """ Creates a custom dataloader for speech transcription. Arguments: manifest_filename (str): Path to manifest file audio_length (float): Length of longest audio clip (seconds) transcript_length (int): Length of longest transcription sample_freq_hz (int): Sample rate of audio files (hertz) frame_length (float): Length of window for spectrogram calculation (seconds) frame_stride (float): Stride for spectrogram calculation (seconds) feature_type (str): Feature space for audio num_filters (int): Number of mel-frequency bands alphabet (str): Alphabet for the character map batch_size (int): Size of a single batch cache_root (str): Path to dataloader cache directory num_batches (int): Number of batches to load. Defaults to infinite single_iteration (bool): Sets "iteration_mode" to "ONCE" seed (int): Random seed for dataloader. Also turns off shuffling. """ if cache_root is None: cache_root = get_data_cache_or_nothing('deepspeech2-cache/') feats_config = dict(type="audio", sample_freq_hz=sample_freq_hz, max_duration="{} seconds".format(audio_length), frame_length="{} seconds".format(frame_length), frame_stride="{} seconds".format(frame_stride), feature_type=feature_type, num_filters=num_filters, emit_length=True) # Transcript transformation parameters transcripts_config = dict(type="char_map", alphabet=alphabet, max_length=transcript_length, emit_length=True) config = {'manifest_filename': manifest_filename, 'batch_size': batch_size, 'etl': [feats_config, transcripts_config], 'cache_directory': cache_root} if seed is not None: config["shuffle_enable"] = False config["shuffle_manifest"] = False config["random_seed"] = seed if num_batches is not None: config["iteration_mode"] = "COUNT" config["iteration_mode_count"] = num_batches elif single_iteration is True: config["iteration_mode"] = "ONCE" return SpeechTranscriptionLoader(config)
def common_config(manifest_file, batch_size, valid_set=False): if (dataset == "cifar10"): # Define Cache cache_root = get_data_cache_or_nothing('cifar10-cache/') # Define image properties image_config = {"type": "image", "height": 32, "width": 32} # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "padding": 4, "crop_enable": False, "flip_enable": True } # Don't enable augmentations if it is test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } return { 'manifest_filename': manifest_file, 'manifest_root': os.path.dirname(manifest_file), 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } elif (dataset == "i1k"): # Define cache cache_root = get_data_cache_or_nothing("i1k-cache/") # Define image properties image_config = {"type": "image", "height": 224, "width": 224} # Define label properties label_config = {"type": "label", "binary": False} # Define Augmentations augmentation = { "type": "image", "scale": (1, 1), "do_area_scale": True, "center": False } # Dont do augemtations on test set if (valid_set): return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config] } # Do augmentations on training set return { 'manifest_filename': manifest_file, 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/", 'batch_size': batch_size, 'block_size': 5000, 'cache_directory': cache_root, 'etl': [image_config, label_config], 'augmentation': [augmentation] } else: print("Unkown dataset.Choose either cifar10 or i1k dataset") exit()