Exemplo n.º 1
0
def common_config(manifest_file, manifest_root, batch_size):
    '''
    Common configuration file for aeon loader

    manifest_file(str): Name of the manifest file
    manifest_root(str): Path for the manifest file
    batch_size(int): Batch size used for training
    '''

    cache_root = get_data_cache_or_nothing('ucf-cache/')
    video_config = {
        'type': "video",
        'max_frame_count': 16,
        'frame': {
            'height': 112,
            'width': 112
        }
    }

    label_config = {'type': "label", 'binary': True}

    augmentation_config = {'type': 'image', 'scale': [0.875, 0.875]}

    configs = {
        'manifest_filename': manifest_file,
        'manifest_root': manifest_root,
        'batch_size': batch_size,
        'block_size': 5000,
        'augmentation': [augmentation_config],
        'cache_directory': cache_root,
        'etl': [video_config, label_config]
    }

    return configs
Exemplo n.º 2
0
def common_config(manifest_file, batch_size):
    cache_root = get_data_cache_or_nothing('mnist-cache/')

    return {'manifest_filename': manifest_file,
            'manifest_root': os.path.dirname(manifest_file),
            'minibatch_size': batch_size,
            'macrobatch_size': 25000,
            'cache_directory': cache_root,
            'type': 'image,label',
            'image': {'height': 28,
                      'width': 28,
                      'channels': 1},
            'label': {'binary': False}}
Exemplo n.º 3
0
    def common_config(manifest_file, batch_size, subset_pct):
        cache_root = get_data_cache_or_nothing('lsun-cache/')

        image_config = {"type": "image",
                        "height": 64,
                        "width": 64}
        label_config = {"type": "label",
                        "binary": False}

        return {'manifest_filename': manifest_file,
                'manifest_root': os.path.dirname(manifest_file),
                'subset_fraction': float(subset_pct / 100.0),
                'batch_size': batch_size,
                'cache_directory': cache_root,
                'etl': [image_config, label_config]}
Exemplo n.º 4
0
    def common_config(manifest_file, batch_size):
        cache_root = get_data_cache_or_nothing('cifar10-cache/')

        image_config = {"type": "image", "height": 32, "width": 32}
        label_config = {"type": "label", "binary": False}
        augmentation = {"type": "image", "scale": [0.8, 0.8]}

        return {
            'manifest_filename': manifest_file,
            'manifest_root': os.path.dirname(manifest_file),
            'batch_size': batch_size,
            'block_size': 5000,
            'cache_directory': cache_root,
            'etl': [image_config, label_config],
            'augmentation': [augmentation]
        }
Exemplo n.º 5
0
def common_config(manifest_file, batch_size):
    cache_root = get_data_cache_or_nothing('cifar10-cache/')

    return {
        'manifest_filename': manifest_file,
        'manifest_root': os.path.dirname(manifest_file),
        'minibatch_size': batch_size,
        'macrobatch_size': 5000,
        'type': 'image,label',
        'cache_directory': cache_root,
        'image': {
            'height': 32,
            'width': 32,
            'scale': [0.8, 0.8]
        },
        'label': {
            'binary': False
        }
    }
Exemplo n.º 6
0
    def common_config(manifest_file, batch_size, dataset=dataset):
        if (dataset == "i1k"):
            cache_root = get_data_cache_or_nothing("i1k-cache/")

            image_config = {"type": "image", "height": 299, "width": 299}

            label_config = {"type": "label", "binary": False}

            augmentation = {"type": "image", "flip_enable": True}

            return {
                'manifest_filename': manifest_file,
                'manifest_root': datadir,
                'batch_size': batch_size,
                'block_size': 5000,
                'cache_directory': cache_root,
                'etl': [image_config, label_config],
                'augmentation': [augmentation]
            }
            print("Imagenet")
        else:
            raise NotImplementedError("Only Imagenet 1K is supported")
Exemplo n.º 7
0
 def common_config(manifest_file, batch_size, valid_set=False):
     if (dataset == "cifar10"):
         # Define Cache
         cache_root = get_data_cache_or_nothing('cifar10-cache/')
         # Define image properties
         image_config = {
             "type": "image",
             "channels": 3,
             "height": 32,
             "width": 32
         }
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "padding": 4,
             "crop_enable": False,
             "flip_enable": True
         }
         # Don't enable augmentations if it is test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': os.path.dirname(manifest_file),
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         return {
             'manifest_filename': manifest_file,
             'manifest_root': os.path.dirname(manifest_file),
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     elif (dataset == "i1k"):
         # Define cache
         cache_root = get_data_cache_or_nothing("i1k-cache/")
         # Define image properties
         image_config = {
             "type": "image",
             "channels": 3,
             "height": 224,
             "width": 224
         }
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "center": False,
             "flip_enable": True,
             "scale": [0.08, 1.0],
             "do_area_scale": True,
             "horizontal_distortion": [0.75, 1.33],
             "lighting": [0.0, 0.01],
             "contrast": [0.9, 1.1],
             "brightness": [0.9, 1.1],
             "saturation": [0.9, 1.1]
         }
         # Dont do augemtations on test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         # Do augmentations on training set
         return {
             'manifest_filename': manifest_file,
             'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     elif (dataset == "cifar100"):
         # Define Cache
         cache_root = get_data_cache_or_nothing('cifar100-cache/')
         # Define image properties
         image_config = {
             "type": "image",
             "channels": 3,
             "height": 32,
             "width": 32
         }
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "padding": 4,
             "crop_enable": False,
             "flip_enable": True
         }
         # Don't enable augmentations if it is test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': os.path.dirname(manifest_file),
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         return {
             'manifest_filename': manifest_file,
             'manifest_root': os.path.dirname(manifest_file),
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     elif (dataset == "i1k100"):
         # Define cache
         cache_root = get_data_cache_or_nothing("i1k-cache/")
         # Define image properties
         image_config = {
             "type": "image",
             "channels": 3,
             "height": 224,
             "width": 224
         }
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "center": False,
             "flip_enable": True,
             "scale": [0.08, 1.0],
             "do_area_scale": True,
             "horizontal_distortion": [0.75, 1.33],
             "lighting": [0.0, 0.01],
             "contrast": [0.9, 1.1],
             "brightness": [0.9, 1.1],
             "saturation": [0.9, 1.1]
         }
         # Dont do augemtations on test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         # Do augmentations on training set
         return {
             'manifest_filename': manifest_file,
             'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     else:
         raise NameError("Unkown dataset.Choose correct dataset")
Exemplo n.º 8
0
def make_aeon_dataloader(manifest_filename, audio_length, transcript_length,
                         sample_freq_hz=16000, frame_length=.025, frame_stride=.01,
                         feature_type='mfsc', num_filters=13,
                         alphabet="_'ABCDEFGHIJKLMNOPQRSTUVWXYZ ",
                         batch_size=32, cache_root=None, num_batches=None,
                         single_iteration=False, seed=None):

    """
    Creates a custom dataloader for speech transcription.

    Arguments:
        manifest_filename (str): Path to manifest file
        audio_length (float): Length of longest audio clip (seconds)
        transcript_length (int): Length of longest transcription
        sample_freq_hz (int): Sample rate of audio files (hertz)
        frame_length (float): Length of window for spectrogram calculation (seconds)
        frame_stride (float): Stride for spectrogram calculation (seconds)
        feature_type (str): Feature space for audio
        num_filters (int): Number of mel-frequency bands
        alphabet (str): Alphabet for the character map
        batch_size (int): Size of a single batch
        cache_root (str): Path to dataloader cache directory
        num_batches (int): Number of batches to load. Defaults to infinite
        single_iteration (bool): Sets "iteration_mode" to "ONCE"
        seed (int): Random seed for dataloader. Also turns off shuffling.
    """

    if cache_root is None:
        cache_root = get_data_cache_or_nothing('deepspeech2-cache/')

    feats_config = dict(type="audio",
                        sample_freq_hz=sample_freq_hz,
                        max_duration="{} seconds".format(audio_length),
                        frame_length="{} seconds".format(frame_length),
                        frame_stride="{} seconds".format(frame_stride),
                        feature_type=feature_type,
                        num_filters=num_filters,
                        emit_length=True)

    # Transcript transformation parameters
    transcripts_config = dict(type="char_map",
                              alphabet=alphabet,
                              max_length=transcript_length,
                              emit_length=True)

    config = {'manifest_filename': manifest_filename,
              'batch_size': batch_size,
              'etl': [feats_config, transcripts_config],
              'cache_directory': cache_root}

    if seed is not None:
        config["shuffle_enable"] = False
        config["shuffle_manifest"] = False
        config["random_seed"] = seed

    if num_batches is not None:
        config["iteration_mode"] = "COUNT"
        config["iteration_mode_count"] = num_batches
    elif single_iteration is True:
        config["iteration_mode"] = "ONCE"

    return SpeechTranscriptionLoader(config)
Exemplo n.º 9
0
 def common_config(manifest_file, batch_size, valid_set=False):
     if (dataset == "cifar10"):
         # Define Cache
         cache_root = get_data_cache_or_nothing('cifar10-cache/')
         # Define image properties
         image_config = {"type": "image", "height": 32, "width": 32}
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "padding": 4,
             "crop_enable": False,
             "flip_enable": True
         }
         # Don't enable augmentations if it is test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': os.path.dirname(manifest_file),
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         return {
             'manifest_filename': manifest_file,
             'manifest_root': os.path.dirname(manifest_file),
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     elif (dataset == "i1k"):
         # Define cache
         cache_root = get_data_cache_or_nothing("i1k-cache/")
         # Define image properties
         image_config = {"type": "image", "height": 224, "width": 224}
         # Define label properties
         label_config = {"type": "label", "binary": False}
         # Define Augmentations
         augmentation = {
             "type": "image",
             "scale": (1, 1),
             "do_area_scale": True,
             "center": False
         }
         # Dont do augemtations on test set
         if (valid_set):
             return {
                 'manifest_filename': manifest_file,
                 'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
                 'batch_size': batch_size,
                 'block_size': 5000,
                 'cache_directory': cache_root,
                 'etl': [image_config, label_config]
             }
         # Do augmentations on training set
         return {
             'manifest_filename': manifest_file,
             'manifest_root': "/dataset/aeon/I1K/i1k-extracted/",
             'batch_size': batch_size,
             'block_size': 5000,
             'cache_directory': cache_root,
             'etl': [image_config, label_config],
             'augmentation': [augmentation]
         }
     else:
         print("Unkown dataset.Choose either cifar10 or i1k dataset")
         exit()