def add_feature_data(feature_config, input_df, proc_df, metadata, preprocessing_parameters, backend, skip_save_processed_input): set_default_value(feature_config["preprocessing"], "in_memory", preprocessing_parameters["in_memory"]) if "audio_feature" not in preprocessing_parameters: raise ValueError( "audio_feature dictionary has to be present in preprocessing " "for audio.") if TYPE not in preprocessing_parameters["audio_feature"]: raise ValueError( "type has to be present in audio_feature dictionary " "for audio.") name = feature_config[NAME] column = feature_config[COLUMN] proc_column = feature_config[PROC_COLUMN] src_path = None # this is not super nice, but works both and DFs and lists first_path = "." for first_path in input_df[column]: break if SRC in metadata: src_path = os.path.dirname(os.path.abspath(metadata.get(SRC))) if src_path is None and not os.path.isabs(first_path): raise ValueError("Audio file paths must be absolute") num_audio_utterances = len(input_df[feature_config[COLUMN]]) padding_value = preprocessing_parameters["padding_value"] normalization_type = preprocessing_parameters["norm"] feature_dim = metadata[name]["feature_dim"] max_length = metadata[name]["max_length"] audio_feature_dict = preprocessing_parameters["audio_feature"] audio_file_length_limit_in_s = preprocessing_parameters[ "audio_file_length_limit_in_s"] if num_audio_utterances == 0: raise ValueError( "There are no audio files in the dataset provided.") if feature_config[PREPROCESSING]["in_memory"]: audio_features = AudioFeatureMixin._process_in_memory( input_df[feature_config[NAME]], src_path, audio_feature_dict, feature_dim, max_length, padding_value, normalization_type, audio_file_length_limit_in_s, backend, ) proc_df[proc_column] = audio_features else: backend.check_lazy_load_supported(feature_config) return proc_df
def populate_defaults(output_feature): # If Loss is not defined, set an empty dictionary set_default_value(output_feature, LOSS, {}) # Populate the default values for LOSS if they aren't defined already set_default_values( output_feature[LOSS], { TYPE: "softmax_cross_entropy", "labels_smoothing": 0, "class_weights": 1, "robust_lambda": 0, "confidence_penalty": 0, "class_similarities_temperature": 0, "weight": 1, }, ) if output_feature[LOSS][TYPE] == "sampled_softmax_cross_entropy": set_default_values( output_feature[LOSS], {"sampler": "log_uniform", "unique": False, "negative_samples": 25, "distortion": 0.75}, ) set_default_values( output_feature, {"top_k": 3, "dependencies": [], "reduce_input": SUM, "reduce_dependencies": SUM} )
def populate_defaults(output_feature): # If Loss is not defined, set an empty dictionary set_default_value(output_feature, LOSS, {}) set_default_values( output_feature[LOSS], { 'robust_lambda': 0, 'confidence_penalty': 0, 'positive_class_weight': 1, 'weight': 1 } ) set_default_value(output_feature[LOSS], 'robust_lambda', 0) set_default_value(output_feature[LOSS], 'confidence_penalty', 0) set_default_value(output_feature[LOSS], 'positive_class_weight', 1) set_default_value(output_feature[LOSS], 'weight', 1) set_default_values( output_feature, { 'threshold': 0.5, 'dependencies': [], 'reduce_input': SUM, 'reduce_dependencies': SUM } )
def populate_defaults(output_feature): # If Loss is not defined, set an empty dictionary set_default_value(output_feature, LOSS, {}) set_default_values( output_feature[LOSS], { "robust_lambda": 0, "confidence_penalty": 0, "positive_class_weight": 1, "weight": 1, }, ) set_default_value(output_feature[LOSS], "robust_lambda", 0) set_default_value(output_feature[LOSS], "confidence_penalty", 0) set_default_value(output_feature[LOSS], "positive_class_weight", 1) set_default_value(output_feature[LOSS], "weight", 1) set_default_values( output_feature, { "threshold": 0.5, "dependencies": [], "reduce_input": SUM, "reduce_dependencies": SUM, }, )
def populate_defaults(output_feature): # If Loss is not defined, set an empty dictionary set_default_value(output_feature, LOSS, {}) # Populate the default values for LOSS if they aren't defined already set_default_values( output_feature[LOSS], { TYPE: 'softmax_cross_entropy', 'labels_smoothing': 0, 'class_weights': 1, 'robust_lambda': 0, 'confidence_penalty': 0, 'class_similarities_temperature': 0, 'weight': 1 }) if output_feature[LOSS][TYPE] == 'sampled_softmax_cross_entropy': set_default_values( output_feature[LOSS], { 'sampler': 'log_uniform', 'unique': False, 'negative_samples': 25, 'distortion': 0.75 }) set_default_values( output_feature, { 'top_k': 3, 'dependencies': [], 'reduce_input': SUM, 'reduce_dependencies': SUM })
def merge_with_defaults(config): config = copy.deepcopy(config) _perform_sanity_checks(config) _set_feature_column(config) _set_proc_column(config) _merge_hyperopt_with_training(config) # ===== Preprocessing ===== config["preprocessing"] = merge_dict(default_preprocessing_parameters, config.get("preprocessing", {})) stratify = config["preprocessing"]["stratify"] if stratify is not None: features = config["input_features"] + config["output_features"] feature_names = {f[COLUMN] for f in features} if stratify not in feature_names: logger.warning("Stratify is not among the features. " "Cannot establish if it is a binary or category") elif [f for f in features if f[COLUMN] == stratify][0][TYPE] not in {BINARY, CATEGORY}: raise ValueError("Stratify feature must be binary or category") # ===== Training ===== set_default_value(config, TRAINING, default_training_params) for param, value in default_training_params.items(): set_default_value(config[TRAINING], param, value) set_default_value( config[TRAINING], "validation_metric", output_type_registry[config["output_features"][0] [TYPE]].default_validation_metric, ) # ===== Training Optimizer ===== optimizer = config[TRAINING]["optimizer"] default_optimizer_params = get_default_optimizer_params(optimizer[TYPE]) for param in default_optimizer_params: set_default_value(optimizer, param, default_optimizer_params[param]) # ===== Input Features ===== for input_feature in config["input_features"]: get_from_registry(input_feature[TYPE], input_type_registry).populate_defaults(input_feature) # ===== Combiner ===== set_default_value(config, "combiner", {TYPE: default_combiner_type}) # ===== Output features ===== for output_feature in config["output_features"]: get_from_registry( output_feature[TYPE], output_type_registry).populate_defaults(output_feature) return config
def merge_with_defaults(model_definition): _perform_sanity_checks(model_definition) # ===== Preprocessing ===== model_definition['preprocessing'] = merge_dict( default_preprocessing_parameters, model_definition.get('preprocessing', {})) stratify = model_definition['preprocessing']['stratify'] if stratify is not None: if stratify not in [ x['name'] for x in model_definition['output_features'] ]: raise ValueError('Stratify must be in output features') if ([ x for x in model_definition['output_features'] if x['name'] == stratify ][0][TYPE] not in [BINARY, CATEGORY]): raise ValueError('Stratify feature must be binary or category') # ===== Model ===== set_default_value(model_definition, 'combiner', {'type': default_combiner_type}) # ===== Training ===== set_default_value(model_definition, TRAINING, default_training_params) for param, value in default_training_params.items(): set_default_value(model_definition[TRAINING], param, value) set_default_value( model_definition[TRAINING], 'validation_metric', output_type_registry[model_definition['output_features'][0] [TYPE]].default_validation_metric) # ===== Training Optimizer ===== optimizer = model_definition[TRAINING]['optimizer'] default_optimizer_params = get_default_optimizer_params(optimizer[TYPE]) for param in default_optimizer_params: set_default_value(optimizer, param, default_optimizer_params[param]) # ===== Input Features ===== for input_feature in model_definition['input_features']: get_from_registry(input_feature[TYPE], input_type_registry).populate_defaults(input_feature) # ===== Output features ===== for output_feature in model_definition['output_features']: get_from_registry( output_feature['type'], output_type_registry).populate_defaults(output_feature) return model_definition
def add_feature_data(feature, input_df, proc_df, metadata, preprocessing_parameters, backend, skip_save_processed_input): set_default_value(feature['preprocessing'], 'in_memory', preprocessing_parameters['in_memory']) if 'audio_feature' not in preprocessing_parameters: raise ValueError( 'audio_feature dictionary has to be present in preprocessing ' 'for audio.') if TYPE not in preprocessing_parameters['audio_feature']: raise ValueError( 'type has to be present in audio_feature dictionary ' 'for audio.') name = feature[NAME] column = feature[COLUMN] proc_column = feature[PROC_COLUMN] src_path = None # this is not super nice, but works both and DFs and lists first_path = '.' for first_path in input_df[column]: break if SRC in metadata: src_path = os.path.dirname(os.path.abspath(metadata.get(SRC))) if src_path is None and not os.path.isabs(first_path): raise ValueError('Audio file paths must be absolute') num_audio_utterances = len(input_df[feature[COLUMN]]) padding_value = preprocessing_parameters['padding_value'] normalization_type = preprocessing_parameters['norm'] feature_dim = metadata[name]['feature_dim'] max_length = metadata[name]['max_length'] audio_feature_dict = preprocessing_parameters['audio_feature'] audio_file_length_limit_in_s = preprocessing_parameters[ 'audio_file_length_limit_in_s'] if num_audio_utterances == 0: raise ValueError( 'There are no audio files in the dataset provided.') if feature[PREPROCESSING]['in_memory']: audio_features = AudioFeatureMixin._process_in_memory( input_df[feature[NAME]], src_path, audio_feature_dict, feature_dim, max_length, padding_value, normalization_type, audio_file_length_limit_in_s, backend) proc_df[proc_column] = audio_features else: backend.check_lazy_load_supported(feature) return proc_df
def merge_with_defaults(config): _perform_sanity_checks(config) _set_feature_column(config) _set_proc_column(config) _merge_hyperopt_with_training(config) # ===== Preprocessing ===== config['preprocessing'] = merge_dict(default_preprocessing_parameters, config.get('preprocessing', {})) stratify = config['preprocessing']['stratify'] if stratify is not None: features = (config['input_features'] + config['output_features']) feature_names = set(f[COLUMN] for f in features) if stratify not in feature_names: logger.warning('Stratify is not among the features. ' 'Cannot establish if it is a binary or category') elif ([f for f in features if f[COLUMN] == stratify][0][TYPE] not in {BINARY, CATEGORY}): raise ValueError('Stratify feature must be binary or category') # ===== Training ===== set_default_value(config, TRAINING, default_training_params) for param, value in default_training_params.items(): set_default_value(config[TRAINING], param, value) set_default_value( config[TRAINING], 'validation_metric', output_type_registry[ config['output_features'][0][TYPE]].default_validation_metric) # ===== Training Optimizer ===== optimizer = config[TRAINING]['optimizer'] default_optimizer_params = get_default_optimizer_params(optimizer[TYPE]) for param in default_optimizer_params: set_default_value(optimizer, param, default_optimizer_params[param]) # ===== Input Features ===== for input_feature in config['input_features']: get_from_registry(input_feature[TYPE], input_type_registry).populate_defaults(input_feature) # ===== Combiner ===== set_default_value(config, 'combiner', {TYPE: default_combiner_type}) # ===== Output features ===== for output_feature in config['output_features']: get_from_registry( output_feature[TYPE], output_type_registry).populate_defaults(output_feature) return config
def update_hyperopt_params_with_defaults(hyperopt_params): from ludwig.hyperopt.execution import executor_registry set_default_value(hyperopt_params, EXECUTOR, {}) set_default_value(hyperopt_params, "split", VALIDATION) set_default_value(hyperopt_params, "output_feature", COMBINED) set_default_value(hyperopt_params, "metric", LOSS) set_default_value(hyperopt_params, "goal", MINIMIZE) set_default_values(hyperopt_params[EXECUTOR], {TYPE: "ray"}) executor = get_from_registry(hyperopt_params[EXECUTOR][TYPE], executor_registry) executor_defaults = { k: v for k, v in executor.__dict__.items() if k in get_class_attributes(executor) } set_default_values( hyperopt_params[EXECUTOR], executor_defaults, )
def populate_defaults(output_feature): # If Loss is not defined, set an empty dictionary set_default_value(output_feature, LOSS, {}) # Populate the default values for LOSS if they aren't defined already set_default_values( output_feature[LOSS], { TYPE: "softmax_cross_entropy", "class_weights": 1, "robust_lambda": 0, "confidence_penalty": 0, "class_similarities_temperature": 0, "weight": 1, }, ) set_default_values( output_feature, { "top_k": 3, "dependencies": [], "reduce_input": SUM, "reduce_dependencies": SUM })
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, {TYPE: "mean_squared_error", "weight": 1}) set_default_value(output_feature[LOSS], TYPE, "mean_squared_error") set_default_value(output_feature[LOSS], "weight", 1) set_default_values( output_feature, { "clip": None, "dependencies": [], "reduce_input": SUM, "reduce_dependencies": SUM, }, )
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, { TYPE: 'mean_squared_error', 'weight': 1 }) set_default_value(output_feature[LOSS], TYPE, 'mean_squared_error') set_default_value(output_feature[LOSS], 'weight', 1) set_default_values( output_feature, { 'clip': None, 'dependencies': [], 'reduce_input': SUM, 'reduce_dependencies': SUM })
def populate_defaults(input_feature): set_default_value(input_feature, TIED, None) set_default_value(input_feature, 'encoder', 'parallel_cnn')
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, {TYPE: SIGMOID_CROSS_ENTROPY, "weight": 1}) set_default_value(output_feature[LOSS], "weight", 1) set_default_value(output_feature[LOSS], "class_weights", None) set_default_value(output_feature, "threshold", 0.5) set_default_value(output_feature, "dependencies", []) set_default_value(output_feature, "reduce_input", SUM) set_default_value(output_feature, "reduce_dependencies", SUM)
def populate_defaults(input_feature): set_default_value(input_feature, TIED, None) set_default_value(input_feature, PREPROCESSING, {})
def add_feature_data(feature, input_df, proc_df, metadata, preprocessing_parameters, backend): set_default_value(feature[PREPROCESSING], 'in_memory', preprocessing_parameters['in_memory']) set_default_value(feature[PREPROCESSING], 'num_processes', preprocessing_parameters['num_processes']) src_path = None if hasattr(input_df, 'src'): src_path = os.path.dirname(os.path.abspath(input_df.src)) num_images = len(input_df) if num_images == 0: raise ValueError('There are no images in the dataset provided.') first_path = next(iter(input_df[feature[COLUMN]])) if src_path is None and not os.path.isabs(first_path): raise ValueError('Image file paths must be absolute') first_path = get_abs_path(src_path, first_path) (should_resize, width, height, num_channels, user_specified_num_channels, first_image) = ImageFeatureMixin._finalize_preprocessing_parameters( preprocessing_parameters, first_path) metadata[feature[NAME]][PREPROCESSING]['height'] = height metadata[feature[NAME]][PREPROCESSING]['width'] = width metadata[feature[NAME]][PREPROCESSING]['num_channels'] = num_channels read_image_and_resize = partial( ImageFeatureMixin._read_image_and_resize, img_width=width, img_height=height, should_resize=should_resize, num_channels=num_channels, resize_method=preprocessing_parameters['resize_method'], user_specified_num_channels=user_specified_num_channels) if feature[PREPROCESSING]['in_memory']: # Number of processes to run in parallel for preprocessing num_processes = feature[PREPROCESSING]['num_processes'] metadata[ feature[NAME]][PREPROCESSING]['num_processes'] = num_processes # Split the dataset into pools only if we have an explicit request to use # multiple processes. In case we have multiple input images use the # standard code anyway. if backend.supports_multiprocessing and (num_processes > 1 or num_images > 1): all_file_paths = [ get_abs_path(src_path, file_path) for file_path in input_df[feature[NAME]] ] with Pool(num_processes) as pool: logger.debug( 'Using {} processes for preprocessing images'.format( num_processes)) proc_df[feature[PROC_COLUMN]] = pool.map( read_image_and_resize, all_file_paths) else: # If we're not running multiple processes and we are only processing one # image just use this faster shortcut, bypassing multiprocessing.Pool.map logger.debug( 'No process pool initialized. Using internal process for preprocessing images' ) proc_df[feature[PROC_COLUMN]] = backend.df_engine.map_objects( input_df[feature[COLUMN]], lambda file_path: read_image_and_resize( get_abs_path(src_path, file_path))) else: backend.check_lazy_load_supported(feature) all_file_paths = [ get_abs_path(src_path, file_path) for file_path in input_df[feature[NAME]] ] data_fp = os.path.splitext(input_df.src)[0] + '.hdf5' mode = 'w' if os.path.isfile(data_fp): mode = 'r+' with h5py.File(data_fp, mode) as h5_file: # todo future add multiprocessing/multithreading image_dataset = h5_file.create_dataset( feature[PROC_COLUMN] + '_data', (num_images, height, width, num_channels), dtype=np.uint8) for i, filepath in enumerate(all_file_paths): image_dataset[i, :height, :width, :] = ( read_image_and_resize(filepath)) h5_file.flush() proc_df[feature[PROC_COLUMN]] = np.arange(num_images) return proc_df
def add_feature_data(feature, dataset_df, dataset, metadata, preprocessing_parameters): set_default_value(feature['preprocessing'], 'in_memory', preprocessing_parameters['in_memory']) if not 'audio_feature' in preprocessing_parameters: raise ValueError( 'audio_feature dictionary has to be present in preprocessing ' 'for audio.') if not TYPE in preprocessing_parameters['audio_feature']: raise ValueError( 'type has to be present in audio_feature dictionary ' 'for audio.') src_path = None # this is not super nice, but works both and DFs and lists first_path = '.' for first_path in dataset_df[feature[NAME]]: break if hasattr(dataset_df, 'src'): src_path = os.path.dirname(os.path.abspath(dataset_df.src)) if src_path is None and not os.path.isabs(first_path): raise ValueError('Audio file paths must be absolute') num_audio_utterances = len(dataset_df) padding_value = preprocessing_parameters['padding_value'] normalization_type = preprocessing_parameters['norm'] feature_name = feature[NAME] feature_dim = metadata[feature_name]['feature_dim'] max_length = metadata[feature_name]['max_length'] audio_feature_dict = preprocessing_parameters['audio_feature'] audio_file_length_limit_in_s = preprocessing_parameters[ 'audio_file_length_limit_in_s'] if num_audio_utterances == 0: raise ValueError( 'There are no audio files in the dataset provided.') audio_stats = { 'count': 0, 'mean': 0, 'var': 0, 'std': 0, 'max': 0, 'min': float('inf'), 'cropped': 0, 'max_length_in_s': audio_file_length_limit_in_s } if feature['preprocessing']['in_memory']: dataset[feature[NAME]] = np.empty( (num_audio_utterances, max_length, feature_dim), dtype=np.float32) for i, path in enumerate(dataset_df[feature[NAME]]): filepath = get_abs_path(src_path, path) audio_feature = AudioFeatureMixin._read_audio_and_transform_to_feature( filepath, audio_feature_dict, feature_dim, max_length, padding_value, normalization_type, audio_stats) dataset[feature[NAME]][i, :, :] = audio_feature audio_stats['std'] = np.sqrt(audio_stats['var'] / float(audio_stats['count'])) print_statistics = ("{} audio files loaded.\n" "Statistics of audio file lengths:\n" "- mean: {:.4f}\n" "- std: {:.4f}\n" "- max: {:.4f}\n" "- min: {:.4f}\n" "- cropped audio_files: {}\n" "Max length was given as {}s").format( audio_stats['count'], audio_stats['mean'], audio_stats['std'], audio_stats['max'], audio_stats['min'], audio_stats['cropped'], audio_stats['max_length_in_s']) logger.debug(print_statistics)
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, {}) set_default_value(output_feature[LOSS], TYPE, MEAN_SQUARED_ERROR) set_default_value(output_feature[LOSS], "weight", 1) set_default_value(output_feature, "reduce_input", None) set_default_value(output_feature, "reduce_dependencies", None) set_default_value(output_feature, "decoder", "projector") set_default_value(output_feature, "dependencies", [])
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, {}) set_default_value(output_feature[LOSS], TYPE, MEAN_SQUARED_ERROR) set_default_value(output_feature[LOSS], 'weight', 1) set_default_value(output_feature, 'reduce_input', None) set_default_value(output_feature, 'reduce_dependencies', None) set_default_value(output_feature, 'decoder', 'projector') set_default_value(output_feature, 'dependencies', [])
def update_hyperopt_params_with_defaults(hyperopt_params): set_default_value(hyperopt_params, STRATEGY, {}) set_default_value(hyperopt_params, EXECUTOR, {}) set_default_value(hyperopt_params, "split", VALIDATION) set_default_value(hyperopt_params, "output_feature", COMBINED) set_default_value(hyperopt_params, "metric", LOSS) set_default_value(hyperopt_params, "goal", MINIMIZE) set_default_values(hyperopt_params[STRATEGY], {TYPE: "random"}) strategy = get_from_registry(hyperopt_params[STRATEGY][TYPE], sampler_registry) strategy_defaults = { k: v for k, v in strategy.__dict__.items() if k in get_class_attributes(strategy) } set_default_values( hyperopt_params[STRATEGY], strategy_defaults, ) set_default_values(hyperopt_params[EXECUTOR], {TYPE: "serial"}) executor = get_from_registry(hyperopt_params[EXECUTOR][TYPE], executor_registry) executor_defaults = { k: v for k, v in executor.__dict__.items() if k in get_class_attributes(executor) } set_default_values( hyperopt_params[EXECUTOR], executor_defaults, )
def populate_defaults(output_feature): set_default_value( output_feature, LOSS, { TYPE: "sequence_softmax_cross_entropy", "class_weights": 1, "robust_lambda": 0, "confidence_penalty": 0, "class_similarities_temperature": 0, "weight": 1, }, ) set_default_value(output_feature[LOSS], "unique", False) set_default_value(output_feature, "decoder", "generator") if output_feature["decoder"] == "tagger": set_default_value(output_feature, "reduce_input", None) set_default_value(output_feature, "dependencies", []) set_default_value(output_feature, "reduce_input", SUM) set_default_value(output_feature, "reduce_dependencies", SUM)
def populate_defaults(input_feature): set_default_value(input_feature, TIED, None) set_default_value(input_feature, "encoder", "parallel_cnn")
def populate_defaults(output_feature): set_default_value( output_feature, LOSS, { TYPE: 'softmax_cross_entropy', 'sampler': None, 'negative_samples': 0, 'distortion': 1, 'labels_smoothing': 0, 'class_weights': 1, 'robust_lambda': 0, 'confidence_penalty': 0, 'class_similarities_temperature': 0, 'weight': 1 }) set_default_value(output_feature[LOSS], TYPE, 'softmax_cross_entropy') set_default_value(output_feature[LOSS], 'labels_smoothing', 0) set_default_value(output_feature[LOSS], 'class_weights', 1) set_default_value(output_feature[LOSS], 'robust_lambda', 0) set_default_value(output_feature[LOSS], 'confidence_penalty', 0) set_default_value(output_feature[LOSS], 'class_similarities_temperature', 0) set_default_value(output_feature[LOSS], 'weight', 1) if output_feature[LOSS][TYPE] == 'sampled_softmax_cross_entropy': set_default_value(output_feature[LOSS], 'sampler', 'log_uniform') set_default_value(output_feature[LOSS], 'negative_samples', 25) set_default_value(output_feature[LOSS], 'distortion', 0.75) else: set_default_value(output_feature[LOSS], 'sampler', None) set_default_value(output_feature[LOSS], 'negative_samples', 0) set_default_value(output_feature[LOSS], 'distortion', 1) set_default_value(output_feature[LOSS], 'unique', False) set_default_value(output_feature, 'decoder', 'generator') if output_feature['decoder'] == 'tagger': set_default_value(output_feature, 'reduce_input', None) set_default_value(output_feature, 'dependencies', []) set_default_value(output_feature, 'reduce_input', SUM) set_default_value(output_feature, 'reduce_dependencies', SUM)
def populate_defaults(input_feature): set_default_value(input_feature, TIED, None) set_default_value(input_feature, "preprocessing", {})
def add_feature_data(feature, dataset_df, data, metadata, preprocessing_parameters): set_default_value(feature['preprocessing'], 'in_memory', preprocessing_parameters['in_memory']) set_default_value(feature['preprocessing'], 'num_processes', preprocessing_parameters['num_processes']) csv_path = None if hasattr(dataset_df, 'csv'): csv_path = os.path.dirname(os.path.abspath(dataset_df.csv)) num_images = len(dataset_df) if num_images == 0: raise ValueError('There are no images in the dataset provided.') # this is not super nice, but works both and DFs and lists first_path = '.' for first_path in dataset_df[feature['name']]: break if csv_path is None and not os.path.isabs(first_path): raise ValueError('Image file paths must be absolute') first_path = get_abs_path(csv_path, first_path) (should_resize, width, height, num_channels, user_specified_num_channels, first_image) = ImageFeatureMixin._finalize_preprocessing_parameters( preprocessing_parameters, first_path) metadata[feature['name']]['preprocessing']['height'] = height metadata[feature['name']]['preprocessing']['width'] = width metadata[ feature['name']]['preprocessing']['num_channels'] = num_channels read_image_and_resize = partial( ImageFeatureMixin._read_image_and_resize, img_width=width, img_height=height, should_resize=should_resize, num_channels=num_channels, resize_method=preprocessing_parameters['resize_method'], user_specified_num_channels=user_specified_num_channels) all_file_paths = [ get_abs_path(csv_path, file_path) for file_path in dataset_df[feature['name']] ] if feature['preprocessing']['in_memory']: # Number of processes to run in parallel for preprocessing num_processes = feature['preprocessing']['num_processes'] metadata[feature['name']]['preprocessing'][ 'num_processes'] = num_processes data[feature['name']] = np.empty( (num_images, height, width, num_channels), dtype=np.uint8) # Split the dataset into pools only if we have an explicit request to use # multiple processes. In case we have multiple input images use the # standard code anyway. if num_processes > 1 or num_images > 1: with Pool(num_processes) as pool: logger.debug( 'Using {} processes for preprocessing images'.format( num_processes)) data[feature['name']] = np.array( pool.map(read_image_and_resize, all_file_paths)) else: # If we're not running multiple processes and we are only processing one # image just use this faster shortcut, bypassing multiprocessing.Pool.map logger.debug( 'No process pool initialized. Using one process for preprocessing images' ) img = read_image_and_resize(all_file_paths[0]) data[feature['name']] = np.array([img]) else: data_fp = os.path.splitext(dataset_df.csv)[0] + '.hdf5' mode = 'w' if os.path.isfile(data_fp): mode = 'r+' with h5py.File(data_fp, mode) as h5_file: # TODO add multiprocessing/multithreading image_dataset = h5_file.create_dataset( feature['name'] + '_data', (num_images, height, width, num_channels), dtype=np.uint8) for i, filepath in enumerate(all_file_paths): image_dataset[i, :height, :width, :] = ( read_image_and_resize(filepath)) data[feature['name']] = np.arange(num_images)
def populate_defaults(input_feature): set_default_value(input_feature, TIED, None)
def add_feature_data(feature, input_df, proc_df, metadata, preprocessing_parameters, backend, skip_save_processed_input): set_default_value(feature['preprocessing'], 'in_memory', preprocessing_parameters['in_memory']) if 'audio_feature' not in preprocessing_parameters: raise ValueError( 'audio_feature dictionary has to be present in preprocessing ' 'for audio.') if TYPE not in preprocessing_parameters['audio_feature']: raise ValueError( 'type has to be present in audio_feature dictionary ' 'for audio.') name = feature[NAME] column = feature[COLUMN] proc_column = feature[PROC_COLUMN] src_path = None # this is not super nice, but works both and DFs and lists first_path = '.' for first_path in input_df[column]: break if hasattr(input_df, 'src'): src_path = os.path.dirname(os.path.abspath(input_df.src)) if src_path is None and not os.path.isabs(first_path): raise ValueError('Audio file paths must be absolute') num_audio_utterances = len(input_df) padding_value = preprocessing_parameters['padding_value'] normalization_type = preprocessing_parameters['norm'] feature_dim = metadata[name]['feature_dim'] max_length = metadata[name]['max_length'] audio_feature_dict = preprocessing_parameters['audio_feature'] audio_file_length_limit_in_s = preprocessing_parameters[ 'audio_file_length_limit_in_s'] if num_audio_utterances == 0: raise ValueError( 'There are no audio files in the dataset provided.') if feature[PREPROCESSING]['in_memory']: audio_features, audio_stats = AudioFeatureMixin._process_in_memory( input_df[feature[NAME]], src_path, audio_feature_dict, feature_dim, max_length, padding_value, normalization_type, audio_file_length_limit_in_s, backend) proc_df[proc_column] = audio_features audio_stats['std'] = np.sqrt(audio_stats['var'] / float(audio_stats['count'])) print_statistics = ("{} audio files loaded.\n" "Statistics of audio file lengths:\n" "- mean: {:.4f}\n" "- std: {:.4f}\n" "- max: {:.4f}\n" "- min: {:.4f}\n" "- cropped audio_files: {}\n" "Max length was given as {}s").format( audio_stats['count'], audio_stats['mean'], audio_stats['std'], audio_stats['max'], audio_stats['min'], audio_stats['cropped'], audio_file_length_limit_in_s) logger.debug(print_statistics) else: backend.check_lazy_load_supported(feature) return proc_df
def populate_defaults(output_feature): set_default_value(output_feature, 'level', 'word') SequenceOutputFeature.populate_defaults(output_feature)
def populate_defaults(output_feature): set_default_value(output_feature, LOSS, { TYPE: SIGMOID_CROSS_ENTROPY, 'weight': 1 }) set_default_value(output_feature[LOSS], 'weight', 1) set_default_value(output_feature[LOSS], 'class_weights', 1) set_default_value(output_feature, 'threshold', 0.5) set_default_value(output_feature, 'dependencies', []) set_default_value(output_feature, 'reduce_input', SUM) set_default_value(output_feature, 'reduce_dependencies', SUM)