def prepare_data_index(self): # Make sure the shuffling and picking of unknowns is deterministic. random.seed(RANDOM_SEED) wanted_words_index = {} for index, wanted_word in enumerate(wanted_words): wanted_words_index[wanted_word] = index + 2 self.data_index = {'validation': [], 'testing': [], 'training': []} unknown_index = {'validation': [], 'testing': [], 'training': []} all_words = {} # Look through all the subfolders to find audio samples search_path = os.path.join(self.data_dir, '*', '*.wav') for wav_path in gfile.Glob(search_path): word = re.search('.*/([^/]+)/.*.wav', wav_path).group(1).lower() # Treat the '_background_noise_' folder as a special case, since we expect # it to contain long audio samples we mix in to improve training. if word == BACKGROUND_NOISE_DIR_NAME: continue all_words[word] = True set_index = which_set(wav_path, validation_percentage, testing_percentage) # If it's a known class, store its detail, otherwise add it to the list # we'll use to train the unknown label. if word in wanted_words_index: self.data_index[set_index].append({ 'label': word, 'file': wav_path }) else: unknown_index[set_index].append({ 'label': word, 'file': wav_path }) if not all_words: raise Exception('No .wavs found at ' + search_path) for index, wanted_word in enumerate(wanted_words): if wanted_word not in all_words: raise Exception('Expected to find ' + wanted_word + ' in labels but only found ' + ', '.join(all_words.keys())) # We need an arbitrary file to load as the input for the silence samples. # It's multiplied by zero later, so the content doesn't matter. silence_wav_path = self.data_index['training'][0]['file'] for set_index in ['validation', 'testing', 'training']: set_size = len(self.data_index[set_index]) silence_size = int(math.ceil(set_size * silence_percentage / 100)) for _ in range(silence_size): self.data_index[set_index].append({ 'label': SILENCE_LABEL, 'file': silence_wav_path }) # Pick some unknowns to add to each partition of the data set. random.shuffle(unknown_index[set_index]) unknown_size = int(math.ceil(set_size * unknown_percentage / 100)) self.data_index[set_index].extend( unknown_index[set_index][:unknown_size]) # Make sure the ordering is random. for set_index in ['validation', 'testing', 'training']: random.shuffle(self.data_index[set_index]) # Prepare the rest of the result data structure. self.words_list = prepare_words_list(wanted_words) self.word_to_index = {} for word in all_words: if word in wanted_words_index: self.word_to_index[word] = wanted_words_index[word] else: self.word_to_index[word] = UNKNOWN_WORD_INDEX self.word_to_index[SILENCE_LABEL] = SILENCE_INDEX
def _delete_ckpt(self): # Remove all checkpoint files. prefix = self._ckpt_path() pattern = prefix + "*" files = gfile.Glob(pattern) map(gfile.Remove, files)
def prepare_data_index(self, silence_percentage, unknown_percentage, wanted_words, validation_percentage, testing_percentage): """Prepares a list of the samples organized by set and label. The training loop needs a list of all the available data, organized by which partition it should belong to, and with ground truth labels attached. This function analyzes the folders below the `data_dir`, figures out the right labels for each file based on the name of the subdirectory it belongs to, and uses a stable hash to assign it to a data set partition. Args: silence_percentage: How much of the resulting data should be background. unknown_percentage: How much should be audio outside the wanted classes. wanted_words: Labels of the classes we want to be able to recognize. validation_percentage: How much of the data set to use for validation. testing_percentage: How much of the data set to use for testing. Returns: Dictionary containing a list of file information for each set partition, and a lookup map for each class to determine its numeric index. Raises: Exception: If expected files are not found. """ # Make sure the shuffling and picking of unknowns is deterministic. random.seed(RANDOM_SEED) wanted_words_index = {} for index, wanted_word in enumerate(wanted_words): wanted_words_index[wanted_word] = index + 2 self.data_index = {'validation': [], 'testing': [], 'training': []} unknown_index = {'validation': [], 'testing': [], 'training': []} all_words = {} # Look through all the subfolders to find audio samples search_path = os.path.join(self.data_dir, '*', '*.wav') for wav_path in gfile.Glob(search_path): word = re.search('.*/([^/]+)/.*.wav', wav_path).group(1).lower() # Treat the '_background_noise_' folder as a special case, since we expect # it to contain long audio samples we mix in to improve training. if word == BACKGROUND_NOISE_DIR_NAME: continue all_words[word] = True set_index = which_set(wav_path, validation_percentage, testing_percentage) # If it's a known class, store its detail, otherwise add it to the list # we'll use to train the unknown label. if word in wanted_words_index: self.data_index[set_index].append({ 'label': word, 'file': wav_path }) else: unknown_index[set_index].append({ 'label': word, 'file': wav_path }) if not all_words: raise Exception('No .wavs found at ' + search_path) for index, wanted_word in enumerate(wanted_words): if wanted_word not in all_words: raise Exception('Expected to find ' + wanted_word + ' in labels but only found ' + ', '.join(all_words.keys())) # We need an arbitrary file to load as the input for the silence samples. # It's multiplied by zero later, so the content doesn't matter. silence_wav_path = self.data_index['training'][0]['file'] for set_index in ['validation', 'testing', 'training']: set_size = len(self.data_index[set_index]) silence_size = int(math.ceil(set_size * silence_percentage / 100)) for _ in range(silence_size): self.data_index[set_index].append({ 'label': SILENCE_LABEL, 'file': silence_wav_path }) # Pick some unknowns to add to each partition of the data set. random.shuffle(unknown_index[set_index]) unknown_size = int(math.ceil(set_size * unknown_percentage / 100)) self.data_index[set_index].extend( unknown_index[set_index][:unknown_size]) # Make sure the ordering is random. for set_index in ['validation', 'testing', 'training']: random.shuffle(self.data_index[set_index]) # Prepare the rest of the result data structure. self.words_list = prepare_words_list(wanted_words) self.word_to_index = {} for word in all_words: if word in wanted_words_index: self.word_to_index[word] = wanted_words_index[word] else: self.word_to_index[word] = UNKNOWN_WORD_INDEX self.word_to_index[SILENCE_LABEL] = SILENCE_INDEX
def read_keyed_batch_examples(file_pattern, batch_size, reader, randomize_input=True, num_epochs=None, queue_capacity=10000, num_threads=1, read_batch_size=1, parse_fn=None, name=None): """Adds operations to read, queue, batch `Example` protos. Given file pattern (or list of files), will setup a queue for file names, read `Example` proto using provided `reader`, use batch queue to create batches of examples of size `batch_size`. All queue runners are added to the queue runners collection, and may be started via `start_queue_runners`. All ops are added to the default graph. Use `parse_fn` if you need to do parsing / processing on single examples. Args: file_pattern: List of files or pattern of file paths containing `Example` records. See `tf.gfile.Glob` for pattern rules. batch_size: An int or scalar `Tensor` specifying the batch size to use. reader: A function or class that returns an object with `read` method, (filename tensor) -> (example tensor). randomize_input: Whether the input should be randomized. num_epochs: Integer specifying the number of times to read through the dataset. If `None`, cycles through the dataset forever. NOTE - If specified, creates a variable that must be initialized, so call `tf.initialize_all_variables()` as shown in the tests. queue_capacity: Capacity for input queue. num_threads: The number of threads enqueuing examples. read_batch_size: An int or scalar `Tensor` specifying the number of records to read at once parse_fn: Parsing function, takes `Example` Tensor returns parsed representation. If `None`, no parsing is done. name: Name of resulting op. Returns: Returns tuple of: - `Tensor` of string keys. - String `Tensor` of batched `Example` proto. Raises: ValueError: for invalid inputs. """ # Retrieve files to read. if isinstance(file_pattern, list): file_names = file_pattern if not file_names: raise ValueError('No files given to dequeue_examples.') else: file_names = list(gfile.Glob(file_pattern)) if not file_names: raise ValueError('No files match %s.' % file_pattern) # Sort files so it will be deterministic for unit tests. They'll be shuffled # in `string_input_producer` if `randomize_input` is enabled. if not randomize_input: file_names = sorted(file_names) # Check input parameters are given and reasonable. if (not queue_capacity) or (queue_capacity <= 0): raise ValueError('Invalid queue_capacity %s.' % queue_capacity) if (batch_size is None) or ( (not isinstance(batch_size, ops.Tensor)) and (batch_size <= 0 or batch_size > queue_capacity)): raise ValueError('Invalid batch_size %s, with queue_capacity %s.' % (batch_size, queue_capacity)) if (read_batch_size is None) or ( (not isinstance(read_batch_size, ops.Tensor)) and (read_batch_size <= 0)): raise ValueError('Invalid read_batch_size %s.' % read_batch_size) if (not num_threads) or (num_threads <= 0): raise ValueError('Invalid num_threads %s.' % num_threads) if (num_epochs is not None) and (num_epochs <= 0): raise ValueError('Invalid num_epochs %s.' % num_epochs) with ops.name_scope(name, 'read_batch_examples', [file_pattern]) as scope: # Setup filename queue with shuffling. with ops.name_scope('file_name_queue') as file_name_queue_scope: file_name_queue = input_ops.string_input_producer( constant_op.constant(file_names, name='input'), shuffle=randomize_input, num_epochs=num_epochs, name=file_name_queue_scope) # Create readers, one per thread and set them to read from filename queue. with ops.name_scope('read'): example_list = [] for _ in range(num_threads): if read_batch_size > 1: keys, examples_proto = reader().read_up_to( file_name_queue, read_batch_size) else: keys, examples_proto = reader().read(file_name_queue) if parse_fn: parsed_examples = parse_fn(examples_proto) # Map keys into example map because batch_join doesn't support # tuple of Tensor + dict. if isinstance(parsed_examples, dict): parsed_examples[KEY_FEATURE_NAME] = keys example_list.append(parsed_examples) else: example_list.append((keys, parsed_examples)) else: example_list.append((keys, examples_proto)) enqueue_many = read_batch_size > 1 if num_epochs is not None: allow_smaller_final_batch = True else: allow_smaller_final_batch = False # Setup batching queue given list of read example tensors. if randomize_input: if isinstance(batch_size, ops.Tensor): min_after_dequeue = int(queue_capacity * 0.4) else: min_after_dequeue = max(queue_capacity - (3 * batch_size), batch_size) queued_examples_with_keys = input_ops.shuffle_batch_join( example_list, batch_size, capacity=queue_capacity, min_after_dequeue=min_after_dequeue, enqueue_many=enqueue_many, name=scope, allow_smaller_final_batch=allow_smaller_final_batch) else: queued_examples_with_keys = input_ops.batch_join( example_list, batch_size, capacity=queue_capacity, enqueue_many=enqueue_many, name=scope, allow_smaller_final_batch=allow_smaller_final_batch) if parse_fn and isinstance(queued_examples_with_keys, dict): queued_keys = queued_examples_with_keys.pop(KEY_FEATURE_NAME) return queued_keys, queued_examples_with_keys return queued_examples_with_keys
def mnist_tfrecord_input(data_dir, training=True, sequence_length=20, img_size=None, batch_size=1, seed=None): """Create input tfrecord tensors and queues. TFRecord: TFRecord(s) are assumed to be placed at `data_dir`. Each sample contains raw uint8 image sequences with key `'img_i'`. Training and validation set are pre-splitted and their corresponding record file have suffix `_trn.tfrecords` or `_val_tfrecords`. Preprocessing: Crop each image to a square one with size `min(ORIGINAL_HEIGHT, ORIGINAL_WIDTH)` and resize (bicubic) to `(IMG_WIDTH, IMG_HEIGHT)`. Normalize pixel value from [0, 255] to [0, 1] Args: data_dir: directory holding TFRecord(s). training: whether to use training or validation data. sequence_length: length of the video sequence. img_size: the (hight, width) of processed img input, if None use original size. batch_size: size of data mimi-batches. seed: random seed for `shuffle_batch` generator. Returns: list of tensors corresponding to images. The images tensor is 5D, batch x time x height x width x 1. Raises: RuntimeError: if no files found. """ file_suffix = '*_trn.tfrecords' if training else '*_val.tfrecords' filenames = gfile.Glob(os.path.join(data_dir, file_suffix)) if not filenames: raise RuntimeError('No data files found.') filename_queue = tf.train.string_input_producer(filenames, shuffle=True) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) image_seq = [] for i in range(sequence_length): # extract image tensor image_name = 'img_{}'.format(i) features = tf.parse_single_example( serialized_example, features={image_name: tf.FixedLenFeature([], tf.string)} ) image = tf.decode_raw(features[image_name], tf.uint8) image = tf.reshape(image, shape=[ORIGINAL_HEIGHT, ORIGINAL_WIDTH, COLOR_CHAN]) # preprocessing crop_size = min(ORIGINAL_HEIGHT, ORIGINAL_WIDTH) image = tf.image.resize_image_with_crop_or_pad(image, crop_size, crop_size) image = tf.reshape(image, [1, crop_size, crop_size, COLOR_CHAN]) if img_size is None: img_size = (ORIGINAL_HEIGHT, ORIGINAL_WIDTH) if img_size[0] != img_size[1]: raise ValueError('Unequal height and width unsupported') image = tf.image.resize_bicubic(image, img_size) image = tf.cast(image, tf.float32) / 255.0 image_seq.append(image) image_seq = tf.concat(axis=0, values=image_seq) image_batch = tf.train.shuffle_batch( tensors=[image_seq], batch_size=batch_size, capacity=100 * batch_size, min_after_dequeue=50 * batch_size, num_threads=batch_size, seed=seed ) return image_batch
def create_image_lists(image_dir, testing_percentage, validation_percentage): if not gfile.Exists(image_dir): tf.logging.error("Image directory '" + image_dir + "' not found.") return None # end if result = {} sub_dirs = [x[0] for x in gfile.Walk(image_dir)] is_root_dir = True for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue # end if dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue # end if # ToDo: This section should be refactored. The right way to do this would be to get a list of the files that are # ToDo: there then append (extend) those, not to get the name except the extension, then append an extension, # ToDo: this (current) way is error prone of the original file has an upper case or mixed case extension extensions = ['jpg', 'jpeg'] file_list = [] tf.logging.info("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(gfile.Glob(file_glob)) # end for if not file_list: tf.logging.warning('No files found') continue # end if if len(file_list) < 20: tf.logging.warning( 'WARNING: Folder has less than 20 images, which may cause issues.' ) elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: tf.logging.warning( 'WARNING: Folder {} has more than {} images. Some images will never be selected.' .format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) # end if label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) training_images = [] testing_images = [] validation_images = [] for file_name in file_list: base_name = os.path.basename(file_name) hash_name = re.sub(r'_nohash_.*$', '', file_name) hash_name_hashed = hashlib.sha1( compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) elif percentage_hash < (testing_percentage + validation_percentage): testing_images.append(base_name) else: training_images.append(base_name) # end if result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
def tearDown(self): prefix = self._iterator_checkpoint_prefix() pattern = prefix + "*" files = gfile.Glob(pattern) map(gfile.Remove, files) super(CheckpointTest, self).tearDown()
def main(_): image_dir = FLAGS.image_file if os.path.isdir(FLAGS.output_dir): if len(os.listdir(FLAGS.output_dir)) > 0: print("WARNING: output folder is not empty") else: sys.exit("output path not defined or does not exist") # For each sufolder (class) ## --> read all the test images in a loop ## --> for each slide: ## Number of Tiles ## Number of Tiles classified in each class ## Average score for each class (first sum then divide) # Read out_filename stats: stats_dict = dict_tiles_stats() # Read the name of the folder (= class names) if FLAGS.tiles_stats == '': create_graph() sub_dirs = [] for item in os.listdir(image_dir): if os.path.isdir(os.path.join(image_dir, item)): sub_dirs.append(os.path.join(image_dir, item)) #sub_dirs = [image_dir] print("sub_dirs:") print(sub_dirs) SlideRootName = '' SlideNames = [] skip = False # get all the label names # For each class for sub_dir in list(sub_dirs): #for sub_dir in list(reversed(sub_dirs)): dir_name = os.path.basename(sub_dir) #extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] extensions = ['jpeg'] file_list = [] print("list the images in folder %s..." % (dir_name)) for extension in extensions: #file_glob = os.path.join(image_dir, dir_name, 'test_*.' + extension) file_glob = os.path.join( image_dir, dir_name, '*' + FLAGS.slide_filter + '*.' + extension) file_list.extend(gfile.Glob(file_glob)) if not file_list: print('No images found') print( os.path.join(image_dir, dir_name, '*' + FLAGS.slide_filter + '*.' + extension)) continue ## Aggregate the results and build heatmaps Start = True NewSlide = True # 1. For each slide, compute the number of good and bad classifications for test_filename in sorted(file_list): #cTileRootName = os.path.basename(test_filename).split('_files')[0] #cTileRootName = os.path.basename(test_filename)[0:50] # remove slide number from image name: cTileRootName = '_'.join( os.path.basename(test_filename).split('_')[0:-2]) # extract coordinates of the tile ixTile = int(os.path.basename(test_filename).split('_')[-2]) iyTile = int( os.path.basename(test_filename).split('_')[-1].split('.')[0]) im2 = imread(test_filename) # check how bif the "re-combined" slide should be (invert col/row because of the swapaxes required) rTile = im2.shape[1] cTile = im2.shape[0] xTile = (ixTile) * (FLAGS.tiles_size - FLAGS.tiles_overlap) yTile = (iyTile) * (FLAGS.tiles_size - FLAGS.tiles_overlap) req_xLength = xTile + rTile req_yLength = yTile + cTile if FLAGS.resample_factor > 0: print("old / new r&cTile") print(rTile, cTile, xTile, yTile, req_xLength, req_yLength) rTile = int(rTile / FLAGS.resample_factor) cTile = int(cTile / FLAGS.resample_factor) #print(rTile, cTile) im2s = scipy.misc.imresize(im2, (cTile, rTile)) rTile = im2s.shape[1] cTile = im2s.shape[0] #ixTile = int(ixTile / FLAGS.resample_factor) #iyTile = int(iyTile / FLAGS.resample_factor) xTile = int(xTile / FLAGS.resample_factor) yTile = int(yTile / FLAGS.resample_factor) req_xLength = xTile + rTile req_yLength = yTile + cTile print(rTile, cTile, xTile, yTile, req_xLength, req_yLength) else: im2s = im2 #print(ixTile, iyTile, rTile, cTile, req_xLength, req_yLength, xTile, yTile) if cTileRootName == SlideRootName: if skip: continue NewSlide = False #if skip: # continue else: # Moved to a new slide print("Analyzing %s" % (cTileRootName)) if Start: Start = False #else: elif skip == False: # For previous the slide which is now finished, compute the averages skip = saveMap(HeatMap_divider, HeatMap_0, WholeSlide_0, SlideRootName, NewSlide) NewSlide = True skip = False SlideRootName = cTileRootName # create a new re-combined slide WholeSlide_0 = np.zeros([req_xLength, req_yLength, 3]) HeatMap_0 = np.zeros([req_xLength, req_yLength, 3]) HeatMap_divider = np.zeros([req_xLength, req_yLength, 3]) # Check score associated with that image: # print("FLAGS.tiles_stats") # print(FLAGS.tiles_stats) # text file with stats from fully retrained network oClass, cmap, current_score = get_inference_from_file( test_filename, cTileRootName, stats_dict) # prepare heatmap print("current score") print(current_score) if current_score < 0: print("No probability found") else: if NewSlide == False: # append to the new re-combined slide WholeSlide_temp = np.zeros([ max(WholeSlide_0.shape[0], req_xLength), max(WholeSlide_0.shape[1], req_yLength), 3 ]) WholeSlide_temp[0:WholeSlide_0.shape[0], 0:WholeSlide_0.shape[1], :] = WholeSlide_0 WholeSlide_temp[xTile:req_xLength, yTile:req_yLength, :] = np.swapaxes( im2s, 0, 1) WholeSlide_0 = WholeSlide_temp del WholeSlide_temp HeatMap_temp = WholeSlide_0 * 0 HeatMap_temp[0:HeatMap_0.shape[0], 0:HeatMap_0.shape[1], :] = HeatMap_0 HeatMap_0 = HeatMap_temp del HeatMap_temp HeatMap_divider_tmp = WholeSlide_0 * 0 HeatMap_divider_tmp[ 0:HeatMap_divider.shape[0], 0:HeatMap_divider.shape[1], :] = HeatMap_divider HeatMap_divider = HeatMap_divider_tmp del HeatMap_divider_tmp #skip = saveMap(HeatMap_divider, HeatMap_0, WholeSlide_0, SlideRootName, label_name, NewSlide) else: WholeSlide_0[xTile:req_xLength, yTile:req_yLength, :] = np.swapaxes( im2s, 0, 1) #heattile = np.ones([512,512]) * current_score heattile = np.ones([req_xLength - xTile, req_yLength - yTile ]) * current_score heattile = cmap(heattile) heattile = heattile[:, :, 0:3] HeatMap_0[xTile:req_xLength, yTile:req_yLength, :] = HeatMap_0[ xTile:req_xLength, yTile:req_yLength, :] + heattile HeatMap_divider[xTile:req_xLength, yTile:req_yLength, :] = HeatMap_divider[ xTile:req_xLength, yTile:req_yLength, :] + 1 skip = saveMap(HeatMap_divider, HeatMap_0, WholeSlide_0, SlideRootName, NewSlide) if skip: continue skip = saveMap(HeatMap_divider, HeatMap_0, WholeSlide_0, SlideRootName, NewSlide)
def run_and_gather_logs(name, test_name, test_args, benchmark_type): """Run the bazel test given by test_name. Gather and return the logs. Args: name: Benchmark target identifier. test_name: A unique bazel target, e.g. "//path/to:test" test_args: A string containing all arguments to run the target with. benchmark_type: A string representing the BenchmarkType enum; the benchmark type for this target. Returns: A tuple (test_results, mangled_test_name), where test_results: A test_log_pb2.TestResults proto test_adjusted_name: Unique benchmark name that consists of benchmark name optionally followed by GPU type. Raises: ValueError: If the test_name is not a valid target. subprocess.CalledProcessError: If the target itself fails. IOError: If there are problems gathering test log output from the test. MissingLogsError: If we couldn't find benchmark logs. """ if not (test_name and six.ensure_str(test_name).startswith("//") and ".." not in test_name and not six.ensure_str(test_name).endswith(":") and not six.ensure_str(test_name).endswith(":all") and not six.ensure_str(test_name).endswith("...") and len(six.ensure_str(test_name).split(":")) == 2): raise ValueError("Expected test_name parameter with a unique test, e.g.: " "--test_name=//path/to:test") test_executable = six.ensure_str(test_name.rstrip()).strip("/").replace( ":", "/") if gfile.Exists(os.path.join("bazel-bin", test_executable)): # Running in standalone mode from core of the repository test_executable = os.path.join("bazel-bin", test_executable) else: # Hopefully running in sandboxed mode test_executable = os.path.join(".", test_executable) test_adjusted_name = name gpu_config = gpu_info_lib.gather_gpu_devices() if gpu_config: gpu_name = gpu_config[0].model gpu_short_name_match = re.search(r"Tesla (K40|K80|P100|V100)", six.ensure_str(gpu_name)) if gpu_short_name_match: gpu_short_name = gpu_short_name_match.group(0) test_adjusted_name = six.ensure_str(name) + "|" + gpu_short_name.replace( " ", "_") temp_directory = tempfile.mkdtemp(prefix="run_and_gather_logs") mangled_test_name = ( six.ensure_str(test_adjusted_name).strip("/").replace("|", "_").replace( "/", "_").replace(":", "_")) test_file_prefix = os.path.join(temp_directory, mangled_test_name) test_file_prefix = "%s." % test_file_prefix try: if not gfile.Exists(test_executable): test_executable_py3 = test_executable + ".python3" if not gfile.Exists(test_executable_py3): raise ValueError("Executable does not exist: %s" % test_executable) test_executable = test_executable_py3 test_args = shlex.split(test_args) # This key is defined in tf/core/util/reporter.h as # TestReporter::kTestReporterEnv. os.environ["TEST_REPORT_FILE_PREFIX"] = test_file_prefix start_time = time.time() subprocess.check_call([test_executable] + test_args) run_time = time.time() - start_time log_files = gfile.Glob("{}*".format(test_file_prefix)) if not log_files: raise MissingLogsError("No log files found at %s." % test_file_prefix) return (process_test_logs( test_adjusted_name, test_name=test_name, test_args=test_args, benchmark_type=benchmark_type, start_time=int(start_time), run_time=run_time, log_files=log_files), test_adjusted_name) finally: try: gfile.DeleteRecursively(temp_directory) except OSError: pass
def get_image_list(image_dir, category, partition): """ Compute list of images. """ file_glob = os.path.join(image_dir, category, partition, '*.jpeg') file_list = gfile.Glob(file_glob) return file_list
def build_tfrecord_single(conf, mode='train', input_files=None, shuffle=True, buffersize=512): """Create input tfrecord tensors. Args: training: training or validation data_files. conf: A dictionary containing the configuration for the experiment Returns: list of tensors corresponding to images, actions, and states. The images tensor is 5D, batch x time x height x width x channels. The state and action tensors are 3D, batch x time x dimension. Raises: RuntimeError: if no files found. """ if 'sdim' in conf: sdim = conf['sdim'] else: sdim = 3 if 'adim' in conf: adim = conf['adim'] else: adim = 4 print('adim', adim) print('sdim', sdim) if input_files is not None: if not isinstance(input_files, list): filenames = [input_files] else: filenames = input_files else: filenames = gfile.Glob(os.path.join(conf['data_dir'], mode) + '/*') if mode == 'val' or mode == 'test': shuffle = False else: shuffle = True if not filenames: raise RuntimeError('No data_files files found.') print('using shuffle: ', shuffle) if shuffle: shuffle_list(filenames) # Reads an image from a file, decodes it into a dense tensor, and resizes it # to a fixed shape. def _parse_function(serialized_example): image_seq, image_main_seq, endeffector_pos_seq, gen_images_seq, gen_states_seq,\ action_seq, object_pos_seq, robot_pos_seq, goal_image = [], [], [], [], [], [], [], [], [] load_indx = list(range(0, conf['sequence_length'], conf['skip_frame'])) print('using frame sequence: ', load_indx) rand_h = tf.random_uniform([1], minval=-0.2, maxval=0.2) rand_s = tf.random_uniform([1], minval=-0.2, maxval=0.2) rand_v = tf.random_uniform([1], minval=-0.2, maxval=0.2) features_name = {} for i in load_indx: image_names = [] if 'view' in conf: cam_ids = [conf['view']] else: if 'ncam' in conf: ncam = conf['ncam'] else: ncam = 1 cam_ids = range(ncam) for icam in cam_ids: image_names.append( str(i) + '/image_view{}/encoded'.format(icam)) features_name[image_names[-1]] = tf.FixedLenFeature([1], tf.string) if 'image_only' not in conf: action_name = str(i) + '/action' endeffector_pos_name = str(i) + '/endeffector_pos' if 'image_only' not in conf: features_name[action_name] = tf.FixedLenFeature([adim], tf.float32) features_name[endeffector_pos_name] = tf.FixedLenFeature( [sdim], tf.float32) if 'test_metric' in conf: robot_pos_name = str(i) + '/robot_pos' object_pos_name = str(i) + '/object_pos' features_name[robot_pos_name] = tf.FixedLenFeature( [conf['test_metric']['robot_pos'] * 2], tf.int64) features_name[object_pos_name] = tf.FixedLenFeature( [conf['test_metric']['object_pos'] * 2], tf.int64) if 'load_vidpred_data' in conf: gen_image_name = str(i) + '/gen_images' gen_states_name = str(i) + '/gen_states' features_name[gen_image_name] = tf.FixedLenFeature([1], tf.string) features_name[gen_states_name] = tf.FixedLenFeature([sdim], tf.float32) features = tf.parse_single_example(serialized_example, features=features_name) images_t = [] for image_name in image_names: image = decode_im(conf, features, image_name) if 'color_augmentation' in conf: # print 'performing color augmentation' image_hsv = tf.image.rgb_to_hsv(image) img_stack = [ tf.unstack(imag, axis=2) for imag in tf.unstack(image_hsv, axis=0) ] stack_mod = [ tf.stack([x[0] + rand_h, x[1] + rand_s, x[2] + rand_v], axis=2) for x in img_stack ] image_rgb = tf.image.hsv_to_rgb(tf.stack(stack_mod)) image = tf.clip_by_value(image_rgb, 0.0, 1.0) images_t.append(image) image_seq.append(tf.stack(images_t, axis=1)) if 'image_only' not in conf: endeffector_pos = tf.reshape(features[endeffector_pos_name], shape=[1, sdim]) endeffector_pos_seq.append(endeffector_pos) action = tf.reshape(features[action_name], shape=[1, adim]) action_seq.append(action) if 'test_metric' in conf: robot_pos = tf.reshape(features[robot_pos_name], shape=[1, 2]) robot_pos_seq.append(robot_pos) object_pos = tf.reshape( features[object_pos_name], shape=[1, conf['test_metric']['object_pos'], 2]) object_pos_seq.append(object_pos) if 'load_vidpred_data' in conf: gen_images_seq.append(decode_im(gen_image_name)) gen_states = tf.reshape(features[gen_states_name], shape=[1, sdim]) gen_states_seq.append(gen_states) return_dict = {} image_seq = tf.concat(values=image_seq, axis=0) image_seq = tf.squeeze(image_seq) if 'use_cam' in conf: image_seq = image_seq[:, conf['use_cam']] return_dict['images'] = image_seq if 'goal_image' in conf: features_name = {} features_name['/goal_image'] = tf.FixedLenFeature([1], tf.string) features = tf.parse_single_example(serialized_example, features=features_name) goal_image = tf.squeeze(decode_im(conf, features, '/goal_image')) return_dict['goal_image'] = goal_image if 'first_last_noarm' in conf: features_name = {} features_name['/first_last_noarm0'] = tf.FixedLenFeature([1], tf.string) features = tf.parse_single_example(serialized_example, features=features_name) first_last_noarm0 = tf.squeeze( decode_im(conf, features, '/first_last_noarm0')) features_name['/first_last_noarm1'] = tf.FixedLenFeature([1], tf.string) features = tf.parse_single_example(serialized_example, features=features_name) first_last_noarm1 = tf.squeeze( decode_im(conf, features, '/first_last_noarm1')) return_dict['first_last_noarm'] = tf.stack( [first_last_noarm0, first_last_noarm1], axis=0) if 'image_only' not in conf: if 'no_touch' in conf: return_dict['endeffector_pos'] = tf.concat( endeffector_pos_seq, 0)[:, :-2] else: return_dict['endeffector_pos'] = tf.concat( endeffector_pos_seq, 0) if 'autograsp' in conf: return_dict['actions'] = tf.concat(action_seq, 0)[:, :-1] else: return_dict['actions'] = tf.concat(action_seq, 0) if 'load_vidpred_data' in conf: return_dict['gen_images'] = gen_images_seq return_dict['gen_states'] = gen_states_seq return return_dict dataset = tf.data.TFRecordDataset(filenames) dataset = dataset.map(_parse_function) if 'max_epoch' in conf: dataset = dataset.repeat(conf['max_epoch']) else: dataset = dataset.repeat() if shuffle: dataset = dataset.shuffle(buffer_size=buffersize) dataset = dataset.batch(conf['batch_size']) iterator = dataset.make_one_shot_iterator() next_element = iterator.get_next() output_element = {} for k in list(next_element.keys()): output_element[k] = tf.reshape( next_element[k], [conf['batch_size']] + next_element[k].get_shape().as_list()[1:]) return output_element
'path_modelo_arquitetura': 'inception_v3.pb', 'input_mean': 128, 'input_std': 128, } graph, operacao_entrada, operacao_saida, rotulos = Carrega_Grafo_e_Rotulos( model_info) # Inicia a sessao with tensorflow.Session(graph=graph) as sessao: grounds_predicts = [['fn', 'label']] labels = ['upright', 'rotated_left', 'rotated_right', 'upside_down'] matriz_numpy = [] for path_imagem in gfile.Glob('test/*'): #for path_imagem in range(1): image = cv2.imread(path_imagem) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) face_dims_expander = tensorflow.expand_dims(image, 0) face_resized = tensorflow.image.resize_bilinear( face_dims_expander, [model_info['input_width'], model_info['input_height']]) face_normalizada = tensorflow.divide( tensorflow.subtract(face_resized, model_info['input_mean']), model_info['input_std']) resultado = sessao.run( operacao_saida.outputs[0], {operacao_entrada.outputs[0]: sessao.run(face_normalizada)})
def prepare_data_index(self, silence_percentage, unknown_percentage, wanted_words, validation_percentage, testing_percentage): random.seed(RANDOM_SEED) wanted_words_index = {} for index, wanted_word in enumerate(wanted_words): wanted_words_index[wanted_word] = index + 2 self.data_index = {'validation': [], 'testing': [], 'training': []} unknown_index = {'validation': [], 'testing': [], 'training': []} all_words = {} search_path = os.path.join(self.data_dir, '*', '*.wav') for wav_path in gfile.Glob(search_path): _, word = os.path.split(os.path.dirname(wav_path)) word = word.lower() if word == BACKGROUND_NOISE_DIR_NAME: continue all_words[word] = True set_index = which_set(wav_path, validation_percentage, testing_percentage) if word in wanted_words_index: self.data_index[set_index].append({ 'label': word, 'file': wav_path }) else: unknown_index[set_index].append({ 'label': word, 'file': wav_path }) if not all_words: raise Exception('No .wavs found at ' + search_path) for index, wanted_word in enumerate(wanted_words): if wanted_word not in all_words: raise Exception('Expected to find ' + wanted_word + ' in labels but only found ' + ', '.join(all_words.keys())) silence_wav_path = self.data_index['training'][0]['file'] for set_index in ['validation', 'testing', 'training']: set_size = len(self.data_index[set_index]) silence_size = int(math.ceil(set_size * silence_percentage / 100)) for _ in range(silence_size): self.data_index[set_index].append({ 'label': SILENCE_LABEL, 'file': silence_wav_path }) random.shuffle(unknown_index[set_index]) unknown_size = int(math.ceil(set_size * unknown_percentage / 100)) self.data_index[set_index].extend( unknown_index[set_index][:unknown_size]) for set_index in ['validation', 'testing', 'training']: random.shuffle(self.data_index[set_index]) self.words_list = prepare_words_list(wanted_words) self.word_to_index = {} for word in all_words: if word in wanted_words_index: self.word_to_index[word] = wanted_words_index[word] else: self.word_to_index[word] = UNKNOWN_WORD_INDEX self.word_to_index[SILENCE_LABEL] = SILENCE_INDEX
gfile.MakeDirs(save_dir) with tf.Session( target="", config=tf.ConfigProto(device_count={"CPU": 2})) as sess: with sess.graph.device("/cpu:0"): v0 = tf.Variable(111, name="v0") with sess.graph.device("/cpu:1"): v1 = tf.Variable(222, name="v1") save = tf.train.Saver({"v0": v0, "v1": v1}, sharded=True, max_to_keep=2) tf.initialize_all_variables().run() self.assertEqual([], save.last_checkpoints) s1 = save.save(sess, os.path.join(save_dir, "s1")) self.assertEqual([s1], save.last_checkpoints) self.assertEquals(2, len(gfile.Glob(s1))) s2 = save.save(sess, os.path.join(save_dir, "s2")) self.assertEqual([s1, s2], save.last_checkpoints) self.assertEquals(2, len(gfile.Glob(s1))) self.assertEquals(2, len(gfile.Glob(s2))) s3 = save.save(sess, os.path.join(save_dir, "s3")) self.assertEqual([s2, s3], save.last_checkpoints) self.assertEquals(0, len(gfile.Glob(s1))) self.assertEquals(2, len(gfile.Glob(s2))) self.assertEquals(2, len(gfile.Glob(s3))) class KeepCheckpointEveryNHoursTest(tf.test.TestCase):
dest="verbose", help="Print helpful data.", default=False) options, args = parser.parse_args() # Get cmdline args verbose = options.verbose exp = options.experiment # Retrieve directory from which to load model load_model_dir = os.path.join(base_model_dir, str(exp)) # Retrieve tfrecords filename tf_filename = None filepaths = gfile.Glob(base_tfrecord_dir) readme_file = os.path.join(load_model_dir, "README.md") with open(readme_file, "r") as f: name = f.readline().strip() keys = name.split("_") # Find tfrecord among all tfrecords using README keys for fp in filepaths: tf_filename = fp for k in keys: if k not in fp: tf_filename = None break if tf_filename: break
def prepare_data_index(self, silence_percentage, unknown_percentage, wanted_words, validation_percentage, testing_percentage): """ 准备按集合和标签组织的样本列表: 训练循环需要一个所有可用数据的列表, 按照它应该属于的数据集划分情况进行组织, 并附上真实标签。 此函数分析“data-dir”下面的文件夹, 根据文件所属子目录的名称为每个文件指定正确的标签, 并使用稳定的哈希将其分配给数据集分区。 Args: silence_percentage: 结果数据中应该有多少是静音数据。 unknown_percentage: 多少设为未知数据。 wanted_words: 我们想要识别的标签类别 validation_percentage: 多少数据集用于验证 testing_percentage: 多少数据集用于测试 Returns: 包含每个集合分区的文件信息列表的字典,以及用于确定其数值索引的每个类的查找映射。 """ random.seed(RANDOM_SEED) wanted_words_index = {} for index, wanted_word in enumerate(wanted_words): wanted_words_index[wanted_word] = index + 2 self.data_index = {'validation': [], 'testing': [], 'training': []} unknown_index = {'validation': [], 'testing': [], 'training': []} all_words = {} # 查看所有子文件夹以查找音频样本 search_path = os.path.join(self.data_dir, '*', '*.wav') for wav_path in gfile.Glob(search_path): _, word = os.path.split(os.path.dirname(wav_path)) word = word.lower() # 添加背景噪音 if word == BACKGROUND_NOISE_DIR_NAME: continue all_words[word] = True set_index = which_set(wav_path, validation_percentage, testing_percentage) # 如果是已知标签,保存它的细节,否则将使用它用于训练未知的标签。 if word in wanted_words_index: self.data_index[set_index].append({ 'label': word, 'file': wav_path }) else: unknown_index[set_index].append({ 'label': word, 'file': wav_path }) if not all_words: raise Exception('No .wavs found at ' + search_path) for index, wanted_word in enumerate(wanted_words): if wanted_word not in all_words: raise Exception('Expected to find ' + wanted_word + ' in labels but only found ' + ', '.join(all_words.keys())) # 我们需要一个任意文件作为静音文件的输入来加载。它后来乘以零,所以内容无关紧要都能变为静音数据集。 silence_wav_path = self.data_index['training'][0]['file'] for set_index in ['validation', 'testing', 'training']: set_size = len(self.data_index[set_index]) silence_size = int(math.ceil(set_size * silence_percentage / 100)) for _ in range(silence_size): self.data_index[set_index].append({ 'label': SILENCE_LABEL, 'file': silence_wav_path }) # 选择一些未知项添加到数据集的训练、验证和测试部分 random.shuffle(unknown_index[set_index]) unknown_size = int(math.ceil(set_size * unknown_percentage / 100)) self.data_index[set_index].extend( unknown_index[set_index][:unknown_size]) # 确保训练顺序是随机的 for set_index in ['validation', 'testing', 'training']: random.shuffle(self.data_index[set_index]) # 准备结果数据结构的其余部分 self.words_list = prepare_words_list(wanted_words) self.word_to_index = {} for word in all_words: if word in wanted_words_index: self.word_to_index[word] = wanted_words_index[word] else: self.word_to_index[word] = UNKNOWN_WORD_INDEX self.word_to_index[SILENCE_LABEL] = SILENCE_INDEX
def prepare_split_data_index(self, wanted_words, split_data): """Prepares a list of the samples organized by set and label. The training loop needs a list of all the available data, organized by which partition it should belong to, and with ground truth labels attached. This function analyzes the folders below the `data_dir`, where `data_dir` has to contain folders (prepared by user): testing training validation _background_noise_ - contains data which are used for adding background noise to training data only Args: wanted_words: Labels of the classes we want to be able to recognize. split_data: True - split data automatically; False - user splits the data Returns: Dictionary containing a list of file information for each set partition, and a lookup map for each class to determine its numeric index. Raises: Exception: If expected files are not found. """ # Make sure the shuffling and picking of unknowns is deterministic. random.seed(RANDOM_SEED) dirs = ['testing', 'training', 'validation'] self.validate_dir_structure(self.data_dir, dirs) wanted_words_index = {} for index, wanted_word in enumerate(wanted_words): wanted_words_index[wanted_word] = index self.words_list = prepare_words_list(wanted_words, split_data) self.data_index = {'validation': [], 'testing': [], 'training': []} for set_index in dirs: all_words = {} # Look through all the subfolders in set_index to find audio samples search_path = os.path.join(os.path.join(self.data_dir, set_index), '*', '*.wav') for wav_path in gfile.Glob(search_path): _, word = os.path.split(os.path.dirname(wav_path)) word = word.lower() # Treat the '_background_noise_' folder as a special case, # it contains long audio samples we mix in to improve training. if word == BACKGROUND_NOISE_DIR_NAME: continue all_words[word] = True # If it's a known class, store its detail, otherwise raise error if word in wanted_words_index: self.data_index[set_index].append({ 'label': word, 'file': wav_path }) else: raise Exception('Unknown word ' + word) if not all_words: raise IOError('No .wavs found at ' + search_path) for index, wanted_word in enumerate(wanted_words): if wanted_word not in all_words: raise IOError('Expected to find ' + wanted_word + ' in labels but only found ' + ', '.join(all_words.keys())) # Make sure the ordering is random. for set_index in ['validation', 'testing', 'training']: random.shuffle(self.data_index[set_index]) # Prepare the rest of the result data structure. self.word_to_index = {} for word in all_words: if word in wanted_words_index: self.word_to_index[word] = wanted_words_index[word] else: raise Exception('Unknown word ' + word)
def minibatch(self, dataset, subset, use_datasets, cache_data, shift_ratio=-1): if shift_ratio < 0: shift_ratio = self.shift_ratio with tf.name_scope('batch_processing'): # Build final results per split. images = [[] for _ in range(self.num_splits)] labels = [[] for _ in range(self.num_splits)] if use_datasets: glob_pattern = dataset.tf_record_pattern(subset) file_names = gfile.Glob(glob_pattern) if not file_names: raise ValueError('Found no files in --data_dir matching: {}' .format(glob_pattern)) ds = tf.data.TFRecordDataset.list_files(file_names) ds = ds.apply( interleave_ops.parallel_interleave( tf.data.TFRecordDataset, cycle_length=10)) if cache_data: ds = ds.take(1).cache().repeat() counter = tf.data.Dataset.range(self.batch_size) counter = counter.repeat() ds = tf.data.Dataset.zip((ds, counter)) ds = ds.prefetch(buffer_size=self.batch_size) ds = ds.shuffle(buffer_size=10000) ds = ds.repeat() ds = ds.apply( batching.map_and_batch( map_func=self.parse_and_preprocess, batch_size=self.batch_size_per_split, num_parallel_batches=self.num_splits)) ds = ds.prefetch(buffer_size=self.num_splits) ds_iterator = ds.make_one_shot_iterator() for d in xrange(self.num_splits): labels[d], images[d] = ds_iterator.get_next() else: record_input = data_flow_ops.RecordInput( file_pattern=dataset.tf_record_pattern(subset), seed=301, parallelism=64, buffer_size=10000, batch_size=self.batch_size, shift_ratio=shift_ratio, name='record_input') records = record_input.get_yield_op() records = tf.split(records, self.batch_size, 0) records = [tf.reshape(record, []) for record in records] for idx in xrange(self.batch_size): value = records[idx] (label, image) = self.parse_and_preprocess(value, idx) split_index = idx % self.num_splits labels[split_index].append(label) images[split_index].append(image) for split_index in xrange(self.num_splits): if not use_datasets: images[split_index] = tf.parallel_stack(images[split_index]) labels[split_index] = tf.concat(labels[split_index], 0) images[split_index] = tf.cast(images[split_index], self.dtype) depth = 3 images[split_index] = tf.reshape( images[split_index], shape=[self.batch_size_per_split, self.height, self.width, depth]) labels[split_index] = tf.reshape(labels[split_index], [self.batch_size_per_split]) return images, labels
def tearDown(self): # Remove all checkpoint files. prefix = self._iterator_checkpoint_prefix() pattern = prefix + "*" files = gfile.Glob(pattern) map(gfile.Remove, files)
def build_tfrecord_input(training=True): """Create input tfrecord tensors. Args: training: training or validation data_files. Returns: list of tensors corresponding to images, actions, and states. The images tensor is 5D, batch x time x height x width x channels. The state and action tensors are 3D, batch x time x dimension. Raises: RuntimeError: if no files found. """ filenames = gfile.Glob(os.path.join(FLAGS.data_dir, '*')) if not filenames: raise RuntimeError('No data_files files found.') index = int(np.floor(FLAGS.train_val_split * len(filenames))) if training: filenames = filenames[:index] else: filenames = filenames[index:] # import pdb; pdb.set_trace() filename_queue = tf.train.string_input_producer(filenames, shuffle=True) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) image_seq, state_seq, action_seq = [], [], [] for i in range(FLAGS.sequence_length): image_name = 'move/' + str(i) + '/image/encoded' action_name = 'move/' + str(i) + '/action' state_name = 'move/' + str(i) + '/state' # print 'reading index', i if FLAGS.use_state: features = { image_name: tf.FixedLenFeature([1], tf.string), action_name: tf.FixedLenFeature([ACION_DIM], tf.float32), state_name: tf.FixedLenFeature([STATE_DIM], tf.float32) } else: features = {image_name: tf.FixedLenFeature([1], tf.string)} features = tf.parse_single_example(serialized_example, features=features) image = tf.decode_raw(features[image_name], tf.uint8) image = tf.reshape( image, shape=[1, ORIGINAL_HEIGHT * ORIGINAL_WIDTH * COLOR_CHAN]) image = tf.reshape( image, shape=[1, ORIGINAL_HEIGHT, ORIGINAL_WIDTH, COLOR_CHAN]) if IMG_HEIGHT != IMG_WIDTH: raise ValueError('Unequal height and width unsupported') crop_size = min(ORIGINAL_HEIGHT, ORIGINAL_WIDTH) # image = tf.image.resize_image_with_crop_or_pad(image, crop_size, crop_size) # image = tf.reshape(image, [1, crop_size, crop_size, COLOR_CHAN]) # image = tf.image.resize_bicubic(image, [IMG_HEIGHT, IMG_WIDTH]) image = tf.cast(image, tf.float32) / 255.0 image_seq.append(image) if FLAGS.use_state: state = tf.reshape(features[state_name], shape=[1, STATE_DIM]) state_seq.append(state) action = tf.reshape(features[action_name], shape=[1, ACION_DIM]) action_seq.append(action) image_seq = tf.concat(0, image_seq) if FLAGS.use_state: state_seq = tf.concat(0, state_seq) action_seq = tf.concat(0, action_seq) [image_batch, action_batch, state_batch] = tf.train.batch([image_seq, action_seq, state_seq], FLAGS.batch_size, num_threads=FLAGS.batch_size, capacity=100 * FLAGS.batch_size) return image_batch, action_batch, state_batch else: image_batch = tf.train.batch([image_seq], FLAGS.batch_size, num_threads=FLAGS.batch_size, capacity=100 * FLAGS.batch_size) zeros_batch_action = tf.zeros( [FLAGS.batch_size, FLAGS.sequence_length, ACION_DIM]) zeros_batch_state = tf.zeros( [FLAGS.batch_size, FLAGS.sequence_length, STATE_DIM]) return image_batch, zeros_batch_action, zeros_batch_state
def prepare_data_index(self): search_path = os.path.join(self.data_dir, '*', '*.wav') self.files = [] for wav_path in gfile.Glob(search_path): self.files.append(wav_path)
def __init__(self, hparams, mode): # self.prepare_inputs() # hparams.sample_rate = 96000 self.batch_size = hparams.batch_size wav_search_path = os.path.join(DATA_DIR, "wav48", '**', '*.wav') ls_input_files = gfile.Glob(wav_search_path) txt_search_path = os.path.join(DATA_DIR, "txt", '**', '*.txt') ls_target_files = gfile.Glob(txt_search_path) ifiles = set([ os.path.basename(os.path.splitext(path)[0]) for path in ls_input_files ]) tfiles = set([ os.path.basename(os.path.splitext(path)[0]) for path in ls_target_files ]) files = list(ifiles - (ifiles - tfiles)) self.ls_input_files = [ os.path.join(DATA_DIR, "wav48", fn.split('_')[0], fn + '.wav') for fn in files ] self.ls_target_files = [ os.path.join(DATA_DIR, "txt", fn.split('_')[0], fn + '.txt') for fn in files ] self.target_files = tf.placeholder(tf.string, shape=[None]) tgt_dataset = tf.data.Dataset.from_tensor_slices(self.target_files) tgt_dataset = tgt_dataset.flat_map( lambda filename: tf.data.TextLineDataset(filename).take(1)) # tgt_dataset = tgt_dataset.map(lambda string: tf.string_split([string], delimiter="").values) tgt_dataset = tgt_dataset.map( lambda str: tf.py_func(self.encode, [str], tf.int64)) tgt_dataset = tgt_dataset.map( lambda phones: (tf.cast(phones, tf.int32), tf.size(phones))) # src_dataset = tf.data.Dataset.from_generator(gen, (tf.float32, tf.int32)) self.input_files = tf.placeholder(tf.string, shape=[None]) src_dataset = tf.data.Dataset.from_tensor_slices(self.input_files) src_dataset = wav_utils.wav_to_features(src_dataset, hparams, 40) if hparams.max_train > 0: src_dataset.take(hparams.max_train) tgt_dataset.take(hparams.max_train) src_tgt_dataset = tf.data.Dataset.zip((src_dataset, tgt_dataset)) src_tgt_dataset.shuffle(1000) self.batched_dataset = src_tgt_dataset def batching_func(x): return x.padded_batch( self.batch_size, padded_shapes=(([None, DCT_COEFFICIENT_COUNT], []), ([None], []))) if hparams.num_buckets > 1: def key_func(src, tgt): bucket_width = 10 # Bucket sentence pairs by the length of their source sentence and target # sentence. bucket_id = tf.maximum(src[1] // bucket_width, tgt[1] // bucket_width) return tf.to_int64(tf.minimum(hparams.num_buckets, bucket_id)) def reduce_func(unused_key, windowed_data): return batching_func(windowed_data) self.batched_dataset = src_tgt_dataset.apply( tf.contrib.data.group_by_window( key_func=key_func, reduce_func=reduce_func, window_size=hparams.batch_size)) self.iterator = self.batched_dataset.make_initializable_iterator()
from sklearn.model_selection import train_test_split from tqdm import tqdm from tensorflow.python.platform import gfile from progress.bar import Bar bottleneck_dir = 'bottlenecks' ### LOAD DATA FROM BOTTLENECKS data_inputs = [] data_labels = [] bottleneck_list = [] file_glob = os.path.join(bottleneck_dir, '*.txt') bottleneck_list.extend(gfile.Glob(file_glob)) for bottleneck_file in bottleneck_list: bottleneck = open(bottleneck_file) bottleneck_string = bottleneck.read() bottleneck_values = [float(x) for x in bottleneck_string.split(',')] data_inputs.append(bottleneck_values) if 'cat' in bottleneck_file: data_labels.append([1, 0]) else: data_labels.append([0, 1]) # Splitting into train, val, and test train_inputs, valtest_inputs, train_labels, valtest_labels = train_test_split(data_inputs, data_labels, test_size=0.3, random_state=42)
def create_image_lists(image_dir, testing_percentage, validation_percentage): """이미지 디렉토리에서 인풋 데이터를 찾아 데이터로 변환한다""" ## image_dir가 존재하지 않는다면 오류 출력 if not gfile.Exists(image_dir): print("Image directory '" + image_dir + "' not found.") return None result = {} ### image_dir 내 하위 디렉토리(label)를 가져온다 sub_dirs = [x[0] for x in gfile.Walk(image_dir)] is_root_dir = True for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue extensions = ['jpg', 'jpeg', 'JPG', 'JPEG','bin'] file_list = [] dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue print("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(gfile.Glob(file_glob)) ## 파일이 없거나 데이터가 작으면 예외 처리 if not file_list: print('No files found') continue if len(file_list) < 20: print("WARNING: Folder has less than 20 images, which may cause issues.") elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: print("WARNING: Folder {} has more than {} images. Some images will never be selected".format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) ## 트레이닝 / 밸리데이션 / 테스트셋으로 나눈다. training_images = [] testing_images = [] validation_images = [] for file_name in file_list: base_name = os.path.basename(file_name) hash_name = re.sub(r'_nohash_.*$', '', file_name) hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) elif percentage_hash < (testing_percentage + validation_percentage): testing_images.append(base_name) else: training_images.append(base_name) result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
def create_image_lists_manual(image_dir, validation_percentage): """Builds a list of training images from the file system. Analyzes the sub folders in the image directory, splits them into stable training, testing, and validation sets, and returns a data structure describing the lists of images for each label and their paths. Args: image_dir: String path to a folder containing subfolders of training images. validation_percentage: Integer percentage of images reserved for validation. Returns: A dictionary containing an entry for each label subfolder, with images split into training, testing, and validation sets within each label. """ ''' ======================================================================= Image_Dir format (V2) ======================================================================= Expected image_dir format (Ver2) img_dir --Class A --Train --- Image001 --- Image002 ..... etc --Test --- Image001 --- Image002 ..... etc --Class B --Train --- Image001 --- Image002 .... --Test --- Image001 --- Image002 ..... etc ''' # Create Sub directory for Training objects sub_dirs = [x[0] for x in walklevel(image_dir, level=1)] is_root_dir = True result = {} sub_dir_iter = 0 print("Reset sub_dir_iter: ", sub_dir_iter) for sub_dir in sub_dirs: sub_dir_iter = sub_dir_iter + 1 print("sub_dir_iter: ", sub_dir_iter) print("curr_sub_dir") print(sub_dir) if is_root_dir: is_root_dir = False continue extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] file_list_train = [] file_list_test = [] dir_name = os.path.basename( sub_dir) #dir_name 0s gonna be used to be image label tf.logging.info("Looking for images in '" + dir_name + "'") # Get a list of files that match the given patterns (pattern: file_glob_train/test + extension) Training = "Training" Testing = "Testing" for extension in extensions: file_glob_train = os.path.join(image_dir, dir_name, Training, '*.' + extension) file_list_train.extend(gfile.Glob(file_glob_train)) file_glob_test = os.path.join(image_dir, dir_name, Testing, '*.' + extension) file_list_test.extend(gfile.Glob(file_glob_test)) # Initialize return result label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) print(label_name) training_images = [] testing_images = [] validation_images = [] # FOR LOOP: TRAINING file_name_iter = 0 print("[TRAIN] Reset file_name_iter: ", file_name_iter) for file_name in file_list_train: file_name_iter = file_name_iter + 1 print("[TRAIN] file_name_iter: ", file_name_iter) base_name = os.path.basename(file_name) # We want to ignore anything after '_nohash_' in the file name when # deciding which set to put an image in, the data set creator has a way of # grouping photos that are close variations of each other. For example # this is used in the plant disease data set to group multiple pictures of # the same leaf. hash_name = re.sub(r'_nohash_.*$', '', file_name) # This looks a bit magical, but we need to decide whether this file should # go into the training, testing, or validation sets, and we want to keep # existing files in the same set even if more files are subsequently # added. # To do that, we need a stable way of deciding based on just the file name # itself, so we do a hash of that and then use that to generate a # probability value that we use to assign it. hash_name_hashed = hashlib.sha1( compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) else: training_images.append(base_name) # FOR LOOP: TESTING file_name_iter = 0 print("[TEST] Reset file_name_iter: ", file_name_iter) for file_name in file_list_test: file_name_iter = file_name_iter + 1 print("[TEST] file_name_iter: ", file_name_iter) base_name = os.path.basename(file_name) testing_images.append(base_name) # Generate result result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
return np.roll(data, 1600) def stretch(self, data, rate=1): input_length = 16000 data = librosa.effects.time_stretch(data, rate) if len(data) > input_length: data = data[:input_length] else: data = np.pad(data, (0, max(0, input_length - len(data))), "constant") return data # Set the path for the target class paths = os.path.join('./2', '*.wav') waves = gfile.Glob(paths) for path in waves: # Create a new instance from AudioAugmentation class aa = AudioAugmentation() # Read and show cat sound data = aa.read_audio_file(path) aa.write_audio_file(path, data) # Adding noise to sound data_noise = aa.add_noise(data) # Shifting the sound data_roll = aa.shift(data)
def create_image_lists(image_dir, testing_percentage, validation_percentage): """file system으로부터 training 이미지들의 list를 만든다. 이미지 디렉토리로부터 sub folder들을 분석하고, 그들을 training, testing, validation sets으로 나눈다. 그리고 각각의 label을 위한 이미지 list와 그들의 경로(path)를 나타내는 자료구조(data structure)를 반환한다. 인수들(Args): image_dir: 이미지들의 subfolder들을 포함한 folder의 String path. testing_percentage: 전체 이미지중 테스트를 위해 사용되는 비율을 나타내는 Integer. validation_percentage: 전체 이미지중 validation을 위해 사용되는 비율을 나타내는 Integer. 반환값들(Returns): 각각의 label subfolder를 위한 enrtry를 포함한 dictionary A dictionary (각각의 label에서 이미지드릉ㄴ training, testing, validation sets으로 나뉘어져 있다.) """ if not gfile.Exists(image_dir): print("Image directory '" + image_dir + "' not found.") return None result = {} sub_dirs = [x[0] for x in gfile.Walk(image_dir)] # root directory는 처음에 온다. 따라서 이를 skip한다. is_root_dir = True for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] file_list = [] dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue print("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(gfile.Glob(file_glob)) if not file_list: print('No files found') continue if len(file_list) < 20: print( 'WARNING: Folder has less than 20 images, which may cause issues.' ) elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: print( 'WARNING: Folder {} has more than {} images. Some images will ' 'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) training_images = [] testing_images = [] validation_images = [] for file_name in file_list: base_name = os.path.basename(file_name) # 어떤 이미지로 리스트를 만들지 결정할때 파일 이름에 "_nohash_"가 포함되어 있으면 이를 무시할 수 있다. # 이를 이용해서, 데이터셋을 만드는 사람은 서로 비슷한 사진들을 grouping할 수있다. # 예를 들어, plant disease를 데이터셋을 만들기 위해서, 여러 장의 같은 잎사귀(leaf)를 grouping할 수 있다. hash_name = re.sub(r'_nohash_.*$', '', file_name) # 이는 일종의 마법처럼 보일 수 있다. 하지만, 우리는 이 파일이 training sets로 갈지, testing sets로 갈지, validation sets로 갈지 결정해야만 한다. # 그리고 우리는 더많은 파일들이 추가되더라도, 같은 set에 이미 존재하는 파일들이 유지되길 원한다. # 그렇게 하기 위해서는, 우리는 파일 이름 그자체로부터 결정하는 안정적인 방법이 있어야만 한다. # 따라서, 우리는 파일 이름을 hash하고, 이를 이를 할당하는데 사용하는 확률을 결정하는데 사용한다. hash_name_hashed = hashlib.sha1( compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) elif percentage_hash < (testing_percentage + validation_percentage): testing_images.append(base_name) else: training_images.append(base_name) result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
def create_image_lists(image_dir, testing_percentage, validation_percentage): """ 从文件系统构建训练图像列表。 分析图像目录中的子文件夹,将它们分成稳定的训练,测试和验证集,并返回描述每个标签及其路径的图像列表的数据结构。 ARGS: image_dir:包含图像子文件夹的文件夹的字符串路径。 testing_percentage:要为测试保留的图像的整数百分比。 validation_percentage:为验证保留的图像的整数百分比。 返回: 包含每个标签子文件夹条目的字典,图像在每个标签内分为训练,测试和验证集。 """ if not gfile.Exists(image_dir): tf.logging.error("Image directory '" + image_dir + "' not found.") return None result = {} sub_dirs = [x[0] for x in gfile.Walk(image_dir)] # The root directory comes first, so skip it. is_root_dir = True for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] file_list = [] dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue #tf.compat.v1.logging.info("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(gfile.Glob(file_glob)) if not file_list: tf.logging.warning('No files found') continue if len(file_list) < 20: tf.logging.warning( 'WARNING: Folder has less than 20 images, which may cause issues.') elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: tf.logging.warning( 'WARNING: Folder {} has more than {} images. Some images will ' 'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) training_images = [] testing_images = [] validation_images = [] for file_name in file_list: base_name = os.path.basename(file_name) hash_name = re.sub(r'_nohash_.*$', '', file_name) # 这看起来有点神奇,但是我们需要确定这个文件 # 进入训练,测试或验证集,并且我们希望 # 将现有文件保留在同一个集合中,即使随后添加了更多文件。 # 为此,我们需要一种基于文件名本身的稳定决策方式, # 因此我们对其进行哈希处理, # 然后使用它来生成我们用来分配它的概率值。 hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) elif percentage_hash < (testing_percentage + validation_percentage): testing_images.append(base_name) else: training_images.append(base_name) result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
def create_image_lists(image_dir, testing_percentage, validation_percentage): """Builds a list of training images from the file system. Analyzes the sub folders in the image directory, splits them into stable training, testing, and validation sets, and returns a data structure describing the lists of images for each label and their paths. Args: image_dir: String path to a folder containing subfolders of images. testing_percentage: Integer percentage of the images to reserve for tests. validation_percentage: Integer percentage of images reserved for validation. Returns: A dictionary containing an entry for each label subfolder, with images split into training, testing, and validation sets within each label. """ if not gfile.Exists(image_dir): print("Image directory '" + image_dir + "' not found.") return None result = {} sub_dirs = [x[0] for x in gfile.Walk(image_dir)] # The root directory comes first, so skip it. is_root_dir = True for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] file_list = [] dir_name = os.path.basename(sub_dir) if dir_name == image_dir: continue print("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) file_list.extend(gfile.Glob(file_glob)) if not file_list: print('No files found') continue if len(file_list) < 20: print( 'WARNING: Folder has less than 20 images, which may cause issues.' ) elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: print( 'WARNING: Folder {} has more than {} images. Some images will ' 'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) training_images = [] testing_images = [] validation_images = [] for file_name in file_list: base_name = os.path.basename(file_name) # We want to ignore anything after '_nohash_' in the file name when # deciding which set to put an image in, the data set creator has a way of # grouping photos that are close variations of each other. For example # this is used in the plant disease data set to group multiple pictures of # the same leaf. hash_name = re.sub(r'_nohash_.*$', '', file_name) # This looks a bit magical, but we need to decide whether this file should # go into the training, testing, or validation sets, and we want to keep # existing files in the same set even if more files are subsequently # added. # To do that, we need a stable way of deciding based on just the file name # itself, so we do a hash of that and then use that to generate a # probability value that we use to assign it. hash_name_hashed = hashlib.sha1( compat.as_bytes(hash_name)).hexdigest() percentage_hash = ((int(hash_name_hashed, 16) % (MAX_NUM_IMAGES_PER_CLASS + 1)) * (100.0 / MAX_NUM_IMAGES_PER_CLASS)) if percentage_hash < validation_percentage: validation_images.append(base_name) elif percentage_hash < (testing_percentage + validation_percentage): testing_images.append(base_name) else: training_images.append(base_name) result[label_name] = { 'dir': dir_name, 'training': training_images, 'testing': testing_images, 'validation': validation_images, } return result
def minibatch(self, dataset, subset, use_datasets, shift_ratio=-1): if shift_ratio < 0: shift_ratio = self.shift_ratio with tf.name_scope('batch_processing'): # Build final results per split. images = [[] for _ in range(self.num_splits)] labels = [[] for _ in range(self.num_splits)] if use_datasets: glob_pattern = dataset.tf_record_pattern(subset) file_names = gfile.Glob(glob_pattern) if not file_names: raise ValueError('Found no files in --data_dir matching: {}' .format(glob_pattern)) ds = tf.contrib.data.TFRecordDataset(file_names) counter = tf.contrib.data.Dataset.range(self.batch_size) counter = counter.repeat() ds = tf.contrib.data.Dataset.zip((ds, counter)) ds = ds.map( self.parse_and_preprocess, num_parallel_calls=self.batch_size) ds = ds.prefetch(buffer_size=self.batch_size) ds = ds.shuffle(buffer_size=10000) ds = ds.repeat() ds_iterator = ds.make_one_shot_iterator() # TODO(jsimsa): Use datasets' batch transformation instead of (see # below) once the transformation implements parallel data copy. # # NOTE: The current implementation does not preserve the order of # elements between the shuffle buffer and the batch. for idx in xrange(self.batch_size): label, image = ds_iterator.get_next() split_index = idx % self.num_splits labels[split_index].append(label) images[split_index].append(image) else: record_input = data_flow_ops.RecordInput( file_pattern=dataset.tf_record_pattern(subset), seed=301, parallelism=64, buffer_size=10000, batch_size=self.batch_size, shift_ratio=shift_ratio, name='record_input') records = record_input.get_yield_op() records = tf.split(records, self.batch_size, 0) records = [tf.reshape(record, []) for record in records] for idx in xrange(self.batch_size): value = records[idx] (label, image) = self.parse_and_preprocess(value, idx) split_index = idx % self.num_splits labels[split_index].append(label) images[split_index].append(image) for split_index in xrange(self.num_splits): images[split_index] = tf.parallel_stack(images[split_index]) labels[split_index] = tf.concat(labels[split_index], 0) images[split_index] = tf.cast(images[split_index], self.dtype) depth = 3 images[split_index] = tf.reshape( images[split_index], shape=[self.batch_size_per_split, self.height, self.width, depth]) labels[split_index] = tf.reshape(labels[split_index], [self.batch_size_per_split]) return images, labels