def get_split(split_name, dataset_dir, file_pattern=None, reader=None): if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/class/coarse_label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/filepath': tf.FixedLenFeature((), tf.string, default_value=''), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(shape=[224, 224, 1], channels=1), 'label': slim.tfexample_decoder.Tensor('image/class/label'), 'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'), 'filepath': slim.tfexample_decoder.Tensor('image/filepath'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) dataset = slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names) coarse_labels_to_names = None if dataset_utils.has_labels(dataset_dir, _COARSE_LABEL_FILENAME): coarse_labels_to_names = dataset_utils.read_label_file( dataset_dir, _COARSE_LABEL_FILENAME) dataset.coarse_labels_to_names = coarse_labels_to_names return dataset
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading cifar100. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if not reader: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/class/coarse_label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]), 'label': slim.tfexample_decoder.Tensor('image/class/label'), 'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading flowers. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, dataset_list_dir='', file_pattern=None, reader=None, modality='rgb', num_samples=1): """Gets a dataset tuple with instructions for reading cifar10. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) # if not file_pattern: # file_pattern = _FILE_PATTERN # file_pattern = os.path.join(dataset_dir, file_pattern % split_name) if split_name == 'train': _LIST = _TRAIN_LIST else: _LIST = _VAL_LIST with open(_LIST, 'r') as fin: data_sources = [ ' '.join([ os.path.join(dataset_dir, el.split()[0]), ] + el.split()[1:]) for el in fin.read().splitlines() ] # Allowing None in the signature so that dataset_factory can use the default. if not reader: reader = readerFn labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=data_sources, reader=reader, decoder=decoderFn(num_samples), num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, division_idx=None, file_pattern=None, reader=None, num_classes=None, num_samples=None): """Gets a dataset tuple with instructions for reading training data. Returns: A 'Dataset' namedtuple. Raises: ValueError: if 'split_name' is not the train split. """ if division_idx is not None: raise ValueError('No division index is needed for the training dataset.') if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/tracklet_id': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/cam_id': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'tracklet_id': slim.tfexample_decoder.Tensor('image/tracklet_id'), 'cam_id': slim.tfexample_decoder.Tensor('image/cam_id'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=num_samples, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): _NUM_CLASSES = len(glob.glob(dataset_dir + '/*/')) ALL_NUM = len(glob.glob(dataset_dir + '/*/*.jpg')) NUM_VAL = int(ALL_NUM * 0.1) SPLITS_TO_SIZES = {'train': ALL_NUM - NUM_VAL, 'validation': NUM_VAL} _FILE_PATTERN = 'mydata_%s_*.tfrecord' _ITEMS_TO_DESCRIPTIONS = { 'image': 'A color image of varying size.', 'label': 'A single integer between 0 and 257', } if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)) } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label') } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, dataset_list_dir='', file_pattern=None, reader=None, modality='rgb', num_samples=1): """Gets a dataset tuple with instructions for reading cifar10. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) # if not file_pattern: # file_pattern = _FILE_PATTERN # file_pattern = os.path.join(dataset_dir, file_pattern % split_name) if split_name == 'train': _LIST = _TRAIN_LIST else: _LIST = _VAL_LIST with open(_LIST, 'r') as fin: data_sources = [ ' '.join([os.path.join(dataset_dir, el.split()[0]),] + el.split()[1:]) for el in fin.read().splitlines() ] # Allowing None in the signature so that dataset_factory can use the default. if not reader: reader = readerFn labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=data_sources, reader=reader, decoder=decoderFn(num_samples), num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, num_samples, dataset_dir, file_pattern=None, reader=None): if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) SPLITS_TO_SIZES[split_name] = num_samples if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def load_model(self): sess = tf.Session() x = tf.placeholder(tf.float32,shape=[None,None,3]) # read dataset info # if dataset_utils.has_labels(self.model_dir): labels_to_names = dataset_utils.read_label_file(self.model_dir) with open(os.path.join(self.model_dir,'info.json')) as f: info_dict = json.load(f) _num_classes = info_dict['num_class']+1 # Select the model # network_fn = nets_factory.get_network_fn( self.conf['infor']['model_name'], num_classes=_num_classes, is_training=False) # Select the preprocessing function # preprocessing_name = self.conf['infor']['model_name'] if self.conf['infor']['preprocessing_name']=='None' else self.conf['infor']['preprocessing_name'] image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = self.conf['infor']['eval_image_size'] if self.conf['infor']['eval_image_size']!='None' else network_fn.default_image_size image = image_preprocessing_fn(x, eval_image_size, eval_image_size) # inference map imgs = tf.placeholder(tf.float32,shape=[None,None,None,3]) logits, _ = network_fn(imgs) predictions = tf.argmax(logits, 1) # load model # saver = tf.train.Saver() params_file = tf.train.latest_checkpoint(self.model_dir) saver.restore(sess, params_file) self.output['x']=x self.output['imgs']=imgs self.output['predictions']=predictions self.output['labels_to_names']=labels_to_names self.output['sess'] = sess self.output['image'] = image
def get_split(self, split_name): splits_to_sizes = self.__get_num_samples__(split_name) if split_name not in ['train', 'validation']: raise ValueError('split name %s was not recognized.' % split_name) file_pattern = self.dataset_name + '_' + split_name + '_*.tfrecord' file_pattern = os.path.join(self.tfrecord_dir, file_pattern) reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(self.tfrecord_dir): labels_to_names = dataset_utils.read_label_file(self.tfrecord_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=splits_to_sizes, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=self.num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None): """Gets a dataset tuple with instructions for reading flowers. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. # if reader is None: # reader = tf.TFRecordReader labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) filenames = tf.gfile.Glob(file_pattern) return Dataset(filenames=filenames, num_samples=SPLITS_TO_SIZES[split_name], num_classes=_NUM_CLASSES, labels_to_names=labels_to_names, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
def get_split_test(split_name, dataset_dir, file_pattern=None, reader=None): if split_name not in SPLITS_TO_SIZES_TEST: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN #file_pattern = os.path.join(dataset_dir, file_pattern % split_name) file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image_id': tf.FixedLenFeature((), tf.string, default_value=''), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'image_id': slim.tfexample_decoder.Tensor('image_id'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES_TEST[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names, shuffle=False)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None, mode=""): """Gets a dataset tuple with instructions for reading MNIST. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in _SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'), 'image/class/label': tf.FixedLenFeature([1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)), 'image/filename': tf.FixedLenFeature((), tf.string, default_value='no_filename'), # 'image/filename': tf.VarLenFeature(dtype=tf.string), } base_size = 128 if mode == "multiple": width = base_size * 2 height = base_size elif mode == "tiny" or mode == "tinygan": width = 32 height = 32 else: width = base_size height = base_size items_to_handlers = { 'image': slim.tfexample_decoder.Image(shape=[height, width, 1], channels=1), 'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]), 'filename': slim.tfexample_decoder.Tensor('image/filename'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=_SPLITS_TO_SIZES[split_name], num_classes=_NUM_CLASSES, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading ImageNet. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in _SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature( (), tf.string, default_value=''), 'image/format': tf.FixedLenFeature( (), tf.string, default_value='jpeg'), 'image/class/label': tf.FixedLenFeature( [], dtype=tf.int64, default_value=-1), 'image/class/text': tf.FixedLenFeature( [], dtype=tf.string, default_value=''), 'image/object/bbox/xmin': tf.VarLenFeature( dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature( dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature( dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature( dtype=tf.float32), 'image/object/class/label': tf.VarLenFeature( dtype=tf.int64), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'label': slim.tfexample_decoder.Tensor('image/class/label'), 'label_text': slim.tfexample_decoder.Tensor('image/class/text'), 'object/bbox': slim.tfexample_decoder.BoundingBox( ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) else: labels_to_names = create_readable_names_for_imagenet_labels() dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=_SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(dataset_name, split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading flowers. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ num_classes, splits_to_sizes, items_to_descriptions = dataset_utils.read_dataset_config_json(dataset_name, dataset_dir) if split_name not in splits_to_sizes: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN % (dataset_name, split_name) file_pattern = os.path.join(dataset_dir, file_pattern) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/name': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/height': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'image/width': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), 'image_name': slim.tfexample_decoder.Tensor('image/name'), 'image_height': slim.tfexample_decoder.Tensor('image/height'), 'image_width': slim.tfexample_decoder.Tensor('image/width'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) def dataset_class_weight(splits_to_sizes, split_name, label_to_class_number): """ Returns sorted list of class ratios """ num_samples = splits_to_sizes[split_name] num_samples_per_class = splits_to_sizes[split_name+'_per_class'] sorted_class_weights = list() sorted_label_name = sorted(label_to_class_number.keys(), key= lambda x: label_to_class_number[x]) for label_name in sorted_label_name: if label_name in num_samples_per_class: norm_class_weight = num_samples / (num_samples_per_class[label_name] * num_classes) sorted_class_weights.append(norm_class_weight) return sorted_class_weights labels_to_names = None if dataset_utils.has_labels(dataset_dir): class_to_label_name, label_to_class_number = dataset_utils.read_label_file(dataset_dir) num_samples=splits_to_sizes[split_name] # if split_name+'_per_class' in splits_to_sizes: sorted_class_weights = dataset_class_weight(splits_to_sizes, split_name, label_to_class_number) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=num_samples, items_to_descriptions=items_to_descriptions, num_classes=num_classes, labels_to_names=label_to_class_number, sorted_class_weights=sorted_class_weights )
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading database. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ images_count, class_count = _get_filenames_and_classes_count(dataset_dir) num_validation = images_count*_PERC_VALIDATION//100 splits_to_sizes = {'train': (images_count-num_validation), 'validation': num_validation} _items_to_descriptions = { 'image': 'A color image of varying size.', 'label': 'A single integer between 0 and ' + str(class_count), } if split_name not in splits_to_sizes: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir + '/tfrecords', file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir + '/tfrecords'): labels_to_names = dataset_utils.read_label_file(dataset_dir + '/tfrecords') return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=splits_to_sizes[split_name], items_to_descriptions=_items_to_descriptions, num_classes=class_count, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern, reader, split_to_sizes, items_to_descriptions, num_classes): """Gets a dataset tuple with instructions for reading Pascal VOC dataset. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in split_to_sizes: raise ValueError('split name %s was not recognized.' % split_name) file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader # Features in Pascal VOC TFRecords. keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/height': tf.FixedLenFeature([1], tf.int64), 'image/width': tf.FixedLenFeature([1], tf.int64), 'image/channels': tf.FixedLenFeature([1], tf.int64), 'image/shape': tf.FixedLenFeature([3], tf.int64), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'shape': slim.tfexample_decoder.Tensor('image/shape'), 'object/bbox': slim.tfexample_decoder.BoundingBox( ['xmin', 'ymin', 'xmax', 'ymax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) # else: # labels_to_names = create_readable_names_for_imagenet_labels() # dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=split_to_sizes[split_name], items_to_descriptions=items_to_descriptions, num_classes=num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading cifar10. instruction(n. 指示; 教育; 用法说明) Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. pattern(n. 花样, 图案; 格局; 形态, 样式; 样品, 样本) It is assumed that the pattern contains a '%s' string so that the split assumed(adj. 假装的; 假定的, 设想的; 假冒的; 被承担的) name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. # signature(n. 签名, 签字, 签署) if not reader: reader = tf.TFRecordReader keys_to_features = { # fixed(adj. 固定的; 不变的; 确定的; 不动的) 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) # handler(n. 操作者; 处理者; 操作装置; 处理机; 教练, 训练员; 管理人, 负责人; # 由某事件使其活跃并管理照顾那个事件的过程 (计算机用语)) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, # source(n. 来源, 根源, 水源; 起点; 原始码 (计算机用语)) reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, # description(n. 描写; 形容; 叙述; 说明书) num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, photos_subdir='photos', tfrecords_subdir='tfrecords', file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading tumblr data. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. photos_subdir: The subdirectory containing the photos. tfrecords_subdir: The subdirectory containing the TFRecords files. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ # if split_name not in SPLITS_TO_SIZES: #raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, tfrecords_subdir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir, photos_subdir): labels_to_names = dataset_utils.read_label_file( dataset_dir, photos_subdir) # Get split size train_valid_filename = os.path.join(dataset_dir, photos_subdir, _TRAIN_VALID_FILENAME) with tf.gfile.Open(train_valid_filename, 'rb') as f: lines = f.read().decode() lines = lines.split('\n') lines = filter(None, lines) train_valid_split = {} for line in lines: index = line.index(':') train_valid_split[line[:index]] = (int)(line[index + 1:]) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=train_valid_split[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=len(labels_to_names), labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): #这个是slim分类中的函数,主要是根据split_name来选择并解析tfrecord文件 """Gets a dataset tuple with instructions for reading cifar10. #返回的就是Dataset数据集格式 Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) #识别文件名格式 # Allowing None in the signature so that dataset_factory can use the default. if not reader: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } #解析文件个结构 items_to_handlers = { 'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) #用TFExampleDecoder解析文件 labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file( dataset_dir) #读取文件类名,映射为字典{‘标签’:名称} return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names) #返回数据类,下面就可以用provider进行数据batch生成
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading ImageNet. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if not file_pattern: file_pattern = os.path.join(dataset_dir, FILE_PATTERN) else: file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # data_source = os.path.join(dataset_dir, 'MOT17_a11.record') # data_source = dataset_dir # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { # 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), # 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), # 'image/height': tf.FixedLenFeature([1], dtype=tf.int64), # 'image/width': tf.FixedLenFeature([1], dtype=tf.int64), # 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), # 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), # 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), # 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), # 'image/object/class/label': tf.VarLenFeature(dtype=tf.int64), # 'image/object/class/label_text': tf.FixedLenFeature((), tf.string, default_value=''), 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/filename': tf.FixedLenFeature((), tf.string, default_value=''), 'image/key/sha256': tf.FixedLenFeature((), tf.string, default_value=''), 'image/source_id': tf.FixedLenFeature((), tf.string, default_value=''), 'image/height': tf.FixedLenFeature((), tf.int64, default_value=1), 'image/width': tf.FixedLenFeature((), tf.int64, default_value=1), # Object boxes and classes. 'image/object/bbox/xmin': tf.VarLenFeature(tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(tf.float32), 'image/object/class/label': tf.VarLenFeature(tf.int64), 'image/object/class/text': tf.VarLenFeature(tf.string), 'image/object/area': tf.VarLenFeature(tf.float32), 'image/object/is_crowd': tf.VarLenFeature(tf.int64), 'image/object/difficult': tf.VarLenFeature(tf.int64), 'image/object/group_of': tf.VarLenFeature(tf.int64), 'image/object/weight': tf.VarLenFeature(tf.float32), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'), # 'object/label_text': slim.tfexample_decoder.Tensor('image/object/class/label_text'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) label_dir = os.path.join(os.getcwd(), 'datasets') labels_to_names = None if dataset_utils.has_labels(label_dir): labels_to_names = dataset_utils.read_label_file2(label_dir) # else: # labels_to_names = create_readable_names_for_imagenet_labels() # dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=ITEMS_TO_DESCRIPTIONS, num_classes=NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None, file_pattern_for_counting=None): ''' Obtains the split - training or validation - to create a Dataset class for feeding the examples into a queue later on. This function will set up the decoder and dataset information all into one Dataset class so that you can avoid the brute work later on. Your file_pattern is very important in locating the files later. INPUTS: - split_name(str): 'train' or 'validation'. Used to get the correct data split of tfrecord files - dataset_dir(str): the dataset directory where the tfrecord files are located - file_pattern(str): the file name structure of the tfrecord files in order to get the correct data - file_pattern_for_counting(str): the string name to identify your tfrecord files for counting OUTPUTS: - dataset (Dataset): A Dataset class object where we can read its various components for easier batch creation later. ''' #First check whether the split_name is train or validation if split_name not in ['train', 'validation']: raise ValueError( 'The split_name %s is not recognized. Please input either train or validation as the split_name' % (split_name)) if file_pattern is None: file_pattern = _FILE_PATTERN if file_pattern_for_counting is None: file_pattern_for_counting = _FILE_PATTERN_FOR_COUNTING #Create the full path for a general file_pattern to locate the tfrecord_files print(file_pattern) print(split_name) print(dataset_dir) file_pattern_path = os.path.join(dataset_dir, file_pattern % (split_name)) #Count the total number of examples in all of these shard num_samples = 0 file_pattern_for_counting = file_pattern_for_counting + '_' + split_name tfrecords_to_count = [ os.path.join(dataset_dir, file) for file in os.listdir(dataset_dir) if file.startswith(file_pattern_for_counting) ] for tfrecord_file in tfrecords_to_count: for record in tf.python_io.tf_record_iterator(tfrecord_file): num_samples += 1 #Create a reader, which must be a TFRecord reader in this case reader = tf.TFRecordReader #Create the keys_to_features dictionary for the decoder keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } #Create the items_to_handlers dictionary for the decoder. items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } #Start to create the decoder decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) #Create the labels_to_name file labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) #Actually create the dataset dataset = slim.dataset.Dataset( data_sources=file_pattern_path, decoder=decoder, reader=reader, num_readers=4, num_samples=num_samples, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS) return dataset
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading MNIST. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in _SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) #filename_queue = tf.train.string_input_producer(['./data/cifar10_train.tfrecord']) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader """ _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/height': tf.FixedLenFeature((), tf.int64), 'image/width': tf.FixedLenFeature((), tf.int64), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)) }) """ keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), #'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'), #'image/height': tf.FixedLenFeature((), tf.int64), #'image/width': tf.FixedLenFeature((), tf.int64), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)) } items_to_handlers = { #'image': slim.tfexample_decoder.Image(shape=[32, 128, 3], channels=3), #'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), #'image': tf.image.decode_png(features['image/encoded'], channels=3) 'image': slim.tfexample_decoder.Image(image_key = 'image/encoded', format_key = 'image/format'), 'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]) } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=_SPLITS_TO_SIZES[split_name], num_classes=_NUM_CLASSES, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, labels_to_names=labels_to_names)
def main(_): if not FLAGS.predict_file: raise ValueError( 'You must supply the predict file path with --predict_file') with tf.Graph().as_default(): tf.logging.set_verbosity(tf.logging.INFO) ############################################################################################################## network_fn = nets_factory.get_network_fn(FLAGS.model_name, num_classes=_NUM_CLASSES, is_training=False) image_size = FLAGS.predict_image_size or network_fn.default_image_size # images, images_raw = load_batch_2(FLAGS.predict_file, height=image_size, width=image_size) image_string = tf.gfile.FastGFile(FLAGS.predict_file, 'rb').read() image = tf.image.decode_jpeg(image_string, channels=3) processed_image = inception_preprocessing.preprocess_image( image, image_size, image_size, is_training=False) processed_images = tf.expand_dims(processed_image, 0) logits, _ = network_fn(processed_images) probabilities = tf.nn.softmax(logits) checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) init_fn = slim.assign_from_checkpoint_fn( checkpoint_path, slim.get_variables_to_restore()) with tf.Session() as sess: with slim.queues.QueueRunners(sess): sess.run(tf.initialize_local_variables()) init_fn(sess) np_images, np_probabilities = sess.run([image, probabilities]) predicted_label = np.argmax(np_probabilities[0, :]) print(predicted_label) labels_to_names = None if dataset_utils.has_labels(FLAGS.dataset_dir): labels_to_names = dataset_utils.read_label_file(FLAGS.dataset_dir) class_name = labels_to_names[predicted_label] print(class_name) if dataset_utils.has_labels(FLAGS.dataset_dir, INFO_FILE): class_names_to_info = read_info_file(FLAGS.dataset_dir) if class_name in merge_info: print('It is the back of a coin, please take a photo of front.') class_name_list = merge_info[class_name] for item in class_name_list: info = class_names_to_info[item.split('_')[0]] print('Maybe the value is {}'.format(info['value'])) else: name_info = class_name.split('_') origin_name = name_info[0] back_or_front = name_info[1] info = class_names_to_info[origin_name] print('value is %s, it is %s of the coin, the other info is %s' % (info['value'], back_or_front, info['info'])) return info['value']
def get_split(split_name, dataset_dir, dataset_str, reader=None, num_classes=180): """Gets a dataset tuple with instructions for reading flowers. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ file_pattern = dataset_str + '_%s_*.tfrecord' file_pattern = os.path.join(dataset_dir, file_pattern % split_name) if 'oriTrn1' in dataset_str: SPLITS_TO_SIZES = {'train': 2592, 'validation': 288} elif 'Trn' in dataset_str: SPLITS_TO_SIZES = {'train': 1944, 'validation': 216} elif 'Tst' in dataset_str or 'Gratings' in dataset_str or 'FiltNoise' in dataset_str or 'FiltIms' in dataset_str: SPLITS_TO_SIZES = {} for bb in range(96): SPLITS_TO_SIZES['batch' + str(bb)] = 90 if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) # assert len(labels_to_names)==num_classes return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='Market1501', file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading flowers. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name)) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image_raw_0' : tf.FixedLenFeature([], tf.string), 'image_raw_1' : tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar 'id_0': tf.FixedLenFeature([], tf.int64), 'id_1': tf.FixedLenFeature([], tf.int64), 'cam_0': tf.FixedLenFeature([], tf.int64), 'cam_1': tf.FixedLenFeature([], tf.int64), 'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'), 'image_height': tf.FixedLenFeature([], tf.int64, default_value=128), 'image_width': tf.FixedLenFeature([], tf.int64, default_value=64), 'real_data': tf.FixedLenFeature([], tf.int64, default_value=1), # 'attrs_0': tf.FixedLenFeature([27], tf.int64), # 'attrs_1': tf.FixedLenFeature([27], tf.int64), # 'attrs_w2v25_0': tf.FixedLenFeature([25*12], tf.float32), # 'attrs_w2v25_1': tf.FixedLenFeature([25*12], tf.float32), # 'attrs_w2v50_0': tf.FixedLenFeature([50*12], tf.float32), # 'attrs_w2v50_1': tf.FixedLenFeature([50*12], tf.float32), # 'attrs_w2v100_0': tf.FixedLenFeature([100*12], tf.float32), # 'attrs_w2v100_1': tf.FixedLenFeature([100*12], tf.float32), # 'attrs_w2v150_0': tf.FixedLenFeature([150*12], tf.float32), # 'attrs_w2v150_1': tf.FixedLenFeature([150*12], tf.float32), 'pose_peaks_0': tf.FixedLenFeature([16*8*18], tf.float32), 'pose_peaks_1': tf.FixedLenFeature([16*8*18], tf.float32), 'pose_peaks_0_rc': tf.FixedLenFeature([18*2], tf.float32), 'pose_peaks_1_rc': tf.FixedLenFeature([18*2], tf.float32), # 'pose_dense_r4_0': tf.FixedLenFeature([128*64*18], tf.float32), # 'pose_dense_r4_1': tf.FixedLenFeature([128*64*18], tf.float32), 'pose_mask_r4_0': tf.FixedLenFeature([128*64*1], tf.int64), 'pose_mask_r4_1': tf.FixedLenFeature([128*64*1], tf.int64), 'shape': tf.FixedLenFeature([1], tf.int64), 'indices_r4_0': tf.VarLenFeature(dtype=tf.int64), 'values_r4_0': tf.VarLenFeature(dtype=tf.float32), 'indices_r4_1': tf.VarLenFeature(dtype=tf.int64), 'values_r4_1': tf.VarLenFeature(dtype=tf.float32), 'pose_subs_0': tf.FixedLenFeature([20], tf.float32), 'pose_subs_1': tf.FixedLenFeature([20], tf.float32), } items_to_handlers = { 'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'), 'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'), 'label': slim.tfexample_decoder.Tensor('label'), 'id_0': slim.tfexample_decoder.Tensor('id_0'), 'id_1': slim.tfexample_decoder.Tensor('id_1'), # 'attrs_0': slim.tfexample_decoder.Tensor('attrs_0',shape=[27]), # 'attrs_1': slim.tfexample_decoder.Tensor('attrs_1',shape=[27]), # 'attrs_w2v25_0': slim.tfexample_decoder.Tensor('attrs_w2v25_0',shape=[25*12]), # 'attrs_w2v25_1': slim.tfexample_decoder.Tensor('attrs_w2v25_1',shape=[25*12]), # 'attrs_w2v50_0': slim.tfexample_decoder.Tensor('attrs_w2v50_0',shape=[50*12]), # 'attrs_w2v50_1': slim.tfexample_decoder.Tensor('attrs_w2v50_1',shape=[50*12]), # 'attrs_w2v100_0': slim.tfexample_decoder.Tensor('attrs_w2v100_0',shape=[100*12]), # 'attrs_w2v100_1': slim.tfexample_decoder.Tensor('attrs_w2v100_1',shape=[100*12]), # 'attrs_w2v150_0': slim.tfexample_decoder.Tensor('attrs_w2v150_0',shape=[150*12]), # 'attrs_w2v150_1': slim.tfexample_decoder.Tensor('attrs_w2v150_1',shape=[150*12]), 'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0',shape=[16*8*18]), 'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1',shape=[16*8*18]), 'pose_peaks_0_rc': slim.tfexample_decoder.Tensor('pose_peaks_0_rc',shape=[18*2]), 'pose_peaks_1_rc': slim.tfexample_decoder.Tensor('pose_peaks_1_rc',shape=[18*2]), # 'pose_dense_r4_0': slim.tfexample_decoder.Tensor('pose_dense_r4_0',shape=[128*64*18]), # 'pose_dense_r4_1': slim.tfexample_decoder.Tensor('pose_dense_r4_1',shape=[128*64*18]), 'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0',shape=[128*64*1]), 'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1',shape=[128*64*1]), 'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False), 'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False), 'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]), 'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) print('load pn_pairs_num ......') fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p') with open(fpath,'r') as f: pn_pairs_num = pickle.load(f) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=pn_pairs_num, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
rbboxes = np_methods.bboxes_clip(rbbox_img, rbboxes) rclasses, rscores, rbboxes = np_methods.bboxes_sort(rclasses, rscores, rbboxes, top_k=400) rclasses, rscores, rbboxes = np_methods.bboxes_nms( rclasses, rscores, rbboxes, nms_threshold=nms_threshold) # Resize bboxes to original image shape. Note: useless for Resize.WARP! rbboxes = np_methods.bboxes_resize(rbbox_img, rbboxes) return rclasses, rscores, rbboxes dataset_dir = os.path.join(os.getcwd(), 'datasets') labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file2(dataset_dir, filename='labels_voc.txt') # Test on some demo image and visualize output. path = '/home/ace19/dl_data/MOT/MOT17/test/sample-test/' # path = './detection_image/coco/' image_names = sorted(os.listdir(path)) for image in image_names: img = mpimg.imread(path + image) # img = mpimg.imread(path + image_names[-5]) start_time = time.time() rclasses, rscores, rbboxes = process_image(img)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading MNIST. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in _SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'), # 'image/class/label': tf.FixedLenFeature( # [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)), # Changed to varlen to be compatible with the rest of the multi-label framework. 'image/class/label': tf.VarLenFeature(tf.int64, ), } num_channels = 3 if hasattr(tf.flags.FLAGS, 'color_space') and tf.flags.FLAGS.color_space == "gray": num_channels = 1 items_to_handlers = { 'image': slim.tfexample_decoder.Image( shape=[_IMAGE_HW, _IMAGE_HW, num_channels], channels=num_channels), # 'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]), # Took off shape to be compatible with the rest of the multi-label framework. 'label': slim.tfexample_decoder.Tensor('image/class/label', ), } items_to_handlers['source'] = items_to_handlers['image'] items_to_handlers['target'] = items_to_handlers['label'] items_to_handlers['conditional_labels'] = items_to_handlers['label'] decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=_SPLITS_TO_SIZES[split_name], num_classes=_NUM_CLASSES, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, labels_to_names=labels_to_names, items_used=[ 'image', 'label', 'source', 'target', 'conditional_labels' ], items_need_preprocessing=[ 'image', 'label', 'source', 'target', 'conditional_labels' ], has_source=True, )
def get_split(train_name, split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading flowers. Args: train_name: A train name. split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in ["train", "validation"]: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = train_name + '_%s_*.tfrecord' file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) num_file = os.path.join(dataset_dir, "train_eval_num.txt") with tf.gfile.Open(num_file, 'rb') as f: num_des = f.readline() num_des = num_des.decode(encoding='UTF-8', errors='strict') data = eval(num_des) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=data[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=data["classes_num"], labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading cifar10. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN # # The format is as below # file_pattern = 'flowers_%s_*.tfrecord' # file_pattern % split_name = file_pattern = 'flowers_train_*.tfrecord' file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'pair/speech': tf.FixedLenFeature((), tf.string, default_value=''), 'pair/mouth': tf.FixedLenFeature((), tf.string, default_value=''), 'speech/format': tf.FixedLenFeature((), tf.string, default_value='raw'), 'mouth/format': tf.FixedLenFeature((), tf.string, default_value='raw'), 'pair/class/label': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/channel_speech': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/feature_speech': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/frame_speech': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/channel_mouth': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/height_mouth': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), 'pair/width_mouth': tf.FixedLenFeature( [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { # 'image': slim.tfexample_decoder.Image(shape=[1,2000,1]), # 'lip': slim.tfexample_decoder.Image(shape=[1, 2000, 1]), 'speech': slim.tfexample_decoder.Image(format_key='speech/format', image_key='pair/speech', shape=[13,15,1], channels=1), 'mouth': slim.tfexample_decoder.Image(format_key='mouth/format', image_key='pair/mouth', shape=[47,73,9], channels=9), # 'speech': slim.tfexample_decoder.Tensor(tensor_key='pair/speech', shape=[13,15,1]), # 'mouth': slim.tfexample_decoder.Tensor(tensor_key='mouth/format', shape=[47,73,9]), 'speech_format': slim.tfexample_decoder.Tensor('speech/format'), 'mouth_format': slim.tfexample_decoder.Tensor('mouth/format'), 'label': slim.tfexample_decoder.Tensor('pair/class/label'), 'channel_speech': slim.tfexample_decoder.Tensor('pair/channel_speech'), 'feature_speech': slim.tfexample_decoder.Tensor('pair/feature_speech'), 'frame_speech': slim.tfexample_decoder.Tensor('pair/frame_speech'), 'channel_mouth': slim.tfexample_decoder.Tensor('pair/channel_mouth'), 'height_mouth': slim.tfexample_decoder.Tensor('pair/height_mouth'), 'width_mouth': slim.tfexample_decoder.Tensor('pair/width_mouth'), } decoder = slim.tfexample_decoder.TFExampleDecoder( keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading ImageNet. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in _SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1), 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, default_value=''), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'image/object/class/label': tf.VarLenFeature(dtype=tf.int64), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'label': slim.tfexample_decoder.Tensor('image/class/label'), 'label_text': slim.tfexample_decoder.Tensor('image/class/text'), 'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if LOAD_READABLE_NAMES: if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) else: labels_to_names = create_readable_names_for_imagenet_labels() dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=_SPLITS_TO_SIZES[split_name], items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='DeepFashion', file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading DeepFashion. Args: split_name: A train/validation split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/validation split. """ if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name)) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image_raw_0': tf.FixedLenFeature([], tf.string), 'image_raw_1': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar 'id_0': tf.FixedLenFeature([], tf.int64), 'id_1': tf.FixedLenFeature([], tf.int64), 'cam_0': tf.FixedLenFeature([], tf.int64), 'cam_1': tf.FixedLenFeature([], tf.int64), 'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'), 'image_height': tf.FixedLenFeature([], tf.int64, default_value=256), 'image_width': tf.FixedLenFeature([], tf.int64, default_value=256), 'real_data': tf.FixedLenFeature([], tf.int64, default_value=1), 'pose_peaks_0': tf.FixedLenFeature([16*16*18], tf.float32), 'pose_peaks_1': tf.FixedLenFeature([16*16*18], tf.float32), 'pose_mask_r4_0': tf.FixedLenFeature([256*256*1], tf.int64), 'pose_mask_r4_1': tf.FixedLenFeature([256*256*1], tf.int64), 'shape': tf.FixedLenFeature([1], tf.int64), 'indices_r4_0': tf.VarLenFeature(dtype=tf.int64), 'values_r4_0': tf.VarLenFeature(dtype=tf.float32), 'indices_r4_1': tf.VarLenFeature(dtype=tf.int64), 'values_r4_1': tf.VarLenFeature(dtype=tf.float32), 'pose_subs_0': tf.FixedLenFeature([20], tf.float32), 'pose_subs_1': tf.FixedLenFeature([20], tf.float32), } items_to_handlers = { 'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'), 'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'), 'label': slim.tfexample_decoder.Tensor('label'), 'id_0': slim.tfexample_decoder.Tensor('id_0'), 'id_1': slim.tfexample_decoder.Tensor('id_1'), 'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0', shape=[16*16*18]), 'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1', shape=[16*16*18]), 'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0', shape=[256*256*1]), 'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1', shape=[256*256*1]), 'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False), 'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False), 'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]), 'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p') print('load pn_pairs_num ......:', fpath) with open(fpath,'r') as f: pn_pairs_num = pickle.load(f) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=pn_pairs_num, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=_NUM_CLASSES, labels_to_names=labels_to_names)
def get_split_multiphase_multislice_mask(split_name, dataset_dir, file_pattern, reader, split_to_sizes, items_to_descriptions, num_classes): if split_name not in split_to_sizes: raise ValueError('split name %s was not recognized.' % split_name) print(split_name) # file_pattern = os.path.join(dataset_dir, file_pattern % split_name) file_pattern = os.path.join(dataset_dir, file_pattern) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader # Features in Pascal VOC TFRecords. keys_to_features = { 'image/nc/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/art/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/pv/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/mask/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='PNG'), 'image/height': tf.FixedLenFeature([1], tf.int64), 'image/width': tf.FixedLenFeature([1], tf.int64), 'image/channels': tf.FixedLenFeature([1], tf.int64), 'image/shape': tf.FixedLenFeature([3], tf.int64), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), } items_to_handlers = { 'nc_image': slim.tfexample_decoder.Image('image/nc/encoded', 'image/format'), 'art_image': slim.tfexample_decoder.Image('image/art/encoded', 'image/format'), 'pv_image': slim.tfexample_decoder.Image('image/pv/encoded', 'image/format'), 'mask_image': slim.tfexample_decoder.Image('image/mask/encoded', 'image/format', channels=1), 'shape': slim.tfexample_decoder.Tensor('image/shape'), 'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/oriented_bbox/x1': slim.tfexample_decoder.Tensor('image/object/bbox/xmin'), 'object/oriented_bbox/x2': slim.tfexample_decoder.Tensor('image/object/bbox/xmax'), 'object/oriented_bbox/x3': slim.tfexample_decoder.Tensor('image/object/bbox/xmax'), 'object/oriented_bbox/x4': slim.tfexample_decoder.Tensor('image/object/bbox/xmin'), 'object/oriented_bbox/y1': slim.tfexample_decoder.Tensor('image/object/bbox/ymin'), 'object/oriented_bbox/y2': slim.tfexample_decoder.Tensor('image/object/bbox/ymin'), 'object/oriented_bbox/y3': slim.tfexample_decoder.Tensor('image/object/bbox/ymax'), 'object/oriented_bbox/y4': slim.tfexample_decoder.Tensor('image/object/bbox/ymax'), 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) # else: # labels_to_names = create_readable_names_for_imagenet_labels() # dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=split_to_sizes[split_name], items_to_descriptions=items_to_descriptions, num_classes=num_classes, labels_to_names=labels_to_names)
images, labels = tf.train.batch([image, label], batch_size=BATCH_SIZE, allow_smaller_final_batch=True) # get the model prediction network_fn = nets_factory.get_network_fn("cifarnet", num_classes=10, is_training=False) # run the image through the model # predictions,_ = lenet.lenet(images) predictions, _ = network_fn(images) # convert prediction values for each class into single class prediction predictions = tf.to_int64(tf.argmax(predictions, 1)) if dataset_utils.has_labels(kaggle_test): print("fffffffff") labels_to_names = dataset_utils.read_label_file(kaggle_test) print(labels_to_names) saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() saver.restore(sess, checkpoint_path) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) predicted_lables = np.zeros(300000) p = [] for i in range(1): print(i) predicted_lables[i * BATCH_SIZE:(i + 1) *
def get_split(split_name, dataset_dir, file_pattern, reader, items_to_descriptions, num_classes): """Gets a dataset tuple with instructions for reading Pascal VOC dataset. Args: split_name: A trainval/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if split_name not in ['trainval', 'test']: raise ValueError('split name %s was not recognized.' % split_name) file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader # Features in Pascal VOC TFRecords. keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/height': tf.FixedLenFeature([1], tf.int64), 'image/width': tf.FixedLenFeature([1], tf.int64), 'image/channels': tf.FixedLenFeature([1], tf.int64), 'image/shape': tf.FixedLenFeature([3], tf.int64), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'shape': slim.tfexample_decoder.Tensor('image/shape'), 'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) # else: # labels_to_names = create_readable_names_for_imagenet_labels() # dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=split_to_sizes[split_name], items_to_descriptions=items_to_descriptions, num_classes=num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """Gets a dataset tuple with instructions for reading Pascal VOC dataset. Args: split_name: A train/test split name. dataset_dir: The base directory of the dataset sources. file_pattern: The file pattern to use when matching the dataset sources. It is assumed that the pattern contains a '%s' string so that the split name can be inserted. reader: The TensorFlow reader type. Returns: A `Dataset` namedtuple. Raises: ValueError: if `split_name` is not a valid train/test split. """ if not file_pattern: file_pattern = FILE_PATTERN if split_name not in SPLITS_TO_SIZES: raise ValueError('split name %s was not recognized.' % split_name) file_pattern = os.path.join(dataset_dir, file_pattern % split_name) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader # Features in Pascal VOC TFRecords. keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/height': tf.FixedLenFeature([1], tf.int64), 'image/width': tf.FixedLenFeature([1], tf.int64), 'image/channels': tf.FixedLenFeature([1], tf.int64), 'image/shape': tf.FixedLenFeature([3], tf.int64), 'object/label': tf.VarLenFeature(dtype=tf.int64), 'object/truncated': tf.VarLenFeature(dtype=tf.float32), 'object/occluded': tf.VarLenFeature(dtype=tf.int64), 'object/alpha': tf.VarLenFeature(dtype=tf.float32), 'object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'object/dimensions/height': tf.VarLenFeature(dtype=tf.float32), 'object/dimensions/width': tf.VarLenFeature(dtype=tf.float32), 'object/dimensions/length': tf.VarLenFeature(dtype=tf.float32), 'object/location/x': tf.VarLenFeature(dtype=tf.float32), 'object/location/y': tf.VarLenFeature(dtype=tf.float32), 'object/location/z': tf.VarLenFeature(dtype=tf.float32), 'object/rotation_y': tf.VarLenFeature(dtype=tf.float32), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'shape': slim.tfexample_decoder.Tensor('image/shape'), 'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('object/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) # else: # labels_to_names = create_readable_names_for_imagenet_labels() # dataset_utils.write_label_file(labels_to_names, dataset_dir) return slim.dataset.Dataset( data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=SPLITS_TO_SIZES[split_name], items_to_descriptions=ITEMS_TO_DESCRIPTIONS, num_classes=NUM_CLASSES, labels_to_names=labels_to_names ) # Copyright 2015 Paul Balanca. All Rights Reserved.