def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/class/coarse_label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/filepath': tf.FixedLenFeature((), tf.string, default_value=''),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[224, 224, 1], channels=1),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'),
'filepath': slim.tfexample_decoder.Tensor('image/filepath'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
dataset = slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
coarse_labels_to_names = None
if dataset_utils.has_labels(dataset_dir, _COARSE_LABEL_FILENAME):
coarse_labels_to_names = dataset_utils.read_label_file(
dataset_dir, _COARSE_LABEL_FILENAME)
dataset.coarse_labels_to_names = coarse_labels_to_names
return dataset
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar100.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/class/coarse_label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name,
dataset_dir,
dataset_list_dir='',
file_pattern=None,
reader=None,
modality='rgb',
num_samples=1):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# if not file_pattern:
# file_pattern = _FILE_PATTERN
# file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if split_name == 'train':
_LIST = _TRAIN_LIST
else:
_LIST = _VAL_LIST
with open(_LIST, 'r') as fin:
data_sources = [
' '.join([
os.path.join(dataset_dir,
el.split()[0]),
] + el.split()[1:]) for el in fin.read().splitlines()
]
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = readerFn
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=data_sources,
reader=reader,
decoder=decoderFn(num_samples),
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, division_idx=None,
file_pattern=None, reader=None, num_classes=None, num_samples=None):
"""Gets a dataset tuple with instructions for reading training data.
Returns:
A 'Dataset' namedtuple.
Raises:
ValueError: if 'split_name' is not the train split.
"""
if division_idx is not None:
raise ValueError('No division index is needed for the training dataset.')
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/tracklet_id': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/cam_id': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'tracklet_id': slim.tfexample_decoder.Tensor('image/tracklet_id'),
'cam_id': slim.tfexample_decoder.Tensor('image/cam_id'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
_NUM_CLASSES = len(glob.glob(dataset_dir + '/*/'))
ALL_NUM = len(glob.glob(dataset_dir + '/*/*.jpg'))
NUM_VAL = int(ALL_NUM * 0.1)
SPLITS_TO_SIZES = {'train': ALL_NUM - NUM_VAL, 'validation': NUM_VAL}
_FILE_PATTERN = 'mydata_%s_*.tfrecord'
_ITEMS_TO_DESCRIPTIONS = {
'image': 'A color image of varying size.',
'label': 'A single integer between 0 and 257',
}
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64))
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label')
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, dataset_list_dir='', file_pattern=None, reader=None, modality='rgb', num_samples=1):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# if not file_pattern:
# file_pattern = _FILE_PATTERN
# file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if split_name == 'train':
_LIST = _TRAIN_LIST
else:
_LIST = _VAL_LIST
with open(_LIST, 'r') as fin:
data_sources = [
' '.join([os.path.join(dataset_dir, el.split()[0]),] + el.split()[1:])
for el in fin.read().splitlines()
]
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = readerFn
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=data_sources,
reader=reader,
decoder=decoderFn(num_samples),
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name,
num_samples,
dataset_dir,
file_pattern=None,
reader=None):
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
SPLITS_TO_SIZES[split_name] = num_samples
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def load_model(self):
sess = tf.Session()
x = tf.placeholder(tf.float32,shape=[None,None,3])
# read dataset info #
if dataset_utils.has_labels(self.model_dir):
labels_to_names = dataset_utils.read_label_file(self.model_dir)
with open(os.path.join(self.model_dir,'info.json')) as f:
info_dict = json.load(f)
_num_classes = info_dict['num_class']+1
# Select the model #
network_fn = nets_factory.get_network_fn(
self.conf['infor']['model_name'],
num_classes=_num_classes,
is_training=False)
# Select the preprocessing function #
preprocessing_name = self.conf['infor']['model_name'] if self.conf['infor']['preprocessing_name']=='None' else self.conf['infor']['preprocessing_name']
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=False)
eval_image_size = self.conf['infor']['eval_image_size'] if self.conf['infor']['eval_image_size']!='None' else network_fn.default_image_size
image = image_preprocessing_fn(x, eval_image_size, eval_image_size)
# inference map
imgs = tf.placeholder(tf.float32,shape=[None,None,None,3])
logits, _ = network_fn(imgs)
predictions = tf.argmax(logits, 1)
# load model #
saver = tf.train.Saver()
params_file = tf.train.latest_checkpoint(self.model_dir)
saver.restore(sess, params_file)
self.output['x']=x
self.output['imgs']=imgs
self.output['predictions']=predictions
self.output['labels_to_names']=labels_to_names
self.output['sess'] = sess
self.output['image'] = image
def get_split(self, split_name):
splits_to_sizes = self.__get_num_samples__(split_name)
if split_name not in ['train', 'validation']:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = self.dataset_name + '_' + split_name + '_*.tfrecord'
file_pattern = os.path.join(self.tfrecord_dir, file_pattern)
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(self.tfrecord_dir):
labels_to_names = dataset_utils.read_label_file(self.tfrecord_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=splits_to_sizes,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=self.num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
# if reader is None:
# reader = tf.TFRecordReader
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
filenames = tf.gfile.Glob(file_pattern)
return Dataset(filenames=filenames,
num_samples=SPLITS_TO_SIZES[split_name],
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
def get_split_test(split_name, dataset_dir, file_pattern=None, reader=None):
if split_name not in SPLITS_TO_SIZES_TEST:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
#file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image_id': tf.FixedLenFeature((), tf.string, default_value=''),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'image_id': slim.tfexample_decoder.Tensor('image_id'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES_TEST[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names,
shuffle=False)
def get_split(split_name,
dataset_dir,
file_pattern=None,
reader=None,
mode=""):
"""Gets a dataset tuple with instructions for reading MNIST.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='raw'),
'image/class/label':
tf.FixedLenFeature([1],
tf.int64,
default_value=tf.zeros([1], dtype=tf.int64)),
'image/filename':
tf.FixedLenFeature((), tf.string, default_value='no_filename'),
# 'image/filename': tf.VarLenFeature(dtype=tf.string),
}
base_size = 128
if mode == "multiple":
width = base_size * 2
height = base_size
elif mode == "tiny" or mode == "tinygan":
width = 32
height = 32
else:
width = base_size
height = base_size
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[height, width, 1],
channels=1),
'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]),
'filename': slim.tfexample_decoder.Tensor('image/filename'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
num_classes=_NUM_CLASSES,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature(
(), tf.string, default_value=''),
'image/format': tf.FixedLenFeature(
(), tf.string, default_value='jpeg'),
'image/class/label': tf.FixedLenFeature(
[], dtype=tf.int64, default_value=-1),
'image/class/text': tf.FixedLenFeature(
[], dtype=tf.string, default_value=''),
'image/object/bbox/xmin': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(
dtype=tf.float32),
'image/object/class/label': tf.VarLenFeature(
dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'label_text': slim.tfexample_decoder.Tensor('image/class/text'),
'object/bbox': slim.tfexample_decoder.BoundingBox(
['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
else:
labels_to_names = create_readable_names_for_imagenet_labels()
dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(dataset_name, split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
num_classes, splits_to_sizes, items_to_descriptions = dataset_utils.read_dataset_config_json(dataset_name, dataset_dir)
if split_name not in splits_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN % (dataset_name, split_name)
file_pattern = os.path.join(dataset_dir, file_pattern)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/name': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/height': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/width': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'image_name': slim.tfexample_decoder.Tensor('image/name'),
'image_height': slim.tfexample_decoder.Tensor('image/height'),
'image_width': slim.tfexample_decoder.Tensor('image/width'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
def dataset_class_weight(splits_to_sizes, split_name, label_to_class_number):
"""
Returns sorted list of class ratios
"""
num_samples = splits_to_sizes[split_name]
num_samples_per_class = splits_to_sizes[split_name+'_per_class']
sorted_class_weights = list()
sorted_label_name = sorted(label_to_class_number.keys(), key= lambda x: label_to_class_number[x])
for label_name in sorted_label_name:
if label_name in num_samples_per_class:
norm_class_weight = num_samples / (num_samples_per_class[label_name] * num_classes)
sorted_class_weights.append(norm_class_weight)
return sorted_class_weights
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
class_to_label_name, label_to_class_number = dataset_utils.read_label_file(dataset_dir)
num_samples=splits_to_sizes[split_name]
# if split_name+'_per_class' in splits_to_sizes:
sorted_class_weights = dataset_class_weight(splits_to_sizes, split_name, label_to_class_number)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples,
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=label_to_class_number,
sorted_class_weights=sorted_class_weights
)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading database.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
images_count, class_count = _get_filenames_and_classes_count(dataset_dir)
num_validation = images_count*_PERC_VALIDATION//100
splits_to_sizes = {'train': (images_count-num_validation), 'validation': num_validation}
_items_to_descriptions = {
'image': 'A color image of varying size.',
'label': 'A single integer between 0 and ' + str(class_count),
}
if split_name not in splits_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir + '/tfrecords', file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir + '/tfrecords'):
labels_to_names = dataset_utils.read_label_file(dataset_dir + '/tfrecords')
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=splits_to_sizes[split_name],
items_to_descriptions=_items_to_descriptions,
num_classes=class_count,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern, reader,
split_to_sizes, items_to_descriptions, num_classes):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in split_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape': slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox': slim.tfexample_decoder.BoundingBox(
['xmin', 'ymin', 'xmax', 'ymax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_to_sizes[split_name],
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar10.
instruction(n. 指示; 教育; 用法说明)
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
pattern(n. 花样, 图案; 格局; 形态, 样式; 样品, 样本)
It is assumed that the pattern contains a '%s' string so that the split
assumed(adj. 假装的; 假定的, 设想的; 假冒的; 被承担的)
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
# signature(n. 签名, 签字, 签署)
if not reader:
reader = tf.TFRecordReader
keys_to_features = { # fixed(adj. 固定的; 不变的; 确定的; 不动的)
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
# handler(n. 操作者; 处理者; 操作装置; 处理机; 教练, 训练员; 管理人, 负责人;
# 由某事件使其活跃并管理照顾那个事件的过程 (计算机用语))
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
# source(n. 来源, 根源, 水源; 起点; 原始码 (计算机用语))
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
# description(n. 描写; 形容; 叙述; 说明书)
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name,
dataset_dir,
photos_subdir='photos',
tfrecords_subdir='tfrecords',
file_pattern=None,
reader=None):
"""Gets a dataset tuple with instructions for reading tumblr data.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
photos_subdir: The subdirectory containing the photos.
tfrecords_subdir: The subdirectory containing the TFRecords files.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
# if split_name not in SPLITS_TO_SIZES:
#raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, tfrecords_subdir,
file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir, photos_subdir):
labels_to_names = dataset_utils.read_label_file(
dataset_dir, photos_subdir)
# Get split size
train_valid_filename = os.path.join(dataset_dir, photos_subdir,
_TRAIN_VALID_FILENAME)
with tf.gfile.Open(train_valid_filename, 'rb') as f:
lines = f.read().decode()
lines = lines.split('\n')
lines = filter(None, lines)
train_valid_split = {}
for line in lines:
index = line.index(':')
train_valid_split[line[:index]] = (int)(line[index + 1:])
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=train_valid_split[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=len(labels_to_names),
labels_to_names=labels_to_names)
def get_split(split_name,
dataset_dir,
file_pattern=None,
reader=None): #这个是slim分类中的函数，主要是根据split_name来选择并解析tfrecord文件
"""Gets a dataset tuple with instructions for reading cifar10. #返回的就是Dataset数据集格式
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir,
file_pattern % split_name) #识别文件名格式
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
} #解析文件个结构
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers) #用TFExampleDecoder解析文件
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(
dataset_dir) #读取文件类名，映射为字典{‘标签’：名称}
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names) #返回数据类，下面就可以用provider进行数据batch生成
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if not file_pattern:
file_pattern = os.path.join(dataset_dir, FILE_PATTERN)
else:
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# data_source = os.path.join(dataset_dir, 'MOT17_a11.record')
# data_source = dataset_dir
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
# 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
# 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
# 'image/height': tf.FixedLenFeature([1], dtype=tf.int64),
# 'image/width': tf.FixedLenFeature([1], dtype=tf.int64),
# 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
# 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
# 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
# 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
# 'image/object/class/label': tf.VarLenFeature(dtype=tf.int64),
# 'image/object/class/label_text': tf.FixedLenFeature((), tf.string, default_value=''),
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string,
default_value='jpeg'),
'image/filename': tf.FixedLenFeature((), tf.string, default_value=''),
'image/key/sha256': tf.FixedLenFeature((), tf.string,
default_value=''),
'image/source_id': tf.FixedLenFeature((), tf.string, default_value=''),
'image/height': tf.FixedLenFeature((), tf.int64, default_value=1),
'image/width': tf.FixedLenFeature((), tf.int64, default_value=1),
# Object boxes and classes.
'image/object/bbox/xmin': tf.VarLenFeature(tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(tf.float32),
'image/object/class/label': tf.VarLenFeature(tf.int64),
'image/object/class/text': tf.VarLenFeature(tf.string),
'image/object/area': tf.VarLenFeature(tf.float32),
'image/object/is_crowd': tf.VarLenFeature(tf.int64),
'image/object/difficult': tf.VarLenFeature(tf.int64),
'image/object/group_of': tf.VarLenFeature(tf.int64),
'image/object/weight': tf.VarLenFeature(tf.float32),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/class/label'),
# 'object/label_text': slim.tfexample_decoder.Tensor('image/object/class/label_text'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
label_dir = os.path.join(os.getcwd(), 'datasets')
labels_to_names = None
if dataset_utils.has_labels(label_dir):
labels_to_names = dataset_utils.read_label_file2(label_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=ITEMS_TO_DESCRIPTIONS,
num_classes=NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name,
dataset_dir,
file_pattern=None,
reader=None,
file_pattern_for_counting=None):
'''
Obtains the split - training or validation - to create a Dataset class for feeding the examples into a queue later on. This function will
set up the decoder and dataset information all into one Dataset class so that you can avoid the brute work later on.
Your file_pattern is very important in locating the files later.
INPUTS:
- split_name(str): 'train' or 'validation'. Used to get the correct data split of tfrecord files
- dataset_dir(str): the dataset directory where the tfrecord files are located
- file_pattern(str): the file name structure of the tfrecord files in order to get the correct data
- file_pattern_for_counting(str): the string name to identify your tfrecord files for counting
OUTPUTS:
- dataset (Dataset): A Dataset class object where we can read its various components for easier batch creation later.
'''
#First check whether the split_name is train or validation
if split_name not in ['train', 'validation']:
raise ValueError(
'The split_name %s is not recognized. Please input either train or validation as the split_name'
% (split_name))
if file_pattern is None:
file_pattern = _FILE_PATTERN
if file_pattern_for_counting is None:
file_pattern_for_counting = _FILE_PATTERN_FOR_COUNTING
#Create the full path for a general file_pattern to locate the tfrecord_files
print(file_pattern)
print(split_name)
print(dataset_dir)
file_pattern_path = os.path.join(dataset_dir, file_pattern % (split_name))
#Count the total number of examples in all of these shard
num_samples = 0
file_pattern_for_counting = file_pattern_for_counting + '_' + split_name
tfrecords_to_count = [
os.path.join(dataset_dir, file) for file in os.listdir(dataset_dir)
if file.startswith(file_pattern_for_counting)
]
for tfrecord_file in tfrecords_to_count:
for record in tf.python_io.tf_record_iterator(tfrecord_file):
num_samples += 1
#Create a reader, which must be a TFRecord reader in this case
reader = tf.TFRecordReader
#Create the keys_to_features dictionary for the decoder
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
#Create the items_to_handlers dictionary for the decoder.
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
#Start to create the decoder
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
#Create the labels_to_name file
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
#Actually create the dataset
dataset = slim.dataset.Dataset(
data_sources=file_pattern_path,
decoder=decoder,
reader=reader,
num_readers=4,
num_samples=num_samples,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
return dataset
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading MNIST.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
#filename_queue = tf.train.string_input_producer(['./data/cifar10_train.tfrecord'])
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
"""
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example, features={
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/height': tf.FixedLenFeature((), tf.int64),
'image/width': tf.FixedLenFeature((), tf.int64),
'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64))
})
"""
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
#'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
#'image/height': tf.FixedLenFeature((), tf.int64),
#'image/width': tf.FixedLenFeature((), tf.int64),
'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64))
}
items_to_handlers = {
#'image': slim.tfexample_decoder.Image(shape=[32, 128, 3], channels=3),
#'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
#'image': tf.image.decode_png(features['image/encoded'], channels=3)
'image': slim.tfexample_decoder.Image(image_key = 'image/encoded', format_key = 'image/format'),
'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[])
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
num_classes=_NUM_CLASSES,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
labels_to_names=labels_to_names)
def main(_):
if not FLAGS.predict_file:
raise ValueError(
'You must supply the predict file path with --predict_file')
with tf.Graph().as_default():
tf.logging.set_verbosity(tf.logging.INFO)
##############################################################################################################
network_fn = nets_factory.get_network_fn(FLAGS.model_name,
num_classes=_NUM_CLASSES,
is_training=False)
image_size = FLAGS.predict_image_size or network_fn.default_image_size
# images, images_raw = load_batch_2(FLAGS.predict_file, height=image_size, width=image_size)
image_string = tf.gfile.FastGFile(FLAGS.predict_file, 'rb').read()
image = tf.image.decode_jpeg(image_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(
image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
logits, _ = network_fn(processed_images)
probabilities = tf.nn.softmax(logits)
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_path, slim.get_variables_to_restore())
with tf.Session() as sess:
with slim.queues.QueueRunners(sess):
sess.run(tf.initialize_local_variables())
init_fn(sess)
np_images, np_probabilities = sess.run([image, probabilities])
predicted_label = np.argmax(np_probabilities[0, :])
print(predicted_label)
labels_to_names = None
if dataset_utils.has_labels(FLAGS.dataset_dir):
labels_to_names = dataset_utils.read_label_file(FLAGS.dataset_dir)
class_name = labels_to_names[predicted_label]
print(class_name)
if dataset_utils.has_labels(FLAGS.dataset_dir, INFO_FILE):
class_names_to_info = read_info_file(FLAGS.dataset_dir)
if class_name in merge_info:
print('It is the back of a coin, please take a photo of front.')
class_name_list = merge_info[class_name]
for item in class_name_list:
info = class_names_to_info[item.split('_')[0]]
print('Maybe the value is {}'.format(info['value']))
else:
name_info = class_name.split('_')
origin_name = name_info[0]
back_or_front = name_info[1]
info = class_names_to_info[origin_name]
print('value is %s, it is %s of the coin, the other info is %s' %
(info['value'], back_or_front, info['info']))
return info['value']
def get_split(split_name,
dataset_dir,
dataset_str,
reader=None,
num_classes=180):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
file_pattern = dataset_str + '_%s_*.tfrecord'
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if 'oriTrn1' in dataset_str:
SPLITS_TO_SIZES = {'train': 2592, 'validation': 288}
elif 'Trn' in dataset_str:
SPLITS_TO_SIZES = {'train': 1944, 'validation': 216}
elif 'Tst' in dataset_str or 'Gratings' in dataset_str or 'FiltNoise' in dataset_str or 'FiltIms' in dataset_str:
SPLITS_TO_SIZES = {}
for bb in range(96):
SPLITS_TO_SIZES['batch' + str(bb)] = 90
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# assert len(labels_to_names)==num_classes
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='Market1501', file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name))
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image_raw_0' : tf.FixedLenFeature([], tf.string),
'image_raw_1' : tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar
'id_0': tf.FixedLenFeature([], tf.int64),
'id_1': tf.FixedLenFeature([], tf.int64),
'cam_0': tf.FixedLenFeature([], tf.int64),
'cam_1': tf.FixedLenFeature([], tf.int64),
'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'),
'image_height': tf.FixedLenFeature([], tf.int64, default_value=128),
'image_width': tf.FixedLenFeature([], tf.int64, default_value=64),
'real_data': tf.FixedLenFeature([], tf.int64, default_value=1),
# 'attrs_0': tf.FixedLenFeature([27], tf.int64),
# 'attrs_1': tf.FixedLenFeature([27], tf.int64),
# 'attrs_w2v25_0': tf.FixedLenFeature([25*12], tf.float32),
# 'attrs_w2v25_1': tf.FixedLenFeature([25*12], tf.float32),
# 'attrs_w2v50_0': tf.FixedLenFeature([50*12], tf.float32),
# 'attrs_w2v50_1': tf.FixedLenFeature([50*12], tf.float32),
# 'attrs_w2v100_0': tf.FixedLenFeature([100*12], tf.float32),
# 'attrs_w2v100_1': tf.FixedLenFeature([100*12], tf.float32),
# 'attrs_w2v150_0': tf.FixedLenFeature([150*12], tf.float32),
# 'attrs_w2v150_1': tf.FixedLenFeature([150*12], tf.float32),
'pose_peaks_0': tf.FixedLenFeature([16*8*18], tf.float32),
'pose_peaks_1': tf.FixedLenFeature([16*8*18], tf.float32),
'pose_peaks_0_rc': tf.FixedLenFeature([18*2], tf.float32),
'pose_peaks_1_rc': tf.FixedLenFeature([18*2], tf.float32),
# 'pose_dense_r4_0': tf.FixedLenFeature([128*64*18], tf.float32),
# 'pose_dense_r4_1': tf.FixedLenFeature([128*64*18], tf.float32),
'pose_mask_r4_0': tf.FixedLenFeature([128*64*1], tf.int64),
'pose_mask_r4_1': tf.FixedLenFeature([128*64*1], tf.int64),
'shape': tf.FixedLenFeature([1], tf.int64),
'indices_r4_0': tf.VarLenFeature(dtype=tf.int64),
'values_r4_0': tf.VarLenFeature(dtype=tf.float32),
'indices_r4_1': tf.VarLenFeature(dtype=tf.int64),
'values_r4_1': tf.VarLenFeature(dtype=tf.float32),
'pose_subs_0': tf.FixedLenFeature([20], tf.float32),
'pose_subs_1': tf.FixedLenFeature([20], tf.float32),
}
items_to_handlers = {
'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'),
'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'),
'label': slim.tfexample_decoder.Tensor('label'),
'id_0': slim.tfexample_decoder.Tensor('id_0'),
'id_1': slim.tfexample_decoder.Tensor('id_1'),
# 'attrs_0': slim.tfexample_decoder.Tensor('attrs_0',shape=[27]),
# 'attrs_1': slim.tfexample_decoder.Tensor('attrs_1',shape=[27]),
# 'attrs_w2v25_0': slim.tfexample_decoder.Tensor('attrs_w2v25_0',shape=[25*12]),
# 'attrs_w2v25_1': slim.tfexample_decoder.Tensor('attrs_w2v25_1',shape=[25*12]),
# 'attrs_w2v50_0': slim.tfexample_decoder.Tensor('attrs_w2v50_0',shape=[50*12]),
# 'attrs_w2v50_1': slim.tfexample_decoder.Tensor('attrs_w2v50_1',shape=[50*12]),
# 'attrs_w2v100_0': slim.tfexample_decoder.Tensor('attrs_w2v100_0',shape=[100*12]),
# 'attrs_w2v100_1': slim.tfexample_decoder.Tensor('attrs_w2v100_1',shape=[100*12]),
# 'attrs_w2v150_0': slim.tfexample_decoder.Tensor('attrs_w2v150_0',shape=[150*12]),
# 'attrs_w2v150_1': slim.tfexample_decoder.Tensor('attrs_w2v150_1',shape=[150*12]),
'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0',shape=[16*8*18]),
'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1',shape=[16*8*18]),
'pose_peaks_0_rc': slim.tfexample_decoder.Tensor('pose_peaks_0_rc',shape=[18*2]),
'pose_peaks_1_rc': slim.tfexample_decoder.Tensor('pose_peaks_1_rc',shape=[18*2]),
# 'pose_dense_r4_0': slim.tfexample_decoder.Tensor('pose_dense_r4_0',shape=[128*64*18]),
# 'pose_dense_r4_1': slim.tfexample_decoder.Tensor('pose_dense_r4_1',shape=[128*64*18]),
'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0',shape=[128*64*1]),
'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1',shape=[128*64*1]),
'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False),
'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False),
'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]),
'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
print('load pn_pairs_num ......')
fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p')
with open(fpath,'r') as f:
pn_pairs_num = pickle.load(f)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=pn_pairs_num,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
rbboxes = np_methods.bboxes_clip(rbbox_img, rbboxes)
rclasses, rscores, rbboxes = np_methods.bboxes_sort(rclasses,
rscores,
rbboxes,
top_k=400)
rclasses, rscores, rbboxes = np_methods.bboxes_nms(
rclasses, rscores, rbboxes, nms_threshold=nms_threshold)
# Resize bboxes to original image shape. Note: useless for Resize.WARP!
rbboxes = np_methods.bboxes_resize(rbbox_img, rbboxes)
return rclasses, rscores, rbboxes
dataset_dir = os.path.join(os.getcwd(), 'datasets')
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file2(dataset_dir,
filename='labels_voc.txt')
# Test on some demo image and visualize output.
path = '/home/ace19/dl_data/MOT/MOT17/test/sample-test/'
# path = './detection_image/coco/'
image_names = sorted(os.listdir(path))
for image in image_names:
img = mpimg.imread(path + image)
# img = mpimg.imread(path + image_names[-5])
start_time = time.time()
rclasses, rscores, rbboxes = process_image(img)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading MNIST.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
# 'image/class/label': tf.FixedLenFeature(
# [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)),
# Changed to varlen to be compatible with the rest of the multi-label framework.
'image/class/label': tf.VarLenFeature(tf.int64, ),
}
num_channels = 3
if hasattr(tf.flags.FLAGS,
'color_space') and tf.flags.FLAGS.color_space == "gray":
num_channels = 1
items_to_handlers = {
'image':
slim.tfexample_decoder.Image(
shape=[_IMAGE_HW, _IMAGE_HW, num_channels], channels=num_channels),
# 'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]),
# Took off shape to be compatible with the rest of the multi-label framework.
'label':
slim.tfexample_decoder.Tensor('image/class/label', ),
}
items_to_handlers['source'] = items_to_handlers['image']
items_to_handlers['target'] = items_to_handlers['label']
items_to_handlers['conditional_labels'] = items_to_handlers['label']
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
num_classes=_NUM_CLASSES,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
labels_to_names=labels_to_names,
items_used=[
'image', 'label', 'source', 'target', 'conditional_labels'
],
items_need_preprocessing=[
'image', 'label', 'source', 'target', 'conditional_labels'
],
has_source=True,
)
def get_split(train_name,
split_name,
dataset_dir,
file_pattern=None,
reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
train_name: A train name.
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in ["train", "validation"]:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = train_name + '_%s_*.tfrecord'
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
num_file = os.path.join(dataset_dir, "train_eval_num.txt")
with tf.gfile.Open(num_file, 'rb') as f:
num_des = f.readline()
num_des = num_des.decode(encoding='UTF-8', errors='strict')
data = eval(num_des)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=data[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=data["classes_num"],
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
# # The format is as below
# file_pattern = 'flowers_%s_*.tfrecord'
# file_pattern % split_name = file_pattern = 'flowers_train_*.tfrecord'
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'pair/speech': tf.FixedLenFeature((), tf.string, default_value=''),
'pair/mouth': tf.FixedLenFeature((), tf.string, default_value=''),
'speech/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
'mouth/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
'pair/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/channel_speech': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/feature_speech': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/frame_speech': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/channel_mouth': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/height_mouth': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'pair/width_mouth': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
# 'image': slim.tfexample_decoder.Image(shape=[1,2000,1]),
# 'lip': slim.tfexample_decoder.Image(shape=[1, 2000, 1]),
'speech': slim.tfexample_decoder.Image(format_key='speech/format', image_key='pair/speech', shape=[13,15,1], channels=1),
'mouth': slim.tfexample_decoder.Image(format_key='mouth/format', image_key='pair/mouth', shape=[47,73,9], channels=9),
# 'speech': slim.tfexample_decoder.Tensor(tensor_key='pair/speech', shape=[13,15,1]),
# 'mouth': slim.tfexample_decoder.Tensor(tensor_key='mouth/format', shape=[47,73,9]),
'speech_format': slim.tfexample_decoder.Tensor('speech/format'),
'mouth_format': slim.tfexample_decoder.Tensor('mouth/format'),
'label': slim.tfexample_decoder.Tensor('pair/class/label'),
'channel_speech': slim.tfexample_decoder.Tensor('pair/channel_speech'),
'feature_speech': slim.tfexample_decoder.Tensor('pair/feature_speech'),
'frame_speech': slim.tfexample_decoder.Tensor('pair/frame_speech'),
'channel_mouth': slim.tfexample_decoder.Tensor('pair/channel_mouth'),
'height_mouth': slim.tfexample_decoder.Tensor('pair/height_mouth'),
'width_mouth': slim.tfexample_decoder.Tensor('pair/width_mouth'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
'image/class/text':
tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/object/bbox/xmin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/class/label':
tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'label':
slim.tfexample_decoder.Tensor('image/class/label'),
'label_text':
slim.tfexample_decoder.Tensor('image/class/text'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if LOAD_READABLE_NAMES:
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
else:
labels_to_names = create_readable_names_for_imagenet_labels()
dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='DeepFashion', file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading DeepFashion.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name))
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image_raw_0': tf.FixedLenFeature([], tf.string),
'image_raw_1': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar
'id_0': tf.FixedLenFeature([], tf.int64),
'id_1': tf.FixedLenFeature([], tf.int64),
'cam_0': tf.FixedLenFeature([], tf.int64),
'cam_1': tf.FixedLenFeature([], tf.int64),
'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'),
'image_height': tf.FixedLenFeature([], tf.int64, default_value=256),
'image_width': tf.FixedLenFeature([], tf.int64, default_value=256),
'real_data': tf.FixedLenFeature([], tf.int64, default_value=1),
'pose_peaks_0': tf.FixedLenFeature([16*16*18], tf.float32),
'pose_peaks_1': tf.FixedLenFeature([16*16*18], tf.float32),
'pose_mask_r4_0': tf.FixedLenFeature([256*256*1], tf.int64),
'pose_mask_r4_1': tf.FixedLenFeature([256*256*1], tf.int64),
'shape': tf.FixedLenFeature([1], tf.int64),
'indices_r4_0': tf.VarLenFeature(dtype=tf.int64),
'values_r4_0': tf.VarLenFeature(dtype=tf.float32),
'indices_r4_1': tf.VarLenFeature(dtype=tf.int64),
'values_r4_1': tf.VarLenFeature(dtype=tf.float32),
'pose_subs_0': tf.FixedLenFeature([20], tf.float32),
'pose_subs_1': tf.FixedLenFeature([20], tf.float32),
}
items_to_handlers = {
'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'),
'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'),
'label': slim.tfexample_decoder.Tensor('label'),
'id_0': slim.tfexample_decoder.Tensor('id_0'),
'id_1': slim.tfexample_decoder.Tensor('id_1'),
'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0', shape=[16*16*18]),
'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1', shape=[16*16*18]),
'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0', shape=[256*256*1]),
'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1', shape=[256*256*1]),
'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False),
'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False),
'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]),
'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p')
print('load pn_pairs_num ......:', fpath)
with open(fpath,'r') as f:
pn_pairs_num = pickle.load(f)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=pn_pairs_num,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split_multiphase_multislice_mask(split_name, dataset_dir, file_pattern,
reader, split_to_sizes,
items_to_descriptions, num_classes):
if split_name not in split_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
print(split_name)
# file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/nc/encoded': tf.FixedLenFeature((), tf.string,
default_value=''),
'image/art/encoded': tf.FixedLenFeature((),
tf.string,
default_value=''),
'image/pv/encoded': tf.FixedLenFeature((), tf.string,
default_value=''),
'image/mask/encoded': tf.FixedLenFeature((),
tf.string,
default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='PNG'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'nc_image':
slim.tfexample_decoder.Image('image/nc/encoded', 'image/format'),
'art_image':
slim.tfexample_decoder.Image('image/art/encoded', 'image/format'),
'pv_image':
slim.tfexample_decoder.Image('image/pv/encoded', 'image/format'),
'mask_image':
slim.tfexample_decoder.Image('image/mask/encoded',
'image/format',
channels=1),
'shape':
slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/oriented_bbox/x1':
slim.tfexample_decoder.Tensor('image/object/bbox/xmin'),
'object/oriented_bbox/x2':
slim.tfexample_decoder.Tensor('image/object/bbox/xmax'),
'object/oriented_bbox/x3':
slim.tfexample_decoder.Tensor('image/object/bbox/xmax'),
'object/oriented_bbox/x4':
slim.tfexample_decoder.Tensor('image/object/bbox/xmin'),
'object/oriented_bbox/y1':
slim.tfexample_decoder.Tensor('image/object/bbox/ymin'),
'object/oriented_bbox/y2':
slim.tfexample_decoder.Tensor('image/object/bbox/ymin'),
'object/oriented_bbox/y3':
slim.tfexample_decoder.Tensor('image/object/bbox/ymax'),
'object/oriented_bbox/y4':
slim.tfexample_decoder.Tensor('image/object/bbox/ymax'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/bbox/label'),
'object/difficult':
slim.tfexample_decoder.Tensor('image/object/bbox/difficult'),
'object/truncated':
slim.tfexample_decoder.Tensor('image/object/bbox/truncated'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_to_sizes[split_name],
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=labels_to_names)
images, labels = tf.train.batch([image, label],
batch_size=BATCH_SIZE,
allow_smaller_final_batch=True)
# get the model prediction
network_fn = nets_factory.get_network_fn("cifarnet",
num_classes=10,
is_training=False)
# run the image through the model
# predictions,_ = lenet.lenet(images)
predictions, _ = network_fn(images)
# convert prediction values for each class into single class prediction
predictions = tf.to_int64(tf.argmax(predictions, 1))
if dataset_utils.has_labels(kaggle_test):
print("fffffffff")
labels_to_names = dataset_utils.read_label_file(kaggle_test)
print(labels_to_names)
saver = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
saver.restore(sess, checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
predicted_lables = np.zeros(300000)
p = []
for i in range(1):
print(i)
predicted_lables[i * BATCH_SIZE:(i + 1) *
def get_split(split_name, dataset_dir, file_pattern, reader,
items_to_descriptions, num_classes):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
split_name: A trainval/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in ['trainval', 'test']:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string,
default_value='jpeg'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape':
slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/bbox/label'),
'object/difficult':
slim.tfexample_decoder.Tensor('image/object/bbox/difficult'),
'object/truncated':
slim.tfexample_decoder.Tensor('image/object/bbox/truncated'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_to_sizes[split_name],
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if not file_pattern:
file_pattern = FILE_PATTERN
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'object/label': tf.VarLenFeature(dtype=tf.int64),
'object/truncated': tf.VarLenFeature(dtype=tf.float32),
'object/occluded': tf.VarLenFeature(dtype=tf.int64),
'object/alpha': tf.VarLenFeature(dtype=tf.float32),
'object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'object/dimensions/height': tf.VarLenFeature(dtype=tf.float32),
'object/dimensions/width': tf.VarLenFeature(dtype=tf.float32),
'object/dimensions/length': tf.VarLenFeature(dtype=tf.float32),
'object/location/x': tf.VarLenFeature(dtype=tf.float32),
'object/location/y': tf.VarLenFeature(dtype=tf.float32),
'object/location/z': tf.VarLenFeature(dtype=tf.float32),
'object/rotation_y': tf.VarLenFeature(dtype=tf.float32),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape':
slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('object/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=ITEMS_TO_DESCRIPTIONS,
num_classes=NUM_CLASSES,
labels_to_names=labels_to_names
) # Copyright 2015 Paul Balanca. All Rights Reserved.
