def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/class/coarse_label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/filepath': tf.FixedLenFeature((), tf.string, default_value=''),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[224, 224, 1], channels=1),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'),
'filepath': slim.tfexample_decoder.Tensor('image/filepath'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
dataset = slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
coarse_labels_to_names = None
if dataset_utils.has_labels(dataset_dir, _COARSE_LABEL_FILENAME):
coarse_labels_to_names = dataset_utils.read_label_file(
dataset_dir, _COARSE_LABEL_FILENAME)
dataset.coarse_labels_to_names = coarse_labels_to_names
return dataset
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar100.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/class/coarse_label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'coarse_label': slim.tfexample_decoder.Tensor('image/class/coarse_label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(
split_name,
dataset_dir,
file_pattern, #the pattern contains a '%s' string so that the split name can be inserted
reader,
split_to_sizes=SPLITS_TO_SIZES,
items_to_description=ITEMS_TO_DESCRIPTIONS,
num_classes=NUM_CLASSES):
if split_name not in split_to_sizes:
raise ValueError('split name %s not recognized' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string,
default_value='jpeg'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape':
slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/bbox/label'),
'object/difficult':
slim.tfexample_decoder.Tensor('image/object/bbox/difficult'),
'object/truncated':
slim.tfexample_decoder.Tensor('image/object/bbox/truncated'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_name = None
if dataset_utils.has_labels(dataset_dir):
labels_to_name = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_to_sizes[split_name],
items_to_descriptions=items_to_description,
num_classes=num_classes,
labels_to_name=labels_to_name)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(channels=1),
#'image': slim.tfexample_decoder.Image(shape=[IMAGE_SEZE, IMAGE_SEZE, 1], channels=1),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern, reader, split_to_sizes, items_to_descriptions, num_classes):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in split_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
reader = tf.TFRecordReader if reader is None else reader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape': slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox': slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'),
'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'),
'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder,
num_samples=split_to_sizes[split_name], items_to_descriptions=items_to_descriptions,
num_classes=num_classes, labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, num_data, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources. It
is assumed that the pattern contains a '%s' string so that the split name
can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
# if split_name not in _SPLITS_TO_SIZES:
# raise ValueError('split name %s was not recognized.' % split_name)
print("file_pattern : {}".format(file_pattern))
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[128, 128, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
labels_file = os.path.join(dataset_dir, LABELS_FILENAME)
if tf.gfile.Exists(labels_file):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
print("labels_to_names : {}".format(labels_to_names))
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_data,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=len(labels_to_names),
labels_to_names=labels_to_names)
def get_split(split_name,
dataset_dir,
dataset_list_dir='',
file_pattern=None,
reader=None,
modality='rgb',
num_samples=1):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# if not file_pattern:
# file_pattern = _FILE_PATTERN
# file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if split_name == 'train':
_LIST = _TRAIN_LIST
else:
_LIST = _VAL_LIST
with open(_LIST, 'r') as fin:
data_sources = [
' '.join([
os.path.join(dataset_dir,
el.split()[0]),
] + el.split()[1:]) for el in fin.read().splitlines()
]
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = readerFn
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=data_sources,
reader=reader,
decoder=decoderFn(num_samples),
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def load_dataset(self, dataset_dir): self._has_dataset = False self._labels_to_names = dataset_utils.read_label_file(dataset_dir) self._number_of_classes = len(self._labels_to_names) if(self._number_of_classes > 0): self._has_dataset = True return(self._has_dataset)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
_NUM_CLASSES = len(glob.glob(dataset_dir + '/*/'))
ALL_NUM = len(glob.glob(dataset_dir + '/*/*.jpg'))
NUM_VAL = int(ALL_NUM * 0.1)
SPLITS_TO_SIZES = {'train': ALL_NUM - NUM_VAL, 'validation': NUM_VAL}
_FILE_PATTERN = 'mydata_%s_*.tfrecord'
_ITEMS_TO_DESCRIPTIONS = {
'image': 'A color image of varying size.',
'label': 'A single integer between 0 and 257',
}
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64))
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label')
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, division_idx=None,
file_pattern=None, reader=None, num_classes=None, num_samples=None):
"""Gets a dataset tuple with instructions for reading training data.
Returns:
A 'Dataset' namedtuple.
Raises:
ValueError: if 'split_name' is not the train split.
"""
if division_idx is not None:
raise ValueError('No division index is needed for the training dataset.')
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/tracklet_id': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/cam_id': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'tracklet_id': slim.tfexample_decoder.Tensor('image/tracklet_id'),
'cam_id': slim.tfexample_decoder.Tensor('image/cam_id'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
assert file_pattern is None, file_pattern
assert reader is None, reader
source_pattern = _source_pattern(split_name, dataset_dir)
example_count = _example_count_from_sources(source_pattern)
labels = dataset_utils.read_label_file(dataset_dir)
return tf.contrib.slim.dataset.Dataset(
data_sources=source_pattern,
reader=tf.TFRecordReader,
decoder=_decoder(),
num_samples=example_count,
items_to_descriptions=_item_descriptions(),
num_classes=len(labels),
labels_to_names=labels)
def get_split(split_name, dataset_dir, dataset_list_dir='', file_pattern=None, reader=None, modality='rgb', num_samples=1):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# if not file_pattern:
# file_pattern = _FILE_PATTERN
# file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if split_name == 'train':
_LIST = _TRAIN_LIST
else:
_LIST = _VAL_LIST
with open(_LIST, 'r') as fin:
data_sources = [
' '.join([os.path.join(dataset_dir, el.split()[0]),] + el.split()[1:])
for el in fin.read().splitlines()
]
# Allowing None in the signature so that dataset_factory can use the default.
if not reader:
reader = readerFn
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=data_sources,
reader=reader,
decoder=decoderFn(num_samples),
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name,
num_samples,
dataset_dir,
file_pattern=None,
reader=None):
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
SPLITS_TO_SIZES[split_name] = num_samples
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def load_model(self):
sess = tf.Session()
x = tf.placeholder(tf.float32,shape=[None,None,3])
# read dataset info #
if dataset_utils.has_labels(self.model_dir):
labels_to_names = dataset_utils.read_label_file(self.model_dir)
with open(os.path.join(self.model_dir,'info.json')) as f:
info_dict = json.load(f)
_num_classes = info_dict['num_class']+1
# Select the model #
network_fn = nets_factory.get_network_fn(
self.conf['infor']['model_name'],
num_classes=_num_classes,
is_training=False)
# Select the preprocessing function #
preprocessing_name = self.conf['infor']['model_name'] if self.conf['infor']['preprocessing_name']=='None' else self.conf['infor']['preprocessing_name']
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=False)
eval_image_size = self.conf['infor']['eval_image_size'] if self.conf['infor']['eval_image_size']!='None' else network_fn.default_image_size
image = image_preprocessing_fn(x, eval_image_size, eval_image_size)
# inference map
imgs = tf.placeholder(tf.float32,shape=[None,None,None,3])
logits, _ = network_fn(imgs)
predictions = tf.argmax(logits, 1)
# load model #
saver = tf.train.Saver()
params_file = tf.train.latest_checkpoint(self.model_dir)
saver.restore(sess, params_file)
self.output['x']=x
self.output['imgs']=imgs
self.output['predictions']=predictions
self.output['labels_to_names']=labels_to_names
self.output['sess'] = sess
self.output['image'] = image
def get_split(self, split_name):
splits_to_sizes = self.__get_num_samples__(split_name)
if split_name not in ['train', 'validation']:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = self.dataset_name + '_' + split_name + '_*.tfrecord'
file_pattern = os.path.join(self.tfrecord_dir, file_pattern)
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(self.tfrecord_dir):
labels_to_names = dataset_utils.read_label_file(self.tfrecord_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=splits_to_sizes,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=self.num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
# if reader is None:
# reader = tf.TFRecordReader
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
filenames = tf.gfile.Glob(file_pattern)
return Dataset(filenames=filenames,
num_samples=SPLITS_TO_SIZES[split_name],
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
def get_split_test(split_name, dataset_dir, file_pattern=None, reader=None):
if split_name not in SPLITS_TO_SIZES_TEST:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
#file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image_id': tf.FixedLenFeature((), tf.string, default_value=''),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'image_id': slim.tfexample_decoder.Tensor('image_id'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES_TEST[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names,
shuffle=False)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar10.
instruction(n. 指示; 教育; 用法说明)
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
pattern(n. 花样, 图案; 格局; 形态, 样式; 样品, 样本)
It is assumed that the pattern contains a '%s' string so that the split
assumed(adj. 假装的; 假定的, 设想的; 假冒的; 被承担的)
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
# signature(n. 签名, 签字, 签署)
if not reader:
reader = tf.TFRecordReader
keys_to_features = { # fixed(adj. 固定的; 不变的; 确定的; 不动的)
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(shape=[32, 32, 3]),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
# handler(n. 操作者; 处理者; 操作装置; 处理机; 教练, 训练员; 管理人, 负责人;
# 由某事件使其活跃并管理照顾那个事件的过程 (计算机用语))
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
# source(n. 来源, 根源, 水源; 起点; 原始码 (计算机用语))
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
# description(n. 描写; 形容; 叙述; 说明书)
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
desc_path = os.path.join(dataset_dir, "desc.json")
if not os.path.isfile(desc_path):
raise FileNotFoundError(
str.format('Describe file ({0}) not exists!',
os.path.abspath(desc_path)))
with open(desc_path) as desc_file:
desc = json.load(desc_file)
SPLITS_TO_SIZES = desc['splits_to_sizes']
_NUM_CLASSES = desc['number_of_class']
_FILE_PATTERN = desc['file_pattern']
_ITEMS_TO_DESCRIPTIONS = desc['descriptions']
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
# if split_name not in SPLITS_TO_SIZES:
# raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
num_samples = None
num_classes = None
for file in os.listdir(dataset_dir):
if fnmatch.fnmatch(file, 'labels_*.txt'):
split = str(file).split('_')
num_samples = {'train': int(split[2]), 'validation': int(split[3])}
num_classes = int(split[1])
labels_to_names = dataset_utils.read_label_file(dataset_dir,
filename=str(file))
# if dataset_utils.has_labels(dataset_dir):
# labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature(
(), tf.string, default_value=''),
'image/format': tf.FixedLenFeature(
(), tf.string, default_value='jpeg'),
'image/class/label': tf.FixedLenFeature(
[], dtype=tf.int64, default_value=-1),
'image/class/text': tf.FixedLenFeature(
[], dtype=tf.string, default_value=''),
'image/object/bbox/xmin': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(
dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(
dtype=tf.float32),
'image/object/class/label': tf.VarLenFeature(
dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'label_text': slim.tfexample_decoder.Tensor('image/class/text'),
'object/bbox': slim.tfexample_decoder.BoundingBox(
['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
else:
labels_to_names = create_readable_names_for_imagenet_labels()
dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='DeepFashion', file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading DeepFashion.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name))
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image_raw_0': tf.FixedLenFeature([], tf.string),
'image_raw_1': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar
'id_0': tf.FixedLenFeature([], tf.int64),
'id_1': tf.FixedLenFeature([], tf.int64),
'cam_0': tf.FixedLenFeature([], tf.int64),
'cam_1': tf.FixedLenFeature([], tf.int64),
'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'),
'image_height': tf.FixedLenFeature([], tf.int64, default_value=256),
'image_width': tf.FixedLenFeature([], tf.int64, default_value=256),
'real_data': tf.FixedLenFeature([], tf.int64, default_value=1),
'pose_peaks_0': tf.FixedLenFeature([16*16*18], tf.float32),
'pose_peaks_1': tf.FixedLenFeature([16*16*18], tf.float32),
'pose_mask_r4_0': tf.FixedLenFeature([256*256*1], tf.int64),
'pose_mask_r4_1': tf.FixedLenFeature([256*256*1], tf.int64),
'shape': tf.FixedLenFeature([1], tf.int64),
'indices_r4_0': tf.VarLenFeature(dtype=tf.int64),
'values_r4_0': tf.VarLenFeature(dtype=tf.float32),
'indices_r4_1': tf.VarLenFeature(dtype=tf.int64),
'values_r4_1': tf.VarLenFeature(dtype=tf.float32),
'pose_subs_0': tf.FixedLenFeature([20], tf.float32),
'pose_subs_1': tf.FixedLenFeature([20], tf.float32),
}
items_to_handlers = {
'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'),
'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'),
'label': slim.tfexample_decoder.Tensor('label'),
'id_0': slim.tfexample_decoder.Tensor('id_0'),
'id_1': slim.tfexample_decoder.Tensor('id_1'),
'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0', shape=[16*16*18]),
'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1', shape=[16*16*18]),
'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0', shape=[256*256*1]),
'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1', shape=[256*256*1]),
'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False),
'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False),
'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]),
'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p')
print('load pn_pairs_num ......:', fpath)
with open(fpath,'r') as f:
pn_pairs_num = pickle.load(f)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=pn_pairs_num,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def execute(checkpoint_path, model_no):
if FLAGS.num_validation == 0:
print("FLAGS.num_validation is 0, no need to validation")
return None
if checkpoint_path == None:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
print("Begin validation_confusion_matrix %s" %
format(datetime.now().isoformat()))
t1 = time()
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
labels_to_names = dataset_utils.read_label_file(FLAGS.dataset_dir)
num_classes = len(labels_to_names)
def decode(serialized_example):
feature = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
features = tf.parse_single_example(serialized_example,
features=feature)
# image
image_string = features['image/encoded']
image = tf.image.decode_jpeg(image_string, channels=3)
image = image_preprocessing_fn(image, FLAGS.train_image_size,
FLAGS.train_image_size)
# label
label = features['image/class/label']
label = tf.one_hot(label, num_classes)
return image, label
def input_iter(filenames, batch_size, num_epochs):
if not num_epochs:
num_epochs = 1
dataset = tf.data.TFRecordDataset(filenames,
num_parallel_reads=FLAGS.num_readers)
dataset = dataset.map(decode)
dataset = dataset.repeat(num_epochs)
dataset = dataset.batch(batch_size)
# dataset = dataset.shuffle(buffer_size=NUM_IMAGES)
iterator = dataset.make_one_shot_iterator()
return iterator
with tf.Graph().as_default() as graph:
tf_global_step = slim.get_or_create_global_step()
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=False)
eval_image_size = FLAGS.train_image_size
x = tf.placeholder(tf.float32,
[None, eval_image_size, eval_image_size, 3])
y_ = tf.placeholder(tf.float32, [None, num_classes])
logits, endpoints = network_fn(x)
predictions_key = "Predictions"
if FLAGS.model_name.startswith("resnet"):
predictions_key = "predictions"
y = endpoints[predictions_key]
test_labels = tf.argmax(y_, 1, name="label")
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
input_dir = []
for i in range(5):
data_file = os.path.join(
FLAGS.dataset_dir,
"garbage_validation_0000%d-of-00005.tfrecord") % i
input_dir.append(data_file)
iter = input_iter(input_dir, batch_size, 1)
next_batch = iter.get_next()
saver = tf.train.Saver(
var_list=variables_to_restore) #Same as slim.get_variables()
init1 = tf.global_variables_initializer()
init2 = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(init1)
sess.run(init2)
saver.restore(sess, checkpoint_path)
images, labels = sess.run(next_batch)
predictions = sess.run(fetches=y, feed_dict={x: images})
predictions = np.squeeze(predictions)
ids = sess.run(test_labels, feed_dict={y_: labels})
errorList = []
v_records = []
for i in range(batch_size):
prediction = predictions[i]
top_k = prediction.argsort()[-5:][::-1]
if ids[i] != top_k[0]:
errorList.append(str(ids[i]) + ":" + str(top_k[0]))
v_record = str(ids[i]) + " " + labels_to_names[ids[i]] + " => "
#print(ids[i], labels_to_names[ids[i]], "=> ", end='')
for id in top_k:
human_string = labels_to_names[id]
score = prediction[id]
v_record = v_record + str(
id) + ":" + human_string + "(P=" + str(score) + "), "
#print('%d:%s(P=%.5f), ' % (id, human_string, score), end='')
print(v_record)
v_records.append(v_record)
print(errorList)
errorid_filename = os.path.join(FLAGS.inference_dir,
model_no + "_error.csv")
print("Write file: %s ..." % errorid_filename)
with tf.gfile.Open(errorid_filename, 'w') as f:
for idmap in errorList:
f.write('%s\n' % (idmap))
validation_record_filename = os.path.join(
FLAGS.inference_dir, model_no + "_validation_record.txt")
print("Write file: %s ..." % validation_record_filename)
with tf.gfile.Open(validation_record_filename, 'w') as f:
for v_rec in v_records:
f.write('%s\n' % (v_rec))
sess.close()
t2 = time()
print("End validation_confusion_matrix %d s" % (t2 - t1))
sys.stdout.flush()
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading cifar10.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
print(file_pattern)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_name = None
if dataset_utils.has_labels(dataset_dir):
labels_to_name = dataset_utils.read_label_file(dataset_dir)
num_samples = int(open(dataset_dir+'/'+split_name+'.txt','r').read())
print ('num_samples: '+str(num_samples))
number_classes = int((open(dataset_dir+'/labels.txt','r').read()).split('\n')[-2].split(':')[0])+1
print ('num_classes: '+str(number_classes))
dat = slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
slim.dataset.Dataset.num_classes = property(lambda self: number_classes)
slim.dataset.Dataset.labels_to_names = property(lambda self: labels_to_names)
return dat
def get_split(split_name,
dataset_dir,
dataset_str,
reader=None,
num_classes=180):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
file_pattern = dataset_str + '_%s_*.tfrecord'
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
if 'oriTrn1' in dataset_str:
SPLITS_TO_SIZES = {'train': 2592, 'validation': 288}
elif 'Trn' in dataset_str:
SPLITS_TO_SIZES = {'train': 1944, 'validation': 216}
elif 'Tst' in dataset_str or 'Gratings' in dataset_str or 'FiltNoise' in dataset_str or 'FiltIms' in dataset_str:
SPLITS_TO_SIZES = {}
for bb in range(96):
SPLITS_TO_SIZES['batch' + str(bb)] = 90
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# assert len(labels_to_names)==num_classes
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
return res
'''
with tf.Graph().as_default():
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# set your image
#imgPath = '/mnt/notebook/kihong/models/slim/images/dog.jpg'
imgPath = 'images/dog2.jpg'
testImage_string = tf.gfile.FastGFile(imgPath, 'rb').read()
testImage = tf.image.decode_jpeg(testImage_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(testImage, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
logits, _ = inception_v3.inception_v3(processed_images, num_classes=257, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'model.ckpt-500'), slim.get_model_variables('InceptionV3'))
with tf.Session() as sess:
init_fn(sess)
np_image, probabilities = sess.run([processed_images, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x: x[1])]
names = dataset_utils.read_label_file(dataset_dir)
#names = caltect256.create_readable_names_for_caltect256_labels()
for i in range(15):
index = sorted_inds[i]
print((probabilities[index], names[index]))
def get_split(split_name, dataset_dir, dataset_name,
file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
# load dataset info from file
dataset_info_file_path = os.path.join(dataset_dir)
# dataset_info_file_path = os.path.join(dataset_dir, dataset_name + '.info')
with open(dataset_info_file_path, 'r') as f:
contents = f.read().split('\n')
splits_to_sizes = {}
for line in contents:
info = line.split(':')
splits_to_sizes[info[0]] = int(info[1])
num_classes = splits_to_sizes['label']
if split_name not in splits_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir,
file_pattern % (dataset_name, split_name))
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=splits_to_sizes[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)
def export(checkpoint_path, modelNo):
print("Begin exporting %s" % format(datetime.now().isoformat()))
saved_model_dir = "SavedModel"
inference_dir = os.environ['MODEL_INFERENCE_PATH']
export_dir = os.path.join(inference_dir, saved_model_dir, modelNo,
"SavedModel")
print("The path of saved model: %s" % export_dir)
if tf.gfile.Exists(export_dir):
print('Saved model folder already exist. Delete it firstly.')
if (export_dir.endswith(saved_model_dir)):
tf.gfile.DeleteRecursively(export_dir)
if (checkpoint_path == None):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
print("checkpoint_path: %s" % checkpoint_path)
with tf.Graph().as_default() as graph:
tf_global_step = slim.get_or_create_global_step()
labels_to_names = dataset_utils.read_label_file(FLAGS.dataset_dir)
num_classes = len(labels_to_names)
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=False)
input_shape = [None, FLAGS.train_image_size, FLAGS.train_image_size, 3]
input_tensor = tf.placeholder(name='input_1',
dtype=tf.float32,
shape=input_shape)
predictions_key = "Predictions"
if FLAGS.model_name.startswith("resnet"):
logits, endpoints = network_fn(input_tensor)
predictions_key = "predictions"
elif FLAGS.model_name.startswith("inception"):
logits, endpoints = network_fn(input_tensor,
create_aux_logits=False)
elif FLAGS.model_name.startswith("nasnet_mobile"):
logits, endpoints = network_fn(input_tensor, use_aux_head=0)
predictions = endpoints[predictions_key]
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(
var_list=variables_to_restore) #Same as slim.get_variables()
init1 = tf.global_variables_initializer()
init2 = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(init1)
sess.run(init2)
saver.restore(sess, checkpoint_path)
#uninitialized_variables = [str(v, 'utf-8') for v in set(sess.run(tf.report_uninitialized_variables()))]
#print(uninitialized_variables)
#tf.graph_util.convert_variables_to_constants()
print("Exporting saved model to: %s" % export_dir)
prediction_signature = predict_signature_def(
inputs={'input_1': input_tensor},
outputs={'output': predictions})
signature_def_map = {
tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
prediction_signature
}
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
builder.add_meta_graph_and_variables(
sess,
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
clear_devices=True,
main_op=None, #Suggest tf.tables_initializer()?
strip_default_attrs=False) #Suggest True?
builder.save()
sess.close()
print("Done exporting %s" % format(datetime.now().isoformat()))
output_sum = output_sum / len(output_list)
# apply softmax function (Predicitions)
predictions_plant = tf.nn.softmax(output_sum)
predictions_bin = tf.nn.softmax(bin_outputs)
image_ids = media_list[0]
plant = tf.argmax(predictions_bin, 1)
with tf.Session() as sess:
predictions_plant, predictions_bin, plant = sess.run(
[predictions_plant, predictions_bin, plant])
run_file = 'sabanci_run1.txt'
labels_to_class_names = dataset_utils.read_label_file('/plantclef2015')
labels_filename = os.path.join(final_eval_dir, run_file)
rejected_noplants = 0
rejected_plants = 0
with tf.gfile.Open(labels_filename, 'w') as f:
for i in range(len(image_ids)):
if plant[i]: # 1=plant 0=nonplant
if max(predictions_plant[i, :]) >= 0.4: # threshold
current_image = predictions_plant[i, :]
for j in range(len(current_image)):
f.write('%s;%s;%f\n' %
(image_ids[i], labels_to_class_names[j],
predictions_plant[i, j]))
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
'image/class/text':
tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/object/bbox/xmin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/class/label':
tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'label':
slim.tfexample_decoder.Tensor('image/class/label'),
'label_text':
slim.tfexample_decoder.Tensor('image/class/text'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
else:
labels_to_names = create_readable_names_for_imagenet_labels()
dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(split_name, dataset_dir, data_name='Market1501', file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % (data_name, split_name))
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image_raw_0' : tf.FixedLenFeature([], tf.string),
'image_raw_1' : tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64), # For FixedLenFeature, [] means scalar
'id_0': tf.FixedLenFeature([], tf.int64),
'id_1': tf.FixedLenFeature([], tf.int64),
'cam_0': tf.FixedLenFeature([], tf.int64),
'cam_1': tf.FixedLenFeature([], tf.int64),
'image_format': tf.FixedLenFeature([], tf.string, default_value='jpg'),
'image_height': tf.FixedLenFeature([], tf.int64, default_value=128),
'image_width': tf.FixedLenFeature([], tf.int64, default_value=64),
'real_data': tf.FixedLenFeature([], tf.int64, default_value=1),
# 'attrs_0': tf.FixedLenFeature([27], tf.int64),
# 'attrs_1': tf.FixedLenFeature([27], tf.int64),
# 'attrs_w2v25_0': tf.FixedLenFeature([25*12], tf.float32),
# 'attrs_w2v25_1': tf.FixedLenFeature([25*12], tf.float32),
# 'attrs_w2v50_0': tf.FixedLenFeature([50*12], tf.float32),
# 'attrs_w2v50_1': tf.FixedLenFeature([50*12], tf.float32),
# 'attrs_w2v100_0': tf.FixedLenFeature([100*12], tf.float32),
# 'attrs_w2v100_1': tf.FixedLenFeature([100*12], tf.float32),
# 'attrs_w2v150_0': tf.FixedLenFeature([150*12], tf.float32),
# 'attrs_w2v150_1': tf.FixedLenFeature([150*12], tf.float32),
'pose_peaks_0': tf.FixedLenFeature([16*8*18], tf.float32),
'pose_peaks_1': tf.FixedLenFeature([16*8*18], tf.float32),
'pose_peaks_0_rc': tf.FixedLenFeature([18*2], tf.float32),
'pose_peaks_1_rc': tf.FixedLenFeature([18*2], tf.float32),
# 'pose_dense_r4_0': tf.FixedLenFeature([128*64*18], tf.float32),
# 'pose_dense_r4_1': tf.FixedLenFeature([128*64*18], tf.float32),
'pose_mask_r4_0': tf.FixedLenFeature([128*64*1], tf.int64),
'pose_mask_r4_1': tf.FixedLenFeature([128*64*1], tf.int64),
'shape': tf.FixedLenFeature([1], tf.int64),
'indices_r4_0': tf.VarLenFeature(dtype=tf.int64),
'values_r4_0': tf.VarLenFeature(dtype=tf.float32),
'indices_r4_1': tf.VarLenFeature(dtype=tf.int64),
'values_r4_1': tf.VarLenFeature(dtype=tf.float32),
'pose_subs_0': tf.FixedLenFeature([20], tf.float32),
'pose_subs_1': tf.FixedLenFeature([20], tf.float32),
}
items_to_handlers = {
'image_raw_0': slim.tfexample_decoder.Image(image_key='image_raw_0', format_key='image_format'),
'image_raw_1': slim.tfexample_decoder.Image(image_key='image_raw_1', format_key='image_format'),
'label': slim.tfexample_decoder.Tensor('label'),
'id_0': slim.tfexample_decoder.Tensor('id_0'),
'id_1': slim.tfexample_decoder.Tensor('id_1'),
# 'attrs_0': slim.tfexample_decoder.Tensor('attrs_0',shape=[27]),
# 'attrs_1': slim.tfexample_decoder.Tensor('attrs_1',shape=[27]),
# 'attrs_w2v25_0': slim.tfexample_decoder.Tensor('attrs_w2v25_0',shape=[25*12]),
# 'attrs_w2v25_1': slim.tfexample_decoder.Tensor('attrs_w2v25_1',shape=[25*12]),
# 'attrs_w2v50_0': slim.tfexample_decoder.Tensor('attrs_w2v50_0',shape=[50*12]),
# 'attrs_w2v50_1': slim.tfexample_decoder.Tensor('attrs_w2v50_1',shape=[50*12]),
# 'attrs_w2v100_0': slim.tfexample_decoder.Tensor('attrs_w2v100_0',shape=[100*12]),
# 'attrs_w2v100_1': slim.tfexample_decoder.Tensor('attrs_w2v100_1',shape=[100*12]),
# 'attrs_w2v150_0': slim.tfexample_decoder.Tensor('attrs_w2v150_0',shape=[150*12]),
# 'attrs_w2v150_1': slim.tfexample_decoder.Tensor('attrs_w2v150_1',shape=[150*12]),
'pose_peaks_0': slim.tfexample_decoder.Tensor('pose_peaks_0',shape=[16*8*18]),
'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1',shape=[16*8*18]),
'pose_peaks_0_rc': slim.tfexample_decoder.Tensor('pose_peaks_0_rc',shape=[18*2]),
'pose_peaks_1_rc': slim.tfexample_decoder.Tensor('pose_peaks_1_rc',shape=[18*2]),
# 'pose_dense_r4_0': slim.tfexample_decoder.Tensor('pose_dense_r4_0',shape=[128*64*18]),
# 'pose_dense_r4_1': slim.tfexample_decoder.Tensor('pose_dense_r4_1',shape=[128*64*18]),
'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0',shape=[128*64*1]),
'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1',shape=[128*64*1]),
'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False),
'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False),
'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]),
'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
print('load pn_pairs_num ......')
fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p')
with open(fpath,'r') as f:
pn_pairs_num = pickle.load(f)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=pn_pairs_num,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)
def get_split(dataset_name, split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
num_classes, splits_to_sizes, items_to_descriptions = dataset_utils.read_dataset_config_json(dataset_name, dataset_dir)
if split_name not in splits_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN % (dataset_name, split_name)
file_pattern = os.path.join(dataset_dir, file_pattern)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/name': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/height': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
'image/width': tf.FixedLenFeature(
[], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
'image_name': slim.tfexample_decoder.Tensor('image/name'),
'image_height': slim.tfexample_decoder.Tensor('image/height'),
'image_width': slim.tfexample_decoder.Tensor('image/width'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
def dataset_class_weight(splits_to_sizes, split_name, label_to_class_number):
"""
Returns sorted list of class ratios
"""
num_samples = splits_to_sizes[split_name]
num_samples_per_class = splits_to_sizes[split_name+'_per_class']
sorted_class_weights = list()
sorted_label_name = sorted(label_to_class_number.keys(), key= lambda x: label_to_class_number[x])
for label_name in sorted_label_name:
if label_name in num_samples_per_class:
norm_class_weight = num_samples / (num_samples_per_class[label_name] * num_classes)
sorted_class_weights.append(norm_class_weight)
return sorted_class_weights
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
class_to_label_name, label_to_class_number = dataset_utils.read_label_file(dataset_dir)
num_samples=splits_to_sizes[split_name]
# if split_name+'_per_class' in splits_to_sizes:
sorted_class_weights = dataset_class_weight(splits_to_sizes, split_name, label_to_class_number)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=num_samples,
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=label_to_class_number,
sorted_class_weights=sorted_class_weights
)
def final_evaluation(label_dir, dataset_dir, filename="predictions.txt"):
number_images = 100 #8000
correct = 0 # TODO REMOVE
#[ 2.49181414 -0.33259141 -1.27059281 -1.52844727 2.57813239 -1.20948148
checkpoint_paths = [
"mydata/resnet_finetuned_plantclef2015_1/model.ckpt-145552",
"mydata/resnet_finetuned_plantclef2015_1/model.ckpt-145552",
"mydata/resnet_finetuned_plantclef2015_1/model.ckpt-145552"
]
output_list = []
labels_list = []
for index in range(len(checkpoint_paths)):
with tf.Graph().as_default():
dataset = dataVisualisation.get_split('train_set3', dataset_dir)
data_provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=8000,
common_queue_min=0)
image_raw, label = data_provider.get(['image', 'label'])
"""
imaget = my_resnet_preprocessing.preprocess_image(image_raw, 224, 224, is_training=False)
image,augmented_image1,augmented_image2, augmented_image3, augmented_image4,augmented_image5, labels = tf.train.batch([imaget,imaget,imaget,imaget,imaget,imaget, label], batch_size=1,
num_threads=1,
capacity=12 * 1)
"""
image, augmented_image1, augmented_image2, augmented_image3, augmented_image4, augmented_image5 = my_resnet_preprocessing.preprocess_for_final_run(
image_raw, 224, 224)
image, augmented_image1, augmented_image2, augmented_image3, augmented_image4, augmented_image5, labels = tf.train.batch(
[
image, augmented_image1, augmented_image2,
augmented_image3, augmented_image4, augmented_image5, label
],
batch_size=1,
num_threads=1,
capacity=2 * 1)
logits1 = resNetClassifier.my_cnn(image,
is_training=False,
dropout_rate=1)
logits2 = resNetClassifier.my_cnn(augmented_image1,
is_training=False,
dropout_rate=1)
logits3 = resNetClassifier.my_cnn(augmented_image2,
is_training=False,
dropout_rate=1)
logits4 = resNetClassifier.my_cnn(augmented_image3,
is_training=False,
dropout_rate=1)
logits5 = resNetClassifier.my_cnn(augmented_image4,
is_training=False,
dropout_rate=1)
logits6 = resNetClassifier.my_cnn(augmented_image5,
is_training=False,
dropout_rate=1)
total_output = np.empty([number_images * 1, dataset.num_classes])
total_labels = np.empty([number_images * 1], dtype=np.int32)
offset = 0
#init_fn = slim.assign_from_checkpoint_fn(checkpoint_paths[index], slim.get_model_variables())
with tf.Session() as sess:
coord = tf.train.Coordinator()
saver = tf.train.Saver()
saver.restore(sess, checkpoint_paths[index])
#init_fn(sess)
merged = tf.summary.merge_all()
test_writer = tf.summary.FileWriter(
'/home/lolek/FlowerIdentificationM/models/slim/mydata/test/'
+ '/train')
visualize_writer = tf.summary.FileWriter(
'/home/lolek/FlowerIdentificationM/models/slim/mydata/test/'
+ '/visualize')
#for v in tf.trainable_variables():
# print(v.name)
#print_tensors_in_checkpoint_file(checkpoint_paths[index], tensor_name =None, all_tensors=False)
#var = [v for v in tf.trainable_variables() if v.name == "my_fc_1/weights:0"]
#print("my_fc_1/weights \n", sess.run(var))
#print_tensors_in_checkpoint_file(checkpoint_paths[index], tensor_name ='my_fc_1/weights', all_tensors=False)
#var2 = [v for v in tf.trainable_variables() if v.name == "resnet_v2_50/block1/unit_1/bottleneck_v2/conv3/weights:0"]
#print("resnet_v2_50/block1/unit_1/bottleneck_v2/conv3/weights \n", sess.run(var2))
#print_tensors_in_checkpoint_file(checkpoint_paths[index], tensor_name ='resnet_v2_50/block1/unit_1/bottleneck_v2/conv3/weights', all_tensors=False)
# Visualize Kernels
tf.get_variable_scope().reuse_variables()
weights = tf.get_variable("resnet_v2_50/conv1/weights")
grid = kernel_visualization.put_kernels_on_grid(weights)
sum1 = tf.summary.image('conv1/kernels', grid, max_outputs=1)
_, summary1 = sess.run([merged, sum1])
visualize_writer.add_summary(summary1, 2)
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(number_images):
print('step: %d/%d' % (i + 1, number_images))
"""
image_t1, image_t2 = sess.run([image, augmented_image1])
plt.figure()
plt.imshow(image_t1[0, :, :, :].astype(np.uint8))
plt.title('title')
plt.axis('off')
plt.show()
plt.figure()
plt.imshow(image_t2[0, :, :, :].astype(np.uint8))
plt.title('title')
plt.axis('off')
plt.show()
"""
sum1 = tf.summary.image('final_eval_whole_image1', image)
sum2 = tf.summary.image('final_eval_image_center1',
augmented_image1)
sum3 = tf.summary.image('final_eval_top_left1',
augmented_image2)
sum4 = tf.summary.image('final_eval_bottom_left1',
augmented_image3)
sum5 = tf.summary.image('final_eval_top_right1',
augmented_image4)
sum6 = tf.summary.image('final_eval_bottom_right1',
augmented_image5)
_, summary1, summary2, summary3, summary4, summary5, summary6 = sess.run(
[merged, sum1, sum2, sum3, sum4, sum5, sum6])
test_writer.add_summary(summary1, 1)
test_writer.add_summary(summary2, 1)
test_writer.add_summary(summary3, 1)
test_writer.add_summary(summary4, 1)
test_writer.add_summary(summary5, 1)
test_writer.add_summary(summary6, 1)
logit1, logit2, logit3, logit4, logit5, logit6, media_id = sess.run(
[
logits1, logits2, logits3, logits4, logits5,
logits6, labels
])
print(media_id, " ", np.argmax(logit1[0]), " ",
np.argmax(logit2[0]), " ", np.argmax(logit3[0]), " ",
np.argmax(logit4[0]), " ", np.argmax(logit5[0]), " ",
np.argmax(logit6[0]))
#print(np.amax(logit1[0]), " ",np.amax(logit2[0]), " ",np.amax(logit3[0]), " ",np.amax(logit4[0]), " ",np.amax(logit5[0]), " ",np.amax(logit6[0]))
#print(len(logit1[0]))
media_id = media_id[0]
logits = tuple(
max(i, j) for i, j in zip(logit1[0], logit2[0]))
logits = tuple(
max(i, j) for i, j in zip(logits, logit3[0]))
logits = tuple(
max(i, j) for i, j in zip(logits, logit4[0]))
logits = tuple(
max(i, j) for i, j in zip(logits, logit5[0]))
logits = tuple(
max(i, j) for i, j in zip(logits, logit6[0]))
"""
logits = tuple(i + j for i, j in zip(logit1[0], logit2[0]))
logits = tuple(i + j for i, j in zip(logits, logit3[0]))
logits = tuple(i + j for i, j in zip(logits, logit4[0]))
logits = tuple(i + j for i, j in zip(logits, logit5[0]))
logits = tuple(i + j for i, j in zip(logits, logit6[0]))
logits = [x / 6 for x in logits]
"""
logits = numpy_softmax(logits)
#print(np.argmax(logits))
#print(len(logits))
#first_prediction = np.argmax(logits)
#print(first_prediction," ", media_id)
#print(np.argmax(logits))
#logits = logit1
#print(np.amax(logits))
#pred = [np.argmax(logit1[0]),np.argmax(logit2[0]),np.argmax(logit3[0]), np.argmax(logit4[0]), np.argmax(logit5[0]), np.argmax(logit6[0] )]
if media_id == np.argmax(logits): # TODO REMOVE
correct = correct + 1 # TODO REMOVE
#print(correct) # TODO REMOVE
#print(media_id, " ", np.argmax(logit1)," ",np.argmax(logit2)," ",np.argmax(logit3), " ",np.argmax(logit4), " ",np.argmax(logit5), " ",np.argmax(logit6) )
total_output[offset:offset + 1] = logits
total_labels[offset:offset + 1] = media_id
offset += 1
coord.request_stop()
coord.join(threads)
output_list.append(total_output)
labels_list.append(total_labels)
print(correct) # TODO REMOVE
for i in range(len(output_list)):
logits = tf.cast(tf.constant(output_list[i]), dtype=tf.float32)
predictions = tf.nn.softmax(logits)
labels = tf.constant(labels_list[i])
top1_op = tf.nn.in_top_k(predictions, labels, 1)
top5_op = tf.nn.in_top_k(predictions, labels, 5)
with tf.Session() as sess:
top1, top5 = sess.run([top1_op, top5_op])
print('Top 1 accuracy: %f' % (np.sum(top1) / float(number_images)))
print('Top 5 accuracy: %f' % (np.sum(top5) / float(number_images)))
for i in range(number_images):
image_id = labels_list[0][i]
prediction1 = np.amax(output_list[0][i])
prediction2 = np.amax(output_list[1][i])
prediction3 = np.amax(output_list[2][i])
print(prediction1, " ", prediction2, " ", prediction3)
# Find best class with highest prediction (softmax) score
if prediction1 > prediction2:
if prediction1 > prediction3:
prediction = np.argmax(output_list[0][i])
probability = prediction1
else:
prediction = np.argmax(output_list[2][i])
probability = prediction3
else:
if prediction2 > prediction3:
prediction = np.argmax(output_list[1][i])
probability = prediction2
else:
prediction = np.argmax(output_list[2][i])
probability = prediction3
class_id = dataset_utils.read_label_file(label_dir)[prediction]
image_id = dataset_utils.read_label_file(label_dir)[
image_id] # TODO REMOVE!!!
print('<%s;%s;%f>\n' % (image_id, class_id, probability))
# Save the predictions
labels_filename = os.path.join(label_dir, filename)
with tf.gfile.Open(labels_filename, 'a') as f:
f.write('<%s;%s;%f>\n' %
(image_id, class_id,
probability)) # <ImageId;ClassId;Probability>
def get_split(split_name, dataset_dir, file_pattern, reader,
split_to_sizes, items_to_descriptions, num_classes):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in split_to_sizes:
raise ValueError('split name %s was not recognized.' % split_name)
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'shape': slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox': slim.tfexample_decoder.BoundingBox(
['xmin', 'ymin', 'xmax', 'ymax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
# else:
# labels_to_names = create_readable_names_for_imagenet_labels()
# dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(
data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_to_sizes[split_name],
items_to_descriptions=items_to_descriptions,
num_classes=num_classes,
labels_to_names=labels_to_names)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
#######################
# Config model_deploy #
#######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
######################
# Select the network #
######################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=True)
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
with tf.device(deploy_config.inputs_device()):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
train_image_size = FLAGS.train_image_size or network_fn.default_image_size
print(train_image_size)
image = image_preprocessing_fn(image, train_image_size,
train_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
labels = slim.one_hot_encoding(
labels, dataset.num_classes - FLAGS.labels_offset)
batch_queue = slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * deploy_config.num_clones)
images, labels = batch_queue.dequeue()
print(images, labels)
logits, end_points = network_fn(images)
labels_to_class_names = dataset_utils.read_label_file(
FLAGS.dataset_dir, filename='labels.txt')
print(labels_to_class_names)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
images_np, labels_np = sess.run([images, labels])
print(images_np.shape, labels_np.shape)
for i in range(10):
image_np, label_np = sess.run([images, labels])
# plt.imshow(image_np[0,:,:,:])
# plt.title('label name:'+str(labels_to_class_names[np.argmax(label_np[0])]))
# plt.show()
cv2.imshow(
'label name:',
cv2.cvtColor(image_np[0, :, :, :], cv2.COLOR_RGB2BGR))
print(labels_to_class_names[np.argmax(label_np[0])])
cv2.waitKey(0)
coord.request_stop()
coord.join(threads)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading flowers.
Args:
split_name: A train/validation split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/validation split.
"""
# 字典匹配查找key，根据train和validation关键字划分tfrecord文件
if split_name not in SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern: #tfrecord文件名可以指定，如果没有，则默认是'satellite_%s_*.tfrecord'
file_pattern = _FILE_PATTERN
# #'satellite_train_*.tfrecord' 'satellite_validation_*.tfrecord'
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
# 定义了图片的默认格式 。 收集的卫星图片的格式为 jpg 图片，因此修改为 jpg 。
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)