def __init__(self,
tensor_key,
label_map_proto_file,
shape_keys=None,
shape=None,
default_value=''):
"""Initializes the LookupTensor handler.
Simply calls a vocabulary (most often, a label mapping) lookup.
Args:
tensor_key: the name of the `TFExample` feature to read the tensor from.
label_map_proto_file: File path to a text format LabelMapProto message
mapping class text to id.
shape_keys: Optional name or list of names of the TF-Example feature in
which the tensor shape is stored. If a list, then each corresponds to
one dimension of the shape.
shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is
reshaped accordingly.
default_value: The value used when the `tensor_key` is not found in a
particular `TFExample`.
Raises:
ValueError: if both `shape_keys` and `shape` are specified.
"""
name_to_id = label_map_util.get_label_map_dict(label_map_proto_file,
use_display_name=False)
# We use a default_value of -1, but we expect all labels to be contained
# in the label map.
try:
# Dynamically try to load the tf v2 lookup, falling back to contrib
lookup = tf.compat.v2.lookup
hash_table_class = tf.compat.v2.lookup.StaticHashTable
except AttributeError:
lookup = contrib_lookup
hash_table_class = contrib_lookup.HashTable
name_to_id_table = hash_table_class(
initializer=lookup.KeyValueTensorInitializer(
keys=tf.constant(list(name_to_id.keys())),
values=tf.constant(list(name_to_id.values()), dtype=tf.int64)),
default_value=-1)
display_name_to_id = label_map_util.get_label_map_dict(
label_map_proto_file, use_display_name=True)
# We use a default_value of -1, but we expect all labels to be contained
# in the label map.
display_name_to_id_table = hash_table_class(
initializer=lookup.KeyValueTensorInitializer(
keys=tf.constant(list(display_name_to_id.keys())),
values=tf.constant(list(display_name_to_id.values()),
dtype=tf.int64)),
default_value=-1)
self._name_to_id_table = name_to_id_table
self._display_name_to_id_table = display_name_to_id_table
super(_ClassTensorHandler, self).__init__(tensor_key, shape_keys,
shape, default_value)
def main(_):
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
data_dir = FLAGS.data_dir
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
logging.info('Reading dataset.')
examples_path = '/home/wangshiyao/Documents/data/imagenet/gen_list/combine_train_list.txt'
annotations_dir = '/home/wangshiyao/Documents/data/imagenet/'
examples_list = dataset_util.read_examples_list(examples_path)
num_label = [0] * 31
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
if int(idx) % 100 == 0:
print(idx, num_label)
path = os.path.join(annotations_dir, example)
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, example, FLAGS.data_dir,
label_map_dict, FLAGS.set, num_label)
#writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
logging.info('Reading from Pet dataset.')
image_dir = os.path.join('annotations', 'images')
annotations_dir = os.path.join('annotations', 'labels')
examples_path = os.path.join(annotations_dir, 'trainval.txt')
examples_list = dataset_util.read_examples_list(examples_path)
label_map_dict = label_map_util.get_label_map_dict(
os.path.join(annotations_dir, 'label_map.pbtxt'))
# Test images are not included in the downloaded data set, so we shall perform
# our own split.
random.seed(42)
random.shuffle(examples_list)
num_examples = len(examples_list)
num_train = int(0.7 * num_examples)
train_examples = examples_list[:num_train]
val_examples = examples_list[num_train:]
logging.info('%d training and %d validation examples.',
len(train_examples), len(val_examples))
train_output_path = 'train.record'
val_output_path = 'val.record'
create_tf_record(train_output_path, label_map_dict, annotations_dir,
image_dir, train_examples)
create_tf_record(val_output_path, label_map_dict, annotations_dir,
image_dir, val_examples)
if not os.path.exists(os.path.join(MODEL_DIR, 'model.ckpt.index')):
download_base_model()
def main(_):
data_dir = FLAGS.data_dir
annotations_dir = os.path.join(data_dir, 'labels')
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
img_path = os.path.join(data_dir, 'img')
examples_list = glob.glob(img_path + '/*.jpg')
# examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
example = example.split('/')[-1].split('.')[0]
path = os.path.join(annotations_dir, example + '.xml')
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(
data=data,
dataset_directory=os.path.join(data_dir),
label_map_dict=label_map_dict,
ignore_difficult_instances=FLAGS.ignore_difficult_instances,
image_subdirectory='img')
writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
"""
This function contains all actual work that will be done
by tf.app.run() call in '__main__'
"""
#print('\tFLAGS.label_map_path:', FLAGS.label_map_path)
#print('\tFLAGS.data_dir: "%s"' % FLAGS.data_dir)
#print('\tFLAGS.output_path: "%s"' % FLAGS.output_path)
if not os.path.isdir(FLAGS.data_dir):
print("\tthe specified data_dir does not exist")
return
if not FLAGS.output_path:
#print("\toutput_path is not specified")
if FLAGS.data_dir.endswith('/'):
FLAGS.data_dir = FLAGS.data_dir[:-1]
FLAGS.output_path = os.path.split(FLAGS.data_dir)[1] + '.tfrecord'
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
amls_list = []
for root, dirs, files in os.walk(FLAGS.data_dir):
for file in files:
if file.endswith(".aml"):
aml_path = os.path.join(root, file)
amls_list.append(aml_path)
create_tf_record(FLAGS.data_dir, amls_list, label_map_dict,
FLAGS.output_path)
def main(_):
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
if FLAGS.year not in YEARS:
raise ValueError('year must be in : {}'.format(YEARS))
data_dir = FLAGS.data_dir
years = ['VOC2007', 'VOC2012']
if FLAGS.year != 'merged':
years = [FLAGS.year]
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
# returns a dictionary of label names to id. {"dog": 1}
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
for year in years:
logging.info('Reading from PASCAL %s dataset.', year)
examples_path = os.path.join(data_dir, year, 'ImageSets', 'Main',
FLAGS.set + '.txt')
annotations_dir = os.path.join(data_dir, year, FLAGS.annotations_dir)
examples_list = dataset_utils.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_utils.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,
FLAGS.ignore_difficult_instances)
writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
#writer = tf.python_io.TFRecordWriter(FLAGS.output_path) # .record格式的文件输出位置（按照教程中的数据进行就可以了，不要随意更改）
writer = tf.python_io.TFRecordWriter(
"D:\\program file\\python program1\\untitled\\objection detection\\payment_detection\\data\\test.record"
) # .record格式的文件输出位置
#label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path) # 数字与标签之间的转化文件存储路径
label_map_dict = label_map_util.get_label_map_dict(
'D:\\program file\\python program1\\untitled\\objection detection\\payment_detection\\training\\payment_label_map.pbtxt'
) # 数字与标签之间的转化文件存储路径
#path = os.path.join(os.getcwd(), 'images') # 图片的存储位置
path = os.path.join(
'D:\\program file\\python program1\\untitled\\objection detection\\payment_detection',
'images')
#print(os.getcwd())
#examples = pd.read_csv(FLAGS.csv_input) # csv文FLAGS.label_map_path件的输出位置
#examples = pd.read_csv(os.getcwd())
examples = pd.read_csv(
'D:\\program file\\python program1\\untitled\\objection detection\\payment_detection\\data\\payment_labels.csv'
)
grouped = split(examples, 'filename')
for group in grouped:
tf_example = create_tf_example(group, label_map_dict, path)
writer.write(tf_example.SerializeToString())
writer.close()
#output_path = os.path.join(os.getcwd(), FLAGS.output_path)
output_path = os.path.join(
"D:\\program file\\python program1\\untitled\\objection detection\\payment_detection\\data"
)
print('Successfully created the TFRecords: {}'.format(output_path))
def main(_):
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
data_dir = FLAGS.data_dir
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
print ("output_path is :")
print(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
logging.info('Reading from VID dataset.')
examples_path = os.path.join(data_dir,'ImageSets', 'VID','list'
,FLAGS.set + '_list.txt')
annotations_dir = os.path.join(data_dir, FLAGS.annotations_dir, 'VID', FLAGS.set)
examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example)
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,
FLAGS.set)
writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
data_dir = FLAGS.data_dir
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
logging.info('Reading from Pet dataset.')
image_dir = os.path.join(data_dir, 'images')
annotations_dir = os.path.join(data_dir, 'annotations')
examples_path = os.path.join(annotations_dir, 'trainval.txt')
examples_list = dataset_util.read_examples_list(examples_path)
# Test images are not included in the downloaded data set, so we shall perform
# our own split.
random.seed(42)
random.shuffle(examples_list)
num_examples = len(examples_list)
num_train = int(0.8 * num_examples)
train_examples = examples_list[:num_train]
val_examples = examples_list[num_train:]
logging.info('%d training and %d validation examples.',
len(train_examples), len(val_examples))
train_output_path = os.path.join(FLAGS.output_dir, 'hand_train.record')
val_output_path = os.path.join(FLAGS.output_dir, 'hand_val.record')
create_tf_record(train_output_path, label_map_dict, annotations_dir,
image_dir, train_examples)
create_tf_record(val_output_path, label_map_dict, annotations_dir,
image_dir, val_examples)
def main(_):
data_dir = FLAGS.data_dir
print('run the here')
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
logging.info('Reading from Pet dataset.')
image_dir = data_dir
# annotations_dir = os.path.join(data_dir, 'annotations')
examples_path = os.path.join(data_dir, 'xml_random.txt')
examples_list = dataset_util.read_examples_list(examples_path)
#
# # Test images are not included in the downloaded data set, so we shall perform
# # our own split.
# random.seed(42)
# random.shuffle(examples_list)
num_examples = len(examples_list)
num_train = int(1 * num_examples)
train_examples = examples_list[:num_train]
# val_examples = examples_list[num_train:]
# logging.info('%d training and %d validation examples.',
# len(train_examples), len(val_examples))
#
train_output_path = os.path.join(FLAGS.output_dir, 'object_train.record')
print('train_output_path: {}'.format(train_output_path))
print('image_dir: {}'.format(image_dir))
# val_output_path = os.path.join(FLAGS.output_dir, 'pet_val.record')
len_train_examples = 'train_examples number: {}'.format(
len(train_examples))
print(len_train_examples)
create_tf_record(train_output_path, label_map_dict, image_dir,
train_examples)
def main(_):
# if FLAGS.set not in SETS:
# raise ValueError('set must be in : {}'.format(SETS))
# if FLAGS.year not in YEARS:
# raise ValueError('year must be in : {}'.format(YEARS))
data_dir = 'E:/computerscience/my projects/humanoid/VOCdevkit'
years = ['VOC2012']
# if FLAGS.year != 'merged':
# years = [FLAGS.year]
writer = tf.python_io.TFRecordWriter('pascal_train.record')
label_map_dict = label_map_util.get_label_map_dict(
'data/pascal_label_map.pbtxt')
for year in years:
logging.info('Reading from PASCAL %s dataset.', year)
examples_path = os.path.join(data_dir, year, 'ImageSets', 'Main',
'aeroplane_' + 'train' + '.txt')
annotations_dir = os.path.join(data_dir, year, 'Annotations')
examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, data_dir, label_map_dict)
print(tf_example)
writer.write(tf_example.SerializeToString())
break
writer.close()
def main(_):
label_map = label_map_util.get_label_map_dict(input_label_map)
all_box_annotations = pd.read_csv(input_box_annotations_csv)
all_images = tf.gfile.Glob(os.path.join(input_images_directory, '*.jpg'))
all_image_ids = [
os.path.splitext(os.path.basename(v))[0] for v in all_images
]
all_image_ids = pd.DataFrame({'ImageID': all_image_ids})
all_annotations = pd.concat([all_box_annotations, all_image_ids])
count = 0
class_count_list = []
for i in range(500):
class_count_list.append(0)
with open(ouput_file, "w") as f_out:
for counter, image_data in enumerate(
all_annotations.groupby('ImageID')):
image_id, image_annotations = image_data
filtered_data_frame = image_annotations[
image_annotations.LabelName.isin(label_map)]
if len(filtered_data_frame) == 0:
continue
filtered_data_frame_boxes = filtered_data_frame[
~filtered_data_frame.YMin.isnull()]
class_id_np = filtered_data_frame_boxes.LabelName.map(
lambda x: label_map[x]).as_matrix()
is_ok = False
for item in class_id_np:
class_count_list[item - 1] = class_count_list[item - 1] + 1
if class_count_list[item - 1] < 500:
is_ok = True
break
if not is_ok:
continue
image_path = os.path.join(input_images_directory,
image_id + '.jpg')
#img_size=cv2.imread(image_path).shape
ymin_np = filtered_data_frame_boxes.YMin.as_matrix()
xmin_np = filtered_data_frame_boxes.XMin.as_matrix()
ymax_np = filtered_data_frame_boxes.YMax.as_matrix()
xmax_np = filtered_data_frame_boxes.XMax.as_matrix()
if len(ymin_np) < 1:
continue
count = count + 1
boxes_str = ''
for i in range(len(ymin_np)):
box_str = str(int(xmin_np[i])) + ',' + str(int(
ymin_np[i])) + ',' + str(int(xmax_np[i])) + ',' + str(
int(ymax_np[i])) + ',' + str(int(class_id_np[i] - 1))
boxes_str = boxes_str + ' ' + box_str
if box_str == '':
continue
line_str = image_path + boxes_str + '\n'
f_out.write(line_str)
if count % 1000 == 0:
print(count)
def get_categories_from_map_file(label_map_path):
label_map = get_label_map_dict(label_map_path)
categories = []
for label_key in label_map:
categories.append({
'name': label_key,
'id': label_map[label_key]
})
return categories
def image2tfrecord(image_dir, tfrecord_path, label_map_path ):
with tf.gfile.GFile("../data/annotation_test.xml", 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = recursive_parse_xml_to_dict(xml)['annotation']
label_map_dict = get_label_map_dict(label_map_path)
example = dict_to_tf_example(data=data, dataset_directory=image_dir,
label_map_dict=label_map_dict,
image_subdirectory="")
with tf.python_io.TFRecordWriter(tfrecord_path) as tf_writer:
tf_writer.write(example.SerializeToString())
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
required_flags = [
'input_box_annotations_csv', 'input_images_directory', 'input_label_map',
'output_tf_record_path_prefix'
]
for flag_name in required_flags:
if not getattr(FLAGS, flag_name):
raise ValueError('Flag --{} is required'.format(flag_name))
label_map = label_map_util.get_label_map_dict(FLAGS.input_label_map)
all_box_annotations = pd.read_csv(FLAGS.input_box_annotations_csv)
if FLAGS.input_image_label_annotations_csv:
all_label_annotations = pd.read_csv(FLAGS.input_image_label_annotations_csv)
all_label_annotations.rename(
columns={'Confidence': 'ConfidenceImageLabel'}, inplace=True)
else:
all_label_annotations = None
all_images = tf.gfile.Glob(
os.path.join(FLAGS.input_images_directory, '*.jpg'))
all_image_ids = [os.path.splitext(os.path.basename(v))[0] for v in all_images]
all_image_ids = pd.DataFrame({'ImageID': all_image_ids})
all_annotations = pd.concat(
[all_box_annotations, all_image_ids, all_label_annotations])
tf.logging.log(tf.logging.INFO, 'Found %d images...', len(all_image_ids))
with contextlib2.ExitStack() as tf_record_close_stack:
output_tfrecords = open_sharded_output_tfrecords(
tf_record_close_stack, FLAGS.output_tf_record_path_prefix,
FLAGS.num_shards)
for counter, image_data in enumerate(all_annotations.groupby('ImageID')):
tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 1000,
counter)
image_id, image_annotations = image_data
# In OID image file names are formed by appending ".jpg" to the image ID.
image_path = os.path.join(FLAGS.input_images_directory, image_id + '.jpg')
if not os.path.exists(image_path):
continue
with tf.gfile.Open(image_path, 'rb') as image_file:
encoded_image = image_file.read()
tf_example = oid_tfrecord_creation.tf_example_from_annotations_data_frame(
image_annotations, label_map, encoded_image)
if tf_example:
shard_idx = int(image_id, 16) % FLAGS.num_shards
output_tfrecords[shard_idx].write(tf_example.SerializeToString())
def load_model():
# figure out how many classes
label_map_dict = label_map_util.get_label_map_dict(PASCAL_LABEL_MAP_FILE)
num_classes = len(label_map_dict)
logger.info('num_of_classes in pascal_label_map: {}'.format(num_classes))
# Load label map
label_map = label_map_util.load_labelmap(PASCAL_LABEL_MAP_FILE)
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes=num_classes, use_display_name=True)
category_index = label_map_util.create_category_index(categories)
# Load a frozen Tensorflow model into memory
model_path = Model.get_model_path(MODEL_NAME)
logger.info('getting model {} from {}'.format(MODEL_NAME, model_path))
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(model_path, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
# Get handles to input and output tensors
ops = tf.get_default_graph().get_operations()
all_tensor_names = {
output.name
for op in ops for output in op.outputs
}
tensor_dict = {}
for key in [
'num_detections', 'detection_boxes', 'detection_scores',
'detection_classes'
]:
tensor_name = key + ':0'
if tensor_name in all_tensor_names:
tensor_dict[key] = tf.get_default_graph(
).get_tensor_by_name(tensor_name)
image_tensor = tf.get_default_graph().get_tensor_by_name(
'image_tensor:0')
sess = tf.Session(graph=detection_graph)
return {
'session': sess,
'image_tensor': image_tensor,
'tensor_dict': tensor_dict,
'category_index': category_index
}
def serialize_labels(label_map_path, images_dir):
label_map_dict = {}
if os.path.exists(label_map_path):
label_map_dict = label_map_util.get_label_map_dict(label_map_path)
images = os.listdir(images_dir)
labels = set()
for image in images:
label = image.split('_')[0]
labels.add(label)
labels = sorted(list(labels))
for label_name in labels:
if label_name not in label_map_dict:
label_id = len(label_map_dict) + 1
label_map_dict[label_name] = label_id
serialize_to_pbtxt(label_map_path, label_map_dict)
def get_data(train_data_dir, label_map_path):
label_map = get_label_map_dict(
label_map_path) # lable_map[name:id] id begin with 1
image_path_list = []
image_label_list = []
train_data_num = 0
for cur_folder, sub_folders, sub_files in os.walk(train_data_dir):
for file in sub_files:
if file.endswith('jpg'):
train_data_num += 1
image_path_list.append(os.path.join(cur_folder, file))
image_label_list.append(
label_map[os.path.split(cur_folder)[-1]])
print('train_image_num:', train_data_num)
data_tuple = (image_path_list, image_label_list)
return data_tuple
def main(_):
image_dir = FLAGS.image_dir
annotation_path = FLAGS.annotation_file
annotation_fid = open(annotation_path)
output_dir = os.path.dirname (FLAGS.output)
output_name = os.path.basename(FLAGS.output)
output_name, output_ext = os.path.splitext(output_name)
with open(annotation_path) as annotation_fid:
annotations = yaml.safe_load(annotation_fid)
annotation_size = len(annotations)
# Shuffle
annotation_keys = annotations.keys()
np.random.shuffle(annotation_keys)
# Train:Validation
training_size = int(annotation_size * FLAGS.train_val_ratio)
training_data = annotation_keys[:training_size]
validataion_data = annotation_keys[training_size:]
use_display_name = True if FLAGS.category_name == 'display_name' else False
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_file, use_display_name=use_display_name)
dataset = [
('_train', training_data),
('_val', validataion_data),
]
for suffix, data_list in dataset:
if not data_list: # Data is empty
continue
output = os.path.join(output_dir, output_name + suffix + output_ext)
with tf.python_io.TFRecordWriter(output) as writer:
for idx, key in enumerate(data_list):
# Print Status
if idx % 100 == 0:
print('On image %d of %d'% (idx, training_size))
tf_example = dict_to_tf_example(annotations[key], image_dir, label_map_dict)
writer.write(tf_example.SerializeToString())
print("Success. ")
def main(_):
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
# TODO(user): Write code to read in your dataset to examples variable
frames = parse_pascal_voc_groundtruth(FLAGS.annotation_dir)
if FLAGS.imageset_txt != '':
train_set = np.genfromtxt(FLAGS.imageset_txt, dtype=np.int)
else:
train_set = None
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
for frame in frames.values():
if train_set is None or int(frame['frame_id']) in train_set:
tf_example = create_tf_example(frame, label_map_dict)
writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
print(FLAGS.data_dir)
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
#if FLAGS.year not in YEARS:
# raise ValueError('year must be in : {}'.format(YEARS))
data_dir = FLAGS.data_dir
#years = ['VOC2007', 'VOC2012']
#if FLAGS.year != 'merged':
years = [FLAGS.year]
ACTIONSET = ['tfrecord', 'imageset']
if FLAGS.action not in ACTIONSET:
raise ValueError('action must be in : {}'.format(ACTIONSET))
if FLAGS.action == 'tfrecord':
pass
elif FLAGS.action == 'imageset':
gen_image_set(FLAGS.data_dir, FLAGS.year, FLAGS.imageset)
return
writer = tf.io.TFRecordWriter(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
for year in years:
logging.info('Reading from PASCAL %s dataset.', year)
examples_path = os.path.join(data_dir, year, 'ImageSets', 'Main',
FLAGS.imageset + '_' + FLAGS.set + '.txt')
annotations_dir = os.path.join(data_dir, year, FLAGS.annotations_dir)
examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
with tf.io.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str.encode('utf-8'))
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir,
label_map_dict,
FLAGS.ignore_difficult_instances)
writer.write(tf_example.SerializeToString())
writer.close()
def load_models(self, model_list):
#Todo: ensure existance
self.models = model_list
#determine categories
for m in self.models:
#get each models label dict
m['categories'] = label_map_util.get_label_map_dict( m['paths']['labels'], use_display_name=m['use_display_name'] )
#go through models, for each unique category list all models that can identify, use first as evaluation model
for m in self.models:
for key in m['categories']:
if key in self.categories:
self.categories[key]['models'].append(m['name'])
else:
self.categories[key] = {'models' : [m['name']], 'evaluation_model' : m['name']}
def main(_):
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
dataset_path = FLAGS.dataset_path
label_d = label_map_util.get_label_map_dict(FLAGS.label_map_path)
# Annotation file paths
bbox_file = os.path.join(dataset_path, 'Anno/list_bbox.txt')
cat_cloth_file = os.path.join(dataset_path, 'Anno/list_category_cloth.txt')
cat_img_file = os.path.join(dataset_path, 'Anno/list_category_img.txt')
stage_file = os.path.join(dataset_path, 'Eval/list_eval_partition.txt')
# Read annotation files
bbox_df = pd.read_csv(bbox_file, sep='\s+', skiprows=1)
cat_cloth_df = pd.read_csv(cat_cloth_file, sep='\s+', skiprows=1)
cat_img_df = pd.read_csv(cat_img_file, sep='\s+', skiprows=1)
stage_df = pd.read_csv(stage_file, sep='\s+', skiprows=1)
# Merge dfs
cat_cloth_df["category_label"] = cat_cloth_df.index + 1
cat_df = cat_img_df.merge(cat_cloth_df, how='left', on='category_label')
examples_df = cat_df.merge(bbox_df, how='left', on='image_name')
examples_df = examples_df.merge(stage_df, how='left', on='image_name')
# Select train, val or test images
examples_df = examples_df[examples_df["evaluation_status"] ==
FLAGS.evaluation_status]
# Shuffle
examples_df = examples_df.sample(frac=1).reset_index(drop=True)
none_counter = 0
for irow, example in examples_df.iterrows():
tf_example = create_tf_example(example,
path_root=os.path.join(
dataset_path, 'Img/'),
LABEL_DICT=label_d)
if tf_example is not None:
writer.write(tf_example.SerializeToString())
else:
none_counter += 1
print("Skipped %d images." % none_counter)
writer.close()
def analyze_and_return(url):
IMAGE_SIZE = (12, 8)
category_index_2 = label_map_util.get_label_map_dict(PATH_TO_LABELS)
fixed_category_index = {v: k for k,v in category_index_2.items()}
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
# Definite input and output Tensors for detection_graph
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Each box represents a part of the image where a particular object was detected.
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
# Each score represent how level of confidence for each of the objects.
# Score is shown on the result image, together with the class label.
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
# Don't want a for-loop in final version
response = requests.get(url)
image = Image.open(BytesIO(response.content)) #change later so raw image file, rather than file path
# the array based representation of the image will be used later in order to prepare the
# result image with boxes and labels on it.
image_np = load_image_into_numpy_array(image)
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(boxes, scores, classes, num) = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
# Visualization of the results of a detection.
# vis_util.visualize_boxes_and_labels_on_image_array(
# image_np,
# np.squeeze(boxes),
# np.squeeze(classes).astype(np.int32),
# np.squeeze(scores),
# category_index,
# use_normalized_coordinates=True,
# line_thickness=8)
(im_width, im_height) = image.size
#vis_util.save_image_array_as_png(image_np,sys.argv[1])
#plt.figure(figsize=IMAGE_SIZE)
#plt.imsave(imgout,image_np)
#print(np.squeeze(boxes))
#print(boxes)
#print(np.squeeze(classes).astype(np.int32))
return prepare_json(fixed_category_index,im_width,im_height,np.squeeze(boxes),np.squeeze(scores),np.squeeze(classes).astype(np.int32))
def test_get_label_map_dict(self):
label_map_string = """
item {
id:2
name:'cat'
}
item {
id:1
name:'dog'
}
"""
label_map_path = os.path.join(self.get_temp_dir(), 'label_map.pbtxt')
with tf.gfile.Open(label_map_path, 'wb') as f:
f.write(label_map_string)
label_map_dict = label_map_util.get_label_map_dict(label_map_path)
self.assertEqual(label_map_dict['dog'], 1)
self.assertEqual(label_map_dict['cat'], 2)
def detect_images(func_detect_image,
func_read_img,
path_to_input_images,
path_to_output_images=None,
path_to_output_csv=None,
output_PIL=True):
PATH_TO_LABELS = os.path.join('data', 'mscoco_label_map.pbtxt')
labels_to_names = label_map_util.get_label_map_dict(PATH_TO_LABELS,
use_display_name=True)
image_filenames = [
f for f in listdir(path_to_input_images)
if isfile(join(path_to_input_images, f))
]
data = []
image_filenames = sorted(image_filenames)
for i, image_filename in tqdm(enumerate(image_filenames)):
input_filename = os.path.join(path_to_input_images, image_filename)
output_filename = os.path.join(path_to_output_images,
'processed_' + image_filename)
try:
image = func_read_img(input_filename)
except:
print('{} is not a valid image file'.format(image_filename))
continue
output_image, detection = func_detect_image(image)
data.append(detection)
if output_PIL:
output_image.save(output_filename)
else:
plt.figure()
plt.axis('off')
plt.imsave(output_filename, output_image)
output_csv = os.path.join(path_to_output_csv, 'results.csv')
df = pd.DataFrame(np.array(data))
df.to_csv(output_csv, header=list(labels_to_names.values()), index=False)
def main(_):
logging.info('Prepare process samples in {}'.format(FLAGS.data_dir))
data_dir = FLAGS.data_dir
years = list(map(lambda x: x.strip(), str(FLAGS.year).split(',')))
label_map_file = FLAGS.label_map_path
if not os.path.exists(label_map_file):
label_map_file = os.path.join(data_dir, 'label_map.pbtxt')
if not os.path.exists(label_map_file):
raise FileExistsError('label map file not exist.')
label_map_dict = label_map_util.get_label_map_dict(label_map_file)
output_path = FLAGS.output_path
if not output_path:
output_path = os.path.basename(os.path.dirname(data_dir + os.sep)) + '.tfrecord'
logging.info('Prepare write samples to {}'.format(output_path))
writer = tf.io.TFRecordWriter(output_path)
for year in years:
logging.info('Reading from PASCAL %s dataset.', year)
examples_path = gen_image_set(FLAGS.data_dir, year)
examples_list = dataset_util.read_examples_list(examples_path)
annotations_dir = os.path.join(data_dir, year, FLAGS.annotations_dir)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
with tf.io.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str.encode('utf-8'))
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir, year, label_map_dict,
FLAGS.ignore_difficult_instances)
writer.write(tf_example.SerializeToString())
writer.close()
def main(_):
"""
Main function for generating wider face TFRecord
"""
train_data_dir = FLAGS.train_data_dir # train_data_dir: ./WIDER_train/images
val_data_dir = FLAGS.val_data_dir # val_data_dir: ./WIDER_val/images
train_examples_path = FLAGS.train_examples_path # train_examples_path: ./examples_files/train_examples.txt
val_examples_path = FLAGS.val_examples_path # val_examples_path: ./examples_files/val_examples.txt
# TODO: check read_examples_list here
train_examples = dataset_util.read_examples_list(
train_examples_path
) # train_example: a file contains the all names of the images
# (e.g. 0--Parade/0_Parade_marchingband_1_849.jpg)
val_examples = dataset_util.read_examples_list(
val_examples_path
) # val_example: a file contains the all names of the images
# (e.g. 0--Parade/0_Parade_marchingband_1_465.jpg)
train_anno_path = FLAGS.train_anno_path
val_anno_path = FLAGS.val_anno_path
train_anno_dict = ra.read_anno(train_anno_path)
val_anno_dict = ra.read_anno(val_anno_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
random.seed(42)
random.shuffle(train_examples)
random.shuffle(val_examples)
# output_dir: ./tfrecord_data
train_output_path = os.path.join(FLAGS.output_dir,
'wider_faces_train.record')
val_output_path = os.path.join(FLAGS.output_dir, 'wider_faces_val.record')
create_tf_record(train_output_path, FLAGS.num_shards, label_map_dict,
train_anno_dict, train_data_dir, train_examples)
create_tf_record(val_output_path, FLAGS.num_shards, label_map_dict,
val_anno_dict, val_data_dir, val_examples)
def main(_):
data_dir = FLAGS.data_dir
label_map_dict = label_map_util.get_label_map_dict(
FLAGS.label_map_path) #mapping the labels to index
logging.info('Reading from Pet dataset.')
image_dir = os.path.join(data_dir, 'images') #Image directory
annotations_dir = os.path.join(data_dir,
'annotations') #Annotation diector
examples_path = os.path.join(
annotations_dir, 'trainval.txt') #example names and conventions
examples_list = dataset_util.read_examples_list(
examples_path
) #readinf this as a list . Help to identify image and index
# Test images are not included in the downloaded data set, so we shall perform
# our own split.
random.seed(42)
random.shuffle(examples_list)
num_examples = len(examples_list)
num_train = int(0.7 * num_examples) #deviding the training
train_examples = examples_list[:num_train] # Train VAl
val_examples = examples_list[num_train:] #val
logging.info('%d training and %d validation examples.',
len(train_examples), len(val_examples))
train_output_path = os.path.join(
FLAGS.output_dir, 'pet_train.record') #output path to create records
val_output_path = os.path.join(FLAGS.output_dir, 'pet_val.record')
create_tf_record(
train_output_path,
label_map_dict,
annotations_dir, #creating the tf records
image_dir,
train_examples)
create_tf_record(val_output_path, label_map_dict, annotations_dir,
image_dir, val_examples)
def main(_):
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
data_dir = FLAGS.data_dir
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
print('Reading from PASCAL dataset.')
examples_path = os.path.join(data_dir, 'ImageSets', 'Main',
FLAGS.set + '.txt')
if FLAGS.include_segment_class or FLAGS.include_segment_object:
examples_path = os.path.join(data_dir, 'ImageSets', 'Segmentation',
FLAGS.set + '.txt')
annotations_dir = os.path.join(data_dir, FLAGS.annotations_dir)
examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
mask_filename = None
if FLAGS.include_segment_class or FLAGS.include_segment_object:
mask_filename = example + ".png"
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,
FLAGS.ignore_difficult_instances, mask_filename=mask_filename,
include_segment_class=FLAGS.include_segment_class,
include_segment_object=FLAGS.include_segment_object)
writer.write(tf_example.SerializeToString())
writer.close()