def test_create_metadata_file_for_testing(self):
test_helper.get_mat_test_metadata()
data_pack = []
for i, data in enumerate(
metadata_generator(test_helper.test_mat_metadata_file)):
data_pack.append(data)
self.assertEqual(data_pack[0].label, '19')
self.assertEqual(data_pack[20].label, '2')
self.assertEqual(data_pack[24].label, '601')
self.assertEqual(data_pack[24].filename, '25.png')
def _create_test_data():
mat_metadata_file = get_mat_test_metadata()
filenames, labels, bboxes = [], [], []
for i, data in enumerate(metadata_generator(mat_metadata_file)):
filenames.append(data.filename)
labels.append(data.label)
bboxes.append(data.bbox())
write_tf_records(filenames, labels, 5, bboxes, train_data_dir_path,
test_data_file)
def _create_test_data():
mat_metadata_file = get_mat_test_metadata()
filenames, labels, bboxes, sep_bboxes = [], [], [], []
for i, data in enumerate(metadata_generator(mat_metadata_file)):
filenames.append(data.filename)
labels.append(data.label)
bboxes.append(data.bbox())
sep_bboxes.append([[int(bb.left), int(bb.top), int(bb.width), int(bb.height)] for bb in data.bboxes])
write_tf_records(filenames, labels, 5, bboxes, sep_bboxes, 'full', train_data_dir_path, test_data_file)
write_tf_records(filenames, labels, 5, bboxes, sep_bboxes, 'number_0', train_data_dir_path, test_data_file_number_0)
write_pickle(bboxes, filenames, labels, sep_bboxes, test_metadata_file)
def test_metadata_generator(self):
gen = metadata_generator(test_helper.train_mat_metadata_file)
sampled = [gen.__next__() for i in range(30)]
self.assertIsInstance(sampled[0], Data)
self.assertEqual(
sampled[0],
Data('1.png',
[BBox('1', 77, 246, 81, 219),
BBox('9', 81, 323, 96, 219)]))
self.assertEqual(
sampled[20],
Data('21.png',
[BBox(label=2, top=6.0, left=72.0, width=52.0, height=85.0)]))
def parse_data(mat_metadata_file_path,
max_number_length,
data_dir_path,
rand_box_count=5):
metadata = metadata_generator(mat_metadata_file_path)
filenames, labels, bboxes, sep_bboxes = [], [], [], []
for i, md in enumerate(metadata):
if len(md.label) > max_number_length:
tf.logging.info(
'ignore record, label too long: label=%s, filename=%s',
md.label, md.filename)
continue
bbox, sep_bbox_list, size = fix_bboxes(md, data_dir_path)
sep_bbox_list = [[int(v) for v in bb] for bb in sep_bbox_list]
if any([is_valid_bbox(bb, size) for bb in [bbox] + sep_bbox_list]):
tf.logging.info(
'ignore failed to fix record %s(%s), bad bbox. bbox=%s, sep_bbox_list=%s',
md.filename, size, bbox, sep_bbox_list)
continue
if rand_box_count != 0:
rand_bboxes = random_bbox(rand_box_count, bbox, size)
for i in range(rand_box_count):
filenames.append(md.filename)
labels.append(md.label)
bboxes.append(rand_bboxes[i])
sep_bboxes.append(sep_bbox_list)
else:
filenames.append(md.filename)
labels.append(md.label)
bboxes.append(bbox)
sep_bboxes.append(sep_bbox_list)
if i % 1000 == 0 and i > 0:
tf.logging.info('readed count: %s', i)
rand_idxes = list(range(len(filenames)))
random.shuffle(rand_idxes)
rand_filenames, rand_labels, rand_bboxes, rand_sep_bboxes = [], [], [], []
for i in rand_idxes:
rand_filenames.append(filenames[i])
rand_labels.append(labels[i])
rand_bboxes.append(bboxes[i])
rand_sep_bboxes.append(sep_bboxes[i])
return rand_filenames, rand_labels, rand_bboxes, rand_sep_bboxes
def parse_data(mat_metadata_file_path,
max_number_length,
data_dir_path,
rand_box_count=5):
metadata = metadata_generator(mat_metadata_file_path)
filenames, labels, bboxes = [], [], []
for i, md in enumerate(metadata):
if len(md.label) > max_number_length:
tf.logging.info(
'ignore record, label too long: label=%s, filename=%s',
md.label, md.filename)
continue
bbox, size = fix_bbox(md, data_dir_path)
if any([j < 0
for j in bbox]) or bbox[0] >= size[0] or bbox[1] >= size[1]:
tf.logging.info(
'ignore failed to fix record %s, bad bbox. bbox=%s',
md.filename, bbox)
continue
if rand_box_count != 0:
rand_bboxes = random_bbox(rand_box_count, bbox, size)
for i in range(rand_box_count):
filenames.append(md.filename)
labels.append(md.label)
bboxes.append(rand_bboxes[i])
else:
filenames.append(md.filename)
labels.append(md.label)
bboxes.append(bbox)
if i % 1000 == 0 and i > 0:
tf.logging.info('readed count: %s', i)
rand_idxes = list(range(len(filenames)))
random.shuffle(rand_idxes)
rand_filenames, rand_labels, rand_bboxes = [], [], []
for i in rand_idxes:
rand_filenames.append(filenames[i])
rand_labels.append(labels[i])
rand_bboxes.append(bboxes[i])
return rand_filenames, rand_labels, rand_bboxes