def main_train_test(ini, common_info, logger=None):
# Init. local variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
img_size, batch, epoch, data_yaml_path, model_yaml_path, model_weight_path, rst_dir_name = \
ini['img_size'], ini['batch'], ini['epoch'], ini['data_yaml_path'], ini['model_yaml_path'], ini['model_weight_path'], ini['rst_dir_name']
train_cmd = ini['train_py_cmd']
train_args = [
'--img', img_size, '--batch', batch, '--epoch', epoch, '--data',
data_yaml_path, '--cfg', model_yaml_path, '--weight',
model_weight_path, '--name', rst_dir_name
]
for arg in train_args:
train_cmd += ''.join([' ', arg])
logger.info(" [TRAIN] # Train shell cmd : {}".format(train_cmd))
subprocess.call(train_cmd, shell=True,
cwd=ini['train_root_path']) # train by python
# subprocess.call(train_cmd, shell=True, cwd=ini['train_root_path']) # train by shell
return True
def main_generate(ini, common_info, logger=None):
# Init. local variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cg.folder_exists(vars['gt_path'], create_=True)
img_fnames = sorted(
cg.get_filenames(vars['img_path'], extensions=ig.IMG_EXTENSIONS))
ann_fnames = sorted(
cg.get_filenames(vars['ann_path'], extensions=jg.META_EXTENSION))
logger.info(
" [GENERATE] # Total file number to be processed: {:d}.".format(
len(img_fnames)))
for idx, img_fname in enumerate(img_fnames):
_, img_core_name, img_ext = cg.split_fname(img_fname)
img = ig.imread(img_fname, color_fmt='RGB')
# Load json
ann_fname = ann_fnames[idx]
_, ann_core_name, _ = cg.split_fname(ann_fname)
ann_core_name = ann_core_name.replace('.jpg', '')
if ann_core_name == img_core_name:
with open(ann_fname) as json_file:
json_data = json.load(json_file)
objects = json_data['objects']
# pprint.pprint(objects)
bboxes = []
texts = []
for obj in objects:
class_name = obj['classTitle']
if class_name != common_info['dataset_type'].lower():
continue
[x1, y1], [x2, y2] = obj['points']['exterior']
text = obj['description']
x_min, y_min, x_max, y_max = int(min(x1, x2)), int(min(
y1, y2)), int(max(x1, x2)), int(max(y1, y2))
if x_max - x_min <= 0 or y_max - y_min <= 0:
continue
rect4 = ic.convert_rect2_to_rect4([x_min, x_max, y_min, y_max])
bboxes.append(rect4)
texts.append(text)
file_utils.saveResult(img_file=img_core_name,
img=img,
boxes=bboxes,
texts=texts,
dirname=vars['gt_path'])
logger.info(" [GENERATE-OCR] # Generated to {} ({:d}/{:d})".format(
vars['gt_path'] + img_core_name + '.txt', (idx + 1),
len(img_fnames)))
logger.info(" # {} in {} mode finished.".format(_this_basename_, GENERATE))
return True
def main(args):
ini = cg.get_ini_parameters(args.ini_fname)
common_info = {}
for key, val in ini['COMMON'].items():
common_info[key] = val
logger = cl.setup_logger_with_ini(ini['LOGGER'],
logging_=args.logging_,
console_=args.console_logging_)
if args.op_mode == PREPROCESS_ALL:
# Preprocess ko, math dataset
dataset_types = KO_MATH.split('_')
for dataset_type in dataset_types:
# Reload ini
ini = cg.get_ini_parameters(
f'craft_learn_{dataset_type.lower()}.ini')
common_info = {}
for key, val in ini['COMMON'].items():
common_info[key] = val
# Init. local variables
vars = {}
for key, val in ini[PREPROCESS_ALL].items():
vars[key] = cs.replace_string_from_dict(val, common_info)
# Run generate & split
tgt_dir_names = vars['tgt_dir_names'].replace(' ', '').split(',')
for tgt_dir_name in tgt_dir_names:
common_info['tgt_dir_name'] = tgt_dir_name
main_generate(ini[GENERATE], common_info, logger=logger)
main_split(ini[SPLIT], common_info, logger=logger)
# Run merge
main_merge(ini[MERGE], common_info, logger=logger)
elif args.op_mode == GENERATE:
main_generate(ini[GENERATE], common_info, logger=logger)
elif args.op_mode == SPLIT:
main_split(ini[SPLIT], common_info, logger=logger)
elif args.op_mode == MERGE:
main_merge(ini[MERGE], common_info, logger=logger)
elif args.op_mode == TRAIN:
main_train(ini[TRAIN], common_info, logger=logger)
elif args.op_mode == TEST:
main_test(ini[TEST], common_info, logger=logger)
elif args.op_mode == TRAIN_TEST:
ret, model_dir = main_train(ini[TRAIN], common_info, logger=logger)
main_test(ini[TEST], common_info, logger=logger)
print(" # Trained model directory is {}".format(model_dir))
elif args.op_mode == SPLIT_TEXTLINE:
main_split_textline(ini[SPLIT_TEXTLINE], common_info, logger=logger)
else:
print(" @ Error: op_mode, {}, is incorrect.".format(args.op_mode))
return True
def main_train(ini, common_info, logger=None):
# Init. path variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cuda_ids = vars['cuda_ids'].split(',')
latest_model_dir = cg.get_model_dir(root_dir=vars['root_model_path'],
model_file=vars['model_name'],
version='latest')
latest_model_path = os.path.join(latest_model_dir, vars['model_name'])
train_args = [
'--tgt_class',
common_info['tgt_class'],
'--img_path',
vars['train_img_path'],
'--gt_path',
vars['train_gt_path'],
'--pretrain_model_path',
latest_model_path,
'--cuda',
vars['cuda'],
'--cuda_ids',
cuda_ids,
'--resume',
vars['resume'],
'--valid_epoch',
vars['valid_epoch'],
'--batch_size',
vars['batch_size'],
'--learning_rate',
vars['learning_rate'],
'--momentum',
vars['momentum'],
'--weight_decay',
vars['weight_decay'],
'--gamma',
vars['gamma'],
'--num_workers',
vars['num_workers'],
]
train.main(train.parse_arguments(train_args), logger=logger)
return True
def main_split_textline(ini, common_info, logger=None):
# Init. path variables
global box_color, rst_dir_name
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
except_dir_names = vars['except_dir_names'].replace(' ', '').split(',')
img_mode = vars['img_mode']
link_, copy_, border_, save_detect_box_img_, save_refine_box_img_ = \
cs.string_to_boolean(vars['link_']), cs.string_to_boolean(vars['copy_']), cs.string_to_boolean(vars['border_']), \
cs.string_to_boolean(vars['save_detect_box_img_']), cs.string_to_boolean(vars['save_refine_box_img_'])
# Init. CRAFT
gpu_ = cs.string_to_boolean(vars['cuda'])
device = torch.device('cuda' if (
torch.cuda.is_available() and gpu_) else 'cpu')
ko_model_dir, math_model_dir = cg.get_model_dir(root_dir=vars['ko_model_path'], model_file=vars['ko_model_name']), \
cg.get_model_dir(root_dir=vars['math_model_path'], model_file=vars['math_model_name'])
ko_detector = get_detector(os.path.join(ko_model_dir,
vars['ko_model_name']),
device,
quantize=False)
math_detector = get_detector(os.path.join(math_model_dir,
vars['math_model_name']),
device,
quantize=False)
easyocr_ini = cg.get_ini_parameters(
os.path.join(_this_folder_, vars['ocr_ini_fname']))
craft_params = load_craft_parameters(easyocr_ini['CRAFT'])
project = sly.Project(directory=vars['textline_dataset_path'],
mode=sly.OpenMode.READ,
block_directories=except_dir_names)
# Preprocess datasets
if link_:
link_or_copy_datasets(src_dir_path=vars['textline_dataset_path'],
dst_dir_path=vars['refine_dataset_path'],
dir_names=project.datasets.keys(),
except_dir_names=except_dir_names,
tgt_dir_name='img/',
mode=LINK,
logger=logger)
if copy_:
link_or_copy_datasets(src_dir_path=vars['textline_dataset_path'],
dst_dir_path=vars['refine_dataset_path'],
dir_names=project.datasets.keys(),
except_dir_names=except_dir_names,
tgt_dir_name='ann/',
mode=COPY,
logger=logger)
# Load and split textlines
for dataset in project:
sly.logger.info('Processing dataset: {}/{}'.format(
project.name, dataset.name))
for item_idx, item_name in enumerate(dataset):
item_paths = dataset.get_item_paths(item_name)
ann, json_data = sly.Annotation.load_json_file(
item_paths.ann_path, project.meta)
raw_img = sly.image.read(item_paths.img_path)
draw_detect_img, draw_refine_img = raw_img.copy(), raw_img.copy()
# Draw textline box
for label in ann.labels:
if label.obj_class.name == textline:
label.geometry.draw_contour(
draw_detect_img,
color=label.obj_class.color,
config=label.obj_class.geometry_config,
thickness=3)
label.geometry.draw_contour(
draw_refine_img,
color=label.obj_class.color,
config=label.obj_class.geometry_config,
thickness=3)
draw_detect_img = ((draw_detect_img.astype(np.uint16) +
raw_img.astype(np.uint16)) / 2).astype(
np.uint8)
draw_refine_img = ((draw_refine_img.astype(np.uint16) +
raw_img.astype(np.uint16)) / 2).astype(
np.uint8)
# Get textline ground truths
gt_objs = []
for idx, label in reversed(list(enumerate(ann.labels))):
if label.obj_class.name != textline:
crop_img = raw_img[
label.geometry.top:label.geometry.bottom,
label.geometry.left:label.geometry.right]
if crop_img.size == 0:
continue
crop_box, ret_ = ip.get_binary_area_coordinates_by_threshold(
crop_img, min_thresh=127, max_thresh=255)
if ret_:
crop_box_obj = ic.Box(crop_box)
proc_box = ic.calc_global_box_pos_in_box(
g_box=[
label.geometry.left, label.geometry.right,
label.geometry.top, label.geometry.bottom
],
box=crop_box_obj.rect2,
format='rect2')
min_x, max_x, min_y, max_y = proc_box
# debug_img = raw_img[min_y:max_y, min_x:max_x]
# print('Prev geo. : ', ann.labels[idx].geometry.left, ann.labels[idx].geometry.right, ann.labels[idx].geometry.top, ann.labels[idx].geometry.bottom)
# Remove label
ann = ann.delete_label(label)
# update coordinates
crop_labels = label.crop(
sly.Rectangle(min_y, min_x, max_y,
max_x)) # top, left, bottom, right,
for crop_label in crop_labels:
ann = ann.add_label(crop_label)
# print('Next geo. : ', ann.labels[idx].geometry.left, ann.labels[idx].geometry.right, ann.labels[idx].geometry.top, ann.labels[idx].geometry.bottom)
continue
# Remove textline object
ann = ann.delete_label(label)
if (label.geometry.right - label.geometry.left) <= 0 or (
label.geometry.bottom - label.geometry.top) <= 0:
continue
gt_box = ic.Box(
box=[[label.geometry.left, label.geometry.top],
[label.geometry.right, label.geometry.bottom]])
gt_obj = object.Object(name=textline,
box_obj=gt_box,
description=label.description.strip())
gt_objs.append(gt_obj)
# Get predict results
pred_objs = []
for detector in [ko_detector, math_detector]:
tgt_class = ko if (detector is ko_detector) else (math if (
detector is math_detector) else 'None')
# # Make border
border_margin = 0
if border_:
border_color = ig.WHITE
border_margin = 30
raw_img = cv2.copyMakeBorder(raw_img,
border_margin,
border_margin,
border_margin,
border_margin,
cv2.BORDER_CONSTANT,
value=border_color)
boxes = get_textbox(
detector,
raw_img,
canvas_size=craft_params['canvas_size'],
mag_ratio=craft_params['mag_ratio'],
text_threshold=craft_params['text_threshold'],
link_threshold=craft_params['link_threshold'],
low_text=craft_params['low_text'],
poly=False,
device=device,
optimal_num_chars=True)
if border_:
boxes = [
np.array([
box[0] - border_margin, box[1] - border_margin,
box[2] - border_margin, box[3] - border_margin,
box[4] - border_margin, box[5] - border_margin,
box[6] - border_margin, box[7] - border_margin
]) for box in boxes
]
horizontal_list, _ = group_text_box(
boxes, craft_params['slope_ths'],
craft_params['ycenter_ths'], craft_params['height_ths'],
craft_params['width_ths'], craft_params['add_margin'])
for h_box in horizontal_list:
pred_box = ic.Box(
box=[[h_box[0], h_box[2]], [h_box[1], h_box[3]]])
pred_obj = object.Object(name=tgt_class,
box_obj=pred_box,
description='')
pred_objs.append(pred_obj)
if tgt_class == ko:
box_color = ig.BROWN
if tgt_class == math:
box_color = ig.MAGENTA
draw_detect_img = ip.draw_box_on_img(draw_detect_img,
pred_box.flat_box,
color=box_color,
thickness=2)
# Save result image
ko_model_epoch, math_model_epoch = vars['ko_model_name'].split('_')[-1].replace('.pth', ''), \
vars['math_model_name'].split('_')[-1].replace('.pth', '')
rst_dir_name = f'{ko}_' + ko_model_epoch + '_' + f'{math}_' + math_model_epoch
rst_dir_path = os.path.join(vars['rst_path'], rst_dir_name,
'draw_box')
if save_detect_box_img_:
cg.folder_exists(rst_dir_path, create_=True)
ig.imwrite(
draw_detect_img,
os.path.join(rst_dir_path, f'[{img_mode}] ' + item_name))
# Compare GT. & PRED.
refine_gts = refine_ground_truths_by_predict_values(
gt_objs, pred_objs, raw_img) # test input : GTS, PREDS
# Draw refined boxes & texts
if save_refine_box_img_:
for (rf_box, rf_text, rf_class) in refine_gts:
rf_rect2 = ic.convert_rect4_to_rect2(rf_box)
x_min, x_max, y_min, y_max = rf_rect2
box = (x_min, y_min, x_max, y_max)
if rf_class == ko:
box_color = ig.BROWN
if rf_class == math:
box_color = ig.MAGENTA
# Draw boxes
draw_refine_img = ip.draw_box_on_img(draw_refine_img,
box,
color=box_color,
thickness=3)
# Draw texts (for 한글)
pil_img = Image.fromarray(draw_refine_img)
draw = ImageDraw.Draw(pil_img)
font = cg.KOR_FONT
margin_x, margin_y = 10, 45
draw.text(xy=((x_min + 1 + margin_x),
(y_min + 1 + margin_y)),
text=rf_text,
font=font,
fill=box_color)
draw_refine_img = np.array(pil_img)
rst_dir_path = os.path.join(vars['rst_path'], rst_dir_name,
'refine_box')
cg.folder_exists(rst_dir_path, create_=True)
ig.imwrite(
draw_refine_img,
os.path.join(rst_dir_path, f'[{img_mode}] ' + item_name))
# Insert refine_gts to json
if len(ann.labels) > 0:
obj_id = ann.labels[-1].geometry.sly_id + 1
else:
obj_id = 0
refine_json_data, refine_obj_id = update_json_from_results(
ann.to_json(), obj_id, [ko, math], refine_gts)
# Save refined json
rst_ann_fname = item_paths.ann_path.replace(
vars['textline_dataset_path'], vars['refine_dataset_path'])
with open(rst_ann_fname, 'w', encoding='utf-8') as f:
json.dump(refine_json_data, f, ensure_ascii=False, indent=4)
sly.logger.info('[{}/{}] Refined json path : {}'.format(
item_idx + 1, len(dataset), rst_ann_fname))
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True
def main_merge(ini, common_info, logger=None):
# Init. path variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cg.folder_exists(vars['total_dataset_path'], create_=True)
datasets = [
dataset for dataset in os.listdir(vars['dataset_path'])
if (dataset != 'total') and ('meta.json' not in dataset)
]
sort_datasets = sorted(datasets, key=lambda x: (int(x.split('_')[0])))
lower_dataset_type = common_info['dataset_type'].lower(
) if common_info['dataset_type'] != TEXTLINE else ''
if lower_dataset_type:
tgt_dir = 'craft_{}_gt'.format(lower_dataset_type)
else:
tgt_dir = 'craft_gt'
if len(sort_datasets) != 0:
for dir_name in sort_datasets:
src_train_path, src_test_path = os.path.join(vars['dataset_path'], dir_name, TRAIN.lower()), \
os.path.join(vars['dataset_path'], dir_name, TEST.lower())
src_train_img_path, src_train_gt_path = os.path.join(src_train_path, 'img/'), \
os.path.join(src_train_path, tgt_dir + '/')
src_test_img_path, src_test_gt_path = os.path.join(src_test_path, 'img/'), \
os.path.join(src_test_path, tgt_dir + '/')
dst_train_path, dst_test_path = os.path.join(vars['total_dataset_path'], TRAIN.lower()), \
os.path.join(vars['total_dataset_path'], TEST.lower())
dst_train_img_path, dst_train_gt_path = os.path.join(dst_train_path, 'img/'), \
os.path.join(dst_train_path, tgt_dir + '/')
dst_test_img_path, dst_test_gt_path = os.path.join(dst_test_path, 'img/'), \
os.path.join(dst_test_path, tgt_dir + '/')
if cg.folder_exists(dst_train_img_path) and cg.folder_exists(dst_train_gt_path) and \
cg.folder_exists(dst_test_img_path) and cg.folder_exists(dst_test_gt_path):
logger.info(" # Already {} is exist".format(
vars['total_dataset_path']))
else:
cg.folder_exists(dst_train_img_path,
create_=True), cg.folder_exists(
dst_train_gt_path, create_=True)
cg.folder_exists(dst_test_img_path,
create_=True), cg.folder_exists(
dst_test_gt_path, create_=True)
# Apply symbolic link for gt & img path
for op_mode in [TRAIN, TEST]:
if op_mode == TRAIN:
src_img_path, src_gt_path = src_train_img_path, src_train_gt_path
dst_img_path, dst_gt_path = dst_train_img_path, dst_train_gt_path
elif op_mode == TEST:
src_img_path, src_gt_path = src_test_img_path, src_test_gt_path
dst_img_path, dst_gt_path = dst_test_img_path, dst_test_gt_path
# link img_path
img_sym_cmd = 'ln "{}"* "{}"'.format(
src_img_path, dst_img_path) # to all files
subprocess.call(img_sym_cmd, shell=True)
logger.info(" # Link img files {}\n{}->{}.".format(
src_img_path, MARGIN, dst_img_path))
# link gt_path
gt_sym_cmd = 'ln "{}"* "{}"'.format(
src_gt_path, dst_gt_path) # to all files
subprocess.call(gt_sym_cmd, shell=True)
logger.info(" # Link gt files {}\n{}->{}.".format(
src_gt_path, MARGIN, dst_gt_path))
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True
def main_split(ini, common_info, logger=None):
# Init. path variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cg.folder_exists(vars['img_path'], create_=False)
cg.folder_exists(vars['gt_path'], create_=False)
cg.folder_exists(vars['train_path'], create_=False)
cg.folder_exists(vars['test_path'], create_=False)
train_ratio = float(vars['train_ratio'])
test_ratio = round(1.0 - train_ratio, 2)
lower_dataset_type = common_info['dataset_type'].lower(
) if common_info['dataset_type'] != TEXTLINE else ''
if lower_dataset_type:
tgt_dir = 'craft_{}_gt'.format(lower_dataset_type)
else:
tgt_dir = 'craft_gt'
gt_list = sorted(
cg.get_filenames(vars['gt_path'], extensions=cg.TEXT_EXTENSIONS))
train_gt_list, test_gt_list = train_test_split(gt_list,
train_size=train_ratio,
random_state=2000)
train_img_list, test_img_list = [gt_path.replace(tgt_dir, 'img').replace('.txt', '.jpg').replace('gt_', '') for
gt_path in train_gt_list], \
[gt_path.replace(tgt_dir, 'img').replace('.txt', '.jpg').replace('gt_', '') for
gt_path in test_gt_list]
train_img_path, test_img_path = os.path.join(vars['train_path'],
'img/'), os.path.join(
vars['test_path'], 'img/')
train_gt_path, test_gt_path = os.path.join(vars['train_path'],
tgt_dir + '/'), os.path.join(
vars['test_path'],
tgt_dir + '/')
cg.folder_exists(train_img_path,
create_=True), cg.folder_exists(test_img_path,
create_=True)
cg.folder_exists(train_gt_path,
create_=True), cg.folder_exists(test_gt_path,
create_=True)
# Apply symbolic link for gt & img path
if len(gt_list) != 0:
for op_mode in [TRAIN, TEST]:
if op_mode == TRAIN:
gt_list = train_gt_list
img_list = train_img_list
gt_link_path = train_gt_path
img_link_path = train_img_path
elif op_mode == TEST:
gt_list = test_gt_list
img_list = test_img_list
gt_link_path = test_gt_path
img_link_path = test_img_path
# link gt files
for gt_path in gt_list:
gt_sym_cmd = 'ln "{}" "{}"'.format(
gt_path, gt_link_path) # to all files
subprocess.call(gt_sym_cmd, shell=True)
logger.info(" # Link gt files {}\n{}->{}.".format(
gt_list[0], MARGIN, gt_link_path))
# link img files
for img_path in img_list:
img_sym_cmd = 'ln "{}" "{}"'.format(
img_path, img_link_path) # to all files
subprocess.call(img_sym_cmd, shell=True)
logger.info(" # Link img files {}\n{}->{}.".format(
img_list[0], MARGIN, img_link_path))
print(" # (train, test) = ({:d}, {:d}) -> {:d} % ".format(
len(train_gt_list), len(test_gt_list),
int(
float(len(train_gt_list)) /
float(len(train_gt_list) + len(test_gt_list)) * 100)))
return True
def main_crop(ini, common_info, logger=None):
# Init. local variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cg.folder_exists(ini['img_path'], create_=True)
raw_path = os.path.join(_project_folder_, ini['raw_path'])
ann_path = os.path.join(_project_folder_, ini['ann_path'])
raw_fnames = sorted(
cg.get_filenames(raw_path, extensions=ig.IMG_EXTENSIONS))
ann_fnames = sorted(
cg.get_filenames(ann_path, extensions=jg.META_EXTENSION))
logger.info(" [CROP] # Total file number to be processed: {:d}.".format(
len(raw_fnames)))
for idx, raw_fname in enumerate(raw_fnames):
logger.info(" [CROP] # Processing {} ({:d}/{:d})".format(
raw_fname, (idx + 1), len(raw_fnames)))
_, raw_core_name, raw_ext = cg.split_fname(raw_fname)
img = ig.imread(raw_fname, color_fmt='RGB')
# Load json
ann_fname = ann_fnames[idx]
_, ann_core_name, _ = cg.split_fname(ann_fname)
if ann_core_name == raw_core_name + raw_ext:
with open(ann_fname) as json_file:
json_data = json.load(json_file)
objects = json_data['objects']
# pprint.pprint(objects)
# Extract crop position
object_cnt = 0
for obj in objects:
class_name = obj['classTitle']
if class_name != ini['object_class']:
continue
[x1, y1], [x2, y2] = obj['points']['exterior']
x_min, y_min, x_max, y_max = int(min(x1, x2)), int(min(
y1, y2)), int(max(x1, x2)), int(max(y1, y2))
if x_max - x_min <= 0 or y_max - y_min <= 0:
continue
try:
crop_img = img[y_min:y_max, x_min:x_max]
except TypeError:
logger.error(" [CROP] # Crop error : {}".format(raw_fname))
logger.error(" [CROP] # Error pos : {}, {}, {}, {}".format(
x_min, x_max, y_min, y_max))
pass
# Save cropped image
rst_fpath = os.path.join(
_project_folder_, ini['img_path'] + raw_core_name + '_' +
str(object_cnt) + raw_ext)
ig.imwrite(crop_img, rst_fpath)
object_cnt += 1
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True
def main_split(ini, common_info, logger=None):
# Init. local variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
cg.folder_exists(vars['img_path'], create_=False)
if cg.file_exists(vars['train_path']):
print(" @ Warning: train text file path, {}, already exists".format(
vars["train_path"]))
ans = input(" % Proceed (y/n) ? ")
if ans.lower() != 'y':
sys.exit()
if cg.file_exists(vars['val_path']):
print(" @ Warning: test text file path, {}, already exists".format(
vars["val_path"]))
ans = input(" % Proceed (y/n) ? ")
if ans.lower() != 'y':
sys.exit()
# Apply symbolic link for img path
raw_path = os.path.join(_project_folder_, vars['raw_path'])
img_path = os.path.join(_project_folder_, vars['img_path'])
cg.folder_exists(img_path, create_=True)
img_fnames = sorted(
cg.get_filenames(img_path, extensions=ig.IMG_EXTENSIONS))
if len(img_fnames) == 0:
sym_cmd = "ln -s {} {}".format(raw_path + '*',
img_path) # to all files
subprocess.call(sym_cmd, shell=True)
img_fnames = sorted(
cg.get_filenames(img_path, extensions=ig.IMG_EXTENSIONS))
train_ratio = float(vars['train_ratio'])
test_ratio = (1.0 - train_ratio)
train_img_list, test_img_list = train_test_split(img_fnames,
test_size=test_ratio,
random_state=2000)
# Save train.txt file
train_path = os.path.join(_project_folder_, vars['train_path'])
with open(train_path, 'w') as f:
f.write('\n'.join(train_img_list) + '\n')
val_path = os.path.join(_project_folder_, vars['val_path'])
with open(val_path, 'w') as f:
f.write('\n'.join(test_img_list) + '\n')
logger.info(" [SPLIT] # Train : Test ratio -> {} : {}".format(
train_ratio, test_ratio))
logger.info(" [SPLIT] # Train : Test size -> {} : {}".format(
len(train_img_list), len(test_img_list)))
# Modify yaml file
ref_yaml_path = os.path.join(_project_folder_, vars['ref_yaml_path'])
with open(ref_yaml_path, 'r') as f:
data = yaml.safe_load(f)
data['train'] = os.path.join(_project_folder_, vars['train_path'])
data['val'] = os.path.join(_project_folder_, vars['val_path'])
data['names'] = common_info['obj_names'].replace(' ', '').split(',')
data['nc'] = len(data['names'])
# Save yaml file
rst_yaml_path = os.path.join(_project_folder_, vars['rst_yaml_path'])
with open(rst_yaml_path, 'w') as f:
yaml.dump(data, f)
pprint(data)
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True
def main_generate(ini, common_info, logger=None):
# Init. local variables
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
label_path = os.path.join(_project_folder_, vars['label_path'])
cg.folder_exists(label_path, create_=True)
raw_path = os.path.join(_project_folder_, vars['raw_path'])
ann_path = os.path.join(_project_folder_, vars['ann_path'])
raw_fnames = sorted(
cg.get_filenames(raw_path, extensions=ig.IMG_EXTENSIONS))
ann_fnames = sorted(
cg.get_filenames(ann_path, extensions=jg.META_EXTENSION))
logger.info(
" [GENERATE] # Total file number to be processed: {:d}.".format(
len(raw_fnames)))
for idx, raw_fname in enumerate(raw_fnames):
_, raw_core_name, raw_ext = cg.split_fname(raw_fname)
img = ig.imread(raw_fname, color_fmt='RGB')
h, w, c = img.shape
# Load json
ann_fname = ann_fnames[idx]
_, ann_core_name, _ = cg.split_fname(ann_fname)
if ann_core_name == raw_core_name + raw_ext:
with open(ann_fname) as json_file:
json_data = json.load(json_file)
objects = json_data['objects']
# pprint.pprint(objects)
# Extract crop position
obj_names = common_info['obj_names'].replace(' ', '').split(',')
obj_type = common_info['obj_type']
for obj in objects:
obj_name = obj['classTitle']
if obj_name not in obj_names:
continue
class_num = ObjInfo(obj_type, obj_name).get_class_number()
[x1, y1], [x2, y2] = obj['points']['exterior']
x_min, y_min, x_max, y_max = int(min(x1, x2)), int(min(
y1, y2)), int(max(x1, x2)), int(max(y1, y2))
if x_max - x_min <= 0 or y_max - y_min <= 0:
continue
# Save object info to COCO format
rst_fpath = os.path.join(
_project_folder_, vars['label_path'] + raw_core_name + '.txt')
class_no, x_center, y_center, width, height = \
str(class_num), str(((x_max+x_min)/2) / w), str(((y_max+y_min)/2) / h), str((x_max-x_min)/w), str((y_max-y_min)/h)
if cg.file_exists(rst_fpath):
logger.info(
" [GENERATE] # File already exist {} ({:d}/{:d})".format(
rst_fpath, (idx + 1), len(raw_fnames)))
else:
with open(rst_fpath, 'a') as f:
strResult = "{} {} {} {} {}\r\n".format(
class_no, x_center, y_center, width, height)
f.write(strResult)
logger.info(
" [GENERATE] # File is saved {} ({:d}/{:d})".format(
rst_fpath, (idx + 1), len(raw_fnames)))
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True