def combined_json(
detection_json_dir,
classification_json_dir):
""" Combine results from detector and classifier
@param detection_json_dir: directory to detector results
@param classification_json_dir: directory to classifier results
"""
for json_file in tools.find_files(detection_json_dir):
json_data = data.parse_json(json_file)
annotations = json_data['annotations']
for i in range(len(annotations)):
classification_json_file = os.path.join(
classification_json_dir,
os.path.basename(json_file).split('.')[0]
+ '_' + str(i) + '.json')
if os.path.exists(classification_json_file):
classification_annotations = data.parse_json(
classification_json_file)['annotations'][0]
annotations[i]['type'] = 'detection_classification'
annotations[i]['label'] += '_' + \
classification_annotations['label']
annotations[i]['labinfo'] = classification_annotations[
'labinfo']
json_data['annotations'] = annotations
out_json = os.path.join(
os.path.dirname(json_file),
os.path.basename(json_file).split('.')[0] + '_combined.json')
with open(out_json, 'w') as wf:
json.dump(json_data, wf)
def find_images(dir_path=None, walkin=True, keyword=None):
"""Find images under a directory
Keyword arguments:
dir_path -- path of the directory to check (default: '.')
keyword -- keyword used to filter images (default: None)
walkin -- True to list recursively (default: True)
@return output: a list of images found
"""
if dir_path is not None and os.path.isfile(dir_path):
return [dir_path]
return tools.find_files(dir_path=dir_path,
keyword=keyword,
walkin=walkin,
suffix=('.jpg', '.png', '.JPEG', '.bmp', '.gif'))
def create_tf_record(self, json_dir, tfrecord_path):
writer = tf.python_io.TFRecordWriter(tfrecord_path)
check_keys = ["folder", "filename", "annotations"]
for json_file in tools.find_files(json_dir, walkin=False):
try:
json_content = tools.parse_json(json_file)
except BaseException:
print("Fail to open %s" % json_file)
continue
for key in check_keys:
if key not in json_content.keys():
print("%s is not found in %s" % (key, json_file))
continue
tf_example = self.lab_format_to_tf_example(json_content)
writer.write(tf_example.SerializeToString())
writer.close()
def main_process(self):
""" main process """
if not os.path.isdir(self.input_data):
self.logger.error("%s is not a valid folder" % self.input_data)
self.terminate_flag = True
if self.overwrite:
output = open(self.output_file, 'w')
else:
if tools.check_exist(self.output_file):
output = open(self.output_file, 'a')
else:
output = open(self.output_file, 'w')
print("LearnerYOLO: creating %s" % self.output_file)
check_keys = ["folder", "filename", "annotations"]
for json_file in tools.find_files(self.input_data, walkin=False):
try:
json_content = tools.parse_json(json_file)
except BaseException:
self.logger.error("Fail to open %s" % json_file)
continue
for key in check_keys:
if key not in json_content.keys():
self.logger.error("%s is not found in %s" %
(key, json_file))
continue
folder = json_content["folder"]
filename = json_content["filename"]
# FIXME
# folder = folder.replace("results", "labeled_data")
# folder = folder.replace("_tmp", "")
in_img_path = os.path.join(folder, filename)
out_img_path = os.path.join(self.img_folder, filename)
o_file_path = os.path.join(
self.label_folder,
tools.remove_extension(filename) + '.txt')
o_file = open(o_file_path, 'w')
annos = json_content["annotations"]
size, pix = image.get_img_info(in_img_path)
h = float(size[0])
w = float(size[1])
for anno in annos:
X = []
Y = []
cls = anno["label"]
if cls not in self.classes:
self.logger.debug("%s is not in the selected class" % cls)
continue
cls_id = self.classes.index(cls)
X = [anno["left"], anno["right"]]
Y = [anno["top"], anno["bottom"]]
bb = self.convert((w, h), X, Y)
o_file.write(
str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
o_file.close()
self.logger.info("Link %s to %s" % (in_img_path, out_img_path))
os.symlink(in_img_path, out_img_path)
output.write(out_img_path + '\n')
output.close()
# FIXME darknet env has to be well prepared and fix the classes now
train_path = os.path.join(self.darknet_path, "train.txt")
if train_path != self.output_file:
copyfile(self.output_file, train_path)
os.chdir(self.darknet_path)
cmd = ("./darknet detector train cfg/dt42.data"
" cfg/tiny-yolo-voc-dt42.cfg darknet.weights.13 -gpus 1")
self.logger.info("Running %s " % cmd)
output = subprocess.check_output(["bash", "-c", cmd])
self.results = {
"root_directory": self.darknet_path,
"weight_file": "backup_dt42/yolo-voc-dt42_final.weights",
"data_file": "cfg/dt42.data",
"names_file": "data/dt42.names",
"cfg_file": "cfg/tiny-yolo-voc-dt42.cfg"
}
def main():
""" main function to run pipeline """
args = get_args()
logger = logging.getLogger('launcher')
log_level = logging.WARNING
if args.verbosity == 1:
log_level = logging.INFO
elif args.verbosity >= 2:
log_level = logging.DEBUG
formatter = logging.Formatter('[launcher] %(levelname)s %(message)s')
console = logging.StreamHandler()
console.setFormatter(formatter)
logger.setLevel(log_level)
logger.addHandler(console)
logger.info('lab_flag is %r' % args.lab_flag)
pipeline = dt42pl.Pipeline(args.pipeline_config,
dyda_config_path=args.dyda_config_path,
parent_result_folder=args.output,
verbosity=args.verbosity,
lab_flag=args.lab_flag,
force_run_skip=args.force_run_skip)
if args.read_frame:
fr = frame_reader.FrameReader()
# looping over input data paths
logger.info('Running Reader and Selector for frames')
data_list = args.data_list
if args.json_list:
logger.warning('json_list will replace -d/--data_list argument')
data_list = args.json_list
force_snapshot = False
if args.force_snapshot:
force_snapshot = True
bfile_list = False
if args.direct_input:
fpaths = data_list
elif os.path.isfile(data_list):
if tools.check_ext(data_list, ".json"):
fpaths = tools.parse_json(data_list, 'utf-8')
else:
fpaths = tools.txt_to_list(data_list)
bfile_list = True
elif os.path.isdir(data_list):
fpaths = []
bfile_list = False
else:
logger.error("Something wrong with data_list input, please check")
sys.exit(0)
ignore_keys = []
if len(args.ignore_key) > 1:
ignore_keys = args.ignore_key.split(',')
all_pass = False
if args.check_output:
logger.debug(args.ref_output)
if os.path.isdir(args.ref_output):
fn_list = sorted(
tools.find_files(dir_path=args.ref_output,
keyword=None,
suffix=('.json'),
walkin=True))
ref_output = []
for fn in fn_list:
ref_output.append(tools.parse_json(fn, 'utf-8'))
elif os.path.isfile(args.ref_output):
ref_output = tools.parse_json(args.ref_output, 'utf-8')
else:
logger.error("Something wrong with reference output, please check")
sys.exit(0)
all_pass = True
benchmark = False
if args.benchmark:
benchmark = True
if bfile_list and args.loop_over_input and args.multi_channels:
for fi in range(len(fpaths)):
ext_data = []
ext_meta = []
for ci in range(len(fpaths[fi])):
full_path = fpaths[fi][ci]
logger.debug(full_path)
if args.read_frame:
logger.debug('Reading frame for producing binary input')
fr.reset()
fr.input_data = [full_path]
fr.run()
ext_data.append(fr.output_data[0])
else:
ext_data.append(full_path)
ext_meta.append(fpaths[fi][ci])
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(ext_data,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
if not isinstance(pipeline.output, list):
tar_list = [pipeline.output]
ref_list = [ref_output[fi]]
else:
tar_list = pipeline.output
ref_list = ref_output[fi]
for ci, tar_data in enumerate(tar_list):
all_pass = check_result(tar_data,
ref_list[ci],
full_path,
all_pass,
ignore_keys=ignore_keys)
elif bfile_list and args.loop_over_input:
counter = 0
wrong = 0
for fi in range(len(fpaths)):
counter = counter + 1
full_path = fpaths[fi]
logger.debug(full_path)
base_name = tools.remove_extension(full_path,
return_type='base-only')
# Assign external data and metadata
if args.do_not_pack:
ext_data = full_path
else:
ext_data = [full_path]
if args.read_frame:
logger.debug('Reading frame for producing binary input')
fr.reset()
fr.input_data = [full_path]
fr.run()
ext_data = fr.output_data[0]
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(ext_data,
base_name=base_name,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output,
ref_output[fi],
full_path,
all_pass,
ignore_keys=ignore_keys)
elif bfile_list:
ext_meta = []
if args.read_meta:
logger.debug('Reading json for producing binary meta')
for full_path in fpaths:
if args.repeated_metadata_path == '':
meta_path = tools.remove_extension(full_path) + '.json'
else:
meta_path = args.repeated_metadata_path
try:
ext_meta.append(tools.parse_json(meta_path, 'utf-8'))
except BaseException:
logger.error('Fail to parse %s' % meta_path)
sys.exit(0)
pipeline.run(fpaths,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output,
ref_output,
fpaths,
all_pass,
ignore_keys=ignore_keys)
else:
full_path = data_list
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(full_path,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output[0],
ref_output[0],
fpaths,
all_pass,
ignore_keys=ignore_keys)
if args.check_output is True and all_pass is True:
print("Pass all test data in input data list.")
print("Lab pipeline launcher completes successfully")