def test_main_process(self):
# pull test data from gitlab
config_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'79187835280b7adc8149e5e9a8e6ec9c/DeterminatorTargetLabel.config'
dyda_config = lab_tools.pull_json_from_gitlab(config_url)
input_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'4cdb639a0f0862f504e3223713da1c49/'\
'DeterminatorTargetLabel_input.json'
input_data = lab_tools.pull_json_from_gitlab(input_url)
output_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'fd45424cee6c68c7de28fc16f6af734d/'\
'DeterminatorTargetLabel_output.json'
output_data = lab_tools.pull_json_from_gitlab(output_url)
# initialization
determinator_ = DeterminatorTargetLabel(dyda_config_path=dyda_config)
# run determinator
determinator_.reset()
determinator_.metadata[0] = tools.remove_extension(
input_data["filename"], 'base-only')
determinator_.input_data.append(input_data)
determinator_.run()
# compare results with reference
ref_path = output_data
tar_path = determinator_.results[0]
report = dict_comparator.get_diff(ref_path, tar_path)
self.assertEqual(report['extra_field'], [])
self.assertEqual(report['missing_field'], [])
self.assertEqual(report['mismatch_val'], [])
def test_main_process(self):
# pull test data from gitlab
config_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'2b588582c1ad5b901f6b4cc1c8f81e15/dyda.config'
dyda_config = lab_tools.pull_json_from_gitlab(config_url)
input_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'41ed033f30b132a0a48d7ba214f75c41/00000015.json'
input_data = lab_tools.pull_json_from_gitlab(input_url)
output_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'73b6ab545229971f1149ade93dc1ca8f/00000015.json'
output_data = lab_tools.pull_json_from_gitlab(output_url)
# initialization
determinator_ = DeterminatorConfidenceThreshold(dyda_config)
# run determinator
determinator_.reset()
determinator_.metadata[0] = tools.remove_extension(
input_data["filename"], 'base-only')
determinator_.input_data.append(input_data)
determinator_.run()
# compare results with reference
ref_path = output_data
tar_path = determinator_.results
report = dict_comparator.get_diff(ref_path, tar_path)
self.assertEqual(report['extra_field'], [])
self.assertEqual(report['missing_field'], [])
self.assertEqual(report['mismatch_val'], [])
def read_data(self, input_data, dtype):
"""
Get data from given input_data and dtype and change the
base_name (the first element of metadata) to the base filename without
extension if it is still empty
@param input_data: can be path of the data or a numpy array
@dtype: type of the data
-- image: use opencv to read the data file
-- array: assign input_data directly to DataReaderBase.
"""
self.results['data_type'] = dtype
opened_data = np.empty(shape=(0, 0))
if dtype == 'image':
opened_data = self.read_image(input_data)
self.results['data_path'].append(input_data)
base_name = tools.remove_extension(input_data,
return_type='base-only')
if self.metadata[0] == "":
self.metadata[0] = base_name
elif dtype == 'array':
opened_data = input_data
else:
print('[dyda] Error: Type %s is not supported' % dtype)
self.terminate_flag = True
self.output_data.append(opened_data)
def test_main_process(self):
# pull test data from gitlab
input_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'06e852b64cd24cdb017a56c540e195db/FrameSelectorDownsampleFirst_input_list.json'
input_list = lab_tools.pull_json_from_gitlab(input_url)
output_url = 'https://gitlab.com/DT42/galaxy42/dt42-dyda/uploads/'\
'6ce81dd346759aef1d8f4cc81675662e/FrameSelectorDownsampleFirst_output_list.json'
output_list = lab_tools.pull_json_from_gitlab(output_url)
# initialization
frame_selector_ = FrameSelectorDownsampleFirst()
for i in range(len(input_list)):
# run frame_selector
frame_selector_.reset()
frame_selector_.metadata[0] = tools.remove_extension(
input_list[i]['filename'], 'base-only')
frame_selector_.run()
# compare results with reference
ref_data = output_list[i]
tar_data = frame_selector_.results
report = dict_comparator.get_diff(ref_data, tar_data)
self.assertEqual(report['extra_field'], [])
self.assertEqual(report['missing_field'], [])
self.assertEqual(report['mismatch_val'], [])
def read_meta_single(args, logger, full_path):
ext_meta = []
if args.read_meta:
logger.debug('Reading json for producing binary meta')
if args.repeated_metadata_path == '':
meta_path = tools.remove_extension(full_path) + '.json'
else:
meta_path = args.repeated_metadata_path
try:
ext_meta = tools.parse_json(meta_path, 'utf-8')
except BaseException:
logger.error('Fail to parse %s' % meta_path)
sys.exit(0)
return ext_meta
def main_process(self):
""" Main function called by the external code """
encoding = None
if "encoding" in self.param.keys():
encoding = self.param["encoding"]
if isinstance(self.input_data, str):
json_path = self.check_folder(self.input_data)
self.results = self.get_metadata(json_path)
self.metadata[0] = tools.remove_extension(self.input_data,
return_type='base-only')
elif isinstance(self.input_data, list):
for json_path in self.input_data:
json_path = self.check_folder(json_path)
self.results.append(self.get_metadata(json_path))
return True
def main_process(self):
frame_selector_result = self.input_data[0]
inferencer_result = self.input_data[1]
ref_data_name = frame_selector_result['ref_data_name']
base_name = frame_selector_result['base_name']
self.result_list.append(inferencer_result)
found = False
# The (-1)s below means process the list from bottom to top
# in order to make sure the index is unchanged after pop some elements.
for i in range(len(self.result_list) - 1, -1, -1):
if found:
self.result_list.pop(i)
elif 'filename' in self.result_list[i].keys():
filename = tools.remove_extension(
self.result_list[i]['filename'])
if filename == ref_data_name:
found = True
self.results = copy.deepcopy(self.result_list[i])
self.results['filename'] = base_name
def main_process(self):
""" main process """
if not os.path.isdir(self.input_data):
self.logger.error("%s is not a valid folder" % self.input_data)
self.terminate_flag = True
if self.overwrite:
output = open(self.output_file, 'w')
else:
if tools.check_exist(self.output_file):
output = open(self.output_file, 'a')
else:
output = open(self.output_file, 'w')
print("LearnerYOLO: creating %s" % self.output_file)
check_keys = ["folder", "filename", "annotations"]
for json_file in tools.find_files(self.input_data, walkin=False):
try:
json_content = tools.parse_json(json_file)
except BaseException:
self.logger.error("Fail to open %s" % json_file)
continue
for key in check_keys:
if key not in json_content.keys():
self.logger.error("%s is not found in %s" %
(key, json_file))
continue
folder = json_content["folder"]
filename = json_content["filename"]
# FIXME
# folder = folder.replace("results", "labeled_data")
# folder = folder.replace("_tmp", "")
in_img_path = os.path.join(folder, filename)
out_img_path = os.path.join(self.img_folder, filename)
o_file_path = os.path.join(
self.label_folder,
tools.remove_extension(filename) + '.txt')
o_file = open(o_file_path, 'w')
annos = json_content["annotations"]
size, pix = image.get_img_info(in_img_path)
h = float(size[0])
w = float(size[1])
for anno in annos:
X = []
Y = []
cls = anno["label"]
if cls not in self.classes:
self.logger.debug("%s is not in the selected class" % cls)
continue
cls_id = self.classes.index(cls)
X = [anno["left"], anno["right"]]
Y = [anno["top"], anno["bottom"]]
bb = self.convert((w, h), X, Y)
o_file.write(
str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
o_file.close()
self.logger.info("Link %s to %s" % (in_img_path, out_img_path))
os.symlink(in_img_path, out_img_path)
output.write(out_img_path + '\n')
output.close()
# FIXME darknet env has to be well prepared and fix the classes now
train_path = os.path.join(self.darknet_path, "train.txt")
if train_path != self.output_file:
copyfile(self.output_file, train_path)
os.chdir(self.darknet_path)
cmd = ("./darknet detector train cfg/dt42.data"
" cfg/tiny-yolo-voc-dt42.cfg darknet.weights.13 -gpus 1")
self.logger.info("Running %s " % cmd)
output = subprocess.check_output(["bash", "-c", cmd])
self.results = {
"root_directory": self.darknet_path,
"weight_file": "backup_dt42/yolo-voc-dt42_final.weights",
"data_file": "cfg/dt42.data",
"names_file": "data/dt42.names",
"cfg_file": "cfg/tiny-yolo-voc-dt42.cfg"
}
def main():
""" main function to run pipeline """
args = get_args()
logger = logging.getLogger('launcher')
log_level = logging.WARNING
if args.verbosity == 1:
log_level = logging.INFO
elif args.verbosity >= 2:
log_level = logging.DEBUG
formatter = logging.Formatter('[launcher] %(levelname)s %(message)s')
console = logging.StreamHandler()
console.setFormatter(formatter)
logger.setLevel(log_level)
logger.addHandler(console)
logger.info('lab_flag is %r' % args.lab_flag)
pipeline = dt42pl.Pipeline(args.pipeline_config,
dyda_config_path=args.dyda_config_path,
parent_result_folder=args.output,
verbosity=args.verbosity,
lab_flag=args.lab_flag,
force_run_skip=args.force_run_skip)
if args.read_frame:
fr = frame_reader.FrameReader()
# looping over input data paths
logger.info('Running Reader and Selector for frames')
data_list = args.data_list
if args.json_list:
logger.warning('json_list will replace -d/--data_list argument')
data_list = args.json_list
force_snapshot = False
if args.force_snapshot:
force_snapshot = True
bfile_list = False
if args.direct_input:
fpaths = data_list
elif os.path.isfile(data_list):
if tools.check_ext(data_list, ".json"):
fpaths = tools.parse_json(data_list, 'utf-8')
else:
fpaths = tools.txt_to_list(data_list)
bfile_list = True
elif os.path.isdir(data_list):
fpaths = []
bfile_list = False
else:
logger.error("Something wrong with data_list input, please check")
sys.exit(0)
ignore_keys = []
if len(args.ignore_key) > 1:
ignore_keys = args.ignore_key.split(',')
all_pass = False
if args.check_output:
logger.debug(args.ref_output)
if os.path.isdir(args.ref_output):
fn_list = sorted(
tools.find_files(dir_path=args.ref_output,
keyword=None,
suffix=('.json'),
walkin=True))
ref_output = []
for fn in fn_list:
ref_output.append(tools.parse_json(fn, 'utf-8'))
elif os.path.isfile(args.ref_output):
ref_output = tools.parse_json(args.ref_output, 'utf-8')
else:
logger.error("Something wrong with reference output, please check")
sys.exit(0)
all_pass = True
benchmark = False
if args.benchmark:
benchmark = True
if bfile_list and args.loop_over_input and args.multi_channels:
for fi in range(len(fpaths)):
ext_data = []
ext_meta = []
for ci in range(len(fpaths[fi])):
full_path = fpaths[fi][ci]
logger.debug(full_path)
if args.read_frame:
logger.debug('Reading frame for producing binary input')
fr.reset()
fr.input_data = [full_path]
fr.run()
ext_data.append(fr.output_data[0])
else:
ext_data.append(full_path)
ext_meta.append(fpaths[fi][ci])
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(ext_data,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
if not isinstance(pipeline.output, list):
tar_list = [pipeline.output]
ref_list = [ref_output[fi]]
else:
tar_list = pipeline.output
ref_list = ref_output[fi]
for ci, tar_data in enumerate(tar_list):
all_pass = check_result(tar_data,
ref_list[ci],
full_path,
all_pass,
ignore_keys=ignore_keys)
elif bfile_list and args.loop_over_input:
counter = 0
wrong = 0
for fi in range(len(fpaths)):
counter = counter + 1
full_path = fpaths[fi]
logger.debug(full_path)
base_name = tools.remove_extension(full_path,
return_type='base-only')
# Assign external data and metadata
if args.do_not_pack:
ext_data = full_path
else:
ext_data = [full_path]
if args.read_frame:
logger.debug('Reading frame for producing binary input')
fr.reset()
fr.input_data = [full_path]
fr.run()
ext_data = fr.output_data[0]
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(ext_data,
base_name=base_name,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output,
ref_output[fi],
full_path,
all_pass,
ignore_keys=ignore_keys)
elif bfile_list:
ext_meta = []
if args.read_meta:
logger.debug('Reading json for producing binary meta')
for full_path in fpaths:
if args.repeated_metadata_path == '':
meta_path = tools.remove_extension(full_path) + '.json'
else:
meta_path = args.repeated_metadata_path
try:
ext_meta.append(tools.parse_json(meta_path, 'utf-8'))
except BaseException:
logger.error('Fail to parse %s' % meta_path)
sys.exit(0)
pipeline.run(fpaths,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output,
ref_output,
fpaths,
all_pass,
ignore_keys=ignore_keys)
else:
full_path = data_list
ext_meta = read_meta_single(args, logger, full_path)
pipeline.run(full_path,
external_meta=ext_meta,
benchmark=benchmark,
force_snapshot=force_snapshot)
if args.check_output:
all_pass = check_result(pipeline.output[0],
ref_output[0],
fpaths,
all_pass,
ignore_keys=ignore_keys)
if args.check_output is True and all_pass is True:
print("Pass all test data in input data list.")
print("Lab pipeline launcher completes successfully")