def test_mask_transformation(self):
"""Keep countours near ROIs: Test that when using the basic mask generated script this tool produces a mask"""
op = IptKeepCountoursNearRois()
op.apply_test_values_overrides(use_cases=("Mask cleanup",))
script = LoosePipeline.load(
"./ipso_phen/ipapi/samples/pipelines/test_cleaners.json"
)
script.add_module(operator=op, target_group="grp_test_cleaners")
wrapper = BaseImageProcessor(
"./ipso_phen/ipapi/samples/images/arabido_small.jpg",
database=None,
)
res = script.execute(src_image=wrapper, silent_mode=True)
self.assertTrue(
res, "Failed to process Keep countours near ROIs with test script"
)
self.assertIsInstance(
wrapper.mask, np.ndarray, "Empty result for Range threshold"
)
self.assertEqual(len(wrapper.mask.shape), 2, "Masks can only have one channel")
self.assertEqual(
np.sum(wrapper.mask[wrapper.mask != 255]),
0,
"Masks values can only be 0 or 255",
)
def test_feature_out(self):
"""Split overlapped ellipses: "Test that when using the basic mask generated script this tool extracts features"""
op = IptSplitOverlappedEllipses()
op.apply_test_values_overrides(use_cases=("Feature extraction", ))
script = LoosePipeline.load(
"./ipso_phen/ipapi/samples/pipelines/test_extractors.json")
script.add_module(operator=op, target_group="grp_test_extractors")
wrapper = BaseImageProcessor(
"./ipso_phen/ipapi/samples/images/arabido_small.jpg",
database=None,
)
res = script.execute(src_image=wrapper, silent_mode=True)
self.assertIsInstance(
op,
IptBaseAnalyzer,
"Split overlapped ellipses must inherit from ipso_phen.ipapi.iptBaseAnalyzer",
)
self.assertTrue(
res,
"Failed to process Split overlapped ellipses with test script")
self.assertNotEqual(
first=len(wrapper.csv_data_holder.data_list),
second=0,
msg="Split overlapped ellipses returned no data",
)
def test_extractors_pipeline(self):
"""Loose pipeline: Test extractors's test pipeline"""
pipeline = LoosePipeline.load(
os.path.join(
self.pipeline_dir_path,
"test_extractors.json",
))
wrapper = BaseImageProcessor(
os.path.join(
os.path.dirname(__file__),
"..",
"ipso_phen",
"ipapi",
"samples",
"images",
"arabido_small.jpg",
),
database=None,
)
res = pipeline.execute(src_image=wrapper, silent_mode=True)
self.assertTrue(
res,
"Failed to process Keep countours near ROIs with test pipeline")
self.assertIsInstance(wrapper.mask, np.ndarray,
"Empty result for Range threshold")
self.assertEqual(len(wrapper.mask.shape), 2,
"Masks can only have one channel")
self.assertEqual(
np.sum(wrapper.mask[wrapper.mask != 255]),
0,
"Masks values can only be 0 or 255",
)
def test_image_transformation(self):
"""Crop: Test that when an image is in an image goes out"""
op = IptCrop()
op.apply_test_values_overrides(use_cases=("Pre processing", ))
wrapper = BaseImageProcessor(
"./ipso_phen/ipapi/samples/images/arabido_small.jpg",
database=None,
)
res = op.process_wrapper(wrapper=wrapper)
self.assertTrue(res, "Failed to process Crop")
self.assertIsInstance(op.result, np.ndarray, "Empty result for Crop")
"""Crop: "Test that when using the basic mask generated script this tool extracts features"""
op = IptCrop()
op.apply_test_values_overrides(use_cases=("Feature extraction", ))
script = LoosePipeline.load(
"./ipso_phen/ipapi/samples/pipelines/test_extractors.json")
script.add_module(operator=op, target_group="grp_test_extractors")
wrapper = BaseImageProcessor(
"./ipso_phen/ipapi/samples/images/arabido_small.jpg",
database=None,
)
res = script.execute(src_image=wrapper, silent_mode=True)
self.assertIsInstance(
op,
IptBaseAnalyzer,
"Crop must inherit from ipso_phen.ipapi.iptBaseAnalyzer",
)
self.assertTrue(res, "Failed to process Crop with test script")
self.assertNotEqual(
first=len(wrapper.csv_data_holder.data_list),
second=0,
msg="Crop returned no data",
)
def test_mask_transformation(self):
"""Clean horizontal noise (Hough method): Test that when using the basic mask generated script this tool produces a mask"""
op = IptCleanHorizontalNoiseHough()
op.apply_test_values_overrides(use_cases=("Mask cleanup", ))
script = LoosePipeline.load(
"./ipso_phen/ipapi/samples/pipelines/test_cleaners.json")
script.add_module(operator=op, target_group="grp_test_cleaners")
wrapper = BaseImageProcessor(
"./ipso_phen/ipapi/samples/images/18HP01U17-CAM11-20180712221558.bmp",
database=None,
)
res = script.execute(src_image=wrapper, silent_mode=True)
self.assertTrue(
res,
"Failed to process Clean horizontal noise (Hough method) with test script",
)
self.assertIsInstance(wrapper.mask, np.ndarray,
"Empty result for Range threshold")
self.assertEqual(len(wrapper.mask.shape), 2,
"Masks can only have one channel")
self.assertEqual(
np.sum(wrapper.mask[wrapper.mask != 255]),
0,
"Masks values can only be 0 or 255",
)
def test_load(self):
"""Loose pipeline: Try loading all test pipelines"""
for file_name in [
"test_cleaners.json",
"sample_pipeline_arabidopsis.json",
"test_extractors.json",
"tutorial.json",
]:
pipeline = LoosePipeline.load(
os.path.join(
self.pipeline_dir_path,
"test_cleaners.json",
))
self.assertIsInstance(pipeline,
LoosePipeline,
msg=f'Failed to load "{file_name}"')
def prepare_process_muncher(progress_callback, abort_callback, **kwargs):
dbi = DbInfo.from_json(
json_data=json.loads(kwargs["database_info"].replace("'", '"'))
)
output_folder = get_user_path(
user_name=kwargs["current_user"],
key="analysis_folder",
extra=dbi.display_name.lower(),
)
database = db_info_to_database(dbi)
pp = PipelineProcessor(
dst_path=output_folder,
overwrite=kwargs["overwrite_existing"],
seed_output=False,
group_by_series=kwargs["generate_series_id"],
store_images=False,
database=database,
)
pp.progress_callback = progress_callback
pp.abort_callback = abort_callback
pp.ensure_root_output_folder()
pp.grab_files_from_data_base(
experiment=dbi.display_name.lower(),
**database.main_selector,
)
pp.script = LoosePipeline.from_json(json_data=kwargs["script"])
if not pp.accepted_files:
return {
"current": 100,
"total": 100,
"status": "No images in task!",
"result": 42,
}
try:
pp.multi_thread = int(kwargs.get("thread_count", 1))
except:
pp.multi_thread = False
return {
"pipeline_processor": pp,
"output_folder": output_folder,
}
def launch(**kwargs):
def exit_error_message(msg: str) -> None:
print(msg)
logger.error(msg)
start = timer()
if "user" in kwargs and "password" in kwargs:
dbp.master_password = kwargs.pop("user"), kwargs.pop("password")
# Script
script = kwargs.get("script", None)
if script is not None and os.path.isfile(script):
with open(script, "r") as f:
script = json.load(f)
kwargs["script"] = script
# Image(s)
image = kwargs.get("image", None)
image_list = kwargs.get("image_list", None)
image_folder = kwargs.get("image_folder", None)
src_count = (1 if "experiment" in kwargs and "database" in kwargs else len(
[src for src in [image, image_list, image_folder] if src is not None]))
if src_count == 0:
exit_error_message("Missing source images")
return 1
elif src_count > 1:
exit_error_message("Too many sources")
return 1
if image is not None:
kwargs["images"] = [image]
elif image_list is not None:
with open(image_list, "r") as f:
kwargs["images"] = [
img.os.replace("\n", "") for img in f.readlines()
]
elif image_folder is not None:
img_lst = ImageList((".jpg", ".tiff", ".png", ".bmp"))
img_lst.add_folder(image_folder)
kwargs["images"] = img_lst.filter(masks=None)
# State
stored_state = kwargs.pop("stored_state", None)
res = restore_state(blob=stored_state, overrides=kwargs)
# Retrieve images
image_list_ = res.get("images", None)
# Build database
db_data = res.get("database_data", None)
database = res.get("database", None)
experiment = res.get("experiment", None)
if db_data is None:
if database is None:
db = None
else:
db = dbf.db_info_to_database(
DbInfo(
display_name=experiment,
target=database,
dbms="pandas",
))
else:
db = dbf.db_info_to_database(dbb.DbInfo(**db_data))
if experiment is not None:
if "sub_folder_name" not in res or not res["sub_folder_name"]:
res["sub_folder_name"] = experiment
if "csv_file_name" not in res or not res["csv_file_name"]:
res["csv_file_name"] = f"{experiment.lower()}_raw_data"
else:
db = dbf.db_info_to_database(dbb.DbInfo(**db_data))
# Retrieve output folder
output_folder_ = res.get("output_folder", None)
if not output_folder_:
exit_error_message("Missing output folder")
return 1
elif res.get("sub_folder_name", ""):
output_folder_ = os.path.join(output_folder_, res["sub_folder_name"],
"")
else:
output_folder_ = os.path.join(output_folder_, "")
force_directories(output_folder_)
csv_file_name = res.get("csv_file_name", None)
if not csv_file_name:
exit_error_message("Missing output file name")
return 1
if IS_USE_MULTI_THREAD and "thread_count" in res:
try:
mpc = int(res["thread_count"])
except Exception as e:
mpc = False
else:
mpc = False
try:
if isinstance(res["script"], str) and os.path.isfile(res["script"]):
script = LoosePipeline.load(res["script"])
elif isinstance(res["script"], dict):
script = LoosePipeline.from_json(json_data=res["script"])
else:
exit_error_message("Failed to load script: Unknown error")
return 1
except Exception as e:
exit_error_message(f"Failed to load script: {repr(e)}")
return 1
# Build pipeline processor
pp = PipelineProcessor(
database=None if db is None else db.copy(),
dst_path=output_folder_,
overwrite=res["overwrite"],
seed_output=res["append_time_stamp"],
group_by_series=res["generate_series_id"],
store_images=False,
)
if not image_list_:
pp.grab_files_from_data_base(
experiment=db.db_info.display_name.lower(),
order_by="plant",
**db.main_selector,
)
else:
pp.accepted_files = image_list_
if res.get("randomize", False) is True:
random.shuffle(pp.accepted_files)
logger.info("Process summary")
logger.info("_______________")
logger.info(f'database: {res.get("database_data", None)}')
logger.info(f'Output folder: {res["output_folder"]}')
logger.info(f'CSV file name: {res["csv_file_name"]}')
logger.info(f'Overwrite data: {res["overwrite"]}')
logger.info(f'Subfolder name: {res["sub_folder_name"]}')
logger.info(f'Append timestamp to root folder: {res["append_time_stamp"]}')
logger.info(f'Generate series ID: {res["generate_series_id"]}')
logger.info(f'Series ID time delta allowed: {res["series_id_time_delta"]}')
logger.info(f'Build annotation ready CSV: {res["build_annotation_csv"]}')
logger.info(f"Images: {len(pp.accepted_files)}")
logger.info(f"Concurrent processes count: {mpc}")
logger.info(f"Script summary: {str(script)}")
if pp.accepted_files is None or len(pp.accepted_files) < 1:
exit_error_message("No images to process")
return 1
pp.progress_callback = kwargs.get("progress_callback", None)
pp.error_callback = kwargs.get("error_callback", None)
pp.ensure_root_output_folder()
pp.script = script
if not pp.accepted_files:
logger.error("Nothing to precess")
return 1
# Process data
groups_to_process = pp.prepare_groups(res["series_id_time_delta"])
if res["build_annotation_csv"]:
try:
if pp.options.group_by_series:
files, luids = map(list, zip(*groups_to_process))
wrappers = [
file_handler_factory(files[i], db)
for i in [luids.index(x) for x in set(luids)]
]
else:
wrappers = [
file_handler_factory(f, db)
for f in tqdm.tqdm(groups_to_process,
desc="Building annotation CSV")
]
pd.DataFrame.from_dict({
"plant": [i.plant for i in wrappers],
"date_time": [i.date_time for i in wrappers],
"disease_index": "",
}).sort_values(by=["plant", "date_time"],
axis=0,
na_position="first",
ascending=True).to_csv(
os.path.join(
pp.options.dst_path,
f"{csv_file_name}_diseaseindex.csv"),
index=False,
)
except Exception as e:
preffix = "FAIL"
logger.exception("Unable to build disease index file")
else:
preffix = "SUCCESS"
logger.info("Built disease index file")
print(f"{preffix} - Disease index file")
groups_to_process_count = len(groups_to_process)
if groups_to_process_count > 0:
pp.multi_thread = mpc
pp.process_groups(groups_list=groups_to_process)
# Merge dataframe
pp.merge_result_files(csv_file_name=csv_file_name + ".csv")
logger.info(
f"Processed {groups_to_process_count} groups/images in {format_time(timer() - start)}"
)
# Build videos
print("Done, see logs for more details")
return 0