def load_image_data(obj_input, resize=1):
"""
Create a DataFreame with image information (e.g. dimensions).
Parameters
----------
obj_input: str or ndarray
can be a path to an image stored on the harddrive OR an array already
loaded to Python.
resize: float
resize factor for the image (1 = 100%, 0.5 = 50%, 0.1 = 10% of
original size). gets stored to a DataFrame column
Returns
-------
image_data: dict
contains image data (+meta data, if selected)
"""
if obj_input.__class__.__name__ == "str":
if os.path.isfile(obj_input):
path = obj_input
elif os.path.isdir(obj_input):
attr = _load_yaml(os.path.join(obj_input, "attributes.yaml"))
path = attr["project"]["raw_path"]
image = Image.open(path)
width, height = image.size
image.close()
image_data = {
"filename": os.path.split(obj_input)[1],
"filepath": obj_input,
"filetype": os.path.splitext(obj_input)[1],
"width": int(width * resize),
"height": int(height * resize),
"size_ratio_original": resize,
}
elif obj_input.__class__.__name__ == "ndarray":
image = obj_input
width, height = image.shape[0:2]
image_data = {
"filename": "unknown",
"filepath": "unknown",
"filetype": "ndarray",
"width": int(width * resize),
"height": int(height * resize),
"size_ratio_original": resize,
}
else:
warnings.warn("Not a valid image file - cannot read image data.")
## issue warnings for large images
if width * height > 125000000:
warnings.warn("Large image - expect slow processing and consider \
resizing.")
elif width * height > 250000000:
warnings.warn("Extremely large image - expect very slow processing \
and consider resizing.")
## return image data
return image_data
def load(self, dirpath=None, save_suffix=None, contours=False, **kwargs):
"""
Autoload function for container: loads results files with given save_suffix
into the container. Can be used manually, but is typically used within the
pype routine.
Parameters
----------
save_suffix : str, optional
suffix to include when looking for files to load
"""
files, loaded = [], []
## data flags
flag_contours = contours
## check dirpath
if (dirpath.__class__.__name__ == "NoneType"
and not self.dirpath.__class__.__name__ == "NoneType"):
dirpath = self.dirpath
if dirpath.__class__.__name__ == "NoneType":
print(
'No save directory ("dirpath") specified - cannot load files.')
return
if not os.path.isdir(dirpath):
print("Directory does not exist - cannot load files.")
return
## check save_suffix
if (save_suffix.__class__.__name__ == "NoneType"
and not self.save_suffix.__class__.__name__ == "NoneType"):
save_suffix = "_" + self.save_suffix
elif not save_suffix.__class__.__name__ == "NoneType":
save_suffix = "_" + save_suffix
else:
save_suffix = ""
# collect
if len(os.listdir(dirpath)) > 0:
for file in os.listdir(dirpath):
if os.path.isfile(os.path.join(dirpath, file)):
if (len(save_suffix) > 0 and save_suffix in file
and not "pype_config" in file):
files.append(file[0:file.rindex("_")])
elif len(save_suffix) == 0:
files.append(file[0:file.rindex(".")])
else:
print("No files found in given directory")
return
## load attributes
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
## drawing
if not hasattr(self, "df_draw"):
attr = _load_yaml(attr_path)
if "drawing" in attr:
self.df_draw = pd.DataFrame(attr["drawing"], index=[0])
loaded.append("drawing loaded from attributes.yaml")
## other data
if not hasattr(self, "df_other_data"):
attr = _load_yaml(attr_path)
if "other" in attr:
self.df_other_data = pd.DataFrame(attr["other"], index=[0])
loaded.append("columns " +
", ".join(list(self.df_other_data)) +
" from attributes.yaml")
## scale
if not hasattr(self, "scale_template_px_mm_ratio"):
attr = _load_yaml(attr_path)
if "scale" in attr:
if "template_px_mm_ratio" in attr["scale"]:
self.scale_template_px_mm_ratio = attr["scale"][
"template_px_mm_ratio"]
loaded.append(
"template scale information loaded from attributes.yaml"
)
if "current_px_mm_ratio" in attr["scale"]:
self.scale_current_px_mm_ratio = attr["scale"][
"current_px_mm_ratio"]
loaded.append(
"current scale information loaded from attributes.yaml"
)
if "template_path" in attr["scale"]:
if os.path.isfile(attr["scale"]["template_path"]):
self.scale_template = cv2.imread(
attr["scale"]["template_path"])
loaded.append(
"template loaded from root directory")
else:
print("cannot read template image")
## contours
if flag_contours:
if not hasattr(self, "df_contours") and "contours" in files:
path = os.path.join(dirpath, "contours" + save_suffix + ".csv")
if os.path.isfile(path):
df = pd.read_csv(path,
converters={"center": ast.literal_eval})
if "x" in df:
df["coords"] = list(zip(df.x, df.y))
coords = df.groupby("contour")["coords"].apply(list)
coords_arr = _contours_tup_array(coords)
df.drop(columns=["coords", "x", "y"], inplace=True)
df = df.drop_duplicates().reset_index()
df["coords"] = pd.Series(coords_arr, index=df.index)
self.df_contours = df
loaded.append("contours" + save_suffix + ".csv")
else:
print(
"Could not load contours - df saved without coordinates."
)
## landmarks
if not hasattr(self, "df_landmarks") and "landmarks" in files:
path = os.path.join(dirpath, "landmarks" + save_suffix + ".csv")
if os.path.isfile(path):
self.df_landmarks = pd.read_csv(path)
loaded.append("landmarks" + save_suffix + ".csv")
## polylines
if not hasattr(self, "df_polylines") and "polylines" in files:
path = os.path.join(dirpath, "polylines" + save_suffix + ".csv")
if os.path.isfile(path):
self.df_polylines = pd.read_csv(path)
loaded.append("polylines" + save_suffix + ".csv")
## masks
if not hasattr(self, "df_masks") and "masks" in files:
path = os.path.join(dirpath, "masks" + save_suffix + ".csv")
if os.path.isfile(path):
self.df_masks = pd.read_csv(path)
loaded.append("masks" + save_suffix + ".csv")
## feedback
if len(loaded) > 0:
print("AUTOLOAD\n- " + "\n- ".join(loaded))
else:
print("Nothing loaded.")
def save_scale(obj_input, overwrite=True, dirpath=None):
"""
Save a created or detected scale to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite csv if it already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ in ["int", "float"]:
scale_current_px_mm_ratio = obj_input
elif obj_input.__class__.__name__ == "container":
if hasattr(obj_input, "scale_current_px_mm_ratio"):
scale_current_px_mm_ratio = obj_input.scale_current_px_mm_ratio
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No scale supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "scale" in attr:
attr["scale"] = {}
## check if file exists
while True:
if "current_px_mm_ratio" in attr["scale"] and flag_overwrite == False:
print("- scale not saved (overwrite=False)")
break
elif "current_px_mm_ratio" in attr["scale"] and flag_overwrite == True:
print("- save scale to attributes (overwriting)")
pass
else:
print("- save scale to attributes")
pass
attr["scale"]["current_px_mm_ratio"] = scale_current_px_mm_ratio
break
_save_yaml(attr, attr_path)
def load_directory(directory_path,
cont=True,
df=True,
meta=True,
resize=1,
save_suffix=None,
**kwargs):
"""
Parameters
----------
directory_path: str or ndarray
path to a phenopype project directory containing raw image, attributes
file, masks files, results df, etc.
cont: bool, optional
should the loaded image (and DataFrame) be returned as a phenopype
container
df: bool, optional
should a DataFrame containing image information (e.g. dimensions)
be returned.
meta: bool, optional
should the DataFrame encompass image meta data (e.g. from exif-data).
This works only when obj_input is a path string to the original file.
resize: float, optional
resize factor for the image (1 = 100%, 0.5 = 50%, 0.1 = 10% of
original size).
save_suffix : str, optional
suffix to append to filename of results files
kwargs:
developer options
Returns
-------
container
A phenopype container is a Python class where loaded images,
dataframes, detected contours, intermediate output, etc. are stored
so that they are available for inspection or storage at the end of
the analysis.
"""
## kwargs
flag_df = df
flag_container = cont
exif_fields = kwargs.get("fields", default_meta_data_fields)
if not exif_fields.__class__.__name__ == "list":
exif_fields = [exif_fields]
## check if directory
if not os.path.isdir(directory_path):
sys.exit("Not a valid phenoype directory - cannot load files.")
## load attributes-file
attr = _load_yaml(os.path.join(directory_path, "attributes.yaml"))
## build
## legacy
paths = [
attr["project"]["raw_path"],
os.path.join(directory_path,
os.path.basename(attr["project"]["raw_path"])),
(glob.glob(os.path.join(directory_path, "raw*"))[0]
if len(glob.glob(os.path.join(directory_path, "raw*"))) > 0 else
os.path.join(directory_path, "raw.jpg"))
]
for path in paths:
if os.path.isfile(path):
image = cv2.imread(path)
break
df_image_data = pd.DataFrame(
{
"filename": attr["image"]["filename"],
"width": attr["image"]["width"],
"height": attr["image"]["height"],
},
index=[0],
)
if "size_ratio_original" in attr["image"]:
df_image_data["size_ratio_original"] = attr["image"][
"size_ratio_original"]
if "scale" in attr:
if "template_px_mm_ratio" in attr["scale"]:
df_image_data["template_px_mm_ratio"] = attr["scale"][
"template_px_mm_ratio"]
if "current_px_mm_ratio" in attr["scale"]:
df_image_data["current_px_mm_ratio"] = attr["scale"][
"current_px_mm_ratio"]
# ## add meta-data
# if flag_meta:
# exif_data_all, exif_data = attr["meta"], {}
# for field in exif_fields:
# if field in exif_data_all:
# exif_data[field] = exif_data_all[field]
# exif_data = dict(sorted(exif_data.items()))
# df_image_data = pd.concat([df_image_data.reset_index(drop=True),
# pd.DataFrame(exif_data, index=[0])], axis=1)
## return
if flag_container == True:
ct = container(image, df_image_data)
ct.dirpath = directory_path
ct.save_suffix = save_suffix
ct.image_data = attr["image"]
# ## other, saved data to pass on
# if "other" in attr:
# ct.df_other = pd.DataFrame(attr["other"], index=[0])
if flag_container == True:
return ct
elif flag_container == False:
if flag_df:
return image, df_image_data
else:
return image
def save_data_entry(obj_input, overwrite=True, dirpath=None):
"""
Save data entry to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite if entry already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ == "DataFrame":
df = obj_input
elif obj_input.__class__.__name__ == "container":
df = obj_input.df_other_data
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No df supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "other" in attr:
attr["other"] = {}
## check if entry exists
while True:
for col in list(df):
if col in attr["other"] and flag_overwrite == False:
print("- column " + col + " not saved (overwrite=False)")
continue
elif col in attr["other"] and flag_overwrite == True:
print("- add column " + col + " (overwriting)")
pass
else:
print("- add column " + col)
pass
attr["other"][col] = df[col][0]
break
_save_yaml(attr, attr_path)
def save_drawing(obj_input, overwrite=True, dirpath=None):
"""
Save drawing coordinates to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite if drawing already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ == "DataFrame":
df = obj_input
elif obj_input.__class__.__name__ == "container":
df = obj_input.df_draw
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No df supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "drawing" in attr:
attr["drawing"] = {}
## check if file exists
while True:
if "drawing" in attr and flag_overwrite == False:
print("- drawing not saved (overwrite=False)")
break
elif "drawing" in attr and flag_overwrite == True:
print("- drawing saved (overwriting)")
pass
elif not "drawing" in attr:
attr["drawing"] = {}
print("- drawing saved")
pass
for idx, row in df.iterrows():
# if not row["coords"] == attr["drawing"]["coords"]:
attr["drawing"] = dict(row)
_save_yaml(attr, attr_path)
break
def edit_config(
self,
name,
step,
function,
**kwargs
):
"""
[new/experimental] Add or edit functions in all configuration files of a project.
Parameters
----------
name: str
name of config-file. this gets appended to all files and serves as and
identifier of a specific analysis pipeline
step: str
name of the step the function is in
function: str
name of the function
"""
## kwargs
flag_checked = False
## go through project directories
for directory in self.dirpaths:
dirname = os.path.basename(directory)
## save config
preset_path = os.path.join(
self.root_dir, directory, "pype_config_" + name + ".yaml"
)
if os.path.isfile(preset_path):
config = _load_yaml(preset_path)
ordered_steps = ["preprocessing",
"segmentation",
"measurement",
"visualization",
"export"
]
if not step in config.keys():
new_config = ordereddict([("image", ordereddict(config["image"]))])
new_config.update(ordereddict([("pype", ordereddict(config["pype"]))]))
for ordered_step in ordered_steps:
if ordered_step in config:
new_config.update(ordereddict([(ordered_step, config[ordered_step])]))
elif not ordered_step in config and ordered_step == step:
new_config.update(ordereddict([(ordered_step, [function] )]))
else:
new_config = copy.deepcopy(config)
if not function in new_config[step]:
new_config[step].append(function)
if flag_checked == False:
_show_yaml(new_config)
check = input("This is what the new config may look like (can differ beteeen files) - proceed?")
if check in ["True", "true", "y", "yes"]:
flag_checked = True
_save_yaml(new_config, preset_path)
print("New config saved for " + dirname)
else:
print("User check failed - aborting.")
return
def add_scale(self, reference_image, overwrite=False, **kwargs):
"""
Add pype configuration presets to all project directories.
Parameters
----------
reference_image: str
name of template image, either project directory or file link. template
image gets stored in root directory, and information appended to all
attributes files in the project directories
overwrite: bool, optional
overwrite option, if a given pype config-file already exist
template: bool, optional
should a template for scale detection be created. with an existing
template, phenopype can try to find a reference card in a given image,
measure its dimensions, and adjust pixel-to-mm-ratio and colour space
"""
## kwargs
flag_overwrite = overwrite
test_params = kwargs.get("test_params", {})
## load template image
if reference_image.__class__.__name__ == "str":
if os.path.isfile(reference_image):
reference_image = cv2.imread(reference_image)
elif os.path.isdir(os.path.join(self.data_dir, reference_image)):
attr = _load_yaml(
os.path.join(self.data_dir, reference_image, "attributes.yaml")
)
reference_image = cv2.imread(
os.path.join(self.root_dir, attr["project"]["raw_path"])
)
elif reference_image in self.dirnames:
attr = _load_yaml(
os.path.join(self.data_dir, reference_image, "attributes.yaml")
)
reference_image = cv2.imread(attr["project"]["raw_path"])
else:
print("wrong path - cannot load reference image")
return
elif reference_image.__class__.__name__ == "ndarray":
pass
elif reference_image.__class__.__name__ == "int":
reference_image = cv2.imread(self.filepaths[reference_image])
else:
print("wrong type - cannot load reference image")
return
## save template
template_path = os.path.join(self.root_dir, "scale_template.jpg")
while True:
if os.path.isfile(template_path) and flag_overwrite == False:
print(
"- scale template not saved - file already exists (overwrite=False)."
)
break
elif os.path.isfile(template_path) and flag_overwrite == True:
print("- scale template saved under " + template_path + " (overwritten).")
pass
elif not os.path.isfile(template_path):
print("- scale template saved under " + template_path + ".")
pass
## measure scale
px_mm_ratio, df_masks, template = preprocessing.create_scale(
reference_image, template=True, test_params=test_params
)
cv2.imwrite(template_path, template)
break
## save scale information
for directory in self.dirpaths:
attr = _load_yaml(os.path.join(self.root_dir, directory, "attributes.yaml"))
if not "scale" in attr:
print("added scale information to " + attr["project"]["dirname"])
pass
elif "scale" in attr and flag_overwrite:
print(
"added scale information to "
+ attr["project"]["dirname"]
+ " (overwritten)"
)
pass
elif "scale" in attr and not flag_overwrite:
print(
"could not add scale information to "
+ attr["project"]["dirname"]
+ " (overwrite=False)"
)
continue
attr["scale"] = {
"template_px_mm_ratio": px_mm_ratio,
"template_path": template_path,
}
_save_yaml(attr, os.path.join(self.root_dir, directory, "attributes.yaml"))
def add_config(
self,
name,
config_preset=None,
interactive=False,
overwrite=False,
idx=0,
**kwargs
):
"""
Add pype configuration presets to all image folders in the project, either by using
the templates included in the presets folder, or by adding your own templates
by providing a path to a yaml file. Can be tested and modified using the
interactive flag before distributing the config files.
Parameters
----------
name: str
name of config-file. this gets appended to all files and serves as and
identifier of a specific analysis pipeline
preset: str, optional
can be either a string denoting a template name (e.g. preset1, preset2,
landamarking1, ... - in "phenopype/settings/presets.py") or a path to a
compatible yaml file
interactive: bool, optional
start a pype and modify preset before saving it to phenopype directories
overwrite: bool, optional
overwrite option, if a given pype config-file already exist
kwargs:
developer options
"""
## kwargs
flag_interactive = interactive
flag_overwrite = overwrite
## legacy
preset = kwargs.get("preset")
if (
config_preset.__class__.__name__ == "NoneType"
and not preset.__class__.__name__ == "NoneType"
):
config_preset = preset
## load config
if not config_preset.__class__.__name__ == "NoneType" and hasattr(
presets, config_preset
):
config = _create_generic_pype_config(preset=config_preset, config_name=name)
elif not config_preset.__class__.__name__ == "NoneType" and os.path.isfile(
config_preset
):
config = {
"pype": {
"name": name,
"preset": config_preset,
"date_created": datetime.today().strftime("%Y%m%d_%H%M%S"),
}
}
config.update(_load_yaml(config_preset))
print(config)
elif not config_preset.__class__.__name__ == "NoneType" and not hasattr(
presets, config_preset
):
print("Provided preset NOT found - terminating")
return
elif config_preset.__class__.__name__ == "NoneType":
print("No preset provided - defaulting to preset " + default_pype_config)
config = _load_yaml(eval("presets." + default_pype_config))
## modify
if flag_interactive:
image_location = os.path.join(
self.root_dir,
"pype_template_image" + os.path.splitext(self.filenames[idx])[1],
)
copyfile(self.filepaths[idx], image_location)
config_location = os.path.join(
self.root_dir, "pype_config_template-" + name + ".yaml"
)
_save_yaml(config, config_location)
p = pype(
image_location,
name="template-" + name,
config_location=config_location,
presetting=True,
)
config = p.config
## go through project directories
for directory in self.dirpaths:
attr = _load_yaml(os.path.join(self.root_dir, directory, "attributes.yaml"))
pype_preset = {"image": attr["image"]}
pype_preset.update(config)
## save config
preset_path = os.path.join(
self.root_dir, directory, "pype_config_" + name + ".yaml"
)
dirname = attr["project"]["dirname"]
if os.path.isfile(preset_path) and flag_overwrite == False:
print(
"pype_"
+ name
+ ".yaml already exists in "
+ dirname
+ " (overwrite=False)"
)
continue
elif os.path.isfile(preset_path) and flag_overwrite == True:
print("pype_" + name + ".yaml created for " + dirname + " (overwritten)")
_save_yaml(pype_preset, preset_path)
else:
print("pype_" + name + ".yaml created for " + dirname)
_save_yaml(pype_preset, preset_path)
