def save_scale(obj_input, overwrite=True, dirpath=None):
"""
Save a created or detected scale to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite csv if it already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ in ["int", "float"]:
scale_current_px_mm_ratio = obj_input
elif obj_input.__class__.__name__ == "container":
if hasattr(obj_input, "scale_current_px_mm_ratio"):
scale_current_px_mm_ratio = obj_input.scale_current_px_mm_ratio
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No scale supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "scale" in attr:
attr["scale"] = {}
## check if file exists
while True:
if "current_px_mm_ratio" in attr["scale"] and flag_overwrite == False:
print("- scale not saved (overwrite=False)")
break
elif "current_px_mm_ratio" in attr["scale"] and flag_overwrite == True:
print("- save scale to attributes (overwriting)")
pass
else:
print("- save scale to attributes")
pass
attr["scale"]["current_px_mm_ratio"] = scale_current_px_mm_ratio
break
_save_yaml(attr, attr_path)
def save_drawing(obj_input, overwrite=True, dirpath=None):
"""
Save drawing coordinates to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite if drawing already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ == "DataFrame":
df = obj_input
elif obj_input.__class__.__name__ == "container":
df = obj_input.df_draw
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No df supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "drawing" in attr:
attr["drawing"] = {}
## check if file exists
while True:
if "drawing" in attr and flag_overwrite == False:
print("- drawing not saved (overwrite=False)")
break
elif "drawing" in attr and flag_overwrite == True:
print("- drawing saved (overwriting)")
pass
elif not "drawing" in attr:
attr["drawing"] = {}
print("- drawing saved")
pass
for idx, row in df.iterrows():
# if not row["coords"] == attr["drawing"]["coords"]:
attr["drawing"] = dict(row)
_save_yaml(attr, attr_path)
break
def save_data_entry(obj_input, overwrite=True, dirpath=None):
"""
Save data entry to attributes.yaml
Parameters
----------
obj_input : DataFrame or container
input object
overwrite: bool optional
overwrite if entry already exists
dirpath: str, optional
location to save df
"""
## kwargs
flag_overwrite = overwrite
## load df
if obj_input.__class__.__name__ == "DataFrame":
df = obj_input
elif obj_input.__class__.__name__ == "container":
df = obj_input.df_other_data
if (dirpath.__class__.__name__ == "NoneType"
and not obj_input.dirpath.__class__.__name__ == "NoneType"):
dirpath = obj_input.dirpath
else:
print("No df supplied - cannot export results.")
return
## dirpath
if dirpath.__class__.__name__ == "NoneType":
print('No save directory ("dirpath") specified - cannot save result.')
return
else:
if not os.path.isdir(dirpath):
q = input(
"Save folder {} does not exist - create?.".format(dirpath))
if q in ["True", "true", "y", "yes"]:
os.makedirs(dirpath)
else:
print("Directory not created - aborting")
return
## load attributes file
attr_path = os.path.join(dirpath, "attributes.yaml")
if os.path.isfile(attr_path):
attr = _load_yaml(attr_path)
else:
attr = {}
if not "other" in attr:
attr["other"] = {}
## check if entry exists
while True:
for col in list(df):
if col in attr["other"] and flag_overwrite == False:
print("- column " + col + " not saved (overwrite=False)")
continue
elif col in attr["other"] and flag_overwrite == True:
print("- add column " + col + " (overwriting)")
pass
else:
print("- add column " + col)
pass
attr["other"][col] = df[col][0]
break
_save_yaml(attr, attr_path)
def __init__(self, root_dir, overwrite=False):
## kwargs
flag_overwrite = overwrite
## path conversion
root_dir = root_dir.replace(os.sep, "/")
root_dir = os.path.abspath(root_dir)
## feedback
print("--------------------------------------------")
print("Phenopype will create a new project at\n" + root_dir + "\n")
## decision tree if directory exists
while True:
create = input("Proceed? (y/n)\n")
if create == "y" or create == "yes":
if os.path.isdir(root_dir):
if flag_overwrite == True:
rmtree(root_dir, onerror=_del_rw)
print('\n"' + root_dir + '" created (overwritten)')
pass
else:
overwrite = input(
"Warning - project root_dir already exists - overwrite? (y/n)"
)
if overwrite == "y" or overwrite == "yes":
rmtree(root_dir, onerror=_del_rw)
print('\n"' + root_dir + '" created (overwritten)')
pass
else:
print('\n"' + root_dir + '" not created!')
print("--------------------------------------------")
break
else:
pass
else:
print('\n"' + root_dir + '" not created!')
break
## make directories
self.root_dir = root_dir
os.makedirs(self.root_dir)
self.data_dir = os.path.join(self.root_dir, "data")
os.makedirs(self.data_dir)
# ## set working directory
# if not os.path.abspath(root_dir) == os.getcwd():
# os.chdir(root_dir)
# print("Current working directory changed to " + os.path.abspath(root_dir))
# else:
# print("Already in " + os.path.abspath(root_dir))
## generate empty lists
for lst in [
"dirnames",
"dirpaths_rel",
"dirpaths",
"filenames",
"filepaths_rel",
"filepaths",
]:
setattr(self, lst, [])
## global project attributes
project_data = {
"date_created": datetime.today().strftime("%Y%m%d_%H%M%S"),
"date_changed": datetime.today().strftime("%Y%m%d_%H%M%S"),
"root_dir": self.root_dir,
"data_dir": self.data_dir,
}
_save_yaml(project_data, os.path.join(self.root_dir, "attributes.yaml"))
print(
"\nproject attributes written to "
+ os.path.join(self.root_dir, "attributes.yaml")
)
print("--------------------------------------------")
break
def edit_config(
self,
name,
step,
function,
**kwargs
):
"""
[new/experimental] Add or edit functions in all configuration files of a project.
Parameters
----------
name: str
name of config-file. this gets appended to all files and serves as and
identifier of a specific analysis pipeline
step: str
name of the step the function is in
function: str
name of the function
"""
## kwargs
flag_checked = False
## go through project directories
for directory in self.dirpaths:
dirname = os.path.basename(directory)
## save config
preset_path = os.path.join(
self.root_dir, directory, "pype_config_" + name + ".yaml"
)
if os.path.isfile(preset_path):
config = _load_yaml(preset_path)
ordered_steps = ["preprocessing",
"segmentation",
"measurement",
"visualization",
"export"
]
if not step in config.keys():
new_config = ordereddict([("image", ordereddict(config["image"]))])
new_config.update(ordereddict([("pype", ordereddict(config["pype"]))]))
for ordered_step in ordered_steps:
if ordered_step in config:
new_config.update(ordereddict([(ordered_step, config[ordered_step])]))
elif not ordered_step in config and ordered_step == step:
new_config.update(ordereddict([(ordered_step, [function] )]))
else:
new_config = copy.deepcopy(config)
if not function in new_config[step]:
new_config[step].append(function)
if flag_checked == False:
_show_yaml(new_config)
check = input("This is what the new config may look like (can differ beteeen files) - proceed?")
if check in ["True", "true", "y", "yes"]:
flag_checked = True
_save_yaml(new_config, preset_path)
print("New config saved for " + dirname)
else:
print("User check failed - aborting.")
return
def add_scale(self, reference_image, overwrite=False, **kwargs):
"""
Add pype configuration presets to all project directories.
Parameters
----------
reference_image: str
name of template image, either project directory or file link. template
image gets stored in root directory, and information appended to all
attributes files in the project directories
overwrite: bool, optional
overwrite option, if a given pype config-file already exist
template: bool, optional
should a template for scale detection be created. with an existing
template, phenopype can try to find a reference card in a given image,
measure its dimensions, and adjust pixel-to-mm-ratio and colour space
"""
## kwargs
flag_overwrite = overwrite
test_params = kwargs.get("test_params", {})
## load template image
if reference_image.__class__.__name__ == "str":
if os.path.isfile(reference_image):
reference_image = cv2.imread(reference_image)
elif os.path.isdir(os.path.join(self.data_dir, reference_image)):
attr = _load_yaml(
os.path.join(self.data_dir, reference_image, "attributes.yaml")
)
reference_image = cv2.imread(
os.path.join(self.root_dir, attr["project"]["raw_path"])
)
elif reference_image in self.dirnames:
attr = _load_yaml(
os.path.join(self.data_dir, reference_image, "attributes.yaml")
)
reference_image = cv2.imread(attr["project"]["raw_path"])
else:
print("wrong path - cannot load reference image")
return
elif reference_image.__class__.__name__ == "ndarray":
pass
elif reference_image.__class__.__name__ == "int":
reference_image = cv2.imread(self.filepaths[reference_image])
else:
print("wrong type - cannot load reference image")
return
## save template
template_path = os.path.join(self.root_dir, "scale_template.jpg")
while True:
if os.path.isfile(template_path) and flag_overwrite == False:
print(
"- scale template not saved - file already exists (overwrite=False)."
)
break
elif os.path.isfile(template_path) and flag_overwrite == True:
print("- scale template saved under " + template_path + " (overwritten).")
pass
elif not os.path.isfile(template_path):
print("- scale template saved under " + template_path + ".")
pass
## measure scale
px_mm_ratio, df_masks, template = preprocessing.create_scale(
reference_image, template=True, test_params=test_params
)
cv2.imwrite(template_path, template)
break
## save scale information
for directory in self.dirpaths:
attr = _load_yaml(os.path.join(self.root_dir, directory, "attributes.yaml"))
if not "scale" in attr:
print("added scale information to " + attr["project"]["dirname"])
pass
elif "scale" in attr and flag_overwrite:
print(
"added scale information to "
+ attr["project"]["dirname"]
+ " (overwritten)"
)
pass
elif "scale" in attr and not flag_overwrite:
print(
"could not add scale information to "
+ attr["project"]["dirname"]
+ " (overwrite=False)"
)
continue
attr["scale"] = {
"template_px_mm_ratio": px_mm_ratio,
"template_path": template_path,
}
_save_yaml(attr, os.path.join(self.root_dir, directory, "attributes.yaml"))
def add_config(
self,
name,
config_preset=None,
interactive=False,
overwrite=False,
idx=0,
**kwargs
):
"""
Add pype configuration presets to all image folders in the project, either by using
the templates included in the presets folder, or by adding your own templates
by providing a path to a yaml file. Can be tested and modified using the
interactive flag before distributing the config files.
Parameters
----------
name: str
name of config-file. this gets appended to all files and serves as and
identifier of a specific analysis pipeline
preset: str, optional
can be either a string denoting a template name (e.g. preset1, preset2,
landamarking1, ... - in "phenopype/settings/presets.py") or a path to a
compatible yaml file
interactive: bool, optional
start a pype and modify preset before saving it to phenopype directories
overwrite: bool, optional
overwrite option, if a given pype config-file already exist
kwargs:
developer options
"""
## kwargs
flag_interactive = interactive
flag_overwrite = overwrite
## legacy
preset = kwargs.get("preset")
if (
config_preset.__class__.__name__ == "NoneType"
and not preset.__class__.__name__ == "NoneType"
):
config_preset = preset
## load config
if not config_preset.__class__.__name__ == "NoneType" and hasattr(
presets, config_preset
):
config = _create_generic_pype_config(preset=config_preset, config_name=name)
elif not config_preset.__class__.__name__ == "NoneType" and os.path.isfile(
config_preset
):
config = {
"pype": {
"name": name,
"preset": config_preset,
"date_created": datetime.today().strftime("%Y%m%d_%H%M%S"),
}
}
config.update(_load_yaml(config_preset))
print(config)
elif not config_preset.__class__.__name__ == "NoneType" and not hasattr(
presets, config_preset
):
print("Provided preset NOT found - terminating")
return
elif config_preset.__class__.__name__ == "NoneType":
print("No preset provided - defaulting to preset " + default_pype_config)
config = _load_yaml(eval("presets." + default_pype_config))
## modify
if flag_interactive:
image_location = os.path.join(
self.root_dir,
"pype_template_image" + os.path.splitext(self.filenames[idx])[1],
)
copyfile(self.filepaths[idx], image_location)
config_location = os.path.join(
self.root_dir, "pype_config_template-" + name + ".yaml"
)
_save_yaml(config, config_location)
p = pype(
image_location,
name="template-" + name,
config_location=config_location,
presetting=True,
)
config = p.config
## go through project directories
for directory in self.dirpaths:
attr = _load_yaml(os.path.join(self.root_dir, directory, "attributes.yaml"))
pype_preset = {"image": attr["image"]}
pype_preset.update(config)
## save config
preset_path = os.path.join(
self.root_dir, directory, "pype_config_" + name + ".yaml"
)
dirname = attr["project"]["dirname"]
if os.path.isfile(preset_path) and flag_overwrite == False:
print(
"pype_"
+ name
+ ".yaml already exists in "
+ dirname
+ " (overwrite=False)"
)
continue
elif os.path.isfile(preset_path) and flag_overwrite == True:
print("pype_" + name + ".yaml created for " + dirname + " (overwritten)")
_save_yaml(pype_preset, preset_path)
else:
print("pype_" + name + ".yaml created for " + dirname)
_save_yaml(pype_preset, preset_path)
def add_files(
self,
image_dir,
filetypes=default_filetypes,
include=[],
exclude=[],
raw_mode="copy",
search_mode="dir",
unique_mode="path",
overwrite=False,
resize=1,
**kwargs
):
"""
Add files to your project from a directory, can look recursively.
Specify in- or exclude arguments, filetypes, duplicate-action and copy
or link raw files to save memory on the harddrive. For each found image,
a folder will be created in the "data" folder within the projects root
directory. If found images are in subfolders and search_mode is
recursive, the respective phenopype directories will be created with
flattened path as prefix.
E.g., with "raw_files" as folder with the original image files
and "phenopype_proj" as rootfolder:
- raw_files/file.jpg ==> phenopype_proj/data/file.jpg
- raw_files/subdir1/file.jpg ==> phenopype_proj/data/1__subdir1__file.jpg
- raw_files/subdir1/subdir2/file.jpg ==> phenopype_proj/data/2__subdir1__subdir2__file.jpg
Parameters
----------
image_dir: str
path to directory with images
filetypes: list or str, optional
single or multiple string patterns to target files with certain endings.
"default_filetypes" are configured in settings.py
include: list or str, optional
single or multiple string patterns to target certain files to include
exclude: list or str, optional
single or multiple string patterns to target certain files to exclude -
can overrule "include"
raw_mode: {"copy", "link"} str, optional
how should the raw files be passed on to the phenopype directory tree:
"copy" will make a copy of the original file, "link" will only send the
link to the original raw file to attributes, but not copy the actual
file (useful for big files)
search_mode: {"dir", "recursive"}, str, optional
"dir" searches current directory for valid files; "recursive" walks
through all subdirectories
unique_mode: {"filepath", "filename"}, str, optional:
how to deal with image duplicates - "filepath" is useful if identically
named files exist in different subfolders (folder structure will be
collapsed and goes into the filename), whereas filename will ignore
all similar named files after their first occurrence.
kwargs:
developer options
"""
# kwargs
flag_raw_mode = raw_mode
flag_overwrite = overwrite
flag_resize = resize
## path conversion
image_dir = image_dir.replace(os.sep, "/")
image_dir = os.path.abspath(image_dir)
## collect filepaths
filepaths, duplicates = _file_walker(
directory=image_dir,
search_mode=search_mode,
unique_mode=unique_mode,
filetypes=filetypes,
exclude=exclude,
include=include,
)
## feedback
print("--------------------------------------------")
print("phenopype will search for files at\n")
print(image_dir)
print("\nusing the following settings:\n")
print(
"filetypes: "
+ str(filetypes)
+ ", include: "
+ str(include)
+ ", exclude: "
+ str(exclude)
+ ", raw_mode: "
+ str(raw_mode)
+ ", search_mode: "
+ str(search_mode)
+ ", unique_mode: "
+ str(unique_mode)
+ "\n"
)
## loop through files
for filepath in filepaths:
## generate phenopype dir-tree
relpath = os.path.relpath(filepath, image_dir)
depth = relpath.count("\\")
relpath_flat = os.path.dirname(relpath).replace("\\", "__")
if depth > 0:
subfolder_prefix = str(depth) + "__" + relpath_flat + "__"
else:
subfolder_prefix = str(depth) + "__"
dirname = subfolder_prefix + os.path.splitext(os.path.basename(filepath))[0]
dirpath = os.path.join(self.root_dir, "data", dirname)
## make image-specific directories
if os.path.isdir(dirpath) and flag_overwrite == False:
print(
"Found image "
+ relpath
+ " - "
+ dirname
+ " already exists (overwrite=False)"
)
continue
if os.path.isdir(dirpath) and flag_overwrite == True:
rmtree(dirpath, ignore_errors=True, onerror=_del_rw)
print(
"Found image "
+ relpath
+ " - "
+ "phenopype-project folder "
+ dirname
+ " created (overwritten)"
)
os.mkdir(dirpath)
else:
print(
"Found image "
+ relpath
+ " - "
+ "phenopype-project folder "
+ dirname
+ " created"
)
os.mkdir(dirpath)
## load image
image = load_image(filepath, resize=flag_resize)
## copy or link raw files
if flag_raw_mode == "copy":
raw_path = os.path.join(
self.data_dir,
dirname,
"raw" + os.path.splitext(os.path.basename(filepath))[1],
)
if resize < 1:
cv2.imwrite(raw_path, image)
else:
copyfile(filepath, raw_path)
elif flag_raw_mode == "link":
if resize < 1:
warnings.warn("cannot resize image in link mode")
raw_path = filepath
## path reformatting
raw_relpath = os.path.relpath(raw_path, self.root_dir)
raw_relpath = raw_relpath.replace(os.sep, "/")
dir_relpath = os.path.relpath(dirpath, self.root_dir)
dir_relpath = dir_relpath.replace(os.sep, "/")
## collect attribute-data and save
image_data = load_image_data(filepath, flag_resize)
meta_data = load_meta_data(filepath)
project_data = {
"dirname": dirname,
"dirpath": dir_relpath,
"raw_mode": flag_raw_mode,
"raw_path": raw_relpath,
}
if meta_data:
attributes = {
"image": image_data,
"meta": meta_data,
"project": project_data,
}
else:
attributes = {"image": image_data, "project": project_data}
## write attributes file
_save_yaml(
attributes, os.path.join(self.root_dir, dir_relpath, "attributes.yaml")
)
## add to project object
if not dirname in self.dirnames:
## directories
self.dirnames.append(dirname)
self.dirpaths_rel.append(dir_relpath)
self.dirpaths.append(os.path.join(self.root_dir, dir_relpath))
## files
self.filenames.append(image_data["filename"])
self.filepaths_rel.append(raw_relpath)
self.filepaths.append(os.path.join(self.root_dir, raw_relpath))
print("\nFound {} files".format(len(filepaths)))
print("--------------------------------------------")