def load_from_database(self, sftp):
if os.path.isfile(self.cache_file_path):
logger.debug(f"Retrieved from cache: {str(self)}")
return self.load_from_harddrive(self.cache_file_path)
src_img = None
try:
logger.info(f"Downloading {self.name}, please wait...")
try:
with sftp.open(self.blob_path) as file:
file_size = file.stat().st_size
file.prefetch(file_size)
file.set_pipelined()
src_img = cv2.imdecode(
np.fromstring(file.read(), np.uint8),
1,
)
if os.path.isdir(ipso_folders.get_path("mass_storage", False)):
force_directories(os.path.dirname(self.cache_file_path))
cv2.imwrite(self.cache_file_path, src_img)
except Exception as e:
logger.exception(f"FTP error: {repr(e)}")
src_img = self.fix_image(src_image=src_img)
except Exception as e:
logger.exception(f"Failed to download {repr(self)} because {repr(e)}")
return None
else:
logger.info(f"Download succeeded for {self.name}")
return src_img
def __init__(self, prefix):
force_directories("saved_data")
db_path = os.path.join(
ipso_folders.get_path("saved_data", force_creation=True),
f"{prefix}_annotations.db",
)
engine = create_engine(f"sqlite:///{db_path}")
Session = sessionmaker(bind=engine)
self.session = Session()
BaseOrmClass.metadata.create_all(engine)
def cache_file_path(self):
if not self._cache_file_path and self.db_linked is True:
if os.path.isdir(ipso_folders.get_path("mass_storage", False)):
self._cache_file_path = os.path.join(
ipso_folders.get_path("mass_storage", False),
self.robot,
self.experiment,
self.file_name,
)
else:
self._cache_file_path = os.path.join(
ipso_folders.get_path(
key="img_cache",
force_creation=True,
),
self.robot,
self.experiment,
self.file_name,
)
return self._cache_file_path
def load_from_database(self, sftp):
if os.path.isdir(ipso_folders.get_path("mass_storage",
False)) and os.path.isfile(
self.cache_file_path):
logger.debug(f"Retrieved from cache: {str(self)}")
return self.load_from_harddrive(self.cache_file_path)
logger.info(f"Downloading {self.name}, please wait...")
sftp.get(
f"images/{self.robot}/{self.experiment}/{self.file_name}",
self.cache_file_path,
)
logger.info(f"Download succeeded for {self.name}")
return self.load_from_harddrive(override_path=self.cache_file_path)
def process_wrapper(self, **kwargs):
"""
Tensorflow inference:
'Use a Tensorflow model to predict value
Real time: False
Keyword Arguments (in parentheses, argument name):
* Activate tool (enabled): Toggle whether or not tool is active
* Model name without extension (model_name): All models should be located in "tensorflow_models" models folder
* CSV variable name (variable_name):"""
wrapper = self.init_wrapper(**kwargs)
if wrapper is None:
return False
res = False
try:
if self.get_value_of("enabled") == 1:
img = wrapper.current_image
with open(
os.path.join(
ipso_folders.get_path("tensorflow_models", force_creation=False),
f"{self.get_value_of('model_name')}_data.json",
),
"r",
) as f:
model_data = json.load(f)
model = load_model(
os.path.join(
ipso_folders.get_path("tensorflow_models", force_creation=False),
f"{self.get_value_of('model_name')}.h5",
),
custom_objects=model_data["custom_objects"],
)
predictions = model.predict(
{
k: np.array([wrapper.csv_data_holder.data_list[k]])
for k in model_data["vars"]
}
)
self.add_value(
key=self.get_value_of("variable_name"),
value=predictions[0][0] * model_data["max_predicted_var"],
force_add=True,
)
# Write your code here
wrapper.store_image(img, "current_image")
res = True
else:
wrapper.store_image(wrapper.current_image, "current_image")
res = True
except Exception as e:
res = False
logger.error(f"Tensorflow inference FAILED, exception: {repr(e)}")
else:
pass
finally:
return res
import os
import json
import logging
logger = logging.getLogger(os.path.splitext(__name__)[-1].replace(".", ""))
from ipso_phen.ipapi.tools.folders import ipso_folders
dbc_path = os.path.join(
ipso_folders.get_path("db_connect_data", force_creation=False),
"db_connect_data.json",
)
password_overrides = {}
master_password = None
def get_user_and_password(key: str) -> tuple:
if master_password:
return master_password
elif key in password_overrides:
return password_overrides[key]["user"], password_overrides[key]["password"]
elif os.path.isfile(dbc_path):
try:
with open(dbc_path, "r") as f:
dbc = json.load(f)[key]
return dbc["user"], dbc["password"]
except Exception as e:
logger.error(f"Unable to find user and password: {repr(e)}")
return None, None
else:
import sys
import argparse
import logging
import os
import logging
from datetime import datetime as dt
sys.path.insert(0, os.getcwd())
from ipso_phen.ipapi.tools.folders import ipso_folders
logging.basicConfig(
filename=os.path.join(
ipso_folders.get_path("logs", force_creation=True),
f"ipso_cli_{dt.now().strftime('%Y_%m_%d')}.log",
),
filemode="a",
level=logging.INFO,
format="[%(asctime)s - %(name)s - %(levelname)s] - %(message)s",
)
logger = logging.getLogger("IPSO CLI")
logger.info(
"_________________________________________________________________________"
)
logger.info(
"_________________________________________________________________________"
)
logger.info("Launching IPSO CLI")
from ipso_phen.ipapi.base.pipeline_launcher import launch
def process_wrapper(self, **kwargs):
"""
Hough circles detector:
Hough circles detector: Perform a circular Hough transform.
Can generate ROIs
Real time: False
Keyword Arguments (in parentheses, argument name):
* Allow retrieving data from cache (enable_cache): Data will be retrieved only if params are identical.
* ROI name (roi_name):
* Select action linked to ROI (roi_type): no clue
* Select ROI shape (c): no clue
* Target IPT (tool_target): no clue
* Name of ROI to be used (crop_roi_name): Circles will only be detected inside ROI
* Channel (channel):
* Normalize channel (normalize): Normalize channel before edge detection
* Median filter size (odd values only) (median_filter_size):
* Minimal radius to consider (min_radius): All circles smaller than this will be ignored
* Maximal radius to consider (max_radius): All circles bigger than this will be ignored
* Annulus secondary radius delta (annulus_size): Annulus size, 0 means full disc
* Radius granularity (step_radius): Steps for scanning radius
* Maximum number of detected circles (max_peaks): Keeps only n best circles
* Minimum distance between two circles (min_distance): Remove circles that are too close
* Draw line width (line_width):
* Keep only closest, if not, ROI is larger circle (keep_only_one):
* Keep the closest circle closest to (target_position):
* Maximum distance to root position (max_dist_to_root):
* Draw max and min circles (draw_boundaries):
* Draw discarded candidates (draw_candidates):
* Contract/expand circle (expand_circle):
* Edge detection only (edge_only):
* Select edge detection operator (operator):
* Canny's sigma for scikit, aperture for OpenCV (canny_sigma): Sigma.
* Canny's first Threshold (canny_first): First threshold for the hysteresis procedure.
* Canny's second Threshold (canny_second): Second threshold for the hysteresis procedure.
* Kernel size (kernel_size):
* Threshold (threshold): Threshold for kernel based operators
* Apply threshold (apply_threshold):
* Overlay text on top of images (text_overlay): Draw description text on top of images
"""
wrapper = self.init_wrapper(**kwargs)
if wrapper is None:
return False
res = False
try:
# Read params
min_radius = self.get_value_of(
"min_radius",
scale_factor=wrapper.scale_factor,
)
max_radius = self.get_value_of(
"max_radius",
scale_factor=wrapper.scale_factor,
)
step_radius = self.get_value_of(
"step_radius",
scale_factor=wrapper.scale_factor,
)
max_peaks = self.get_value_of("max_peaks")
max_peaks = max_peaks if max_peaks > 0 else np.inf
min_distance = self.get_value_of(
"min_distance",
scale_factor=wrapper.scale_factor,
)
line_width = self.get_value_of(
"line_width",
scale_factor=wrapper.scale_factor,
)
draw_candidates = self.get_value_of("draw_candidates") == 1
input_kind = self.get_value_of("source_selector")
edge_only = self.get_value_of("edge_only") == 1
roi = self.get_ipt_roi(
wrapper=wrapper,
roi_names=[self.get_value_of("crop_roi_name")],
selection_mode="all_named",
)
roi = roi[0] if roi else None
if input_kind == "mask":
img = self.get_mask()
elif input_kind == "current_image":
img = wrapper.current_image
else:
img = None
logger.error(f"Unknown source: {input_kind}")
self.result = None
return
pkl_file = os.path.join(
ipso_folders.get_path("stored_data"),
self.get_short_hash(exclude_list=(
"roi_name",
"roi_type",
"roi_shape",
"tool_target",
"annulus_size",
"line_width",
"keep_only_one",
"target_position",
"max_dist_to_root",
"draw_boundaries",
"draw_candidates",
"expand_circle",
)) + ".pkl",
)
if ((self.get_value_of("enable_cache") == 1) and edge_only is False
and os.path.isfile(pkl_file)):
with open(pkl_file, "rb") as f:
logger.info("Retrieved circle from cache")
accu, cx, cy, radii = pickle.load(f)
else:
# Get the edge
with IptEdgeDetector(wrapper=wrapper,
**self.params_to_dict()) as (
res,
ed,
):
if not res:
return
edges = ed.result
if edge_only is True:
self.result = ed.result
self.demo_image = self.result
return True
if roi is not None:
edges = wrapper.crop_to_roi(
img=edges,
roi=roi,
erase_outside_if_circle=True,
dbg_str="cropped_edges",
)
# Detect circles
hough_radii = np.arange(min_radius, max_radius, step_radius)
hough_res = hough_circle(edges, hough_radii)
# Draw the result
if len(img.shape) == 2:
img = np.dstack((img, img, img))
# Select the most prominent n circles
accu, cx, cy, radii = hough_circle_peaks(
hough_res,
hough_radii,
min_xdistance=min_distance,
min_ydistance=min_distance,
total_num_peaks=max_peaks,
)
if self.get_value_of("enable_cache") == 1:
with open(pkl_file, "wb") as f:
pickle.dump((accu, cx, cy, radii), f)
if roi is not None:
roi = roi.as_rect()
cx += roi.left
cy += roi.top
if self.get_value_of("keep_only_one") == 1:
candidates = [[a, x, y, z]
for a, x, y, z in zip(accu, cx, cy, radii)]
h, w = img.shape[:2]
roi = RectangleRegion(left=0, right=w, top=0, bottom=h)
roi_root = roi.point_at_position(
position=self.get_value_of("target_position"),
round_position=True,
)
min_dist = h * w
min_idx = -1
min_accu = -1
i = 0
colors = ipc.build_color_steps(step_count=len(candidates))
max_dist_to_root = self.get_value_of(
"max_dist_to_root",
scale_factor=wrapper.scale_factor,
)
for c_accu, center_x, center_y, radius in candidates:
if draw_candidates:
cv2.circle(
img,
(center_x, center_y),
radius,
colors[i],
max(1, line_width // 2),
)
cur_dist = roi_root.distance_to(Point(center_x, center_y))
if ((cur_dist < min_dist) and (cur_dist < max_dist_to_root)
and ((cur_dist / min_dist > min_accu / c_accu) or
(min_accu == -1))):
min_dist = cur_dist
min_idx = i
min_accu = c_accu
i += 1
if min_idx >= 0:
self.result = [[
candidates[min_idx][1],
candidates[min_idx][2],
candidates[min_idx][3],
]]
self.result[0][2] += self.get_value_of(
"expand_circle", scale_factor=wrapper.scale_factor)
if self.get_value_of("draw_boundaries") == 1:
cv2.circle(
img,
(roi_root.x, roi_root.y),
min_radius,
ipc.C_RED,
line_width + 4,
)
cv2.circle(
img,
(roi_root.x, roi_root.y),
max_radius,
ipc.C_BLUE,
line_width + 4,
)
else:
self.result = None
else:
self.result = [[x, y, r] for x, y, r in zip(cx, cy, radii)]
if self.result is not None:
colors = ipc.build_color_steps(step_count=len(self.result))
if input_kind == "mask":
img = wrapper.current_image
if len(img.shape) == 2 or (len(img.shape) == 3
and img.shape[2] == 1):
img = np.dstack((img, img, img))
i = 0
annulus_size = self.get_value_of("annulus_size")
for center_x, center_y, radius in self.result:
cv2.circle(img, (center_x, center_y), radius, colors[i],
line_width)
if annulus_size > 0 and radius - annulus_size > 0:
cv2.circle(
img,
(center_x, center_y),
radius - annulus_size,
colors[i],
line_width,
)
i += 1
wrapper.store_image(
image=img,
text="hough_circles",
text_overlay=self.get_value_of("text_overlay") == 1,
)
self.demo_image = img
res = True
except Exception as e:
logger.exception(f'Failed to process {self. name}: "{repr(e)}"')
res = False
else:
pass
finally:
return res
LOCAL_DB = "Local PSQL databases"
MASS_DB = "Mass storage databases"
PHENOSERRE = "Phenoserre databases"
PHENOPSIS = "Phenopsis databases"
TPMP = "TPMP databases"
CUSTOM_DB = "Custom databases"
available_db_dicts = defaultdict(list)
available_db_dicts[DbType.LOCAL_DB] = [
DbInfo(
display_name=name,
target="psql_local",
src_files_path=os.path.join(
ipso_folders.get_path("local_storage", False), name),
dbms="psql",
) for name in os.listdir(ipso_folders.get_path("local_storage"))
if os.path.isdir(
os.path.join(ipso_folders.get_path("local_storage", False), name))
]
if ipso_folders.get_path("mass_storage"):
available_db_dicts[DbType.MASS_DB] = [
DbInfo(
display_name=name,
target="psql_local",
src_files_path=os.path.join(
ipso_folders.get_path("mass_storage", False), name),
dbms="psql",
) for name in os.listdir(ipso_folders.get_path("mass_storage"))
def cache_folder(self):
return ipso_folders.get_path("db_cache", force_creation=True)
def process_wrapper(self, **kwargs):
wrapper = self.init_wrapper(**kwargs)
if wrapper is None:
return False
res = False
try:
# Read params
input_kind = self.get_value_of("source_selector")
edge_only = self.get_value_of("edge_only") == 1
roi = self.get_ipt_roi(
wrapper=wrapper,
roi_names=[self.get_value_of("crop_roi_name")],
selection_mode="all_named",
)
roi = roi[0] if roi else None
if input_kind == "mask":
img = self.get_mask()
elif input_kind == "current_image":
img = wrapper.current_image
else:
img = None
logger.error(f"Unknown source: {input_kind}")
self.result = None
return
pkl_file = os.path.join(
ipso_folders.get_path("stored_data"),
self.get_short_hash(exclude_list=()) + ".pkl",
)
if (
(self.get_value_of("enable_cache") == 1)
and edge_only is False
and os.path.isfile(pkl_file)
):
with open(pkl_file, "rb") as f:
result = pickle.load(f)
else:
# Get the edge
with IptEdgeDetector(wrapper=wrapper, **self.params_to_dict()) as (
res,
ed,
):
if not res:
return
edges = ed.result
if edge_only is True:
self.result = ed.result
self.demo_image = self.result
return True
result = hough_ellipse(
edges,
accuracy=self.get_value_of("hough_accuracy"),
threshold=self.get_value_of("hough_threshold"),
min_size=self.get_value_of("min_radius"),
max_size=self.get_value_of("max_radius"),
)
result.sort(order="accumulator")
if self.get_value_of("enable_cache") == 1:
with open(pkl_file, "wb") as f:
pickle.dump(result, f)
if result is not None:
colors = ipc.build_color_steps(step_count=len(result))
for i, ellipse in enumerate(result):
yc, xc, a, b = [int(round(x)) for x in ellipse[1:5]]
orientation = ellipse[5]
cy, cx = ellipse_perimeter(yc, xc, a, b, orientation)
img[cy, cx] = colors[i]
wrapper.store_image(image=img, text="hough_ellipses")
self.demo_image = img
res = True
except Exception as e:
logger.exception(f'Failed to process {self. name}: "{repr(e)}"')
res = False
else:
pass
finally:
return res