def download_attached(inputs,
path,
suffix="_attached",
do_download=False,
ignore_missing_gt=False):
"""
Download the most recent attached file for each input.
If do_download is False, then the attached file must have the same name (without extension) as the corresponding
input file plus the suffix.
"""
# if do_download is False, need to scan for existing attached files
existing_files, existing_extensions = set(), dict()
for f in os.listdir(path):
if suffix not in f:
continue
name, ext = f.rsplit(suffix + ".")
existing_files.add(name)
existing_extensions[name] = ext
# get path for all inputs
for in_image in inputs:
image = in_image.object
if do_download:
# extract most recent file
files = AttachedFileCollection(image).fetch()
if len(files) == 0:
if ignore_missing_gt:
continue
raise ValueError(
"Missing ground truth attached file for input image '{}'.".
format(in_image.filename))
most_recent = sorted(files.data(),
key=lambda f: int(f.created),
reverse=True)[0]
# download the last file
in_name, _ = split_filename(in_image.filename)
_, ext = split_filename(most_recent.filename)
attached_file = BiaflowsAttachedFile(most_recent,
path,
name_pattern=in_name +
suffix + "." + ext)
most_recent.download(attached_file.filepath)
else:
image_name = os.path.basename(image).rsplit(".")[0]
attached_name = "{}".format(image_name, suffix)
if attached_name not in existing_files:
if ignore_missing_gt:
continue
raise FileNotFoundError(
"Missing attached file for input image '{}'.".format(
image))
attached_file = BiaflowsFilepath(
os.path.join(
path, "{}{}".format(attached_name,
existing_extensions[attached_name])))
in_image.attached.append(attached_file)
def load_model(job, download_path, model_filename="weights.hf5"):
attached_files = AttachedFileCollection(job).fetch()
if not (0 < len(attached_files) < 2):
raise ValueError("More or less than 1 file attached to the Job (found {} file(s)).".format(len(attached_files)))
attached_file = attached_files[0]
if attached_file.filename != model_filename:
raise ValueError(
"Expected model file name is '{}' (found: '{}').".format(model_filename, attached_file.filename))
model_path = os.path.join(download_path, model_filename)
attached_file.download(model_path)
return model_path
def download_attach_files_from_image_group(id_project, folder_path):
# print(self._con.current_user)
# ... Assume we are connected
image_groups = ImageGroupCollection().fetch_with_filter("project", id_project)
for image_group in image_groups:
attached_collection = AttachedFileCollection(image_group).fetch()
if len(attached_collection) > 0:
print(image_group.name)
for attached_file in attached_collection:
print(attached_file.filename)
attached_file.download(path.join(folder_path, attached_file.filename), True)
def _export_metadata(save_object_fn, obj, attached_file_path):
properties = PropertyCollection(obj).fetch()
if len(properties) > 0:
save_object_fn(properties, "properties-object-{}-collection".format(obj.id))
attached_files = AttachedFileCollection(obj).fetch()
if len(attached_files) > 0:
save_object_fn(attached_files, "attached-files-object-{}-collection".format(obj.id))
for attached_file in attached_files:
attached_file.download(os.path.join(attached_file_path, "{filename}"), True)
description = Description(obj).fetch()
if description:
save_object_fn(description, "description-object-{}".format(obj.id))
attached_files = AttachedFileCollection(description).fetch()
if len(attached_files) > 0:
save_object_fn(attached_files, "attached-files-object-{}-collection".format(obj.id))
for attached_file in attached_files:
attached_file.download(os.path.join(attached_file_path, "{filename}"), True)
def download_image_group(image_group, include_attachment, output_folder_raw_image,
output_folder_label_image=None, output_folder_path_attachment=None):
# If optional output directories parameters are None, use the raw directory
if not output_folder_path_attachment:
output_folder_path_attachment = output_folder_raw_image
if not output_folder_label_image:
output_folder_label_image = output_folder_raw_image
# Create directory
if not exists(output_folder_raw_image):
makedirs(output_folder_raw_image)
if output_folder_path_attachment and not exists(output_folder_path_attachment):
makedirs(output_folder_path_attachment)
if output_folder_label_image and not exists(output_folder_label_image):
makedirs(output_folder_label_image)
image_group.download(path.join(output_folder_raw_image, image_group.name), True)
# If a file contain '_lbl.' inside its filename, it will be moved into the output_folder_label_image
regexp = re.compile(r'(.*)_lbl.(.*)')
files = os.listdir(output_folder_raw_image)
for file in files:
if os.path.isfile(path.join(output_folder_raw_image, file)) and regexp.search(file):
print(file)
os.rename(path.join(output_folder_raw_image, file), path.join(output_folder_label_image, file))
continue
else:
continue
if include_attachment:
attached_collection = AttachedFileCollection(image_group).fetch()
# We are supposed to have only 1 attachment per image group for NEUBIAS
# In case there is one more, let's add a number
if len(attached_collection) > 0:
# print(image_group.name)
padding_zero_size = len(str(len(attached_collection)))
#attachment_number = 1
# Sort the attachment by created date
attached_collection = sorted(attached_collection, key=lambda x: x.created)
#Always take the first attachement by default
attached_file=attached_collection[0]
# The attachment file will have the same basename as its associated image group
# We suppose that the file format is .ome.tif
base_image_group_name = image_group.name[:-8]
attachment_file_extension = attached_file.filename[-3:]
attachment_file_name = base_image_group_name + '.' \
+ attachment_file_extension
attached_file.download(path.join(output_folder_path_attachment, attachment_file_name), True)
'''
def download_attach_files_from_image_group(id_project, host, public_key,
private_key, folder_path):
with Cytomine(host=host,
public_key=public_key,
private_key=private_key,
verbose=logging.INFO) as cyto_connection:
print(cyto_connection.current_user)
# ... Assume we are connected
image_groups = ImageGroupCollection().fetch_with_filter(
"project", id_project)
for image_group in image_groups:
attached_collection = AttachedFileCollection(image_group).fetch()
if len(attached_collection) > 0:
print(image_group.name)
for attached_file in attached_collection:
print(attached_file.filename)
attached_file.download(
path.join(folder_path, attached_file.filename), True)
def main(argv):
with CytomineJob.from_cli(argv) as cj:
# use only images from the current project
cj.job.update(progress=1, statusComment="Preparing execution")
# extract images to process
if cj.parameters.cytomine_zoom_level > 0 and (
cj.parameters.cytomine_tile_size != 256
or cj.parameters.cytomine_tile_overlap != 0):
raise ValueError(
"when using zoom_level > 0, tile size should be 256 "
"(given {}) and overlap should be 0 (given {})".format(
cj.parameters.cytomine_tile_size,
cj.parameters.cytomine_tile_overlap))
cj.job.update(
progress=1,
statusComment="Preparing execution (creating folders,...).")
# working path
root_path = str(Path.home())
working_path = os.path.join(root_path, "images")
os.makedirs(working_path, exist_ok=True)
# load training information
cj.job.update(progress=5,
statusComment="Extract properties from training job.")
train_job = Job().fetch(cj.parameters.cytomine_id_job)
properties = PropertyCollection(train_job).fetch().as_dict()
binary = str2bool(properties["binary"].value)
classes = parse_domain_list(properties["classes"].value)
cj.job.update(progress=10, statusComment="Download the model file.")
attached_files = AttachedFileCollection(train_job).fetch()
model_file = attached_files.find_by_attribute("filename",
"model.joblib")
model_filepath = os.path.join(root_path, "model.joblib")
model_file.download(model_filepath, override=True)
pyxit = joblib.load(model_filepath)
# set n_jobs
pyxit.base_estimator.n_jobs = cj.parameters.n_jobs
pyxit.n_jobs = cj.parameters.n_jobs
cj.job.update(progress=45, statusComment="Build workflow.")
builder = SSLWorkflowBuilder()
builder.set_tile_size(cj.parameters.cytomine_tile_size,
cj.parameters.cytomine_tile_size)
builder.set_overlap(cj.parameters.cytomine_tile_overlap)
builder.set_tile_builder(
CytomineTileBuilder(working_path, n_jobs=cj.parameters.n_jobs))
builder.set_logger(StandardOutputLogger(level=Logger.INFO))
builder.set_n_jobs(1)
builder.set_background_class(0)
# value 0 will prevent merging but still requires to run the merging check
# procedure (inefficient)
builder.set_distance_tolerance(2 if cj.parameters.union_enabled else 0)
builder.set_segmenter(
ExtraTreesSegmenter(
pyxit=pyxit,
classes=classes,
prediction_step=cj.parameters.pyxit_prediction_step,
background=0,
min_std=cj.parameters.tile_filter_min_stddev,
max_mean=cj.parameters.tile_filter_max_mean))
workflow = builder.get()
area_checker = AnnotationAreaChecker(
min_area=cj.parameters.min_annotation_area,
max_area=cj.parameters.max_annotation_area)
def get_term(label):
if binary:
if "cytomine_id_predict_term" not in cj.parameters:
return []
else:
return [int(cj.parameters.cytomine_id_predict_term)]
# multi-class
return [label]
zones = extract_images_or_rois(cj.parameters)
for zone in cj.monitor(zones,
start=50,
end=90,
period=0.05,
prefix="Segmenting images/ROIs"):
results = workflow.process(zone)
annotations = AnnotationCollection()
for obj in results:
if not area_checker.check(obj.polygon):
continue
polygon = obj.polygon
if isinstance(zone, ImageWindow):
polygon = affine_transform(
polygon,
[1, 0, 0, 1, zone.abs_offset_x, zone.abs_offset_y])
polygon = change_referential(polygon, zone.base_image.height)
if cj.parameters.cytomine_zoom_level > 0:
zoom_mult = (2**cj.parameters.cytomine_zoom_level)
polygon = affine_transform(
polygon, [zoom_mult, 0, 0, zoom_mult, 0, 0])
annotations.append(
Annotation(location=polygon.wkt,
id_terms=get_term(obj.label),
id_project=cj.project.id,
id_image=zone.base_image.image_instance.id))
annotations.save()
cj.job.update(status=Job.TERMINATED,
status_comment="Finish",
progress=100)
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
is_2d = True
conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=is_2d, **conn.flags)
list_imgs = [int(image.rstrip('.tif')) for image in os.listdir(in_path) if image.endswith('.tif')]
train_job = Job().fetch(conn.parameters.model_to_use)
properties = PropertyCollection(train_job).fetch()
str_terms = ""
for prop in properties:
if prop.fetch(key='id_terms')!=None:
str_terms = prop.fetch(key='id_terms').value
term_list = [int(x) for x in str_terms.split(' ')]
attached_files = AttachedFileCollection(train_job).fetch()
hash_pos = {}
hash_size = {}
for id_term in conn.monitor(term_list, start=10, end=70, period = 0.05, prefix="Finding landmarks for terms..."):
model_file = find_by_attribute(attached_files, "filename", "%d_model.joblib"%id_term)
model_filepath = os.path.join(in_path, "%d_model.joblib"%id_term)
model_file.download(model_filepath, override=True)
cov_file = find_by_attribute(attached_files, 'filename', '%d_cov.joblib'%id_term)
cov_filepath = os.path.join(in_path, "%d_cov.joblib"%id_term)
cov_file.download(cov_filepath, override=True)
parameters_file = find_by_attribute(attached_files, 'filename', '%d_parameters.joblib'%id_term)
parameters_filepath = os.path.join(in_path, '%d_parameters.joblib'%id_term)
parameters_file.download(parameters_filepath, override=True)
model = joblib.load(model_filepath)
[mx, my, cm] = joblib.load(cov_filepath)
parameters_hash = joblib.load(parameters_filepath)
feature_parameters = None
if parameters_hash['feature_type'] in ['haar', 'gaussian']:
fparameters_file = find_by_attribute(attached_files, 'filename', "%d_fparameters.joblib"%id_term)
fparametersl_filepath = os.path.join(in_path, "%d_fparameters.joblib"%id_term)
fparameters_file.download(fparametersl_filepath, override=True)
feature_parameters = joblib.load(fparametersl_filepath)
for id_img in list_imgs:
(x, y, height, width) = searchpoint_cytomine(in_path, id_img, model, mx, my, cm, 1. / (2. ** np.arange(parameters_hash['model_depth'])), parameters_hash['window_size'], parameters_hash['feature_type'], feature_parameters, 'tif', parameters_hash['model_npred'])
if (not id_img in hash_size):
hash_size[id_img] = (height, width)
hash_pos[id_img] = []
hash_pos[id_img].append(((id_term, x, y)))
conn.job.update(status=Job.RUNNING, progress=95, statusComment="Uploading the results...")
for id_img in list_imgs:
(h, w) = hash_size[id_img]
lbl_img = np.zeros((h, w), 'uint8')
for (id_term, x, y) in hash_pos[id_img]:
intx = int(x)
inty = int(y)
if lbl_img[inty, intx] > 0:
(ys, xs) = np.where(lbl_img==0)
dis = np.sqrt((ys-y)**2 + (xs-x)**2)
j = np.argmin(dis)
intx = int(xs[j])
inty = int(ys[j])
lbl_img[inty, intx] = id_term
imwrite(path=os.path.join(out_path, '%d.tif'%id_img), image=lbl_img.astype(np.uint8), is_2d=is_2d)
upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=is_2d, monitor_params={"start": 70, "end": 90, "period": 0.1})
conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main():
with CytomineJob.from_cli(sys.argv) as conn:
base_path = "{}".format(os.getenv("HOME"))
working_path = os.path.join(base_path, str(conn.job.id))
in_path = os.path.join(working_path, "in/")
out_path = os.path.join(working_path, "out/")
tr_working_path = os.path.join(base_path,
str(conn.parameters.model_to_use))
tr_out_path = os.path.join(tr_working_path, "out/")
if not os.path.exists(working_path):
os.makedirs(working_path)
os.makedirs(in_path)
images = ImageInstanceCollection().fetch_with_filter(
"project", conn.parameters.cytomine_id_project)
list_imgs = []
if conn.parameters.images_to_predict == 'all':
for image in images:
list_imgs.append(int(image.id))
image.dump(os.path.join(in_path, '%d.jpg' % (image.id)))
else:
list_imgs = [
int(id_img)
for id_img in conn.parameters.images_to_predict.split(',')
]
for image in images:
if image.id in list_imgs:
image.dump(os.path.join(in_path, '%d.jpg' % (image.id)))
annotation_collection = AnnotationCollection()
train_job = Job().fetch(conn.parameters.model_to_use)
properties = PropertyCollection(train_job).fetch()
str_terms = ""
for prop in properties:
if prop.fetch(key='id_terms') != None:
str_terms = prop.fetch(key='id_terms').value
term_list = [int(x) for x in str_terms.split(' ')]
attached_files = AttachedFileCollection(train_job).fetch()
for id_term in conn.monitor(term_list,
start=10,
end=90,
period=0.05,
prefix="Finding landmarks for terms..."):
model_file = find_by_attribute(attached_files, "filename",
"%d_model.joblib" % id_term)
model_filepath = os.path.join(in_path, "%d_model.joblib" % id_term)
model_file.download(model_filepath, override=True)
cov_file = find_by_attribute(attached_files, 'filename',
'%d_cov.joblib' % id_term)
cov_filepath = os.path.join(in_path, "%d_cov.joblib" % id_term)
cov_file.download(cov_filepath, override=True)
parameters_file = find_by_attribute(
attached_files, 'filename', '%d_parameters.joblib' % id_term)
parameters_filepath = os.path.join(
in_path, '%d_parameters.joblib' % id_term)
parameters_file.download(parameters_filepath, override=True)
model = joblib.load(model_filepath)
[mx, my, cm] = joblib.load(cov_filepath)
parameters_hash = joblib.load(parameters_filepath)
feature_parameters = None
if parameters_hash['feature_type'] in ['haar', 'gaussian']:
fparameters_file = find_by_attribute(
attached_files, 'filename',
"%d_fparameters.joblib" % id_term)
fparametersl_filepath = os.path.join(
in_path, "%d_fparameters.joblib" % id_term)
fparameters_file.download(fparametersl_filepath, override=True)
feature_parameters = joblib.load(fparametersl_filepath)
for id_img in list_imgs:
(x, y) = searchpoint_cytomine(
in_path, id_img, model, mx, my, cm,
1. / (2.**np.arange(parameters_hash['model_depth'])),
parameters_hash['window_size'],
parameters_hash['feature_type'], feature_parameters, 'jpg',
parameters_hash['model_npred'])
circle = Point(x, y)
annotation_collection.append(
Annotation(location=circle.wkt,
id_image=id_img,
id_terms=[id_term],
id_project=conn.parameters.cytomine_id_project))
annotation_collection.save()
def main(argv):
with CytomineJob.from_cli(argv) as cj:
# prepare paths
working_path = str(Path.home())
data_path = os.path.join(working_path, "pred_data")
if not os.path.exists(data_path):
os.makedirs(data_path)
model_filename = "model.pkl"
cj.job.update(progress=5, statusComment="Download model ...")
model_job = Job().fetch(cj.parameters.cytomine_model_job_id)
attached_files = AttachedFileCollection(model_job).fetch_with_filter(
"project", cj.project.id)
if not (0 < len(attached_files) < 2):
raise ValueError(
"More or less than 1 file attached to the Job (found {} file(s))."
.format(len(attached_files)))
attached_file = attached_files[0]
if attached_file.filename != model_filename:
raise ValueError(
"Expected model file name is '{}' (found: '{}').".format(
model_filename, attached_file.filename))
model_path = os.path.join(working_path, model_filename)
attached_file.download(model_path)
# load model
with open(model_path, "rb") as file:
data = pickle.load(file)
model = data["model"]
classifier = data["classifier"]
network = data["network"]
reduction = data["reduction"]
# load and dump annotations
cj.job.update(progress=10, statusComment="Download annotations.")
annotations = get_annotations(
project_id=cj.parameters.cytomine_project_id,
images=parse_list_or_none(cj.parameters.cytomine_images_ids),
users=parse_list_or_none(cj.parameters.cytomine_users_ids),
showWKT=True)
cj.job.update(statusComment="Fetch crops.", progress=15)
n_samples = len(annotations)
x = np.zeros([n_samples], dtype=np.object)
for i, annotation in cj.monitor(enumerate(annotations),
start=15,
end=40,
prefix="Fetch crops",
period=0.1):
file_format = os.path.join(data_path, "{id}.png")
if not annotation.dump(dest_pattern=file_format):
raise ValueError("Download error for annotation '{}'.".format(
annotation.id))
x[i] = file_format.format(id=annotation.id)
available_nets = {
MODEL_RESNET50, MODEL_VGG19, MODEL_VGG16, MODEL_INCEPTION_V3,
MODEL_INCEPTION_RESNET_V2, MODEL_MOBILE, MODEL_DENSE_NET_201,
MODEL_NASNET_LARGE, MODEL_NASNET_MOBILE
}
if network not in available_nets:
raise ValueError(
"Invalid value (='{}'} for parameter 'network'.".format(
network))
if reduction not in {"average_pooling"}:
raise ValueError(
"Invalid value (='{}') for parameter 'reduction'.".format(
reduction))
if classifier not in {"svm"}:
raise ValueError(
"Invalid value (='{}') for parameter 'classifier'.".format(
classifier))
# prepare network
cj.job.update(statusComment="Load neural network '{}'".format(network),
progress=40)
features = PretrainedModelFeatures(model=network,
layer="last",
reduction=reduction,
weights="imagenet")
height, width, _ = features._get_input_shape(network)
loader = ImageLoader(load_size_range=(height, height),
crop_size=height,
random_crop=False)
cj.job.update(statusComment="Transform features.", progress=50)
x_feat = batch_transform(loader,
features,
x,
logger=cj.logger(start=50, end=70,
period=0.1),
batch_size=128)
cj.job.update(statusComment="Prediction with '{}'.".format(classifier),
progress=70)
if hasattr(model, "n_jobs"):
model.n_jobs = cj.parameters.n_jobs
probas = None
if hasattr(model, "predict_proba"):
probas = model.predict_proba(x_feat)
y_pred = model.classes_.take(np.argmax(probas, axis=1), axis=0)
else:
y_pred = model.predict(x_feat)
cj.job.update(statusComment="Upload annotations.", progress=80)
annotation_collection = AnnotationCollection()
for i, annotation in cj.monitor(enumerate(annotations),
start=80,
end=100,
period=0.1,
prefix="Upload annotations"):
annotation_collection.append(
Annotation(location=annotation.location,
id_image=annotation.image,
id_project=annotation.project,
term=[int(y_pred[i])],
rate=float(probas[i])
if probas is not None else 1.0).save())
annotation_collection.save()
cj.job.update(statusComment="Finished.", progress=100)