def main(argv):
# 0. Initialize Cytomine client and job if necessary and parse inputs
with NeubiasJob.from_cli(argv) as nj:
problem_cls = get_discipline(nj, default=CLASS_SPTCNT)
is_2d = False
nj.job.update(status=Job.RUNNING, progress=0, statusComment="Running workflow for problem class '{}' in {}D".format(problem_cls, 2 if is_2d else 3))
# 1. Create working directories on the machine
# 2. Download the images
nj.job.update(progress=0, statusComment="Initialisation...")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)
# 3. Call the image analysis workflow using the run script
nj.job.update(progress=25, statusComment="Launching workflow...")
command = "/usr/bin/xvfb-run java -Xmx6000m -cp /fiji/jars/ij.jar ij.ImageJ --headless --console " \
"-macro macro.ijm \"input={}, output={}, scale={}, radius_xy={}, radius_z={}, noise={}\"".format(in_path, out_path, nj.parameters.ij_scale, nj.parameters.ij_radius_xy, nj.parameters.ij_radius_z, nj.parameters.ij_noise)
return_code = call(command, shell=True, cwd="/fiji") # waits for the subprocess to return
if return_code != 0:
err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
nj.job.update(progress=50, statusComment=err_desc)
raise ValueError(err_desc)
# 4. Upload the annotation and labels to Cytomine
upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
"start": 60, "end": 90, "period": 0.1
})
# 5. Compute and upload the metrics
nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)
# 6. End
nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main():
with NeubiasJob.from_cli(sys.argv[1:]) as nj:
nj.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization...")
# 1. Create working directories on the machine:
# 2. Download (or read) data
problem_cls = get_discipline(nj, default=CLASS_SPTCNT)
is_2d = True
nj.job.update(progress=1, statusComment="Execute workflow on problem class '{}'".format(problem_cls))
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)
# 3. Execute workflow
scale3sens = nj.parameters.icy_scale3sensitivity
nj.job.update(progress=25, statusComment="Launching workflow...")
call("java -cp /icy/lib/ -jar /icy/icy.jar -hl", shell=True, cwd="/icy")
call("java -cp /icy/lib/ -jar /icy/icy.jar -hl -x plugins.adufour.protocols.Protocols "
"protocol=\"/icy/protocols/protocol.protocol\" inputFolder=\"{}\" outputFolder=\"{}\" extension=tif "
"scale2enable=true scale2sensitivity={}".format(in_path, out_path, scale3sens), shell=True, cwd="/icy")
# 3.5 Remove the xml-output files
for p in Path(out_path).glob("*.xml"):
p.unlink()
# 4. Upload the annotation and labels to Cytomine
upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
"start": 60, "end": 90, "period": 0.1
})
# 5. Compute and upload the metrics
nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)
nj.job.update(progress=100, status=Job.TERMINATED, status_comment="Finished.")
def main(argv):
with NeubiasJob.from_cli(argv) as nj:
problem_cls = get_discipline(nj, default=CLASS_PIXCLA)
is_2d = True
nj.job.update(status=Job.RUNNING,
progress=0,
statusComment="Initialisation...")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
problem_cls, nj, **nj.flags)
# 2. Call the image analysis workflow
nj.job.update(progress=10, statusComment="Load model...")
net = load_model("/app/model.pth")
device = torch.device("cpu")
for in_image in Monitor(nj,
in_images,
start=20,
end=75,
period=0.05,
prefix="Apply UNet to input images"):
mask = predict_img(net,
in_image.filepath,
device="cpu",
out_threshold=nj.parameters.threshold)
imwrite(path=os.path.join(out_path, in_image.filename),
image=mask.astype(np.uint8),
is_2d=is_2d)
# 4. Create and upload annotations
nj.job.update(progress=70,
statusComment="Uploading extracted annotation...")
upload_data(problem_cls,
nj,
in_images,
out_path,
**nj.flags,
is_2d=is_2d,
monitor_params={
"start": 70,
"end": 90,
"period": 0.1
})
# 5. Compute and upload the metrics
nj.job.update(
progress=90,
statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path,
**nj.flags)
# 6. End the job
nj.job.update(status=Job.TERMINATED,
progress=100,
statusComment="Finished.")
def main(argv):
# 0. Initialize Cytomine client and job if necessary and parse inputs
with NeubiasJob.from_cli(argv) as nj:
problem_cls = get_discipline(nj, default=CLASS_OBJTRK)
is_2d = False
nj.job.update(status=Job.RUNNING, progress=0, statusComment="Running workflow for problem class '{}' in 2D+t".format(problem_cls))
# 1. Create working directories on the machine
# 2. Download the images
nj.job.update(progress=0, statusComment="Initialisation...")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, **nj.flags)
# 3. Call the image analysis workflow using the run script
nj.job.update(progress=25, statusComment="Launching workflow...")
# DEBUG Workflow not currently working !!
# # CellTrackingChallenge expects the protocol to start with the data folder name
# in_path_folder_name, in_folder_name = os.path.basename(os.path.dirname(in_path)), os.path.basename(in_path)
# shutil.copyfile("/app/proto.protocol", "/app/{}-{}.protocol".format(in_path_folder_name, in_folder_name))
# command = "java -Xmx2048m -jar icy.jar -hl -x plugins.adufour.ctc.CellTrackingChallenge {} {}".format(os.path.dirname(in_path), os.path.basename(in_path))
# return_code = call(command, shell=True, cwd="/app") # waits for the subprocess to return
#
# if return_code != 0:
# err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
# nj.job.update(progress=50, statusComment=err_desc)
# raise ValueError(err_desc)
#
# # move files generated by CellTrackingChallenge into the output folder
# res_path = in_path + "_RES"
# for file in os.listdir(res_path):
# shutil.move(os.path.join(res_path, file), out_path)
# DEBUG copy ground truth in output
for file in os.listdir(gt_path):
outfile = file.replace("_attached", "") if file.endswith(".txt") else file
shutil.copy(os.path.join(gt_path, file), os.path.join(out_path, outfile))
# 4. Upload the annotation and labels to Cytomine
upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
"start": 60, "end": 90, "period": 0.1
})
# 5. Compute and upload the metrics
nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)
# 6. End
nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main(argv):
# 0. Initialize Cytomine client and job
with NeubiasJob.from_cli(argv) as nj:
nj.job.update(status=Job.RUNNING, progress=0, statusComment="Initialisation...")
problem_cls = CLASS_OBJSEG
is_2d = False
# 1. Create working directories on the machine
# 2. Download the images
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)
# 3. Call the image analysis workflow using the run script
nj.job.update(progress=25, statusComment="Launching workflow...")
command = "/usr/bin/xvfb-run java -Xmx6000m -cp /fiji/jars/ij.jar ij.ImageJ --headless --console " \
"-macro macro.ijm \"input={}, output={}, radius={}, min_threshold={}\"".format(in_path, out_path, nj.parameters.ij_radius, nj.parameters.ij_min_threshold)
return_code = call(command, shell=True, cwd="/fiji") # waits for the subprocess to return
if return_code != 0:
err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
nj.job.update(progress=50, statusComment=err_desc)
raise ValueError(err_desc)
# 4. Upload the annotation and labels to Cytomine (annotations are extracted from the mask using
# the AnnotationExporter module)
upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
"start": 60, "end": 90,
"period": 0.1,
"prefix": "Extracting and uploading polygons from masks"
})
# 5. Compute and upload the metrics
nj.job.update(progress=80, statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)
nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)
list_imgs = [int(image.rstrip('.tif')) for image in os.listdir(in_path) if image.endswith('.tif')]
train_job = Job().fetch(conn.parameters.model_to_use)
properties = PropertyCollection(train_job).fetch()
str_terms = ""
for prop in properties:
if prop.fetch(key='id_terms') != None:
str_terms = prop.fetch(key='id_terms').value
term_list = [int(x) for x in str_terms.split(' ')]
attached_files = AttachedFileCollection(train_job).fetch()
feature_file = find_by_attribute(attached_files, "filename", "features.joblib")
feature_filepath = os.path.join(in_path, "features.joblib")
feature_file.download(feature_filepath, override=True)
(h2,v2,h3,v3,sq) = joblib.load(feature_filepath)
coords_file = find_by_attribute(attached_files, "filename", "coords.joblib")
coords_filepath = os.path.join(in_path, "coords.joblib")
coords_file.download(coords_filepath, override=True)
(Xc, Yc) = joblib.load(coords_filepath)
(nims, nldms) = Xc.shape
coords = np.zeros(2 * nldms)
i = 0
for id_term in conn.monitor(term_list, start=10, end=50, period = 0.05, prefix="Building vote maps..."):
model_file = find_by_attribute(attached_files, "filename", "%d_model.joblib" % id_term)
model_filepath = os.path.join(in_path, "%d_model.joblib" % id_term)
model_file.download(model_filepath, override=True)
clf = joblib.load(model_filepath)
mx = np.mean(Xc[:, id_term-1])
my = np.mean(Yc[:, id_term-1])
coords[i] = mx
coords[i+nldms] = my
i+=1
for j in list_imgs:
print(j)
vote_map = build_vote_map(in_path, j, clf, h2, v2, h3, v3, sq, conn.parameters.model_step)
np.savez_compressed('%d_%d_votemap.npy' % (j, id_term), vote_map)
muP_file = find_by_attribute(attached_files, "filename", "muP.joblib")
muP_filepath = os.path.join(in_path, "muP.joblib")
muP_file.download(muP_filepath, override=True)
(mu, P) = joblib.load(muP_filepath)
(nims, nldms) = Xc.shape
for id_img in conn.monitor(list_imgs, start=50, end=80, period = 0.05, prefix="Post-processing..."):
probability_map = np.load('%d_%d_votemap.npy.npz' % (id_img, term_list[0]))['arr_0']
(hpmap,wpmap) = probability_map.shape
probability_volume = np.zeros((hpmap,wpmap,len(term_list)))
probability_volume[:,:,0] = probability_map
for i in range(1,len(term_list)):
id_term = term_list[i]
probability_volume[:, :, i] = np.load('%d_%d_votemap.npy.npz'%(id_img, id_term))['arr_0']
current_R = conn.parameters.model_R_MAX
while current_R >= conn.parameters.model_R_MIN:
coords = np.round(find_best_positions(probability_volume, coords, int(np.round(current_R)))).astype(int)
coords = np.round(fit_shape(mu, P, coords)).astype(int)
current_R = current_R * conn.parameters.model_alpha
x_final = np.round(coords[:nldms])
y_final = np.round(coords[nldms:])
lbl_img = np.zeros((hpmap, wpmap), 'uint8')
for i in range(nldms):
lbl_img[int(y_final[i]), int(x_final[i])] = term_list[i]
imwrite(path=os.path.join(out_path, '%d.tif' % id_img), image=lbl_img.astype(np.uint8), is_2d=True)
upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=True,
monitor_params={"start": 80, "end": 90, "period": 0.1})
conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
xc = x[0]
yr = yc / h
xr = xc / w
xcs[i, id_term - 1] = xc
ycs[i, id_term - 1] = yc
xrs[i, id_term - 1] = xr
yrs[i, id_term - 1] = yr
return np.array(xcs), np.array(ycs), np.array(xrs), np.array(yrs)
if __name__ == "__main__":
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
conn.job.update(status=Job.RUNNING,
statusComment="Initialization of the training phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
problem_cls, conn, is_2d=True, **conn.flags)
tmax = 1
for f in os.listdir(gt_path):
if f.endswith('.tif'):
gt_img = imageio.imread(os.path.join(gt_path, f))
tmax = np.max(gt_img)
break
term_list = range(1, tmax + 1)
tr_im = [
int(id_im)
for id_im in conn.parameters.cytomine_training_images.split(',')
]
DATA = None
REP = None
be = 0
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
is_2d = True
conn.job.update(status=Job.RUNNING,
progress=0,
statusComment="Initialization of the training phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
problem_cls, conn, is_2d=is_2d, **conn.flags)
tmax = 1
for f in os.listdir(gt_path):
if f.endswith('.tif'):
gt_img = imageio.imread(os.path.join(gt_path, f))
tmax = np.max(gt_img)
break
term_list = range(1, tmax + 1)
depths = 1. / (2.**np.arange(conn.parameters.model_depth))
tr_im = [
int(id_im)
for id_im in conn.parameters.cytomine_training_images.split(',')
]
DATA = None
REP = None
be = 0
sfinal = ""
for id_term in term_list:
sfinal += "%d " % id_term
sfinal = sfinal.rstrip(' ')
for id_term in conn.monitor(term_list,
start=10,
end=90,
period=0.05,
prefix="Model building for terms..."):
(xc, yc, xr, yr) = getcoordsim_neubias(gt_path, id_term, tr_im)
nimages = np.max(xc.shape)
mx = np.mean(xr)
my = np.mean(yr)
P = np.zeros((2, nimages))
P[0, :] = xr
P[1, :] = yr
cm = np.cov(P)
passe = False
# additional parameters
feature_parameters = None
if conn.parameters.model_feature_type.lower() == 'gaussian':
std_matrix = np.eye(2) * (
conn.parameters.model_feature_gaussian_std**2)
feature_parameters = np.round(
np.random.multivariate_normal(
[0, 0], std_matrix,
conn.parameters.model_feature_gaussian_n)).astype(int)
elif conn.parameters.model_feature_type.lower() == 'haar':
W = conn.parameters.model_wsize
n = conn.parameters.model_feature_haar_n / (
5 * conn.parameters.model_depth)
h2 = generate_2_horizontal(W, n)
v2 = generate_2_vertical(W, n)
h3 = generate_3_horizontal(W, n)
v3 = generate_3_vertical(W, n)
sq = generate_square(W, n)
feature_parameters = (h2, v2, h3, v3, sq)
for times in range(conn.parameters.model_ntimes):
if times == 0:
rangrange = 0
else:
rangrange = conn.parameters.model_angle
T = build_datasets_rot_mp(
in_path, tr_im, xc, yc, conn.parameters.model_R,
conn.parameters.model_RMAX, conn.parameters.model_P,
conn.parameters.model_step, rangrange,
conn.parameters.model_wsize,
conn.parameters.model_feature_type, feature_parameters,
depths, nimages, 'tif', conn.parameters.model_njobs)
for i in range(len(T)):
(data, rep, img) = T[i]
(height, width) = data.shape
if not passe:
passe = True
DATA = np.zeros((height * (len(T) + 100) *
conn.parameters.model_ntimes, width))
REP = np.zeros(height * (len(T) + 100) *
conn.parameters.model_ntimes)
b = 0
be = height
DATA[b:be, :] = data
REP[b:be] = rep
b = be
be = be + height
REP = REP[0:b]
DATA = DATA[0:b, :]
clf = ExtraTreesClassifier(
n_jobs=conn.parameters.model_njobs,
n_estimators=conn.parameters.model_ntrees)
clf = clf.fit(DATA, REP)
parameters_hash = {}
parameters_hash['cytomine_id_terms'] = sfinal.replace(' ', ',')
parameters_hash['model_R'] = conn.parameters.model_R
parameters_hash['model_RMAX'] = conn.parameters.model_RMAX
parameters_hash['model_P'] = conn.parameters.model_P
parameters_hash['model_npred'] = conn.parameters.model_npred
parameters_hash['model_ntrees'] = conn.parameters.model_ntrees
parameters_hash['model_ntimes'] = conn.parameters.model_ntimes
parameters_hash['model_angle'] = conn.parameters.model_angle
parameters_hash['model_depth'] = conn.parameters.model_depth
parameters_hash['model_step'] = conn.parameters.model_step
parameters_hash['window_size'] = conn.parameters.model_wsize
parameters_hash[
'feature_type'] = conn.parameters.model_feature_type
parameters_hash[
'feature_haar_n'] = conn.parameters.model_feature_haar_n
parameters_hash[
'feature_gaussian_n'] = conn.parameters.model_feature_gaussian_n
parameters_hash[
'feature_gaussian_std'] = conn.parameters.model_feature_gaussian_std
model_filename = joblib.dump(clf,
os.path.join(
out_path,
'%d_model.joblib' % (id_term)),
compress=3)[0]
cov_filename = joblib.dump([mx, my, cm],
os.path.join(
out_path,
'%d_cov.joblib' % (id_term)),
compress=3)[0]
parameter_filename = joblib.dump(
parameters_hash,
os.path.join(out_path, '%d_parameters.joblib' % id_term),
compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job").upload()
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=cov_filename,
domainClassName="be.cytomine.processing.Job").upload()
AttachedFile(
conn.job,
domainIndent=conn.job.id,
filename=parameter_filename,
domainClassName="be.cytomine.processing.Job").upload()
if conn.parameters.model_feature_type == 'haar' or conn.parameters.model_feature_type == 'gaussian':
add_filename = joblib.dump(
feature_parameters,
out_path.rstrip('/') + '/' + '%d_fparameters.joblib' %
(id_term))[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=add_filename,
domainClassName="be.cytomine.processing.Job").upload()
Property(conn.job, key="id_terms", value=sfinal.rstrip(" ")).save()
conn.job.update(progress=100,
status=Job.TERMINATED,
statusComment="Job terminated.")
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
is_2d = True
conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=is_2d, **conn.flags)
list_imgs = [int(image.rstrip('.tif')) for image in os.listdir(in_path) if image.endswith('.tif')]
train_job = Job().fetch(conn.parameters.model_to_use)
properties = PropertyCollection(train_job).fetch()
str_terms = ""
for prop in properties:
if prop.fetch(key='id_terms')!=None:
str_terms = prop.fetch(key='id_terms').value
term_list = [int(x) for x in str_terms.split(' ')]
attached_files = AttachedFileCollection(train_job).fetch()
hash_pos = {}
hash_size = {}
for id_term in conn.monitor(term_list, start=10, end=70, period = 0.05, prefix="Finding landmarks for terms..."):
model_file = find_by_attribute(attached_files, "filename", "%d_model.joblib"%id_term)
model_filepath = os.path.join(in_path, "%d_model.joblib"%id_term)
model_file.download(model_filepath, override=True)
cov_file = find_by_attribute(attached_files, 'filename', '%d_cov.joblib'%id_term)
cov_filepath = os.path.join(in_path, "%d_cov.joblib"%id_term)
cov_file.download(cov_filepath, override=True)
parameters_file = find_by_attribute(attached_files, 'filename', '%d_parameters.joblib'%id_term)
parameters_filepath = os.path.join(in_path, '%d_parameters.joblib'%id_term)
parameters_file.download(parameters_filepath, override=True)
model = joblib.load(model_filepath)
[mx, my, cm] = joblib.load(cov_filepath)
parameters_hash = joblib.load(parameters_filepath)
feature_parameters = None
if parameters_hash['feature_type'] in ['haar', 'gaussian']:
fparameters_file = find_by_attribute(attached_files, 'filename', "%d_fparameters.joblib"%id_term)
fparametersl_filepath = os.path.join(in_path, "%d_fparameters.joblib"%id_term)
fparameters_file.download(fparametersl_filepath, override=True)
feature_parameters = joblib.load(fparametersl_filepath)
for id_img in list_imgs:
(x, y, height, width) = searchpoint_cytomine(in_path, id_img, model, mx, my, cm, 1. / (2. ** np.arange(parameters_hash['model_depth'])), parameters_hash['window_size'], parameters_hash['feature_type'], feature_parameters, 'tif', parameters_hash['model_npred'])
if (not id_img in hash_size):
hash_size[id_img] = (height, width)
hash_pos[id_img] = []
hash_pos[id_img].append(((id_term, x, y)))
conn.job.update(status=Job.RUNNING, progress=95, statusComment="Uploading the results...")
for id_img in list_imgs:
(h, w) = hash_size[id_img]
lbl_img = np.zeros((h, w), 'uint8')
for (id_term, x, y) in hash_pos[id_img]:
intx = int(x)
inty = int(y)
if lbl_img[inty, intx] > 0:
(ys, xs) = np.where(lbl_img==0)
dis = np.sqrt((ys-y)**2 + (xs-x)**2)
j = np.argmin(dis)
intx = int(xs[j])
inty = int(ys[j])
lbl_img[inty, intx] = id_term
imwrite(path=os.path.join(out_path, '%d.tif'%id_img), image=lbl_img.astype(np.uint8), is_2d=is_2d)
upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=is_2d, monitor_params={"start": 70, "end": 90, "period": 0.1})
conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main(argv):
# 0. Initialize Cytomine client and job
with NeubiasJob.from_cli(argv) as nj:
nj.job.update(status=Job.RUNNING,
progress=0,
statusComment='Initialisation...')
problem_cls = CLASS_TRETRC
is_2d = False
threshold_value = nj.parameters.threshold_value
auto_downsampled = nj.parameters.auto_downsampled
# 1. Create working directories on the machine
# 2. Download the images
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
problem_cls, nj, is_2d=is_2d, **nj.flags)
# 3. Call the image analysis workflow using the run script
nj.job.update(progress=25, statusComment='Launching workflow...')
workflow(in_images, out_path, threshold_value, auto_downsampled)
#if return_code != 0:
# err_desc = 'Failed to execute the Vaa3D (return code: {})'.format(return_code)
#nj.job.update(progress=50, statusComment=err_desc)
# raise ValueError(err_desc)
print('files in out_path ' + out_path + ': ')
for file in glob.glob(out_path + '/*'):
print(file)
#files = (glob.glob(in_path+'/*.tif'))
#print('Removing flipped images...')
#for i in range(0,len(files)):
# files[i] = files[i].replace('/in/','/out/')
# print(files[i])
#for out_file in files:
# os.remove(out_file)
# 4. Upload the annotation and labels to Cytomine (annotations are extracted from the mask using
# the AnnotationExporter module
upload_data(problem_cls,
nj,
in_images,
out_path,
**nj.flags,
projection=-1,
is_2d=is_2d,
monitor_params={
"start": 60,
"end": 90,
"period": 0.1,
"prefix":
"Extracting and uploading polygons from masks"
})
#5. Compute and upload the metrics
nj.job.update(
progress=80,
statusComment='Computing and uploading metrics (if necessary)...')
upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path,
**nj.flags)
nj.job.update(status=Job.TERMINATED,
progress=100,
statusComment='Finished.')
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
conn.job.update(progress=0, status=Job.RUNNING, statusComment="Initialization of the training phase...")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)
tmax = 1
for f in os.listdir(gt_path):
if f.endswith('.tif'):
gt_img = imageio.imread(os.path.join(gt_path, f))
tmax = np.max(gt_img)
break
term_list = range(1, tmax + 1)
tr_im = [int(id_im) for id_im in conn.parameters.cytomine_training_images.split(',')]
(xc, yc, xr, yr) = get_neubias_coords(gt_path, tr_im)
(nims, nldms) = xc.shape
Xc = np.zeros((nims, len(term_list)))
Yc = np.zeros(Xc.shape)
for id_term in term_list:
Xc[:, id_term - 1] = xc[:, id_term - 1]
Yc[:, id_term - 1] = yc[:, id_term - 1]
conn.job.update(progress=10, status=Job.RUNNING, statusComment="Building model for phase 1")
(dataset, rep, img, feature_offsets_1) = build_phase_1_model(in_path, image_ids=tr_im, n_jobs=conn.parameters.model_njobs, F=conn.parameters.model_F_P1, R=conn.parameters.model_R_P1, sigma=conn.parameters.model_sigma, delta=conn.parameters.model_delta, P=conn.parameters.model_P, X=Xc, Y=Yc)
clf = SeparateTrees(n_estimators=int(conn.parameters.model_NT_P1), n_jobs=int(conn.parameters.model_njobs))
clf = clf.fit(dataset, rep)
model_filename = joblib.dump(clf, os.path.join(out_path, 'model_phase1.joblib'), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
model_filename = joblib.dump((Xc, Yc), os.path.join(out_path, 'coords.joblib'), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
model_filename = joblib.dump(feature_offsets_1, os.path.join(out_path, 'offsets_phase1.joblib'), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
for id_term in conn.monitor(term_list, start=20, end=80, period=0.05,prefix="Visual model building for terms..."):
(dataset, rep, number, feature_offsets_2) = build_phase_2_model(in_path, image_ids=tr_im, n_jobs=conn.parameters.model_njobs, NT=conn.parameters.model_NT_P2, F=conn.parameters.model_F_P2, R=conn.parameters.model_R_P2, N=conn.parameters.model_ns_P2, sigma=conn.parameters.model_sigma, delta=conn.parameters.model_delta, Xc = Xc[:, id_term-1], Yc = Yc[:, id_term-1])
reg = SeparateTreesRegressor(n_estimators=int(conn.parameters.model_NT_P2), n_jobs=int(conn.parameters.model_njobs))
reg.fit(dataset, rep)
model_filename = joblib.dump(reg, os.path.join(out_path, 'reg_%d_phase2.joblib'%id_term), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
model_filename = joblib.dump(feature_offsets_2, os.path.join(out_path, 'offsets_%d_phase2.joblib' % id_term), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
conn.job.update(progress=90, status=Job.RUNNING, statusComment="Building model for phase 3")
edges = build_edgematrix_phase_3(Xc, Yc, conn.parameters.model_sde, conn.parameters.model_delta, conn.parameters.model_T)
model_filename = joblib.dump(edges, os.path.join(out_path, 'model_edges.joblib'), compress=3)[0]
AttachedFile(
conn.job,
domainIdent=conn.job.id,
filename=model_filename,
domainClassName="be.cytomine.processing.Job"
).upload()
sfinal = ""
for id_term in term_list:
sfinal += "%d " % id_term
sfinal = sfinal.rstrip(' ')
Property(conn.job, key="id_terms", value=sfinal.rstrip(" ")).save()
conn.job.update(progress=100, status=Job.TERMINATED, statusComment="Job terminated.")
def main():
with NeubiasJob.from_cli(sys.argv) as conn:
problem_cls = get_discipline(conn, default=CLASS_LNDDET)
conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)
train_job = Job().fetch(conn.parameters.model_to_use)
properties = PropertyCollection(train_job).fetch()
str_terms = ""
for prop in properties:
if prop.fetch(key='id_terms') != None:
str_terms = prop.fetch(key='id_terms').value
term_list = [int(x) for x in str_terms.split(' ')]
attached_files = AttachedFileCollection(train_job).fetch()
model_file = find_by_attribute(attached_files, "filename", "model_phase1.joblib")
model_filepath = os.path.join(in_path, "model_phase1.joblib")
model_file.download(model_filepath, override=True)
clf = joblib.load(model_filepath)
pr_ims = [int(p) for p in conn.parameters.cytomine_predict_images.split(',')]
tifimg = readimage(in_path, pr_ims[0], image_type='tif')
init_h = 100
init_w = 100
if len(tifimg.shape)==3:
(init_h, init_w, init_d) = tifimg.shape
else:
(init_h, init_w) = tifimg.shape
offset_file = find_by_attribute(attached_files, "filename", "offsets_phase1.joblib")
offset_filepath = os.path.join(in_path, "offsets_phase1.joblib")
offset_file.download(offset_filepath, override=True)
feature_offsets_1 = joblib.load(offset_filepath)
train_parameters = {}
for hashmap in train_job.jobParameters:
train_parameters[hashmap['name']] = hashmap['value']
train_parameters['model_delta'] = float(train_parameters['model_delta'])
train_parameters['model_sde'] = float(train_parameters['model_sde'])
train_parameters['model_T'] = int(train_parameters['model_T'])
for j in conn.monitor(pr_ims, start=10, end=33, period=0.05,prefix="Phase 1 for images..."):
probability_map = probability_map_phase_1(in_path, j, clf, feature_offsets_1, float(train_parameters['model_delta']))
filesave = os.path.join(out_path, 'pmap_%d.npy'%j)
np.savez_compressed(filesave,probability_map)
clf = None
coords_file = find_by_attribute(attached_files, "filename", "coords.joblib")
coords_filepath = os.path.join(in_path, "coords.joblib")
coords_file.download(coords_filepath, override=True)
(Xc, Yc) = joblib.load(coords_filepath)
for j in conn.monitor(pr_ims, start=33, end=66, period=0.05,prefix="Phase 2 for images..."):
filesave = os.path.join(out_path, 'pmap_%d.npy.npz' % j)
probability_map = np.load(filesave)['arr_0']
for id_term in term_list:
reg_file = find_by_attribute(attached_files, "filename", "reg_%d_phase2.joblib"%id_term)
reg_filepath = os.path.join(in_path, "reg_%d_phase2.joblib"%id_term)
reg_file.download(reg_filepath, override=True)
reg = joblib.load(reg_filepath)
off_file = find_by_attribute(attached_files, "filename", 'offsets_%d_phase2.joblib' % id_term)
off_filepath = os.path.join(in_path, 'offsets_%d_phase2.joblib' % id_term)
off_file.download(off_filepath, override=True)
feature_offsets_2 = joblib.load(off_filepath)
probability_map_phase_2 = agregation_phase_2(in_path, j, id_term-1, probability_map, reg, train_parameters['model_delta'], feature_offsets_2, conn.parameters.model_filter_size, conn.parameters.model_beta, conn.parameters.model_n_iterations)
filesave = os.path.join(out_path, 'pmap2_%d_%d.npy' % (j, id_term))
np.savez_compressed(filesave, probability_map_phase_2)
edge_file = find_by_attribute(attached_files, "filename", "model_edges.joblib")
edge_filepath = os.path.join(in_path, "model_edges.joblib")
edge_file.download(edge_filepath, override=True)
edges = joblib.load(edge_filepath)
for j in conn.monitor(pr_ims, start=66, end=90, period=0.05,prefix="Phase 3 for images..."):
filesave = os.path.join(out_path, 'pmap2_%d_%d.npy.npz' % (j, term_list[0]))
probability_map = np.load(filesave)['arr_0']
(hpmap,wpmap) = probability_map.shape
probability_volume = np.zeros((hpmap,wpmap,len(term_list)))
probability_volume[:,:,0] = probability_map
for i in range(1,len(term_list)):
filesave = os.path.join(out_path, 'pmap2_%d_%d.npy.npz' % (j, term_list[i]))
probability_volume[:,:,i] = np.load(filesave)['arr_0']
x_final, y_final = compute_final_solution_phase_3(Xc, Yc, probability_volume, conn.parameters.model_n_candidates, train_parameters['model_sde'], train_parameters['model_delta'], train_parameters['model_T'], edges)
lbl_img = np.zeros((init_h, init_w), 'uint8')
for i in range(x_final.size):
x = min(init_w-1, max(0, int(x_final[i])))
y = min(init_h-1, max(0, int(y_final[i])))
lbl_img[y, x] = term_list[i]
imwrite(path=os.path.join(out_path, '%d.tif' % j), image=lbl_img.astype(np.uint8), is_2d=True)
upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=True, monitor_params={"start": 90, "end": 95, "period": 0.1})
conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")