def main(argv):
    # 0. Initialize Cytomine client and job if necessary and parse inputs
    with NeubiasJob.from_cli(argv) as nj:
        problem_cls = get_discipline(nj, default=CLASS_SPTCNT)
        is_2d = False
        nj.job.update(status=Job.RUNNING, progress=0, statusComment="Running workflow for problem class '{}' in {}D".format(problem_cls, 2 if is_2d else 3))

        # 1. Create working directories on the machine
        # 2. Download the images
        nj.job.update(progress=0, statusComment="Initialisation...")
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)

        # 3. Call the image analysis workflow using the run script
        nj.job.update(progress=25, statusComment="Launching workflow...")
        command = "/usr/bin/xvfb-run java -Xmx6000m -cp /fiji/jars/ij.jar ij.ImageJ --headless --console " \
                  "-macro macro.ijm \"input={}, output={}, scale={}, radius_xy={}, radius_z={}, noise={}\"".format(in_path, out_path, nj.parameters.ij_scale, nj.parameters.ij_radius_xy, nj.parameters.ij_radius_z, nj.parameters.ij_noise)
        return_code = call(command, shell=True, cwd="/fiji")  # waits for the subprocess to return

        if return_code != 0:
            err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
            nj.job.update(progress=50, statusComment=err_desc)
            raise ValueError(err_desc)

        # 4. Upload the annotation and labels to Cytomine
        upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
           "start": 60, "end": 90, "period": 0.1
        })

        # 5. Compute and upload the metrics
        nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)
        
        # 6. End
        nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main():
    with NeubiasJob.from_cli(sys.argv[1:]) as nj:
        nj.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization...")

        # 1. Create working directories on the machine:
        # 2. Download (or read) data
        problem_cls = get_discipline(nj, default=CLASS_SPTCNT)
        is_2d = True
        nj.job.update(progress=1, statusComment="Execute workflow on problem class '{}'".format(problem_cls))
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)

        # 3. Execute workflow
        scale3sens = nj.parameters.icy_scale3sensitivity
        nj.job.update(progress=25, statusComment="Launching workflow...")
        call("java -cp /icy/lib/ -jar /icy/icy.jar -hl", shell=True, cwd="/icy")
        call("java -cp /icy/lib/ -jar /icy/icy.jar -hl -x plugins.adufour.protocols.Protocols "
             "protocol=\"/icy/protocols/protocol.protocol\" inputFolder=\"{}\" outputFolder=\"{}\" extension=tif "
             "scale2enable=true scale2sensitivity={}".format(in_path, out_path, scale3sens), shell=True, cwd="/icy")

        # 3.5 Remove the xml-output files
        for p in Path(out_path).glob("*.xml"):
            p.unlink()
        
        # 4. Upload the annotation and labels to Cytomine
        upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
           "start": 60, "end": 90, "period": 0.1
        })

        # 5. Compute and upload the metrics
        nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)

        nj.job.update(progress=100, status=Job.TERMINATED, status_comment="Finished.")
示例#3
0
def main(argv):
    with NeubiasJob.from_cli(argv) as nj:
        problem_cls = get_discipline(nj, default=CLASS_PIXCLA)
        is_2d = True

        nj.job.update(status=Job.RUNNING,
                      progress=0,
                      statusComment="Initialisation...")
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
            problem_cls, nj, **nj.flags)

        # 2. Call the image analysis workflow
        nj.job.update(progress=10, statusComment="Load model...")
        net = load_model("/app/model.pth")
        device = torch.device("cpu")

        for in_image in Monitor(nj,
                                in_images,
                                start=20,
                                end=75,
                                period=0.05,
                                prefix="Apply UNet to input images"):
            mask = predict_img(net,
                               in_image.filepath,
                               device="cpu",
                               out_threshold=nj.parameters.threshold)

            imwrite(path=os.path.join(out_path, in_image.filename),
                    image=mask.astype(np.uint8),
                    is_2d=is_2d)

        # 4. Create and upload annotations
        nj.job.update(progress=70,
                      statusComment="Uploading extracted annotation...")
        upload_data(problem_cls,
                    nj,
                    in_images,
                    out_path,
                    **nj.flags,
                    is_2d=is_2d,
                    monitor_params={
                        "start": 70,
                        "end": 90,
                        "period": 0.1
                    })

        # 5. Compute and upload the metrics
        nj.job.update(
            progress=90,
            statusComment="Computing and uploading metrics (if necessary)...")
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path,
                       **nj.flags)

        # 6. End the job
        nj.job.update(status=Job.TERMINATED,
                      progress=100,
                      statusComment="Finished.")
示例#4
0
def main(argv):
    # 0. Initialize Cytomine client and job if necessary and parse inputs
    with NeubiasJob.from_cli(argv) as nj:
        problem_cls = get_discipline(nj, default=CLASS_OBJTRK)
        is_2d = False
        nj.job.update(status=Job.RUNNING, progress=0, statusComment="Running workflow for problem class '{}' in 2D+t".format(problem_cls))

        # 1. Create working directories on the machine
        # 2. Download the images
        nj.job.update(progress=0, statusComment="Initialisation...")
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, **nj.flags)

        # 3. Call the image analysis workflow using the run script
        nj.job.update(progress=25, statusComment="Launching workflow...")

        # DEBUG Workflow not currently working !!
        # # CellTrackingChallenge expects the protocol to start with the data folder name
        # in_path_folder_name, in_folder_name = os.path.basename(os.path.dirname(in_path)), os.path.basename(in_path)
        # shutil.copyfile("/app/proto.protocol", "/app/{}-{}.protocol".format(in_path_folder_name, in_folder_name))
        # command = "java -Xmx2048m -jar icy.jar -hl -x plugins.adufour.ctc.CellTrackingChallenge {} {}".format(os.path.dirname(in_path), os.path.basename(in_path))
        # return_code = call(command, shell=True, cwd="/app")  # waits for the subprocess to return
        #
        # if return_code != 0:
        #     err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
        #     nj.job.update(progress=50, statusComment=err_desc)
        #     raise ValueError(err_desc)
        #
        # # move files generated by CellTrackingChallenge into the output folder
        # res_path = in_path + "_RES"
        # for file in os.listdir(res_path):
        #     shutil.move(os.path.join(res_path, file), out_path)

        # DEBUG copy ground truth in output
        for file in os.listdir(gt_path):
            outfile = file.replace("_attached", "") if file.endswith(".txt") else file
            shutil.copy(os.path.join(gt_path, file), os.path.join(out_path, outfile))

        # 4. Upload the annotation and labels to Cytomine
        upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
            "start": 60, "end": 90, "period": 0.1
        })

        # 5. Compute and upload the metrics
        nj.job.update(progress=90, statusComment="Computing and uploading metrics...")
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)

        # 6. End
        nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main(argv):
    # 0. Initialize Cytomine client and job
    with NeubiasJob.from_cli(argv) as nj:
        nj.job.update(status=Job.RUNNING, progress=0, statusComment="Initialisation...")

        problem_cls = CLASS_OBJSEG
        is_2d = False

        # 1. Create working directories on the machine
        # 2. Download the images
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, nj, is_2d=is_2d, **nj.flags)

        # 3. Call the image analysis workflow using the run script
        nj.job.update(progress=25, statusComment="Launching workflow...")
        command = "/usr/bin/xvfb-run java -Xmx6000m -cp /fiji/jars/ij.jar ij.ImageJ --headless --console " \
                  "-macro macro.ijm \"input={}, output={}, radius={}, min_threshold={}\"".format(in_path, out_path, nj.parameters.ij_radius, nj.parameters.ij_min_threshold)
        return_code = call(command, shell=True, cwd="/fiji")  # waits for the subprocess to return

        if return_code != 0:
            err_desc = "Failed to execute the ImageJ macro (return code: {})".format(return_code)
            nj.job.update(progress=50, statusComment=err_desc)
            raise ValueError(err_desc)
            
        # 4. Upload the annotation and labels to Cytomine (annotations are extracted from the mask using
        # the AnnotationExporter module)
        upload_data(problem_cls, nj, in_images, out_path, **nj.flags, is_2d=is_2d, monitor_params={
            "start": 60, "end": 90,
            "period": 0.1,
            "prefix": "Extracting and uploading polygons from masks"
        })

        # 5. Compute and upload the metrics
        nj.job.update(progress=80, statusComment="Computing and uploading metrics (if necessary)...")
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path, **nj.flags)

        nj.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main():
	with NeubiasJob.from_cli(sys.argv) as conn:
		problem_cls = get_discipline(conn, default=CLASS_LNDDET)
		conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
		in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)
		list_imgs = [int(image.rstrip('.tif')) for image in os.listdir(in_path) if image.endswith('.tif')]
		train_job = Job().fetch(conn.parameters.model_to_use)
		properties = PropertyCollection(train_job).fetch()
		str_terms = ""
		for prop in properties:
			if prop.fetch(key='id_terms') != None:
				str_terms = prop.fetch(key='id_terms').value
		term_list = [int(x) for x in str_terms.split(' ')]
		attached_files = AttachedFileCollection(train_job).fetch()

		feature_file = find_by_attribute(attached_files, "filename", "features.joblib")
		feature_filepath = os.path.join(in_path, "features.joblib")
		feature_file.download(feature_filepath, override=True)
		(h2,v2,h3,v3,sq) = joblib.load(feature_filepath)

		coords_file = find_by_attribute(attached_files, "filename", "coords.joblib")
		coords_filepath = os.path.join(in_path, "coords.joblib")
		coords_file.download(coords_filepath, override=True)
		(Xc, Yc) = joblib.load(coords_filepath)
		(nims, nldms) = Xc.shape

		coords = np.zeros(2 * nldms)
		i = 0
		for id_term in conn.monitor(term_list, start=10, end=50, period = 0.05, prefix="Building vote maps..."):
			model_file = find_by_attribute(attached_files, "filename", "%d_model.joblib" % id_term)
			model_filepath = os.path.join(in_path, "%d_model.joblib" % id_term)
			model_file.download(model_filepath, override=True)
			clf = joblib.load(model_filepath)
			mx = np.mean(Xc[:, id_term-1])
			my = np.mean(Yc[:, id_term-1])
			coords[i] = mx
			coords[i+nldms] = my
			i+=1
			for j in list_imgs:
				print(j)
				vote_map = build_vote_map(in_path, j, clf, h2, v2, h3, v3, sq, conn.parameters.model_step)
				np.savez_compressed('%d_%d_votemap.npy' % (j, id_term), vote_map)

		muP_file = find_by_attribute(attached_files, "filename", "muP.joblib")
		muP_filepath = os.path.join(in_path, "muP.joblib")
		muP_file.download(muP_filepath, override=True)
		(mu, P) = joblib.load(muP_filepath)
		(nims, nldms) = Xc.shape
		for id_img in conn.monitor(list_imgs, start=50, end=80, period = 0.05, prefix="Post-processing..."):
			probability_map = np.load('%d_%d_votemap.npy.npz' % (id_img, term_list[0]))['arr_0']
			(hpmap,wpmap) = probability_map.shape
			probability_volume = np.zeros((hpmap,wpmap,len(term_list)))
			probability_volume[:,:,0] = probability_map
			for i in range(1,len(term_list)):
				id_term = term_list[i]
				probability_volume[:, :, i] = np.load('%d_%d_votemap.npy.npz'%(id_img, id_term))['arr_0']
			current_R = conn.parameters.model_R_MAX
			while current_R >= conn.parameters.model_R_MIN:
				coords = np.round(find_best_positions(probability_volume, coords, int(np.round(current_R)))).astype(int)
				coords = np.round(fit_shape(mu, P, coords)).astype(int)
				current_R = current_R * conn.parameters.model_alpha
			x_final = np.round(coords[:nldms])
			y_final = np.round(coords[nldms:])
			lbl_img = np.zeros((hpmap, wpmap), 'uint8')
			for i in range(nldms):
				lbl_img[int(y_final[i]), int(x_final[i])] = term_list[i]
			imwrite(path=os.path.join(out_path, '%d.tif' % id_img), image=lbl_img.astype(np.uint8), is_2d=True)

		upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=True,
					monitor_params={"start": 80, "end": 90, "period": 0.1})
		conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
		upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
		conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
            xc = x[0]
            yr = yc / h
            xr = xc / w
            xcs[i, id_term - 1] = xc
            ycs[i, id_term - 1] = yc
            xrs[i, id_term - 1] = xr
            yrs[i, id_term - 1] = yr
    return np.array(xcs), np.array(ycs), np.array(xrs), np.array(yrs)


if __name__ == "__main__":
    with NeubiasJob.from_cli(sys.argv) as conn:
        problem_cls = get_discipline(conn, default=CLASS_LNDDET)
        conn.job.update(status=Job.RUNNING,
                        statusComment="Initialization of the training phase")
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
            problem_cls, conn, is_2d=True, **conn.flags)
        tmax = 1
        for f in os.listdir(gt_path):
            if f.endswith('.tif'):
                gt_img = imageio.imread(os.path.join(gt_path, f))
                tmax = np.max(gt_img)
                break

        term_list = range(1, tmax + 1)
        tr_im = [
            int(id_im)
            for id_im in conn.parameters.cytomine_training_images.split(',')
        ]
        DATA = None
        REP = None
        be = 0
def main():
    with NeubiasJob.from_cli(sys.argv) as conn:
        problem_cls = get_discipline(conn, default=CLASS_LNDDET)
        is_2d = True
        conn.job.update(status=Job.RUNNING,
                        progress=0,
                        statusComment="Initialization of the training phase")
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
            problem_cls, conn, is_2d=is_2d, **conn.flags)
        tmax = 1
        for f in os.listdir(gt_path):
            if f.endswith('.tif'):
                gt_img = imageio.imread(os.path.join(gt_path, f))
                tmax = np.max(gt_img)
                break

        term_list = range(1, tmax + 1)
        depths = 1. / (2.**np.arange(conn.parameters.model_depth))

        tr_im = [
            int(id_im)
            for id_im in conn.parameters.cytomine_training_images.split(',')
        ]

        DATA = None
        REP = None
        be = 0
        sfinal = ""
        for id_term in term_list:
            sfinal += "%d " % id_term
        sfinal = sfinal.rstrip(' ')
        for id_term in conn.monitor(term_list,
                                    start=10,
                                    end=90,
                                    period=0.05,
                                    prefix="Model building for terms..."):
            (xc, yc, xr, yr) = getcoordsim_neubias(gt_path, id_term, tr_im)
            nimages = np.max(xc.shape)
            mx = np.mean(xr)
            my = np.mean(yr)
            P = np.zeros((2, nimages))
            P[0, :] = xr
            P[1, :] = yr
            cm = np.cov(P)
            passe = False
            # additional parameters
            feature_parameters = None
            if conn.parameters.model_feature_type.lower() == 'gaussian':
                std_matrix = np.eye(2) * (
                    conn.parameters.model_feature_gaussian_std**2)
                feature_parameters = np.round(
                    np.random.multivariate_normal(
                        [0, 0], std_matrix,
                        conn.parameters.model_feature_gaussian_n)).astype(int)
            elif conn.parameters.model_feature_type.lower() == 'haar':
                W = conn.parameters.model_wsize
                n = conn.parameters.model_feature_haar_n / (
                    5 * conn.parameters.model_depth)
                h2 = generate_2_horizontal(W, n)
                v2 = generate_2_vertical(W, n)
                h3 = generate_3_horizontal(W, n)
                v3 = generate_3_vertical(W, n)
                sq = generate_square(W, n)
                feature_parameters = (h2, v2, h3, v3, sq)

            for times in range(conn.parameters.model_ntimes):
                if times == 0:
                    rangrange = 0
                else:
                    rangrange = conn.parameters.model_angle

                T = build_datasets_rot_mp(
                    in_path, tr_im, xc, yc, conn.parameters.model_R,
                    conn.parameters.model_RMAX, conn.parameters.model_P,
                    conn.parameters.model_step, rangrange,
                    conn.parameters.model_wsize,
                    conn.parameters.model_feature_type, feature_parameters,
                    depths, nimages, 'tif', conn.parameters.model_njobs)
                for i in range(len(T)):
                    (data, rep, img) = T[i]
                    (height, width) = data.shape
                    if not passe:
                        passe = True
                        DATA = np.zeros((height * (len(T) + 100) *
                                         conn.parameters.model_ntimes, width))
                        REP = np.zeros(height * (len(T) + 100) *
                                       conn.parameters.model_ntimes)
                        b = 0
                        be = height
                    DATA[b:be, :] = data
                    REP[b:be] = rep
                    b = be
                    be = be + height

            REP = REP[0:b]
            DATA = DATA[0:b, :]

            clf = ExtraTreesClassifier(
                n_jobs=conn.parameters.model_njobs,
                n_estimators=conn.parameters.model_ntrees)
            clf = clf.fit(DATA, REP)

            parameters_hash = {}
            parameters_hash['cytomine_id_terms'] = sfinal.replace(' ', ',')
            parameters_hash['model_R'] = conn.parameters.model_R
            parameters_hash['model_RMAX'] = conn.parameters.model_RMAX
            parameters_hash['model_P'] = conn.parameters.model_P
            parameters_hash['model_npred'] = conn.parameters.model_npred
            parameters_hash['model_ntrees'] = conn.parameters.model_ntrees
            parameters_hash['model_ntimes'] = conn.parameters.model_ntimes
            parameters_hash['model_angle'] = conn.parameters.model_angle
            parameters_hash['model_depth'] = conn.parameters.model_depth
            parameters_hash['model_step'] = conn.parameters.model_step
            parameters_hash['window_size'] = conn.parameters.model_wsize
            parameters_hash[
                'feature_type'] = conn.parameters.model_feature_type
            parameters_hash[
                'feature_haar_n'] = conn.parameters.model_feature_haar_n
            parameters_hash[
                'feature_gaussian_n'] = conn.parameters.model_feature_gaussian_n
            parameters_hash[
                'feature_gaussian_std'] = conn.parameters.model_feature_gaussian_std

            model_filename = joblib.dump(clf,
                                         os.path.join(
                                             out_path,
                                             '%d_model.joblib' % (id_term)),
                                         compress=3)[0]
            cov_filename = joblib.dump([mx, my, cm],
                                       os.path.join(
                                           out_path,
                                           '%d_cov.joblib' % (id_term)),
                                       compress=3)[0]
            parameter_filename = joblib.dump(
                parameters_hash,
                os.path.join(out_path, '%d_parameters.joblib' % id_term),
                compress=3)[0]
            AttachedFile(
                conn.job,
                domainIdent=conn.job.id,
                filename=model_filename,
                domainClassName="be.cytomine.processing.Job").upload()
            AttachedFile(
                conn.job,
                domainIdent=conn.job.id,
                filename=cov_filename,
                domainClassName="be.cytomine.processing.Job").upload()
            AttachedFile(
                conn.job,
                domainIndent=conn.job.id,
                filename=parameter_filename,
                domainClassName="be.cytomine.processing.Job").upload()
            if conn.parameters.model_feature_type == 'haar' or conn.parameters.model_feature_type == 'gaussian':
                add_filename = joblib.dump(
                    feature_parameters,
                    out_path.rstrip('/') + '/' + '%d_fparameters.joblib' %
                    (id_term))[0]
                AttachedFile(
                    conn.job,
                    domainIdent=conn.job.id,
                    filename=add_filename,
                    domainClassName="be.cytomine.processing.Job").upload()

        Property(conn.job, key="id_terms", value=sfinal.rstrip(" ")).save()
        conn.job.update(progress=100,
                        status=Job.TERMINATED,
                        statusComment="Job terminated.")
示例#9
0
def main():
	with NeubiasJob.from_cli(sys.argv) as conn:
		problem_cls = get_discipline(conn, default=CLASS_LNDDET)
		is_2d = True
		conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
		in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=is_2d, **conn.flags)
		list_imgs = [int(image.rstrip('.tif')) for image in os.listdir(in_path) if image.endswith('.tif')]

		train_job = Job().fetch(conn.parameters.model_to_use)
		properties = PropertyCollection(train_job).fetch()
		str_terms = ""
		for prop in properties:
			if prop.fetch(key='id_terms')!=None:
				str_terms = prop.fetch(key='id_terms').value
		term_list = [int(x) for x in str_terms.split(' ')]
		attached_files = AttachedFileCollection(train_job).fetch()

		hash_pos = {}
		hash_size = {}
		for id_term in conn.monitor(term_list, start=10, end=70, period = 0.05, prefix="Finding landmarks for terms..."):
			model_file = find_by_attribute(attached_files, "filename", "%d_model.joblib"%id_term)
			model_filepath = os.path.join(in_path, "%d_model.joblib"%id_term)
			model_file.download(model_filepath, override=True)
			cov_file = find_by_attribute(attached_files, 'filename', '%d_cov.joblib'%id_term)
			cov_filepath = os.path.join(in_path, "%d_cov.joblib"%id_term)
			cov_file.download(cov_filepath, override=True)
			parameters_file = find_by_attribute(attached_files, 'filename', '%d_parameters.joblib'%id_term)
			parameters_filepath = os.path.join(in_path, '%d_parameters.joblib'%id_term)
			parameters_file.download(parameters_filepath, override=True)

			model = joblib.load(model_filepath)
			[mx, my, cm] = joblib.load(cov_filepath)
			parameters_hash = joblib.load(parameters_filepath)
			feature_parameters = None
			if parameters_hash['feature_type'] in ['haar', 'gaussian']:
				fparameters_file = find_by_attribute(attached_files, 'filename', "%d_fparameters.joblib"%id_term)
				fparametersl_filepath = os.path.join(in_path, "%d_fparameters.joblib"%id_term)
				fparameters_file.download(fparametersl_filepath, override=True)
				feature_parameters = joblib.load(fparametersl_filepath)
			for id_img in list_imgs:
				(x, y, height, width) = searchpoint_cytomine(in_path, id_img, model, mx, my, cm, 1. / (2. ** np.arange(parameters_hash['model_depth'])), parameters_hash['window_size'], parameters_hash['feature_type'], feature_parameters, 'tif', parameters_hash['model_npred'])
				if (not id_img in hash_size):
					hash_size[id_img] = (height, width)
					hash_pos[id_img] = []
				hash_pos[id_img].append(((id_term, x, y)))
		conn.job.update(status=Job.RUNNING, progress=95, statusComment="Uploading the results...")
		for id_img in list_imgs:
			(h, w) = hash_size[id_img]
			lbl_img = np.zeros((h, w), 'uint8')
			for (id_term, x, y) in hash_pos[id_img]:
				intx = int(x)
				inty = int(y)
				if lbl_img[inty, intx] > 0:
					(ys, xs) = np.where(lbl_img==0)
					dis = np.sqrt((ys-y)**2 + (xs-x)**2)
					j = np.argmin(dis)
					intx = int(xs[j])
					inty = int(ys[j])
				lbl_img[inty, intx] = id_term
			imwrite(path=os.path.join(out_path, '%d.tif'%id_img), image=lbl_img.astype(np.uint8), is_2d=is_2d)
		upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=is_2d, monitor_params={"start": 70, "end": 90, "period": 0.1})
		conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
		upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
		conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")
def main(argv):
    # 0. Initialize Cytomine client and job
    with NeubiasJob.from_cli(argv) as nj:
        nj.job.update(status=Job.RUNNING,
                      progress=0,
                      statusComment='Initialisation...')
        problem_cls = CLASS_TRETRC
        is_2d = False
        threshold_value = nj.parameters.threshold_value
        auto_downsampled = nj.parameters.auto_downsampled
        # 1. Create working directories on the machine
        # 2. Download the images
        in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(
            problem_cls, nj, is_2d=is_2d, **nj.flags)

        # 3. Call the image analysis workflow using the run script
        nj.job.update(progress=25, statusComment='Launching workflow...')
        workflow(in_images, out_path, threshold_value, auto_downsampled)

        #if return_code != 0:
        #   err_desc = 'Failed to execute the Vaa3D (return code: {})'.format(return_code)
        #nj.job.update(progress=50, statusComment=err_desc)
        #    raise ValueError(err_desc)
        print('files in out_path ' + out_path + ': ')
        for file in glob.glob(out_path + '/*'):
            print(file)

        #files = (glob.glob(in_path+'/*.tif'))
        #print('Removing flipped images...')
        #for i in range(0,len(files)):
        #    files[i] = files[i].replace('/in/','/out/')
        #    print(files[i])
        #for out_file in files:
        #    os.remove(out_file)

        # 4. Upload the annotation and labels to Cytomine (annotations are extracted from the mask using
        # the AnnotationExporter module
        upload_data(problem_cls,
                    nj,
                    in_images,
                    out_path,
                    **nj.flags,
                    projection=-1,
                    is_2d=is_2d,
                    monitor_params={
                        "start": 60,
                        "end": 90,
                        "period": 0.1,
                        "prefix":
                        "Extracting and uploading polygons from masks"
                    })

        #5. Compute and upload the metrics
        nj.job.update(
            progress=80,
            statusComment='Computing and uploading metrics (if necessary)...')
        upload_metrics(problem_cls, nj, in_images, gt_path, out_path, tmp_path,
                       **nj.flags)
        nj.job.update(status=Job.TERMINATED,
                      progress=100,
                      statusComment='Finished.')
def main():
	with NeubiasJob.from_cli(sys.argv) as conn:
		problem_cls = get_discipline(conn, default=CLASS_LNDDET)
		conn.job.update(progress=0, status=Job.RUNNING, statusComment="Initialization of the training phase...")
		in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)

		tmax = 1
		for f in os.listdir(gt_path):
			if f.endswith('.tif'):
				gt_img = imageio.imread(os.path.join(gt_path, f))
				tmax = np.max(gt_img)
				break

		term_list = range(1, tmax + 1)
		tr_im = [int(id_im) for id_im in conn.parameters.cytomine_training_images.split(',')]
		(xc, yc, xr, yr) = get_neubias_coords(gt_path, tr_im)
		(nims, nldms) = xc.shape
		Xc = np.zeros((nims, len(term_list)))
		Yc = np.zeros(Xc.shape)
		for id_term in term_list:
			Xc[:, id_term - 1] = xc[:, id_term - 1]
			Yc[:, id_term - 1] = yc[:, id_term - 1]
		conn.job.update(progress=10, status=Job.RUNNING, statusComment="Building model for phase 1")
		(dataset, rep, img, feature_offsets_1) = build_phase_1_model(in_path, image_ids=tr_im, n_jobs=conn.parameters.model_njobs, F=conn.parameters.model_F_P1, R=conn.parameters.model_R_P1, sigma=conn.parameters.model_sigma, delta=conn.parameters.model_delta, P=conn.parameters.model_P, X=Xc, Y=Yc)
		clf = SeparateTrees(n_estimators=int(conn.parameters.model_NT_P1), n_jobs=int(conn.parameters.model_njobs))
		clf = clf.fit(dataset, rep)
		model_filename = joblib.dump(clf, os.path.join(out_path, 'model_phase1.joblib'), compress=3)[0]
		AttachedFile(
			conn.job,
			domainIdent=conn.job.id,
			filename=model_filename,
			domainClassName="be.cytomine.processing.Job"
		).upload()

		model_filename = joblib.dump((Xc, Yc), os.path.join(out_path, 'coords.joblib'), compress=3)[0]
		AttachedFile(
			conn.job,
			domainIdent=conn.job.id,
			filename=model_filename,
			domainClassName="be.cytomine.processing.Job"
		).upload()

		model_filename = joblib.dump(feature_offsets_1, os.path.join(out_path, 'offsets_phase1.joblib'), compress=3)[0]
		AttachedFile(
			conn.job,
			domainIdent=conn.job.id,
			filename=model_filename,
			domainClassName="be.cytomine.processing.Job"
		).upload()

		for id_term in conn.monitor(term_list, start=20, end=80, period=0.05,prefix="Visual model building for terms..."):
			(dataset, rep, number, feature_offsets_2) = build_phase_2_model(in_path, image_ids=tr_im, n_jobs=conn.parameters.model_njobs, NT=conn.parameters.model_NT_P2, F=conn.parameters.model_F_P2, R=conn.parameters.model_R_P2, N=conn.parameters.model_ns_P2, sigma=conn.parameters.model_sigma, delta=conn.parameters.model_delta, Xc = Xc[:, id_term-1], Yc = Yc[:, id_term-1])
			reg = SeparateTreesRegressor(n_estimators=int(conn.parameters.model_NT_P2), n_jobs=int(conn.parameters.model_njobs))
			reg.fit(dataset, rep)
			model_filename = joblib.dump(reg, os.path.join(out_path, 'reg_%d_phase2.joblib'%id_term), compress=3)[0]
			AttachedFile(
				conn.job,
				domainIdent=conn.job.id,
				filename=model_filename,
				domainClassName="be.cytomine.processing.Job"
			).upload()
			model_filename = joblib.dump(feature_offsets_2, os.path.join(out_path, 'offsets_%d_phase2.joblib' % id_term), compress=3)[0]
			AttachedFile(
				conn.job,
				domainIdent=conn.job.id,
				filename=model_filename,
				domainClassName="be.cytomine.processing.Job"
			).upload()

		conn.job.update(progress=90, status=Job.RUNNING, statusComment="Building model for phase 3")
		edges = build_edgematrix_phase_3(Xc, Yc, conn.parameters.model_sde, conn.parameters.model_delta, conn.parameters.model_T)
		model_filename = joblib.dump(edges, os.path.join(out_path, 'model_edges.joblib'), compress=3)[0]
		AttachedFile(
			conn.job,
			domainIdent=conn.job.id,
			filename=model_filename,
			domainClassName="be.cytomine.processing.Job"
		).upload()

		sfinal = ""
		for id_term in term_list:
			sfinal += "%d " % id_term
		sfinal = sfinal.rstrip(' ')
		Property(conn.job, key="id_terms", value=sfinal.rstrip(" ")).save()
		conn.job.update(progress=100, status=Job.TERMINATED, statusComment="Job terminated.")
示例#12
0
def main():
	with NeubiasJob.from_cli(sys.argv) as conn:
		problem_cls = get_discipline(conn, default=CLASS_LNDDET)
		conn.job.update(status=Job.RUNNING, progress=0, statusComment="Initialization of the prediction phase")
		in_images, gt_images, in_path, gt_path, out_path, tmp_path = prepare_data(problem_cls, conn, is_2d=True, **conn.flags)
		train_job = Job().fetch(conn.parameters.model_to_use)
		properties = PropertyCollection(train_job).fetch()
		str_terms = ""
		for prop in properties:
			if prop.fetch(key='id_terms') != None:
				str_terms = prop.fetch(key='id_terms').value
		term_list = [int(x) for x in str_terms.split(' ')]
		attached_files = AttachedFileCollection(train_job).fetch()
		model_file = find_by_attribute(attached_files, "filename", "model_phase1.joblib")
		model_filepath = os.path.join(in_path, "model_phase1.joblib")
		model_file.download(model_filepath, override=True)
		clf = joblib.load(model_filepath)
		pr_ims = [int(p) for p in conn.parameters.cytomine_predict_images.split(',')]
		tifimg = readimage(in_path, pr_ims[0], image_type='tif')
		init_h = 100
		init_w = 100
		if len(tifimg.shape)==3:
			(init_h, init_w, init_d) = tifimg.shape
		else:
			(init_h, init_w) = tifimg.shape
		offset_file = find_by_attribute(attached_files, "filename", "offsets_phase1.joblib")
		offset_filepath = os.path.join(in_path, "offsets_phase1.joblib")
		offset_file.download(offset_filepath, override=True)
		feature_offsets_1 = joblib.load(offset_filepath)
		train_parameters = {}
		for hashmap in train_job.jobParameters:
			train_parameters[hashmap['name']] = hashmap['value']
		train_parameters['model_delta'] = float(train_parameters['model_delta'])
		train_parameters['model_sde'] = float(train_parameters['model_sde'])
		train_parameters['model_T'] = int(train_parameters['model_T'])
		for j in conn.monitor(pr_ims, start=10, end=33, period=0.05,prefix="Phase 1 for images..."):
			probability_map = probability_map_phase_1(in_path, j, clf, feature_offsets_1, float(train_parameters['model_delta']))
			filesave = os.path.join(out_path, 'pmap_%d.npy'%j)
			np.savez_compressed(filesave,probability_map)

		clf = None

		coords_file = find_by_attribute(attached_files, "filename", "coords.joblib")
		coords_filepath = os.path.join(in_path, "coords.joblib")
		coords_file.download(coords_filepath, override=True)
		(Xc, Yc) = joblib.load(coords_filepath)

		for j in conn.monitor(pr_ims, start=33, end=66, period=0.05,prefix="Phase 2 for images..."):
			filesave = os.path.join(out_path, 'pmap_%d.npy.npz' % j)
			probability_map = np.load(filesave)['arr_0']
			for id_term in term_list:
				reg_file = find_by_attribute(attached_files, "filename", "reg_%d_phase2.joblib"%id_term)
				reg_filepath = os.path.join(in_path, "reg_%d_phase2.joblib"%id_term)
				reg_file.download(reg_filepath, override=True)
				reg = joblib.load(reg_filepath)
				off_file = find_by_attribute(attached_files, "filename", 'offsets_%d_phase2.joblib' % id_term)
				off_filepath = os.path.join(in_path, 'offsets_%d_phase2.joblib' % id_term)
				off_file.download(off_filepath, override=True)
				feature_offsets_2 = joblib.load(off_filepath)
				probability_map_phase_2 = agregation_phase_2(in_path, j, id_term-1, probability_map, reg, train_parameters['model_delta'], feature_offsets_2, conn.parameters.model_filter_size, conn.parameters.model_beta, conn.parameters.model_n_iterations)
				filesave = os.path.join(out_path, 'pmap2_%d_%d.npy' % (j, id_term))
				np.savez_compressed(filesave, probability_map_phase_2)

		edge_file = find_by_attribute(attached_files, "filename", "model_edges.joblib")
		edge_filepath = os.path.join(in_path, "model_edges.joblib")
		edge_file.download(edge_filepath, override=True)
		edges = joblib.load(edge_filepath)
		for j in conn.monitor(pr_ims, start=66, end=90, period=0.05,prefix="Phase 3 for images..."):
			filesave = os.path.join(out_path, 'pmap2_%d_%d.npy.npz' % (j, term_list[0]))
			probability_map = np.load(filesave)['arr_0']
			(hpmap,wpmap) = probability_map.shape
			probability_volume = np.zeros((hpmap,wpmap,len(term_list)))
			probability_volume[:,:,0] = probability_map
			for i in range(1,len(term_list)):
				filesave = os.path.join(out_path, 'pmap2_%d_%d.npy.npz' % (j, term_list[i]))
				probability_volume[:,:,i] = np.load(filesave)['arr_0']
			x_final, y_final = compute_final_solution_phase_3(Xc, Yc, probability_volume, conn.parameters.model_n_candidates, train_parameters['model_sde'], train_parameters['model_delta'], train_parameters['model_T'], edges)
			lbl_img = np.zeros((init_h, init_w), 'uint8')
			for i in range(x_final.size):
				x = min(init_w-1, max(0, int(x_final[i])))
				y = min(init_h-1, max(0, int(y_final[i])))
				lbl_img[y, x] = term_list[i]
			imwrite(path=os.path.join(out_path, '%d.tif' % j), image=lbl_img.astype(np.uint8), is_2d=True)
		upload_data(problem_cls, conn, in_images, out_path, **conn.flags, is_2d=True, monitor_params={"start": 90, "end": 95, "period": 0.1})
		conn.job.update(progress=90, statusComment="Computing and uploading metrics (if necessary)...")
		upload_metrics(problem_cls, conn, in_images, gt_path, out_path, tmp_path, **conn.flags)
		conn.job.update(status=Job.TERMINATED, progress=100, statusComment="Finished.")